From 70fc4f0be53efc66d8fe7c52c303dda0f9fc89df Mon Sep 17 00:00:00 2001 From: Dreagonmon <531486058@qq.com> Date: Wed, 5 Nov 2025 17:36:53 +0800 Subject: [PATCH] vnode --- backup_box/app.py | 33 ++---- backup_box/storage/local_storage.py | 51 +++++++-- backup_box/storage/storage.py | 45 ++++++-- backup_box/storage/vnode.py | 158 +++++++++++++++++++--------- pyproject.toml | 5 +- 5 files changed, 198 insertions(+), 94 deletions(-) diff --git a/backup_box/app.py b/backup_box/app.py index 18f2c64..56fba0c 100644 --- a/backup_box/app.py +++ b/backup_box/app.py @@ -32,31 +32,18 @@ def parse_args(): config.apply_user_config(args.config) print("Args:", args) -def main(): +async def a_main(): parse_args() print("Config:", config.get_config()) print("Hello World") - from .storage.vnode import VFileSystem, new_storage_file, new_storage_dir, dump_storage_item_lines + from .storage.local_storage import LocalStorage from pathlib import PurePosixPath - fs = VFileSystem() - d1 = new_storage_dir("dirt1", [ - new_storage_file("tgn1"), - new_storage_file("tgn2"), - new_storage_file("tgn3"), - new_storage_file("tgn4"), - ]) - d2 = new_storage_dir("dirt2", [ - d1, - new_storage_file("tgn1"), - new_storage_file("tgn2"), - new_storage_file("tgn3"), - new_storage_file("tgn4"), - ]) - d3 = new_storage_dir("dirt3", []) - fs._root["c"].append(d2) - fs._root["c"].append(new_storage_file(n="fsafsa")) - fs._root["c"].append(d3) - print(fs) - fs.remove(PurePosixPath("dirt2/dirt1/tgn1")) - print(fs) + fs = LocalStorage(r"D:\AuberyZhao\BackupBox\backup_box") + async for rt in fs.sync_storage(): + print(rt) + print(fs.dump_fs_tree()) + +def main(): + import asyncio + asyncio.run(a_main()) diff --git a/backup_box/storage/local_storage.py b/backup_box/storage/local_storage.py index f17a8b6..a94b8a3 100644 --- a/backup_box/storage/local_storage.py +++ b/backup_box/storage/local_storage.py @@ -1,15 +1,13 @@ -from collections.abc import Awaitable -from .storage import Storage +from backup_box.storage.storage import AsyncFileReader, AsyncFileWriter from pathlib import PurePath, PurePosixPath from ..config import StorageItem -from .vnode import VFileSystem -from typing import BinaryIO, Literal +from .vnode import VFileSystem, VFile, VDir +from typing import Literal, overload -class LocalStorage(Storage): +class LocalStorage(VFileSystem): def __init__(self, base_path: str) -> None: super().__init__() - self.base = PurePath(base_path) - self.vfs = VFileSystem() + self._base = PurePath(base_path) @classmethod def from_config(cls, cfg: StorageItem) -> "LocalStorage": @@ -17,5 +15,44 @@ class LocalStorage(Storage): raise ValueError("config type mismatch!") return LocalStorage(cfg["path"]) + def _vnode_path_to_syspath(self, *path: VFile | VDir): + filtered = [] + # filter abslute path + for n in path: + name = n["n"].strip() + if name.startswith("/") or name.startswith("\\"): + continue + if name in [".", "..", ""]: + continue + filtered.append(name) + return self._base.joinpath(*filtered) + + async def sync_storage(self): + curr = 0 + total = 1 + parents: list[VDir] = [] + self._root["c"].clear() + node = self._root + yield curr, total + while True: + if node["ty"] == "dir": + pass + if len(parents) <= 0: + break + + async def _on_remove_node(self, node_path: list[VFile | VDir]) -> None: + raise NotImplementedError + + async def _on_create_node(self, node_path: list[VFile | VDir]) -> None: + raise NotImplementedError + + @overload + async def _on_open_node(self, node_path: list[VFile | VDir], mode: Literal['r'] = "r") -> AsyncFileReader: ... + @overload + async def _on_open_node(self, node_path: list[VFile | VDir], mode: Literal['w']) -> AsyncFileWriter: ... + async def _on_open_node(self, node_path: list[VFile | VDir], mode: Literal['r'] | Literal['w'] = "r") -> AsyncFileWriter | AsyncFileReader: + raise NotImplementedError + + diff --git a/backup_box/storage/storage.py b/backup_box/storage/storage.py index 15539cf..0ca14f4 100644 --- a/backup_box/storage/storage.py +++ b/backup_box/storage/storage.py @@ -1,8 +1,25 @@ from ..config import StorageItem from abc import ABC, abstractmethod from pathlib import PurePosixPath -from collections.abc import Awaitable, Iterable, AsyncIterable -from typing import BinaryIO, Literal +from collections.abc import Awaitable, AsyncIterable +from typing import overload, Literal, Protocol, SupportsBytes + +class AsyncFileReader(Protocol): + async def read(self, n: int = -1) -> bytes: + ... + + async def close(self) -> None: + ... + +class AsyncFileWriter(Protocol): + async def write(self, s: SupportsBytes) -> int: + ... + + async def close(self) -> None: + ... + +class AsyncFileLike(AsyncFileReader, AsyncFileWriter): + ... class Storage(ABC): @classmethod @@ -15,46 +32,52 @@ class Storage(ABC): ... @abstractmethod - def sync_storage() -> Iterable[tuple[int,int]] | AsyncIterable[tuple[int,int]]: + def sync_storage(self) -> AsyncIterable[tuple[int,int]]: """ sync storage, yield (current, total) progress """ ... @abstractmethod - def listdir(self, path=PurePosixPath("")) -> list[PurePosixPath] | Awaitable[list[PurePosixPath]]: + def listdir(self, path=PurePosixPath("")) -> AsyncIterable[PurePosixPath]: """ list storage file in 'path' """ ... @abstractmethod - def exists(self, path: PurePosixPath) -> bool | Awaitable[bool]: + def exists(self, path: PurePosixPath) -> Awaitable[bool]: """ if the file or dir is exists """ ... @abstractmethod - def is_dir(self, path: PurePosixPath) -> bool | Awaitable[bool]: + def is_dir(self, path: PurePosixPath) -> Awaitable[bool]: """ if the path is dir """ ... @abstractmethod - def is_file(self, path: PurePosixPath) -> bool | Awaitable[bool]: + def is_file(self, path: PurePosixPath) -> Awaitable[bool]: """ if the path is file """ ... + @overload @abstractmethod - def open(self, path: PurePosixPath, mode: Literal["r", "w"] = "r") -> BinaryIO | Awaitable[BinaryIO]: + def open(self, path: PurePosixPath, mode: Literal["r"] = "r") -> Awaitable[AsyncFileReader]: ... + @overload + @abstractmethod + def open(self, path: PurePosixPath, mode: Literal["w"]) -> Awaitable[AsyncFileWriter]: ... + @abstractmethod + def open(self, path: PurePosixPath, mode: Literal["r", "w"] = "w") -> Awaitable[AsyncFileWriter] | Awaitable[AsyncFileReader]: """ open file-like object for file operation """ ... @abstractmethod - def remove(self, path: PurePosixPath) -> None | Awaitable[None]: + def remove(self, path: PurePosixPath) -> Awaitable[None]: """ delete file or dir from storage, dir should be empty """ ... @abstractmethod - def rmtree(self, path: PurePosixPath) -> None | Awaitable[None]: + def rmtree(self, path: PurePosixPath) -> Awaitable[None]: """ remove dir tree, remove all contents """ ... @abstractmethod - def makedirs(self, path: PurePosixPath, exists_ok=False) -> None | Awaitable[None]: + def makedirs(self, path: PurePosixPath, exists_ok=False) -> Awaitable[None]: """ create dir """ ... diff --git a/backup_box/storage/vnode.py b/backup_box/storage/vnode.py index e7fa252..97b49a6 100644 --- a/backup_box/storage/vnode.py +++ b/backup_box/storage/vnode.py @@ -1,46 +1,30 @@ -from typing import TypedDict, Literal, BinaryIO -from collections.abc import Awaitable +from typing import TypedDict, Literal, overload from pathlib import PurePosixPath from functools import cmp_to_key -from io import BytesIO +from .storage import Storage, AsyncFileWriter, AsyncFileReader +from abc import abstractmethod -class StorageFile(TypedDict): +class VFile(TypedDict): ty: Literal["file"] # type n: str # virtual file name mt: int # modify time sz: int # size -class StorageDir(TypedDict): +class VDir(TypedDict): ty: Literal["dir"] # type n: str # virtual file name - c: list["StorageItem"] # dir content + c: list["VFSItem"] # dir content -StorageItem = StorageFile | StorageDir +VFSItem = VFile | VDir -def new_storage_dir(n="", c=[]) -> StorageDir: - return StorageDir(ty="dir", n=n, c=c) +def new_storage_dir(n="", c=[]) -> VDir: + c = list(c) + return VDir(ty="dir", n=n, c=c) -def new_storage_file(n="", mt=0, sz=0) -> StorageFile: - return StorageFile(ty="file", n=n, mt=mt, sz=sz) +def new_storage_file(n="", mt=0, sz=0) -> VFile: + return VFile(ty="file", n=n, mt=mt, sz=sz) -class VFSError(RuntimeError): - def __init__(self, *args: object) -> None: - super().__init__(*args) - -class FileNotFoundError(VFSError): - def __init__(self, p) -> None: - super().__init__(f"FileNotFound: {p}") - -class NotADirectoryError(VFSError): - def __init__(self, p) -> None: - super().__init__(f"NotADirectory: {p}") - -class DirectoryNotEmptyError(VFSError): - def __init__(self, p) -> None: - super().__init__(f"DirectoryNotEmpty: {p}") - - -def find_vnode(root: StorageDir, path: PurePosixPath) -> list[StorageItem]: +def find_vnode(root: VDir, path: PurePosixPath) -> list[VFSItem]: if len(path.parts) > 0: p = path.parts[0] for n in root["c"]: @@ -55,7 +39,7 @@ def find_vnode(root: StorageDir, path: PurePosixPath) -> list[StorageItem]: raise FileNotFoundError(path) return [root] -def _cmp_storage_item1(item1: StorageItem, item2: StorageItem): +def _cmp_storage_item1(item1: VFSItem, item2: VFSItem): """ cmp, sort dir first, then name """ if item1["ty"] != item2["ty"]: if item1["ty"] == "dir": @@ -69,7 +53,7 @@ def _cmp_storage_item1(item1: StorageItem, item2: StorageItem): return -1 return 0 -def _cmp_storage_item2(item1: StorageItem, item2: StorageItem): +def _cmp_storage_item2(item1: VFSItem, item2: VFSItem): """ cmp, sort file first, then name """ if item1["ty"] != item2["ty"]: if item1["ty"] == "dir": @@ -83,10 +67,10 @@ def _cmp_storage_item2(item1: StorageItem, item2: StorageItem): return -1 return 0 -def sort_storage_items(items: list[StorageItem], dir_first = True): +def sort_storage_items(items: list[VFSItem], dir_first = True): items.sort(key=cmp_to_key(_cmp_storage_item1 if dir_first else _cmp_storage_item2)) -def dump_storage_item_lines(item: StorageItem, indent=0): +def dump_storage_item_lines(item: VFSItem, indent=0): indent_text = " " * indent if item["ty"] == "file": return [f"{indent_text}📄{item["n"]}: size {item["sz"]}"] @@ -98,46 +82,85 @@ def dump_storage_item_lines(item: StorageItem, indent=0): return lines return [] -class VFileSystem(): +class VFileSystem(Storage): def __init__(self) -> None: self._root = new_storage_dir() - def __repr__(self): + def dump_fs_tree(self): return "\n".join(dump_storage_item_lines(self._root)) - def _on_remove_node(self, node_path: list[StorageItem]) -> None | Awaitable[None]: - parents_path = PurePosixPath( *(n["n"] for n in node_path) ) - print("remove:", parents_path) - pass + @abstractmethod + async def _on_remove_node(self, node_path: list[VFSItem]) -> None: + """ remove a file or dir, dir is guaranteed to be empty. """ + path = PurePosixPath( *(n["n"] for n in node_path) ) + print("remove:", path) + ... - def _op_open_node(self, node_path: list[StorageItem]) -> BinaryIO | Awaitable[BinaryIO]: - return BytesIO() + @abstractmethod + async def _on_create_node(self, node_path: list[VFSItem]) -> None: + """ create a file or dir, parent dir is always created first. """ + path = PurePosixPath( *(n["n"] for n in node_path) ) + print("create:", path) + ... - def listdir(self, path=PurePosixPath("")) -> list[PurePosixPath]: + @overload + @abstractmethod + async def _on_open_node(self, node_path: list[VFSItem], mode: Literal["r"] = "r") -> AsyncFileReader: ... + @overload + @abstractmethod + async def _on_open_node(self, node_path: list[VFSItem], mode: Literal["w"]) -> AsyncFileWriter: ... + @abstractmethod + async def _on_open_node(self, node_path: list[VFSItem], mode: Literal["r", "w"] = "r") -> AsyncFileWriter | AsyncFileReader: + print("open", mode, ":", node_path) + ... + + async def listdir(self, path=PurePosixPath("")): nodes = find_vnode(self._root, path) base_path = PurePosixPath( *(n["n"] for n in nodes) ) item = nodes[-1] if item["ty"] != "dir": raise NotADirectoryError(path) sort_storage_items(item["c"], True) - return [ base_path.joinpath(n["n"]) for n in item["c"]] + for n in item["c"]: + yield base_path.joinpath(n["n"]) - def exists(self, path: PurePosixPath) -> bool: + async def exists(self, path: PurePosixPath) -> bool: try: _ = find_vnode(self._root, path) return True except FileNotFoundError: return False - def is_dir(self, path: PurePosixPath) -> bool: + async def is_dir(self, path: PurePosixPath) -> bool: node = find_vnode(self._root, path)[-1] return node["ty"] == "dir" - def is_file(self, path: PurePosixPath) -> bool: + async def is_file(self, path: PurePosixPath) -> bool: node = find_vnode(self._root, path)[-1] return node["ty"] == "file" + + @overload + async def open(self, path: PurePosixPath, mode: Literal["r"] = "r") -> AsyncFileReader: ... + @overload + async def open(self, path: PurePosixPath, mode: Literal["w"]) -> AsyncFileWriter: ... + async def open(self, path: PurePosixPath, mode: Literal["r", "w"] = "r") -> AsyncFileReader | AsyncFileWriter: + try: + nodes = find_vnode(self._root, path) + except FileNotFoundError: + if mode == "r": + raise + # try to create file + nodes = find_vnode(self._root, path.parent) + filenode = new_storage_file(path.name) + if nodes[-1]["ty"] == "dir": + nodes[-1]["c"].append(filenode) + nodes.append(filenode) + await self._on_create_node(nodes) + else: + raise NotADirectoryError(path.parent) + return await self._on_open_node(nodes, mode) - def remove(self, path: PurePosixPath) -> None: + async def remove(self, path: PurePosixPath) -> None: nodes = find_vnode(self._root, path) if len(nodes) == 1: # in the root @@ -149,13 +172,13 @@ class VFileSystem(): last = nodes[-1] if last["ty"] == "dir": if len(last["c"]) > 0: - raise DirectoryNotEmptyError(path) + raise OSError(f"{path} is not empty.") # remove callback - self._on_remove_node(nodes) + await self._on_remove_node(nodes) if parent["ty"] == "dir": parent["c"].remove(last) - def rmtree(self, path: PurePosixPath) -> None: + async def rmtree(self, path: PurePosixPath) -> None: nodes = find_vnode(self._root, path) if len(nodes) == 1: # in the root @@ -180,7 +203,7 @@ class VFileSystem(): # remove callback node_path = list(prefix) node_path.extend(vstack) - self._on_remove_node(node_path) + await self._on_remove_node(node_path) # remove self from parent if parent["ty"] == "dir": parent["c"].remove(last) @@ -197,3 +220,36 @@ class VFileSystem(): if len(vstack) <= 0: break + async def makedirs(self, path: PurePosixPath, exists_ok=False) -> None: + build: list[VFSItem] = [] + # ensure parents dir + parent = self._root + for p in path.parent.parts: + for n in parent["c"]: + if n["n"] == p: + if n["ty"] != "dir": + raise NotADirectoryError(n["n"]) + parent = n + build.append(n) + break + else: + # not found, create dir + node = new_storage_dir(p) + parent["c"].append(node) + parent = node + build.append(node) + await self._on_create_node(build) + # create dir + for n in parent["c"]: + if n["n"] == path.name: + if not exists_ok: + raise FileExistsError(path) + if n["ty"] != "dir": + raise NotADirectoryError(path) + break + else: + node = new_storage_dir(path.name) + parent["c"].append(node) + build.append(node) + await self._on_create_node(build) + diff --git a/pyproject.toml b/pyproject.toml index ac987ff..955f6d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,9 @@ version = "0.0.1" license = "MIT" authors = [{ name="Dreagonmon" }] dependencies = [ - "textual>=6.5.0", - "tomli-w >= 1.2.0", + "textual~=6.5.0", + "tomli-w~=1.2.0", + "aiofiles~=25.1.0" ] requires-python = ">=3.11"