Add WebDAV backup, statistics, progress and dashboard
- WebDAV uploader: concurrency, request and bandwidth limits, retry with backoff, offline catch-up, encrypted manifests, durability tracking - Automatic unique remote directory claimed with a conditional PUT - AES-256-GCM client-side encryption with keyed blob names - /stats, /progress, /progress/stream and a /dashboard page - Own ctypes inotify binding with a compact watch tree (263 MB -> 76 MB RSS) - Directory-path ignore patterns and a forget command - Indexed range queries instead of LIKE for path prefixes Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
retoor
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
ddf09bea88
commit
7e05e3d26c
+307
-174
@@ -7,28 +7,21 @@ import errno
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from asyncinotify import Inotify, Mask, Watch
|
||||
|
||||
from . import inotify as ino
|
||||
from .db import Database
|
||||
from .filters import Filters
|
||||
from .ingest import Coalescer, Rejected, Repository, is_within
|
||||
from .ingest import Coalescer, Rejected, Repository, is_within, tree_range
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DIR_MASK = (
|
||||
Mask.CLOSE_WRITE
|
||||
| Mask.MOVED_TO
|
||||
| Mask.MOVED_FROM
|
||||
| Mask.CREATE
|
||||
| Mask.DELETE
|
||||
| Mask.DELETE_SELF
|
||||
| Mask.MOVE_SELF
|
||||
| Mask.ONLYDIR
|
||||
| Mask.DONT_FOLLOW
|
||||
| Mask.EXCL_UNLINK
|
||||
ino.CLOSE_WRITE | ino.MOVED_TO | ino.MOVED_FROM | ino.CREATE | ino.DELETE
|
||||
| ino.DELETE_SELF | ino.MOVE_SELF | ino.ONLYDIR | ino.DONT_FOLLOW | ino.EXCL_UNLINK
|
||||
)
|
||||
|
||||
NETWORK_FS = frozenset({
|
||||
@@ -93,6 +86,83 @@ def needs_polling(path: str) -> bool:
|
||||
return fstype in NETWORK_FS or fstype.startswith("fuse")
|
||||
|
||||
|
||||
class _Dir:
|
||||
"""A watched directory. Only the name is stored; paths are rebuilt from the parent chain."""
|
||||
|
||||
__slots__ = ("wd", "parent", "name", "children")
|
||||
|
||||
def __init__(self, wd: int, parent: "_Dir | None", name: str):
|
||||
self.wd = wd
|
||||
self.parent = parent
|
||||
self.name = name
|
||||
self.children: dict[str, _Dir] | None = None
|
||||
|
||||
def path(self) -> str:
|
||||
parts = []
|
||||
node: _Dir | None = self
|
||||
while node is not None:
|
||||
parts.append(node.name)
|
||||
node = node.parent
|
||||
return "/".join(reversed(parts))
|
||||
|
||||
def top(self) -> "_Dir":
|
||||
node = self
|
||||
while node.parent is not None:
|
||||
node = node.parent
|
||||
return node
|
||||
|
||||
def attach(self, parent: "_Dir", name: str) -> None:
|
||||
self.parent, self.name = parent, name
|
||||
if parent.children is None:
|
||||
parent.children = {}
|
||||
parent.children[name] = self
|
||||
|
||||
def detach(self) -> None:
|
||||
if self.parent is not None and self.parent.children:
|
||||
self.parent.children.pop(self.name, None)
|
||||
if not self.parent.children:
|
||||
self.parent.children = None
|
||||
self.parent = None
|
||||
|
||||
def walk(self):
|
||||
stack = [self]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
yield node
|
||||
if node.children:
|
||||
stack.extend(node.children.values())
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanProgress:
|
||||
root_id: int
|
||||
reason: str
|
||||
top: str
|
||||
started_at: float = field(default_factory=time.time)
|
||||
dirs_total: int = 0
|
||||
dirs_done: int = 0
|
||||
files_seen: int = 0
|
||||
files_read: int = 0
|
||||
versions_stored: int = 0
|
||||
bytes_stored: int = 0
|
||||
finished_at: float | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
data = asdict(self)
|
||||
elapsed = (self.finished_at or time.time()) - self.started_at
|
||||
data["elapsed_seconds"] = round(elapsed, 1)
|
||||
data["percent"] = (
|
||||
round(100 * self.dirs_done / self.dirs_total, 1) if self.dirs_total else None
|
||||
)
|
||||
rate = self.dirs_done / elapsed if elapsed > 0 else 0
|
||||
remaining = max(0, self.dirs_total - self.dirs_done)
|
||||
data["eta_seconds"] = (
|
||||
round(remaining / rate) if rate > 0 and self.finished_at is None and self.dirs_total else None
|
||||
)
|
||||
data["files_per_second"] = round(self.files_seen / elapsed, 1) if elapsed > 0 else None
|
||||
return data
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -111,21 +181,30 @@ class Monitor:
|
||||
self.poll_interval = poll_interval
|
||||
self.reconcile_interval = reconcile_interval
|
||||
self.max_watch_fraction = max_watch_fraction
|
||||
self.inotify: Inotify | None = None
|
||||
self.watches: dict[str, Watch] = {}
|
||||
self.watch_paths: dict[Watch, str] = {}
|
||||
self.inotify: ino.Inotify | None = None
|
||||
self.by_wd: dict[int, _Dir] = {}
|
||||
self.tops: dict[str, _Dir] = {}
|
||||
self.top_ids: dict[str, int] = {}
|
||||
self.watch_counts: Counter[int] = Counter()
|
||||
self.polled: dict[int, set[str]] = {}
|
||||
self.missing: set[int] = set()
|
||||
self.moves: dict[int, tuple[str, bool, asyncio.TimerHandle]] = {}
|
||||
self.moves: dict[int, tuple[_Dir, str, bool, asyncio.TimerHandle]] = {}
|
||||
self.locks: dict[int, asyncio.Lock] = {}
|
||||
self.tasks: set[asyncio.Task] = set()
|
||||
self.scans: dict[int, ScanProgress] = {}
|
||||
self.last_scans: dict[int, ScanProgress] = {}
|
||||
self.counters: Counter[str] = Counter()
|
||||
self._last_full_reconcile = time.monotonic()
|
||||
|
||||
@property
|
||||
def watch_total(self) -> int:
|
||||
return len(self.by_wd)
|
||||
|
||||
# lifecycle
|
||||
|
||||
async def start(self) -> None:
|
||||
self.inotify = Inotify()
|
||||
self._spawn(self._reader())
|
||||
self.inotify = ino.Inotify()
|
||||
asyncio.get_running_loop().add_reader(self.inotify.fileno(), self._on_readable)
|
||||
for root in self.db.roots():
|
||||
await self._activate(root)
|
||||
self._spawn(self._periodic())
|
||||
@@ -134,14 +213,15 @@ class Monitor:
|
||||
for task in list(self.tasks):
|
||||
task.cancel()
|
||||
await asyncio.gather(*self.tasks, return_exceptions=True)
|
||||
for _, _, handle in self.moves.values():
|
||||
for *_, handle in self.moves.values():
|
||||
handle.cancel()
|
||||
self.moves.clear()
|
||||
if self.inotify is not None:
|
||||
asyncio.get_running_loop().remove_reader(self.inotify.fileno())
|
||||
self.inotify.close()
|
||||
self.inotify = None
|
||||
self.watches.clear()
|
||||
self.watch_paths.clear()
|
||||
self.by_wd.clear()
|
||||
self.tops.clear()
|
||||
|
||||
def _spawn(self, coro) -> asyncio.Task:
|
||||
task = asyncio.get_running_loop().create_task(coro)
|
||||
@@ -159,7 +239,10 @@ class Monitor:
|
||||
def _lock(self, root_id: int) -> asyncio.Lock:
|
||||
return self.locks.setdefault(root_id, asyncio.Lock())
|
||||
|
||||
async def walk_dirs(self, top: str) -> AsyncIterator[str]:
|
||||
def _skip(self, path: str, name: str, root_path: str) -> bool:
|
||||
return self.filters.skip_dir(name, path[len(root_path) + 1:])
|
||||
|
||||
async def walk_dirs(self, top: str, root_path: str) -> AsyncIterator[str]:
|
||||
"""Yield top and every non-ignored directory below it (symlinks not followed)."""
|
||||
stack = [top]
|
||||
seen = 0
|
||||
@@ -169,7 +252,7 @@ class Monitor:
|
||||
try:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
if entry.is_dir(follow_symlinks=False) and not self.filters.dir_ignored(entry.name):
|
||||
if entry.is_dir(follow_symlinks=False) and not self._skip(entry.path, entry.name, root_path):
|
||||
stack.append(entry.path)
|
||||
except OSError:
|
||||
continue
|
||||
@@ -178,7 +261,7 @@ class Monitor:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def count_dirs(self, top: str) -> int:
|
||||
return sum([1 async for _ in self.walk_dirs(top)])
|
||||
return sum([1 async for _ in self.walk_dirs(top, top)])
|
||||
|
||||
async def add_root(self, raw_path: str, force: bool = False) -> dict[str, Any]:
|
||||
path = os.path.realpath(os.path.expanduser(raw_path))
|
||||
@@ -189,8 +272,7 @@ class Monitor:
|
||||
raise RootError("root-overlap", f"{path} is already inside root {root['path']}")
|
||||
if is_within(root["path"], path):
|
||||
raise RootError("root-overlap", f"{path} contains existing root {root['path']}; remove it first")
|
||||
polling = needs_polling(path)
|
||||
if not polling and not force:
|
||||
if not needs_polling(path) and not force:
|
||||
needed = await self.count_dirs(path)
|
||||
limit = max_user_watches()
|
||||
available = limit - watches_in_use()
|
||||
@@ -200,28 +282,40 @@ class Monitor:
|
||||
f"{path} needs {needed} inotify watches but only {available} of {limit} are free; "
|
||||
"choose a smaller directory, raise fs.inotify.max_user_watches, or pass force=true",
|
||||
)
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO roots(path, added_at) VALUES (?, ?)", (path, time.time())
|
||||
cur = self.db.execute("INSERT INTO roots(path, added_at) VALUES (?, ?)", (path, time.time()))
|
||||
root_id = cur.lastrowid
|
||||
# Files and projects kept from an earlier root at this location are re-adopted.
|
||||
lo, hi = tree_range(path)
|
||||
self.db.execute("UPDATE files SET root_id = ? WHERE root_id IS NULL AND path >= ? AND path < ?",
|
||||
(root_id, lo, hi))
|
||||
self.db.execute(
|
||||
"UPDATE projects SET root_id = ? WHERE root_id IS NULL AND (path = ? OR (path >= ? AND path < ?))",
|
||||
(root_id, path, lo, hi),
|
||||
)
|
||||
self.db.audit("root.add", path=path)
|
||||
self.repo.reload_roots()
|
||||
root = self.db.root(cur.lastrowid)
|
||||
root = self.db.root(root_id)
|
||||
await self._activate(root)
|
||||
return self.db.root(root["id"])
|
||||
return self.db.root(root_id)
|
||||
|
||||
async def remove_root(self, root_id: int) -> None:
|
||||
root = self.db.root(root_id)
|
||||
if root is None:
|
||||
raise RootError("not-found", f"root {root_id} does not exist")
|
||||
self._unwatch_tree(root["path"])
|
||||
top = self.tops.get(root["path"])
|
||||
if top is not None:
|
||||
self._unwatch(top)
|
||||
self.top_ids.pop(root["path"], None)
|
||||
self.polled.pop(root_id, None)
|
||||
self.missing.discard(root_id)
|
||||
self.scans.pop(root_id, None)
|
||||
self.db.execute("DELETE FROM roots WHERE id = ?", (root_id,))
|
||||
self.db.audit("root.remove", path=root["path"])
|
||||
self.repo.reload_roots()
|
||||
|
||||
async def _activate(self, root: dict[str, Any]) -> None:
|
||||
root_id, path = root["id"], root["path"]
|
||||
self.top_ids[path] = root_id
|
||||
if not os.path.isdir(path):
|
||||
self.missing.add(root_id)
|
||||
self.db.update_root(root_id, mode="missing")
|
||||
@@ -232,188 +326,209 @@ class Monitor:
|
||||
if needs_polling(path):
|
||||
self.polled[root_id].add(path)
|
||||
else:
|
||||
await self._watch_tree(root_id, path)
|
||||
await self._watch_tree(root_id, path, None, path, path)
|
||||
self._refresh_root_status(root_id)
|
||||
reason = "reconcile" if root["baseline_state"] == "done" else "baseline"
|
||||
self._spawn(self.reconcile(root_id, reason))
|
||||
|
||||
def root_mode(self, root_id: int, path: str) -> str:
|
||||
if root_id in self.missing:
|
||||
return "missing"
|
||||
polled = self.polled.get(root_id, ())
|
||||
if path in polled:
|
||||
return "polling"
|
||||
return "degraded" if polled else "inotify"
|
||||
|
||||
def _refresh_root_status(self, root_id: int) -> None:
|
||||
root = self.db.root(root_id)
|
||||
if root is None:
|
||||
return
|
||||
if root_id in self.missing:
|
||||
mode = "missing"
|
||||
elif root["path"] in self.polled.get(root_id, ()):
|
||||
mode = "polling"
|
||||
elif self.polled.get(root_id):
|
||||
mode = "degraded"
|
||||
else:
|
||||
mode = "inotify"
|
||||
self.db.update_root(
|
||||
root_id,
|
||||
mode=mode,
|
||||
watch_count=sum(1 for p in self.watches if is_within(p, root["path"])),
|
||||
mode=self.root_mode(root_id, root["path"]),
|
||||
watch_count=self.watch_counts[root_id],
|
||||
polled_dirs=len(self.polled.get(root_id, ())),
|
||||
)
|
||||
|
||||
# watches
|
||||
|
||||
def _add_watch(self, path: str) -> bool:
|
||||
if path in self.watches or self.inotify is None:
|
||||
return True
|
||||
try:
|
||||
watch = self.inotify.add_watch(path, DIR_MASK)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENOSPC:
|
||||
raise
|
||||
log.debug("cannot watch %s: %s", path, exc)
|
||||
return False
|
||||
self.watches[path] = watch
|
||||
self.watch_paths[watch] = path
|
||||
return True
|
||||
|
||||
async def _watch_tree(self, root_id: int, top: str) -> None:
|
||||
skip: list[str] = []
|
||||
async for directory in self.walk_dirs(top):
|
||||
if any(is_within(directory, s) for s in skip):
|
||||
continue
|
||||
async def _watch_tree(self, root_id: int, top_path: str, parent: _Dir | None, name: str,
|
||||
root_path: str) -> None:
|
||||
"""Add watches for a directory and every non-ignored directory below it."""
|
||||
assert self.inotify is not None
|
||||
stack: list[tuple[str, _Dir | None, str]] = [(top_path, parent, name)]
|
||||
added = 0
|
||||
while stack:
|
||||
path, parent_node, node_name = stack.pop()
|
||||
try:
|
||||
self._add_watch(directory)
|
||||
wd = self.inotify.add_watch(path, DIR_MASK)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENOSPC:
|
||||
log.warning("inotify watch limit reached; polling %s instead", path)
|
||||
self.polled.setdefault(root_id, set()).add(path)
|
||||
self.counters["watch_limit_hits"] += 1
|
||||
continue
|
||||
if wd in self.by_wd:
|
||||
continue # same directory reached twice (bind mount); keep the first
|
||||
node = _Dir(wd, None, node_name)
|
||||
if parent_node is None:
|
||||
self.tops[path] = node
|
||||
else:
|
||||
node.attach(parent_node, node_name)
|
||||
self.by_wd[wd] = node
|
||||
self.watch_counts[root_id] += 1
|
||||
try:
|
||||
with os.scandir(path) as it:
|
||||
for entry in it:
|
||||
if entry.is_dir(follow_symlinks=False) and not self._skip(entry.path, entry.name, root_path):
|
||||
stack.append((entry.path, node, entry.name))
|
||||
except OSError:
|
||||
log.warning("inotify watch limit reached; polling %s instead", directory)
|
||||
self.polled.setdefault(root_id, set()).add(directory)
|
||||
skip.append(directory)
|
||||
pass
|
||||
added += 1
|
||||
if added % YIELD_EVERY == 0:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
def _unwatch_tree(self, top: str) -> None:
|
||||
for path in [p for p in self.watches if is_within(p, top)]:
|
||||
watch = self.watches.pop(path)
|
||||
self.watch_paths.pop(watch, None)
|
||||
if self.inotify is not None:
|
||||
try:
|
||||
self.inotify.rm_watch(watch)
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
def _unwatch(self, node: _Dir) -> None:
|
||||
top = node.top()
|
||||
root_id = self.top_ids.get(top.name)
|
||||
removed = 0
|
||||
for child in list(node.walk()):
|
||||
if self.by_wd.pop(child.wd, None) is not None:
|
||||
removed += 1
|
||||
if self.inotify is not None:
|
||||
self.inotify.rm_watch(child.wd)
|
||||
if node.parent is None:
|
||||
self.tops.pop(node.name, None)
|
||||
node.detach()
|
||||
if root_id is not None:
|
||||
self.watch_counts[root_id] -= removed
|
||||
|
||||
def _move_watches(self, old: str, new: str) -> None:
|
||||
for path in [p for p in self.watches if is_within(p, old)]:
|
||||
watch = self.watches.pop(path)
|
||||
moved = new + path[len(old):]
|
||||
self.watches[moved] = watch
|
||||
self.watch_paths[watch] = moved
|
||||
def _root_id_of(self, node: _Dir) -> int | None:
|
||||
return self.top_ids.get(node.top().name)
|
||||
|
||||
# events
|
||||
|
||||
async def _reader(self) -> None:
|
||||
assert self.inotify is not None
|
||||
async for event in self.inotify:
|
||||
def _on_readable(self) -> None:
|
||||
if self.inotify is None:
|
||||
return
|
||||
for wd, mask, cookie, name in self.inotify.read():
|
||||
self.counters["events"] += 1
|
||||
try:
|
||||
self._handle(event)
|
||||
self._handle(wd, mask, cookie, name)
|
||||
except Exception:
|
||||
log.exception("failed handling inotify event")
|
||||
|
||||
def _root_id_for(self, path: str) -> int | None:
|
||||
root = self.repo.root_for(path)
|
||||
return root["id"] if root else None
|
||||
|
||||
def _handle(self, event) -> None:
|
||||
mask = event.mask
|
||||
if Mask.Q_OVERFLOW in mask:
|
||||
def _handle(self, wd: int, mask: int, cookie: int, name: str | None) -> None:
|
||||
if mask & ino.Q_OVERFLOW:
|
||||
self.counters["queue_overflows"] += 1
|
||||
log.warning("inotify queue overflow; reconciling all roots")
|
||||
for root in self.db.roots():
|
||||
self._spawn(self.reconcile(root["id"], "overflow"))
|
||||
return
|
||||
watch = event.watch
|
||||
if Mask.IGNORED in mask:
|
||||
path = self.watch_paths.pop(watch, None) if watch is not None else None
|
||||
if path is not None and self.watches.get(path) is watch:
|
||||
del self.watches[path]
|
||||
node = self.by_wd.get(wd)
|
||||
if node is None:
|
||||
return
|
||||
directory = self.watch_paths.get(watch) if watch is not None else None
|
||||
if directory is None:
|
||||
if mask & ino.IGNORED:
|
||||
self.by_wd.pop(wd, None)
|
||||
root_id = self._root_id_of(node)
|
||||
if root_id is not None:
|
||||
self.watch_counts[root_id] -= 1
|
||||
node.detach()
|
||||
return
|
||||
if name is None:
|
||||
if mask & (ino.DELETE_SELF | ino.MOVE_SELF) and node.parent is None:
|
||||
root_id = self.top_ids.get(node.name)
|
||||
log.warning("root %s was removed or moved", node.name)
|
||||
self._unwatch(node)
|
||||
if root_id is not None:
|
||||
self.missing.add(root_id)
|
||||
self._refresh_root_status(root_id)
|
||||
return
|
||||
|
||||
if event.name is None:
|
||||
if mask & (Mask.DELETE_SELF | Mask.MOVE_SELF):
|
||||
root = self.repo.root_for(directory)
|
||||
if root and root["path"] == directory:
|
||||
log.warning("root %s was removed or moved", directory)
|
||||
self._unwatch_tree(directory)
|
||||
self.missing.add(root["id"])
|
||||
self._refresh_root_status(root["id"])
|
||||
is_dir = bool(mask & ino.ISDIR)
|
||||
if mask & ino.MOVED_FROM:
|
||||
handle = asyncio.get_running_loop().call_later(MOVE_PAIR_TIMEOUT, self._moved_out, cookie)
|
||||
self.moves[cookie] = (node, name, is_dir, handle)
|
||||
return
|
||||
|
||||
path = os.path.join(directory, str(event.name))
|
||||
is_dir = Mask.ISDIR in mask
|
||||
|
||||
if Mask.MOVED_FROM in mask:
|
||||
handle = asyncio.get_running_loop().call_later(
|
||||
MOVE_PAIR_TIMEOUT, self._moved_out, event.cookie
|
||||
)
|
||||
self.moves[event.cookie] = (path, is_dir, handle)
|
||||
return
|
||||
|
||||
if Mask.MOVED_TO in mask:
|
||||
origin = self.moves.pop(event.cookie, None)
|
||||
if mask & ino.MOVED_TO:
|
||||
origin = self.moves.pop(cookie, None)
|
||||
if origin is not None:
|
||||
origin[2].cancel()
|
||||
self._moved_in(origin[0] if origin else None, path, is_dir)
|
||||
origin[3].cancel()
|
||||
self._moved_in(origin, node, name, is_dir)
|
||||
return
|
||||
|
||||
path = node.path() + "/" + name
|
||||
if is_dir:
|
||||
if Mask.CREATE in mask:
|
||||
self._new_directory(path)
|
||||
elif Mask.DELETE in mask:
|
||||
self._unwatch_tree(path)
|
||||
if mask & ino.CREATE:
|
||||
self._new_directory(node, name)
|
||||
elif mask & ino.DELETE:
|
||||
child = node.children.get(name) if node.children else None
|
||||
if child is not None:
|
||||
self._unwatch(child)
|
||||
self.repo.mark_deleted_tree(path)
|
||||
return
|
||||
|
||||
if Mask.CLOSE_WRITE in mask:
|
||||
if mask & ino.CLOSE_WRITE:
|
||||
self.capture(path, "monitor", "modified")
|
||||
elif Mask.DELETE in mask:
|
||||
elif mask & ino.DELETE:
|
||||
self.counters["deletes"] += 1
|
||||
self.repo.mark_deleted(path)
|
||||
|
||||
def _new_directory(self, path: str) -> None:
|
||||
root_id = self._root_id_for(path)
|
||||
if root_id is None or any(self.filters.dir_ignored(p) for p in self.repo.relative(path).parts):
|
||||
def _new_directory(self, parent: _Dir, name: str) -> None:
|
||||
root_path = parent.top().name
|
||||
if self._skip(parent.path() + "/" + name, name, root_path):
|
||||
return
|
||||
self._spawn(self._watch_and_scan(root_id, path))
|
||||
root_id = self._root_id_of(parent)
|
||||
if root_id is None:
|
||||
return
|
||||
self._spawn(self._watch_and_scan(root_id, parent, name))
|
||||
|
||||
async def _watch_and_scan(self, root_id: int, path: str) -> None:
|
||||
async def _watch_and_scan(self, root_id: int, parent: _Dir, name: str) -> None:
|
||||
# Watch first, then scan, so files created before the watch existed are not missed.
|
||||
await self._watch_tree(root_id, path)
|
||||
self._refresh_root_status(root_id)
|
||||
path = parent.path() + "/" + name
|
||||
await self._watch_tree(root_id, path, parent, name, parent.top().name)
|
||||
await self.reconcile(root_id, "new-directory", subtree=path)
|
||||
|
||||
def _moved_out(self, cookie: int) -> None:
|
||||
origin = self.moves.pop(cookie, None)
|
||||
if origin is None:
|
||||
return
|
||||
path, is_dir, _ = origin
|
||||
parent, name, is_dir, _ = origin
|
||||
path = parent.path() + "/" + name
|
||||
if is_dir:
|
||||
self._unwatch_tree(path)
|
||||
child = parent.children.get(name) if parent.children else None
|
||||
if child is not None:
|
||||
self._unwatch(child)
|
||||
self.repo.mark_deleted_tree(path)
|
||||
else:
|
||||
self.repo.mark_deleted(path)
|
||||
|
||||
def _moved_in(self, old: str | None, new: str, is_dir: bool) -> None:
|
||||
new_ok = self._root_id_for(new) is not None and not any(
|
||||
self.filters.dir_ignored(p) for p in self.repo.relative(new).parts[: None if is_dir else -1]
|
||||
)
|
||||
def _moved_in(self, origin, parent: _Dir, name: str, is_dir: bool) -> None:
|
||||
new_path = parent.path() + "/" + name
|
||||
old_path = origin[0].path() + "/" + origin[1] if origin else None
|
||||
if is_dir:
|
||||
if old is not None and new_ok and old in self.watches:
|
||||
self._move_watches(old, new)
|
||||
self.repo.rename_tree(old, new)
|
||||
new_ok = not self._skip(new_path, name, parent.top().name)
|
||||
child = None
|
||||
if origin is not None and origin[0].children:
|
||||
child = origin[0].children.get(origin[1])
|
||||
if child is not None and new_ok and self._root_id_of(origin[0]) == self._root_id_of(parent):
|
||||
child.detach()
|
||||
child.attach(parent, name)
|
||||
self.repo.rename_tree(old_path, new_path)
|
||||
self.counters["renames"] += 1
|
||||
return
|
||||
if old is not None:
|
||||
self._unwatch_tree(old)
|
||||
self.repo.mark_deleted_tree(old)
|
||||
if child is not None:
|
||||
self._unwatch(child)
|
||||
if old_path is not None:
|
||||
self.repo.mark_deleted_tree(old_path)
|
||||
if new_ok:
|
||||
self._new_directory(new)
|
||||
self._new_directory(parent, name)
|
||||
return
|
||||
if old is not None and not self.repo.rename(old, new):
|
||||
self.repo.mark_deleted(old)
|
||||
if new_ok:
|
||||
self.capture(new, "monitor", "renamed" if old else "moved-in")
|
||||
if old_path is not None:
|
||||
if self.repo.rename(old_path, new_path):
|
||||
self.counters["renames"] += 1
|
||||
else:
|
||||
self.repo.mark_deleted(old_path)
|
||||
self.capture(new_path, "monitor", "renamed" if old_path else "moved-in")
|
||||
|
||||
def capture(self, path: str, source: str, reason: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
@@ -421,14 +536,13 @@ class Monitor:
|
||||
content, st = self.repo.read_file(path)
|
||||
self.repo.check_content(content)
|
||||
except Rejected as exc:
|
||||
log.debug("skip %s: %s", path, exc.reason)
|
||||
self.counters[f"skipped:{exc.reason}"] += 1
|
||||
return None
|
||||
except FileNotFoundError:
|
||||
except OSError:
|
||||
return None
|
||||
except OSError as exc:
|
||||
log.debug("cannot read %s: %s", path, exc)
|
||||
return None
|
||||
return self.coalescer.submit(path, content, source, reason, st)
|
||||
result = self.coalescer.submit(path, content, source, reason, st)
|
||||
self.counters[f"capture:{result['status']}"] += 1
|
||||
return result
|
||||
|
||||
# scanning
|
||||
|
||||
@@ -438,12 +552,15 @@ class Monitor:
|
||||
if root is None or root_id in self.missing:
|
||||
return 0
|
||||
top = subtree or root["path"]
|
||||
committed = 0
|
||||
seen: set[str] = set()
|
||||
async with self._lock(root_id):
|
||||
progress = ScanProgress(root_id, reason, top)
|
||||
if subtree is None:
|
||||
progress.dirs_total = self.watch_counts[root_id] + len(self.polled.get(root_id, ()))
|
||||
self.scans[root_id] = progress
|
||||
if reason == "baseline":
|
||||
self.db.update_root(root_id, baseline_state="running")
|
||||
async for directory in self.walk_dirs(top):
|
||||
async for directory in self.walk_dirs(top, root["path"]):
|
||||
progress.dirs_done += 1
|
||||
try:
|
||||
entries = list(os.scandir(directory))
|
||||
except OSError:
|
||||
@@ -460,7 +577,7 @@ class Monitor:
|
||||
continue
|
||||
if self.filters.size_reason(st.st_size):
|
||||
continue
|
||||
seen.add(path)
|
||||
progress.files_seen += 1
|
||||
if path in self.coalescer.pending:
|
||||
continue
|
||||
row = self.db.file_by_path(path)
|
||||
@@ -477,24 +594,37 @@ class Monitor:
|
||||
self.repo.check_content(content)
|
||||
except (Rejected, OSError):
|
||||
continue
|
||||
progress.files_read += 1
|
||||
result = self.repo.commit(path, content, "scan", reason, fst)
|
||||
if result["status"] == "committed":
|
||||
committed += 1
|
||||
await asyncio.sleep(0)
|
||||
progress.versions_stored += 1
|
||||
progress.bytes_stored += len(content)
|
||||
if progress.dirs_done % 20 == 0:
|
||||
await asyncio.sleep(0)
|
||||
if progress.dirs_total < progress.dirs_done:
|
||||
progress.dirs_total = progress.dirs_done
|
||||
# Anything indexed as existing but no longer a file on disk is marked deleted.
|
||||
rows = self.db.all(
|
||||
"SELECT path FROM files WHERE root_id = ? AND exists_on_disk = 1", (root_id,)
|
||||
"SELECT path FROM files WHERE root_id = ? AND exists_on_disk = 1 "
|
||||
"AND (path = ? OR (path >= ? AND path < ?))",
|
||||
(root_id, top, *tree_range(top)),
|
||||
)
|
||||
for row in rows:
|
||||
if is_within(row["path"], top) and row["path"] not in seen:
|
||||
if not os.path.isfile(row["path"]):
|
||||
self.repo.mark_deleted(row["path"])
|
||||
for i, row in enumerate(rows):
|
||||
if not os.path.isfile(row["path"]):
|
||||
self.repo.mark_deleted(row["path"])
|
||||
if i % 1000 == 0:
|
||||
await asyncio.sleep(0)
|
||||
fields: dict[str, Any] = {"last_scan_at": time.time()}
|
||||
if reason == "baseline":
|
||||
fields.update(baseline_state="done", baseline_files=committed)
|
||||
fields.update(baseline_state="done", baseline_files=progress.versions_stored)
|
||||
self.db.update_root(root_id, **fields)
|
||||
if committed:
|
||||
log.info("%s scan of %s stored %d versions", reason, top, committed)
|
||||
return committed
|
||||
progress.finished_at = time.time()
|
||||
if subtree is None:
|
||||
self.scans.pop(root_id, None)
|
||||
self.last_scans[root_id] = progress
|
||||
if progress.versions_stored:
|
||||
log.info("%s scan of %s stored %d versions", reason, top, progress.versions_stored)
|
||||
return progress.versions_stored
|
||||
|
||||
async def _periodic(self) -> None:
|
||||
while True:
|
||||
@@ -509,6 +639,9 @@ class Monitor:
|
||||
for root_id, dirs in list(self.polled.items()):
|
||||
for directory in list(dirs):
|
||||
await self.reconcile(root_id, "poll", subtree=directory)
|
||||
for root in self.db.roots():
|
||||
if root["id"] not in self.missing:
|
||||
self._refresh_root_status(root["id"])
|
||||
if time.monotonic() - self._last_full_reconcile >= self.reconcile_interval:
|
||||
self._last_full_reconcile = time.monotonic()
|
||||
for root in self.db.roots():
|
||||
|
||||
Reference in New Issue
Block a user