diff --git a/app/views/filemanager.py b/app/views/filemanager.py index 61c85828..e6903189 100644 --- a/app/views/filemanager.py +++ b/app/views/filemanager.py @@ -1,18 +1,25 @@ """ Admin-only file manager view for browsing the workdir directory. + +Supports three views: + filesystem – navigate the raw workdir tree, each file tagged against DB records + database – list all FileRecord rows, each tagged with on-disk existence + reconcile – show only the delta: orphan disk files and ghost DB records """ import logging import mimetypes -import os from datetime import datetime from pathlib import Path +from typing import Any, Dict, List, Set -from fastapi import Request +from fastapi import Depends, HTTPException, Request from fastapi.responses import FileResponse +from sqlalchemy.orm import Session from app.config import settings -from app.views.base import APIRouter, require_login, templates +from app.models import FileRecord +from app.views.base import APIRouter, get_db, require_login, templates from app.views.settings import require_admin_access logger = logging.getLogger(__name__) @@ -21,6 +28,11 @@ router = APIRouter() _SIZE_UNITS = ["B", "KB", "MB", "GB", "TB"] +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + def _format_size(size_bytes: int) -> str: """Format file size in human-readable form.""" size = float(size_bytes) @@ -39,78 +51,256 @@ def _safe_path(workdir: str, rel_path: str) -> Path: """ base = Path(workdir).resolve() target = (base / rel_path).resolve() - if not str(target).startswith(str(base)): + if not target.is_relative_to(base): raise ValueError("Path traversal detected") return target +def _db_path_set(db: Session) -> Set[str]: + """ + Return the set of all absolute, normalised paths that the DB references + across local_filename, original_file_path, and processed_file_path. + """ + paths: Set[str] = set() + for row in db.query( + FileRecord.local_filename, + FileRecord.original_file_path, + FileRecord.processed_file_path, + ).all(): + for p in row: + if p: + paths.add(str(Path(p).resolve())) + return paths + + +def _file_icon(mime_type: str, is_dir: bool) -> str: + """Return a Font Awesome class for a file/directory.""" + if is_dir: + return "fas fa-folder text-yellow-400" + if mime_type.startswith("image/"): + return "fas fa-file-image text-blue-400" + if mime_type == "application/pdf": + return "fas fa-file-pdf text-red-400" + if "text/" in mime_type: + return "fas fa-file-alt text-gray-400" + if "json" in mime_type: + return "fas fa-file-code text-green-400" + return "fas fa-file text-gray-400" + + +def _scan_dir(target: Path, workdir_base: Path, db_paths: Set[str]) -> List[Dict[str, Any]]: + """ + List one directory level; annotate each file with its DB status. + + db_status values: + "in_db" – path found in DB + "orphan" – on disk but not in DB + "" – directories (not checked against DB) + """ + entries = [] + for item in sorted(target.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())): + try: + stat = item.stat() + except (PermissionError, OSError) as exc: + logger.warning(f"Could not stat {item}: {exc}") + continue + + item_rel = str(item.relative_to(workdir_base)) + mime_type, _ = mimetypes.guess_type(item.name) + mime_type = mime_type or "" + + if item.is_dir(): + db_status = "" + else: + abs_str = str(item.resolve()) + db_status = "in_db" if abs_str in db_paths else "orphan" + + entries.append( + { + "name": item.name, + "rel_path": item_rel, + "abs_path": str(item.resolve()), + "is_dir": item.is_dir(), + "size": _format_size(stat.st_size) if not item.is_dir() else "", + "size_bytes": stat.st_size if not item.is_dir() else 0, + "modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"), + "mime_type": mime_type, + "icon": _file_icon(mime_type, item.is_dir()), + "db_status": db_status, + } + ) + return entries + + +def _walk_all_files(base: Path, db_paths: Set[str]) -> List[Dict[str, Any]]: + """ + Walk the entire workdir tree and return every file (not directory). + Used for the reconciliation view. + """ + entries = [] + for item in sorted(base.rglob("*"), key=lambda p: str(p).lower()): + if item.is_dir(): + continue + try: + stat = item.stat() + except (PermissionError, OSError): + continue + + abs_str = str(item.resolve()) + mime_type, _ = mimetypes.guess_type(item.name) + mime_type = mime_type or "" + entries.append( + { + "name": item.name, + "rel_path": str(item.relative_to(base)), + "abs_path": abs_str, + "is_dir": False, + "size": _format_size(stat.st_size), + "size_bytes": stat.st_size, + "modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"), + "mime_type": mime_type, + "icon": _file_icon(mime_type, False), + "db_status": "in_db" if abs_str in db_paths else "orphan", + } + ) + return entries + + +def _db_records(db: Session, workdir_base: Path) -> List[Dict[str, Any]]: + """ + Return every FileRecord annotated with on-disk existence for each stored path. + """ + rows = [] + for rec in db.query(FileRecord).order_by(FileRecord.id.desc()).all(): + def _check(p: str | None) -> Dict[str, Any]: + if not p: + return {"path": None, "exists": None, "rel": None} + resolved = Path(p).resolve() + exists = resolved.exists() + try: + rel = str(resolved.relative_to(workdir_base)) + except ValueError: + rel = p # outside workdir – show full path + return {"path": p, "exists": exists, "rel": rel} + + local = _check(rec.local_filename) + original = _check(rec.original_file_path) + processed = _check(rec.processed_file_path) + + any_missing = any(info["exists"] is False for info in [local, original, processed]) + + rows.append( + { + "id": rec.id, + "original_filename": rec.original_filename or "—", + "file_size": _format_size(rec.file_size) if rec.file_size else "—", + "mime_type": rec.mime_type or "—", + "created_at": rec.created_at.strftime("%Y-%m-%d %H:%M") if rec.created_at else "—", + "is_duplicate": rec.is_duplicate, + "filehash": (rec.filehash or "")[:12], + "local": local, + "original": original, + "processed": processed, + # overall health flag + "health": "missing" if any_missing else "ok", + } + ) + return rows + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + @router.get("/admin/files") @require_login @require_admin_access -async def filemanager(request: Request): +async def filemanager(request: Request, db: Session = Depends(get_db)): """ - Admin-only file manager for browsing the workdir directory. + Admin-only file manager with three views: + ?view=filesystem (default) – navigate workdir tree + ?view=database – list all DB FileRecord rows + ?view=reconcile – show only deltas (orphans + ghost records) """ workdir = settings.workdir - rel_path = request.query_params.get("path", "") + workdir_base = Path(workdir).resolve() + view = request.query_params.get("view", "filesystem") - # Sanitise the relative path – strip leading slashes / dots - rel_path = rel_path.lstrip("/").lstrip(".") + # Build the DB path set once (used by all views) + db_paths = _db_path_set(db) + # ── Filesystem view ─────────────────────────────────────────────────── + rel_path = request.query_params.get("path", "").lstrip("/").lstrip(".") try: target = _safe_path(workdir, rel_path) except ValueError: logger.warning(f"Path traversal attempt blocked: path={rel_path!r}") - target = Path(workdir).resolve() + target = workdir_base rel_path = "" if not target.exists(): - target = Path(workdir).resolve() + target = workdir_base rel_path = "" - entries = [] - if target.is_dir(): - for item in sorted(target.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())): - try: - stat = item.stat() - item_rel = str(item.relative_to(Path(workdir).resolve())) - mime_type, _ = mimetypes.guess_type(item.name) - entries.append( - { - "name": item.name, - "rel_path": item_rel, - "is_dir": item.is_dir(), - "size": _format_size(stat.st_size) if not item.is_dir() else "", - "size_bytes": stat.st_size if not item.is_dir() else 0, - "modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"), - "mime_type": mime_type or "", - } - ) - except (PermissionError, OSError) as exc: - logger.warning(f"Could not stat {item}: {exc}") + fs_entries: List[Dict[str, Any]] = [] + if view == "filesystem" and target.is_dir(): + fs_entries = _scan_dir(target, workdir_base, db_paths) - # Build breadcrumb trail + # Breadcrumb for filesystem view breadcrumbs = [] if rel_path: - parts = Path(rel_path).parts - accumulated = "" - for part in parts: - accumulated = str(Path(accumulated) / part) if accumulated else part - breadcrumbs.append({"name": part, "path": accumulated}) + accumulated = Path() + for part in Path(rel_path).parts: + accumulated = accumulated / part + breadcrumbs.append({"name": part, "path": str(accumulated)}) - # Parent path for the "go up" link - parent_path = str(Path(rel_path).parent) if rel_path and Path(rel_path).parent != Path(".") else "" - if parent_path == ".": - parent_path = "" + parent_path = "" + if rel_path: + parent = str(Path(rel_path).parent) + parent_path = "" if parent == "." else parent + + # ── Database view ───────────────────────────────────────────────────── + db_records: List[Dict[str, Any]] = [] + if view in ("database", "reconcile"): + db_records = _db_records(db, workdir_base) + + # ── Reconciliation view ─────────────────────────────────────────────── + orphan_files: List[Dict[str, Any]] = [] + ghost_records: List[Dict[str, Any]] = [] + if view == "reconcile": + all_disk = _walk_all_files(workdir_base, db_paths) + orphan_files = [f for f in all_disk if f["db_status"] == "orphan"] + ghost_records = [r for r in db_records if r["health"] == "missing"] + + # ── Summary counts ──────────────────────────────────────────────────── + # Disk file count is only computed for non-filesystem views to avoid + # the overhead of walking the full tree on every directory navigation. + if view in ("database", "reconcile") and workdir_base.exists(): + total_disk = sum(1 for p in workdir_base.rglob("*") if p.is_file()) + else: + total_disk = None # deferred; not shown on filesystem tab header + total_db = db.query(FileRecord).count() return templates.TemplateResponse( "filemanager.html", { "request": request, - "entries": entries, + # view selector + "view": view, + # filesystem tab + "fs_entries": fs_entries, "current_path": rel_path, "parent_path": parent_path, "breadcrumbs": breadcrumbs, + # database tab + "db_records": db_records, + # reconcile tab + "orphan_files": orphan_files, + "ghost_records": ghost_records, + # summary + "total_disk": total_disk, + "total_db": total_db, "workdir": workdir, "app_version": settings.version, }, @@ -130,13 +320,9 @@ async def filemanager_download(request: Request): try: target = _safe_path(workdir, rel_path) except ValueError: - from fastapi import HTTPException - raise HTTPException(status_code=400, detail="Invalid path") if not target.exists() or not target.is_file(): - from fastapi import HTTPException - raise HTTPException(status_code=404, detail="File not found") mime_type, _ = mimetypes.guess_type(target.name) diff --git a/frontend/templates/base.html b/frontend/templates/base.html index 7be7e0cd..4a8715f9 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -86,7 +86,7 @@ - + Status diff --git a/frontend/templates/filemanager.html b/frontend/templates/filemanager.html index 5f7fabb9..db5e88ec 100644 --- a/frontend/templates/filemanager.html +++ b/frontend/templates/filemanager.html @@ -4,106 +4,148 @@ {% block content %}
- -
-

- - File Manager - Admin Only -

-

- Browsing: {{ workdir }} - {% if current_path %}/ {{ current_path }}{% endif %} -

+ +
+
+

+ + File Manager + Admin Only +

+

+ {{ workdir }} +

+
+ +
+ + + {% if total_disk is none %}…{% else %}{{ total_disk }}{% endif %} on disk + + + {{ total_db }} in DB + + {% if total_disk is not none and total_disk != total_db %} + + Delta detected + + {% endif %} +
+
+ + + + + + {% if view == 'filesystem' %} + + +
+ Found in DB + Not in DB (orphan)
- +
- +
- - - - - + + + + + + - + - {% if current_path %} - - + - + {% endif %} - {% if entries %} - {% for entry in entries %} - - + + - - + + - - @@ -111,8 +153,8 @@ {% endfor %} {% else %} - @@ -120,6 +162,201 @@
NameActionsNameDBActions
- + + ..
+ {% if fs_entries %} + {% for entry in fs_entries %} +
{% if entry.is_dir %} - - {% elif entry.mime_type.startswith('image') %} - - {% elif entry.mime_type == 'application/pdf' %} - - {% elif 'text' in entry.mime_type %} - - {% elif 'json' in entry.mime_type %} - - {% else %} - - {% endif %} - - {% if entry.is_dir %} - + {{ entry.name }} {% else %} - {{ entry.name }} + {{ entry.name }} {% endif %} + {% if entry.db_status == 'in_db' %} + + In DB + + {% elif entry.db_status == 'orphan' %} + + Orphan + + {% endif %} + {% if not entry.is_dir %} - Download + class="inline-flex items-center gap-1 px-2.5 py-1 border border-gray-200 text-xs font-medium rounded text-gray-600 bg-white hover:bg-gray-50"> + Download {% endif %}
- + + This directory is empty.
+ {% endif %} + + + {% if view == 'database' %} + + +
+ File exists on disk + File missing from disk + Path not set +
+ + {% if db_records %} +
+ + + + + + + + + + + + + + + {% for rec in db_records %} + + + + + + + {# Helper macro: render a path cell #} + {% for path_info in [rec.local, rec.original, rec.processed] %} + + {% endfor %} + + + + {% endfor %} + +
IDOriginal Filenamelocal_filenameoriginal_file_pathprocessed_file_pathHealth
{{ rec.id }} + {% if rec.is_duplicate %} + dup + {% endif %} + {{ rec.original_filename }} + + {% if path_info.path is none %} + + {% elif path_info.exists %} + + + {{ path_info.rel }} + + {% else %} + + + {{ path_info.rel }} + + {% endif %} + + {% if rec.health == 'ok' %} + + OK + + {% else %} + + Missing + + {% endif %} +
+
+ {% else %} +
+ + No file records found in the database. +
+ {% endif %} + {% endif %} + + + {% if view == 'reconcile' %} + + {% if orphan_files or ghost_records %} +
+ +
+ Delta detected. + Found {{ orphan_files | length }} orphan file(s) on disk with no DB record, + and {{ ghost_records | length }} DB record(s) with missing files on disk. +
+
+ {% else %} +
+ + No delta found — filesystem and database are in sync. +
+ {% endif %} + + +
+

+ + Orphan files (on disk, no DB record) + {{ orphan_files | length }} +

+ {% if orphan_files %} +
+ + + + + + + + + + + + {% for f in orphan_files %} + + + + + + + + {% endfor %} + +
Path (relative to workdir)Actions
{{ f.rel_path }} + + Download + +
+
+ {% else %} +

No orphan files found.

+ {% endif %} +
+ + +
+

+ + Ghost records (in DB, file(s) missing on disk) + {{ ghost_records | length }} +

+ {% if ghost_records %} +
+ + + + + + + + + + + {% for rec in ghost_records %} + + + + + + + {% endfor %} + +
IDOriginal FilenameMissing paths
{{ rec.id }} + {{ rec.original_filename }} + + {% for path_info in [rec.local, rec.original, rec.processed] %} + {% if path_info.exists is sameas false %} +
+ + {{ path_info.rel }} +
+ {% endif %} + {% endfor %} +
+
+ {% else %} +

No ghost records found.

+ {% endif %} +
+ {% endif %}
{% endblock %} +