diff --git a/.env.demo b/.env.demo index 5bf0a14d..d3dc2a46 100644 --- a/.env.demo +++ b/.env.demo @@ -7,6 +7,10 @@ GOTENBERG_URL=http://gotenberg:3000 ALLOW_FILE_DELETE=true # Allow deletion of file records COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2) +# **System Reset / Factory Reset** +# FACTORY_RESET_ON_STARTUP=false # Wipe all user data on every startup (demo/testing only) +# ENABLE_FACTORY_RESET=false # Show the System Reset page in admin UI + # **Logging** # LOG_LEVEL controls the Python root-logger level. # Accepted values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO). diff --git a/app/api/__init__.py b/app/api/__init__.py index 246e47f3..c6d8e379 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -43,6 +43,7 @@ from app.api.shared_links import public_router as shared_links_public_router from app.api.shared_links import router as shared_links_router from app.api.similarity import router as similarity_router from app.api.subscriptions import router as subscriptions_router +from app.api.system_reset import router as system_reset_router from app.api.translation import router as translation_router from app.api.url_upload import router as url_upload_router @@ -97,4 +98,5 @@ router.include_router(audit_logs_router) router.include_router(i18n_router) router.include_router(mobile_router) router.include_router(compliance_router) +router.include_router(system_reset_router) router.include_router(translation_router) diff --git a/app/api/system_reset.py b/app/api/system_reset.py new file mode 100644 index 00000000..5c7ff501 --- /dev/null +++ b/app/api/system_reset.py @@ -0,0 +1,124 @@ +""" +System reset API endpoints for DocuElevate. + +Provides admin-only REST endpoints for: +- Full system reset (wipe all user data) +- Reset with re-import (move originals → reimport folder, wipe, re-ingest) + +Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and +admin privileges. +""" + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.config import settings +from app.database import get_db + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"]) + + +def _require_admin(request: Request) -> dict: + """Ensure the caller is an admin. Raises 403 otherwise.""" + user = request.session.get("user") + if not user or not user.get("is_admin"): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + return user + + +AdminUser = Annotated[dict, Depends(_require_admin)] + + +def _require_feature_enabled() -> None: + """Raise 404 when the factory-reset feature flag is off.""" + if not settings.enable_factory_reset: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.", + ) + + +class ResetRequest(BaseModel): + """Body for system reset endpoints. Requires explicit confirmation.""" + + confirmation: str + + +@router.post("/full") +async def full_reset( + body: ResetRequest, + _admin: AdminUser, + db: Session = Depends(get_db), +) -> dict: + """Wipe all user data (database + work-files). + + The caller must send ``{"confirmation": "DELETE"}`` to proceed. + """ + _require_feature_enabled() + + if body.confirmation != "DELETE": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.', + ) + + from app.utils.system_reset import perform_full_reset + + try: + result = perform_full_reset(db) + except Exception as exc: + logger.exception("Full system reset failed") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"System reset failed: {exc}", + ) from exc + + return {"status": "ok", "result": result} + + +@router.post("/reimport") +async def reset_and_reimport( + body: ResetRequest, + _admin: AdminUser, + db: Session = Depends(get_db), +) -> dict: + """Move original files to a reimport folder, wipe everything, and + configure the reimport folder as a watch folder for automatic + re-ingestion. + + The caller must send ``{"confirmation": "REIMPORT"}`` to proceed. + """ + _require_feature_enabled() + + if body.confirmation != "REIMPORT": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.', + ) + + from app.utils.system_reset import perform_reset_and_reimport + + try: + result = perform_reset_and_reimport(db) + except Exception as exc: + logger.exception("Reset-and-reimport failed") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Reset and reimport failed: {exc}", + ) from exc + + return {"status": "ok", "result": result} + + +@router.get("/status") +async def reset_status(_admin: AdminUser) -> dict: + """Return whether the system reset feature is enabled.""" + return { + "enabled": settings.enable_factory_reset, + "factory_reset_on_startup": settings.factory_reset_on_startup, + } diff --git a/app/config.py b/app/config.py index ac9ab7bb..cad16a06 100644 --- a/app/config.py +++ b/app/config.py @@ -639,6 +639,25 @@ class Settings(BaseSettings): ), ) + # System reset / factory reset settings + factory_reset_on_startup: bool = Field( + default=False, + description=( + "When enabled, DocuElevate wipes all user data (database rows and " + "work-files on disk) on every startup so the instance always comes " + "up in a clean, fresh state. Useful for demo or testing environments. " + "Default: False." + ), + ) + enable_factory_reset: bool = Field( + default=False, + description=( + "Show the 'System Reset' page in the admin UI. When enabled, " + "administrators can trigger a full data wipe or a wipe-and-reimport " + "directly from the web interface. Default: False." + ), + ) + # PDF/A archival conversion settings enable_pdfa_conversion: bool = Field( default=False, diff --git a/app/main.py b/app/main.py index 97bb1d57..a3b56ec4 100644 --- a/app/main.py +++ b/app/main.py @@ -170,6 +170,12 @@ async def lifespan(app: FastAPI): # Startup: Initialize database init_db() # Create tables if they don't exist + # Factory reset on startup — wipe all user data before anything else + if settings.factory_reset_on_startup: + from app.utils.system_reset import perform_startup_reset + + perform_startup_reset() + # Load settings from database after DB initialization from app.database import SessionLocal from app.utils.config_loader import load_settings_from_db diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index e130fbd9..699badf4 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -1900,6 +1900,28 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "factory_reset_on_startup": { + "category": "Feature Flags", + "description": ( + "Wipe all user data on every startup so the instance always starts fresh. " + "Useful for demo/testing environments. Default: False." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "enable_factory_reset": { + "category": "Feature Flags", + "description": ( + "Show the System Reset page in the admin UI. Allows administrators to " + "trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False." + ), + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Backup / Restore "backup_enabled": { "category": "Backup", diff --git a/app/utils/system_reset.py b/app/utils/system_reset.py new file mode 100644 index 00000000..dfeaa5b5 --- /dev/null +++ b/app/utils/system_reset.py @@ -0,0 +1,296 @@ +""" +System reset utilities for DocuElevate. + +Provides functions to: +- Wipe all user data (database rows + work-files on disk) for a fresh start. +- Wipe with re-import: move original files to a dedicated folder, wipe + everything, then let the watch-folder mechanism re-ingest the files. + +Security: All public functions in this module require admin-level access. +They MUST only be invoked from admin-guarded API/view endpoints. +""" + +import logging +import shutil +from pathlib import Path + +from sqlalchemy.orm import Session + +from app.config import settings + +logger = logging.getLogger(__name__) + +# Subdirectories inside *workdir* that contain user-generated data. +# Everything else (app code, static assets, config) is left untouched. +_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups") + +# JSON cache files written by watch-folder / ingest tasks. +_CACHE_FILES = ( + "watch_folder_processed.json", + "ftp_ingest_processed.json", + "sftp_ingest_processed.json", + "dropbox_ingest_processed.json", + "gdrive_ingest_processed.json", + "onedrive_ingest_processed.json", + "nextcloud_ingest_processed.json", + "s3_ingest_processed.json", + "webdav_ingest_processed.json", + "processed_mails.json", + "credential_failures.json", +) + +# The folder name used for storing files prior to re-import. +REIMPORT_FOLDER_NAME = "reimport" + + +def _wipe_workdir_data(workdir: str) -> dict[str, int]: + """Delete user data subdirectories and cache files inside *workdir*. + + Leaves the workdir directory itself intact so the application can + continue to write into it. Also leaves any files that do not belong + to the known data subdirectories or caches. + + Returns: + A dict with counts of deleted directories and files. + """ + workdir_path = Path(workdir) + deleted_dirs = 0 + deleted_files = 0 + + # Remove data subdirectories + for subdir in _USER_DATA_SUBDIRS: + target = workdir_path / subdir + if target.is_dir(): + shutil.rmtree(target) + logger.info("Deleted data directory: %s", target) + deleted_dirs += 1 + + # Remove cache / state JSON files + for cache_file in _CACHE_FILES: + target = workdir_path / cache_file + if target.is_file(): + target.unlink() + logger.info("Deleted cache file: %s", target) + deleted_files += 1 + + # Also remove user_wf_*.json files (per-user watch folder caches) + for f in workdir_path.glob("user_wf_*.json"): + f.unlink() + logger.info("Deleted user watch-folder cache: %s", f) + deleted_files += 1 + + # Remove loose files in workdir root that are user uploads (uuid-named + # files like "a1b2c3d4-…pdf") but NOT application config files. + for entry in workdir_path.iterdir(): + if entry.is_file() and entry.suffix.lower() in { + ".pdf", + ".png", + ".jpg", + ".jpeg", + ".tiff", + ".tif", + ".docx", + ".doc", + ".xlsx", + ".xls", + ".pptx", + ".heic", + ".heif", + ".webp", + ".bmp", + ".gif", + ".txt", + ".rtf", + ".odt", + ".ods", + ".odp", + ".csv", + ".pages", + ".numbers", + ".keynote", + }: + entry.unlink() + logger.info("Deleted loose workdir file: %s", entry) + deleted_files += 1 + + return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files} + + +def _wipe_database(db: Session) -> dict[str, int]: + """Delete all user-generated rows from the database. + + Preserves schema (tables, migrations) and system-seeded rows that will + be re-created on the next startup (subscription plans, default pipeline, + scheduled jobs, compliance templates). + + Returns: + A dict mapping table name → number of rows deleted. + """ + from app.models import ( + AuditLog, + BackupRecord, + DocumentMetadata, + FileProcessingStep, + FileRecord, + InAppNotification, + ProcessingLog, + SavedSearch, + SettingsAuditLog, + SharedLink, + UserImapAccount, + UserIntegration, + UserNotificationPreference, + UserNotificationTarget, + ) + + # Order matters: delete children before parents to respect FK constraints. + tables_to_wipe: list[tuple[str, type]] = [ + ("file_processing_steps", FileProcessingStep), + ("processing_logs", ProcessingLog), + ("shared_links", SharedLink), + ("in_app_notifications", InAppNotification), + ("user_notification_preferences", UserNotificationPreference), + ("user_notification_targets", UserNotificationTarget), + ("user_imap_accounts", UserImapAccount), + ("user_integrations", UserIntegration), + ("saved_searches", SavedSearch), + ("settings_audit_log", SettingsAuditLog), + ("audit_logs", AuditLog), + ("backup_records", BackupRecord), + ("document_metadata", DocumentMetadata), + ("files", FileRecord), + ] + + result: dict[str, int] = {} + for table_name, model in tables_to_wipe: + try: + count = db.query(model).delete() + result[table_name] = count + logger.info("Wiped %d rows from %s", count, table_name) + except Exception: + logger.exception("Failed to wipe table %s during system reset", table_name) + db.rollback() + raise + + db.commit() + return result + + +def perform_full_reset(db: Session) -> dict: + """Perform a complete system reset: wipe database rows + work-files. + + Args: + db: An active SQLAlchemy session. + + Returns: + Summary dict with ``database`` and ``filesystem`` sub-dicts. + """ + logger.warning(">>> SYSTEM RESET: wiping all user data <<<") + + db_result = _wipe_database(db) + fs_result = _wipe_workdir_data(settings.workdir) + + logger.warning(">>> SYSTEM RESET complete <<<") + return {"database": db_result, "filesystem": fs_result} + + +def perform_reset_and_reimport(db: Session) -> dict: + """Move original files to a reimport folder, wipe everything, then + configure the reimport folder as a watch folder for re-ingestion. + + The watch-folder scanner (``scan_all_watch_folders``) will pick up + the files on its next periodic run and process them exactly as if + they had been freshly uploaded — respecting the same backoff + strategy, size limits, and rate limits. + + Args: + db: An active SQLAlchemy session. + + Returns: + Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts. + """ + workdir_path = Path(settings.workdir) + reimport_dir = workdir_path / REIMPORT_FOLDER_NAME + original_dir = workdir_path / "original" + + # 1. Collect original files + files_moved = 0 + reimport_dir.mkdir(parents=True, exist_ok=True) + + if original_dir.is_dir(): + for entry in original_dir.iterdir(): + if entry.is_file(): + # Validate the resolved path stays within original_dir (path traversal guard) + try: + entry.resolve().relative_to(original_dir.resolve()) + except ValueError: + logger.warning("Skipping file outside original dir: %s", entry) + continue + dest = reimport_dir / entry.name + # Avoid overwriting: append counter if name clash + if dest.exists(): + stem = dest.stem + suffix = dest.suffix + counter = 1 + while dest.exists(): + dest = reimport_dir / f"{stem}_{counter}{suffix}" + counter += 1 + shutil.copy2(str(entry), str(dest)) + files_moved += 1 + + logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir) + + # 2. Perform the full reset (wipe DB + other workdir data) + reset_result = perform_full_reset(db) + + # 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS) + # and set up watch folder config to point at it. + _configure_reimport_watch_folder(str(reimport_dir)) + + reset_result["reimport"] = { + "files_moved": files_moved, + "reimport_folder": str(reimport_dir), + } + logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved) + return reset_result + + +def _configure_reimport_watch_folder(reimport_path: str) -> None: + """Append *reimport_path* to the application's watch-folder list. + + The watch-folder scanner uses ``settings.watch_folders`` (a + comma-separated string). We mutate the runtime setting so the + next scan picks up the folder. We also set + ``watch_folder_delete_after_process = True`` so files are cleaned + up after successful processing. + """ + current = getattr(settings, "watch_folders", None) or "" + folders = [f.strip() for f in current.split(",") if f.strip()] + + if reimport_path not in folders: + folders.append(reimport_path) + + # Mutate runtime settings (not persisted to .env — ephemeral) + object.__setattr__(settings, "watch_folders", ",".join(folders)) + object.__setattr__(settings, "watch_folder_delete_after_process", True) + logger.info("Configured reimport watch folder: %s", reimport_path) + + +def perform_startup_reset() -> None: + """Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``. + + Wipes database and filesystem data so the instance starts completely + fresh. Uses its own DB session so it runs before the normal lifespan + seeding logic. + """ + from app.database import SessionLocal + + logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data") + db = SessionLocal() + try: + perform_full_reset(db) + except Exception: + logger.exception("Factory reset on startup failed") + db.rollback() + finally: + db.close() diff --git a/app/views/__init__.py b/app/views/__init__.py index b25a3be8..98100b1a 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -34,6 +34,7 @@ from app.views.share import router as share_router from app.views.shared_links import router as shared_links_router from app.views.status import router as status_router from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages +from app.views.system_reset import router as system_reset_router # System reset / factory reset from app.views.wizard import router as wizard_router # Create a main router that includes all the view routers @@ -67,3 +68,4 @@ router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs router.include_router(audit_logs_router) # Comprehensive audit log viewer router.include_router(help_router) # Built-in help / How-To docs router.include_router(compliance_router) # Compliance templates dashboard +router.include_router(system_reset_router) # System reset / factory reset diff --git a/app/views/base.py b/app/views/base.py index 609ff9fb..0c00b551 100644 --- a/app/views/base.py +++ b/app/views/base.py @@ -94,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None: "allow_signup", getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False), ) + ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False)) req = ctx.get("request") if req is not None: diff --git a/app/views/system_reset.py b/app/views/system_reset.py new file mode 100644 index 00000000..6ef90fca --- /dev/null +++ b/app/views/system_reset.py @@ -0,0 +1,40 @@ +""" +System reset view — admin-only UI page. + +Renders a confirmation-heavy page that allows administrators to: +1. **Full Reset** — wipe all user data (DB + disk) for a fresh start. +2. **Reset & Re-import** — move originals to a reimport folder, wipe, + and let the watch-folder mechanism re-ingest them. + +Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag. +""" + +import logging + +from fastapi import Depends, Request +from fastapi.responses import RedirectResponse, Response +from sqlalchemy.orm import Session + +from app.config import settings +from app.views.base import APIRouter, get_db, require_login, templates +from app.views.settings import require_admin_access + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("/admin/system-reset") +@require_login +@require_admin_access +async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response: + """Render the system reset administration page.""" + if not settings.enable_factory_reset: + return RedirectResponse(url="/settings", status_code=302) + + return templates.TemplateResponse( + "system_reset.html", + { + "request": request, + "factory_reset_on_startup": settings.factory_reset_on_startup, + }, + ) diff --git a/docs/API.md b/docs/API.md index 3d4c7842..29a3dc7a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2405,3 +2405,64 @@ query GetDocument($id: Int!) { } ``` Variables: `{ "id": 42 }` + +## System Reset + +Admin-only endpoints for resetting the system to a clean state. Requires `ENABLE_FACTORY_RESET=True`. + +### GET /api/admin/system-reset/status + +Check whether the system reset feature is enabled. + +**Response (200):** +```json +{ + "enabled": true, + "factory_reset_on_startup": false +} +``` + +### POST /api/admin/system-reset/full + +Wipe all user data (database + work-files). + +**Request:** +```json +{ + "confirmation": "DELETE" +} +``` + +**Response (200):** +```json +{ + "status": "ok", + "result": { + "database": { "files": 42, "processing_logs": 100 }, + "filesystem": { "deleted_dirs": 5, "deleted_files": 12 } + } +} +``` + +### POST /api/admin/system-reset/reimport + +Move original files to a reimport folder, wipe everything, and configure the reimport folder as a watch folder for re-ingestion. + +**Request:** +```json +{ + "confirmation": "REIMPORT" +} +``` + +**Response (200):** +```json +{ + "status": "ok", + "result": { + "database": { "files": 42 }, + "filesystem": { "deleted_dirs": 5, "deleted_files": 12 }, + "reimport": { "files_moved": 42, "reimport_folder": "/workdir/reimport" } + } +} +``` diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 45f4ba13..595dd203 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -17,6 +17,8 @@ Configuration is primarily done through environment variables specified in a `.e | `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` | | `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` | | `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` | +| `FACTORY_RESET_ON_STARTUP` | Wipe all user data on every startup (demo/testing). | `false` | +| `ENABLE_FACTORY_RESET` | Show the System Reset page in the admin UI. | `false` | ### Batch Processing Settings @@ -1856,6 +1858,52 @@ BACKUP_RETAIN_WEEKLY=13 You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables. +## System Reset / Factory Reset + +DocuElevate provides two mechanisms for resetting the system to a clean state. Both are **disabled by default** and must be explicitly enabled. + +### Automatic Reset on Startup + +Set `FACTORY_RESET_ON_STARTUP=true` to wipe all user data (database rows and work-files) every time the application starts. This is useful for demo, testing, or ephemeral environments where you always want a fresh instance. + +```dotenv +FACTORY_RESET_ON_STARTUP=true +``` + +> **Warning:** This destroys all documents, processing history, audit logs, and backups on every restart. Application settings and configuration are preserved. + +### Admin UI Reset Page + +Set `ENABLE_FACTORY_RESET=true` to display the **System Reset** page in the admin navigation menu. From this page, administrators can: + +| Action | Confirmation | Description | +|--------|-------------|-------------| +| **Full Reset** | Type `DELETE` | Wipes all database rows and work-files. The system returns to its initial state. | +| **Reset & Re-import** | Type `REIMPORT` | Copies original files to a `reimport/` folder inside the workdir, wipes everything, then configures the reimport folder as a watch folder so files are automatically re-ingested with the same processing pipeline, rate limits, and backoff strategy as regular uploads. | + +```dotenv +ENABLE_FACTORY_RESET=true +``` + +### API Endpoints + +When `ENABLE_FACTORY_RESET=true`, two admin-only API endpoints are available: + +- `POST /api/admin/system-reset/full` — body: `{"confirmation": "DELETE"}` +- `POST /api/admin/system-reset/reimport` — body: `{"confirmation": "REIMPORT"}` +- `GET /api/admin/system-reset/status` — returns current feature-flag state + +### What Gets Deleted + +| Deleted | Preserved | +|---------|-----------| +| All document records (`files` table) | Application settings (`application_settings` table) | +| Processing logs and steps | User accounts and profiles | +| Audit logs | Subscription plans | +| Backup records | Pipelines and scheduled jobs | +| Original, processed, and temporary files | The workdir directory itself | +| Watch-folder caches and ingestion state | OAuth and integration configuration | + ## Configuration File Location The `.env` file should be placed at the root of the project directory. When using Docker Compose, you can reference it with the `env_file` directive in your `docker-compose.yml`. diff --git a/frontend/templates/base.html b/frontend/templates/base.html index d049cfe2..4cf7c7e2 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -177,6 +177,11 @@ {{ _("nav.backup_restore") }} + {% if enable_factory_reset %} + + {{ _("nav.system_reset") }} + + {% endif %} Audit Logs @@ -445,6 +450,11 @@ {{ _("nav.backup_restore") }} + {% if enable_factory_reset %} + + {{ _("nav.system_reset") }} + + {% endif %} {{ _("nav.status") }} diff --git a/frontend/templates/system_reset.html b/frontend/templates/system_reset.html new file mode 100644 index 00000000..452a8142 --- /dev/null +++ b/frontend/templates/system_reset.html @@ -0,0 +1,261 @@ +{% extends "base.html" %} +{% block title %}{{ _("system_reset.page_title") }} - DocuElevate{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block content %} +
+ + +
+
+

+ + {{ _("system_reset.heading") }} +

+
+

+ {{ _("system_reset.subtitle") }} +

+
+ + + {% if factory_reset_on_startup %} + + {% endif %} + + + + +
+ + +
+
+
+ +
+
+

{{ _("system_reset.full_reset_title") }}

+

{{ _("system_reset.full_reset_subtitle") }}

+
+
+ +
+

{{ _("system_reset.full_reset_desc") }}

+
    +
  • {{ _("system_reset.full_reset_item_db") }}
  • +
  • {{ _("system_reset.full_reset_item_files") }}
  • +
  • {{ _("system_reset.full_reset_item_cache") }}
  • +
  • {{ _("system_reset.full_reset_item_settings_kept") }}
  • +
+
+ +
+ + +

{{ _("system_reset.type_delete_help") }}

+ + +
+
+ + +
+
+
+ +
+
+

{{ _("system_reset.reimport_title") }}

+

{{ _("system_reset.reimport_subtitle") }}

+
+
+ +
+

{{ _("system_reset.reimport_desc") }}

+
    +
  1. {{ _("system_reset.reimport_step_1") }}
  2. +
  3. {{ _("system_reset.reimport_step_2") }}
  4. +
  5. {{ _("system_reset.reimport_step_3") }}
  6. +
+

{{ _("system_reset.reimport_note") }}

+
+ +
+ + +

{{ _("system_reset.type_reimport_help") }}

+ + +
+
+
+ + +
+
+ +
+

+

+      
+
+
+
+ + +{% endblock %} diff --git a/frontend/translations/en.json b/frontend/translations/en.json index 19e3a668..27d117d9 100644 --- a/frontend/translations/en.json +++ b/frontend/translations/en.json @@ -1184,6 +1184,7 @@ "nav.skip_to_content": "Skip to main content", "nav.status": "Status", "nav.subscription": "Subscription", + "nav.system_reset": "System Reset", "nav.toggle_dark_mode": "Toggle dark mode", "nav.toggle_nav": "Toggle navigation menu", "nav.upload": "Upload", @@ -1723,6 +1724,36 @@ "subscription.upgrade_info": "Upgrades take effect immediately. Downgrades are scheduled for the end of your current billing period.", "subscription.upgrade_to_prefix": "Upgrade to", "subscription.usage_heading": "Usage", + "system_reset.danger_desc": "The actions below will permanently destroy data. They cannot be undone. Application settings and configuration are preserved, but all documents, files, processing history, and audit logs will be deleted.", + "system_reset.danger_zone": "Danger Zone — Irreversible Actions", + "system_reset.full_reset_button": "Wipe All Data", + "system_reset.full_reset_desc": "Permanently deletes all user data from the database and removes all work files from disk. The application will be in its initial state after this operation.", + "system_reset.full_reset_item_cache": "All watch-folder caches and ingestion state", + "system_reset.full_reset_item_db": "All document records, processing logs, and audit history", + "system_reset.full_reset_item_files": "All original, processed, and temporary files on disk", + "system_reset.full_reset_item_settings_kept": "Application settings and configuration are preserved", + "system_reset.full_reset_subtitle": "Wipe everything and start fresh", + "system_reset.full_reset_title": "Full System Reset", + "system_reset.heading": "System Reset", + "system_reset.js_error_generic": "An error occurred. Please check the server logs for details.", + "system_reset.js_success_full": "System reset complete. All user data has been wiped.", + "system_reset.js_success_reimport": "Reset complete. Original files have been staged for re-import via the watch folder.", + "system_reset.page_title": "System Reset", + "system_reset.reimport_button": "Reset & Re-import", + "system_reset.reimport_desc": "Copies your original files to a special reimport folder, wipes everything, then lets the watch-folder mechanism re-process them as if they were freshly uploaded.", + "system_reset.reimport_note": "Re-imported files will go through the full processing pipeline with the same rate limits and backoff strategy as regular uploads.", + "system_reset.reimport_step_1": "Original files are copied to a dedicated reimport folder", + "system_reset.reimport_step_2": "All data (database + work files) is wiped clean", + "system_reset.reimport_step_3": "The reimport folder is configured as a watch folder for automatic re-ingestion", + "system_reset.reimport_subtitle": "Wipe and re-process all original files", + "system_reset.reimport_title": "Reset & Re-import", + "system_reset.startup_reset_active": "Factory Reset on Startup is ACTIVE", + "system_reset.startup_reset_desc": "FACTORY_RESET_ON_STARTUP is enabled. All user data is wiped every time the application starts.", + "system_reset.subtitle": "Reset DocuElevate to a clean, fresh state. All user data will be permanently deleted.", + "system_reset.type_delete": "Type DELETE to confirm", + "system_reset.type_delete_help": "You must type the word DELETE in capital letters to enable the reset button.", + "system_reset.type_reimport": "Type REIMPORT to confirm", + "system_reset.type_reimport_help": "You must type the word REIMPORT in capital letters to enable the button.", "terms.cookie_link": "Cookie Policy", "terms.heading": "Terms of Service", "terms.last_updated": "Last Updated:", diff --git a/tests/test_system_reset.py b/tests/test_system_reset.py new file mode 100644 index 00000000..570ebd19 --- /dev/null +++ b/tests/test_system_reset.py @@ -0,0 +1,393 @@ +"""Tests for the system reset feature (app/api/system_reset.py, app/utils/system_reset.py, app/views/system_reset.py).""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.database import Base +from app.models import ( + DocumentMetadata, + FileProcessingStep, + FileRecord, + ProcessingLog, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def reset_workdir(): + """Create a temporary workdir populated with sample user data.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create data subdirectories with dummy files + for subdir in ("original", "processed", "tmp", "pdfa", "backups"): + d = Path(tmpdir) / subdir + d.mkdir() + (d / "sample.pdf").write_bytes(b"%PDF-1.4 fake") + + # Create cache files + for cache in ("watch_folder_processed.json", "ftp_ingest_processed.json"): + (Path(tmpdir) / cache).write_text("{}") + + # Create a per-user watch folder cache + (Path(tmpdir) / "user_wf_42.json").write_text("{}") + + # Create a loose PDF in workdir root + (Path(tmpdir) / "abc123.pdf").write_bytes(b"%PDF-1.4 loose") + + yield tmpdir + + +@pytest.fixture +def reset_db_session(): + """Fresh in-memory database with sample user data rows.""" + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + session = Session() + + # Seed with sample data + fr = FileRecord( + filehash="abc123", + original_filename="test.pdf", + local_filename="uuid.pdf", + file_size=1024, + mime_type="application/pdf", + ) + session.add(fr) + session.flush() + + session.add(ProcessingLog(file_id=fr.id, task_id="t1", step_name="hash_file", status="success")) + session.add(FileProcessingStep(file_id=fr.id, step_name="hash_file", status="success")) + session.add(DocumentMetadata(filename="test.pdf", sender="Alice", recipient="Bob")) + session.commit() + + yield session + + session.close() + Base.metadata.drop_all(bind=engine) + + +# --------------------------------------------------------------------------- +# Unit tests for app/utils/system_reset.py +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestWipeWorkdirData: + """Tests for _wipe_workdir_data().""" + + def test_removes_data_subdirs(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + result = _wipe_workdir_data(reset_workdir) + + # All data subdirectories should be gone + for subdir in ("original", "processed", "tmp", "pdfa", "backups"): + assert not (Path(reset_workdir) / subdir).exists() + + assert result["deleted_dirs"] == 5 + + def test_removes_cache_files(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + result = _wipe_workdir_data(reset_workdir) + + assert not (Path(reset_workdir) / "watch_folder_processed.json").exists() + assert not (Path(reset_workdir) / "ftp_ingest_processed.json").exists() + assert not (Path(reset_workdir) / "user_wf_42.json").exists() + assert result["deleted_files"] >= 3 + + def test_removes_loose_document_files(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + _wipe_workdir_data(reset_workdir) + assert not (Path(reset_workdir) / "abc123.pdf").exists() + + def test_preserves_workdir_directory(self, reset_workdir): + from app.utils.system_reset import _wipe_workdir_data + + _wipe_workdir_data(reset_workdir) + assert Path(reset_workdir).is_dir() + + def test_handles_empty_workdir(self): + """No errors when workdir has no data dirs or caches.""" + from app.utils.system_reset import _wipe_workdir_data + + with tempfile.TemporaryDirectory() as empty_dir: + result = _wipe_workdir_data(empty_dir) + assert result["deleted_dirs"] == 0 + assert result["deleted_files"] == 0 + + +@pytest.mark.unit +class TestWipeDatabase: + """Tests for _wipe_database().""" + + def test_deletes_all_user_data(self, reset_db_session): + from app.utils.system_reset import _wipe_database + + result = _wipe_database(reset_db_session) + + assert result.get("files", 0) >= 1 + assert result.get("processing_logs", 0) >= 1 + assert result.get("file_processing_steps", 0) >= 1 + assert result.get("document_metadata", 0) >= 1 + + def test_tables_are_empty_after_wipe(self, reset_db_session): + from app.utils.system_reset import _wipe_database + + _wipe_database(reset_db_session) + + assert reset_db_session.query(FileRecord).count() == 0 + assert reset_db_session.query(ProcessingLog).count() == 0 + assert reset_db_session.query(FileProcessingStep).count() == 0 + assert reset_db_session.query(DocumentMetadata).count() == 0 + + +@pytest.mark.unit +class TestPerformFullReset: + """Tests for perform_full_reset().""" + + def test_wipes_db_and_filesystem(self, reset_db_session, reset_workdir): + from app.utils.system_reset import perform_full_reset + + with patch("app.utils.system_reset.settings") as mock_settings: + mock_settings.workdir = reset_workdir + result = perform_full_reset(reset_db_session) + + assert "database" in result + assert "filesystem" in result + assert reset_db_session.query(FileRecord).count() == 0 + assert not (Path(reset_workdir) / "original").exists() + + +@pytest.mark.unit +class TestPerformResetAndReimport: + """Tests for perform_reset_and_reimport().""" + + def test_copies_originals_to_reimport_then_wipes(self, reset_db_session, reset_workdir): + from app.utils.system_reset import perform_reset_and_reimport + + with patch("app.utils.system_reset.settings") as mock_settings: + mock_settings.workdir = reset_workdir + mock_settings.watch_folders = "" + mock_settings.watch_folder_delete_after_process = False + result = perform_reset_and_reimport(reset_db_session) + + reimport_dir = Path(reset_workdir) / "reimport" + assert reimport_dir.is_dir() + assert result["reimport"]["files_moved"] >= 1 + + # DB should be wiped + assert reset_db_session.query(FileRecord).count() == 0 + + # Reimport folder should contain the original file + reimport_files = list(reimport_dir.iterdir()) + assert len(reimport_files) >= 1 + + def test_configures_watch_folder(self, reset_db_session, reset_workdir): + from app.utils.system_reset import perform_reset_and_reimport + + with patch("app.utils.system_reset.settings") as mock_settings: + mock_settings.workdir = reset_workdir + mock_settings.watch_folders = "/some/other/folder" + mock_settings.watch_folder_delete_after_process = False + perform_reset_and_reimport(reset_db_session) + + reimport_path = str(Path(reset_workdir) / "reimport") + # watch_folders should now include the reimport path + assert reimport_path in mock_settings.watch_folders + + +@pytest.mark.unit +class TestStartupReset: + """Tests for perform_startup_reset().""" + + def test_startup_reset_calls_full_reset(self): + from app.utils.system_reset import perform_startup_reset + + with patch("app.utils.system_reset.perform_full_reset") as mock_reset: + with patch("app.database.SessionLocal") as mock_sl: + mock_db = mock_sl.return_value + perform_startup_reset() + + mock_reset.assert_called_once_with(mock_db) + mock_db.close.assert_called_once() + + def test_startup_reset_handles_errors(self): + from app.utils.system_reset import perform_startup_reset + + with patch("app.utils.system_reset.perform_full_reset", side_effect=RuntimeError("boom")): + with patch("app.database.SessionLocal") as mock_sl: + mock_db = mock_sl.return_value + # Should not raise + perform_startup_reset() + mock_db.rollback.assert_called_once() + mock_db.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# Integration tests for API endpoints +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +class TestSystemResetApi: + """Tests for the /api/admin/system-reset/ endpoints.""" + + def test_full_reset_requires_admin(self, client): + """Non-admin users get 403.""" + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "DELETE"}, + ) + assert response.status_code == 403 + + def test_full_reset_requires_feature_flag(self, client): + """Returns 404 when ENABLE_FACTORY_RESET is false.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = False + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "DELETE"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + assert response.status_code == 404 + + def test_full_reset_requires_confirmation(self, client): + """Wrong confirmation string gets 400.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "WRONG"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 400 + + def test_reimport_requires_confirmation(self, client): + """Wrong confirmation string gets 400.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + response = client.post( + "/api/admin/system-reset/reimport", + json={"confirmation": "WRONG"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 400 + + def test_status_endpoint(self, client): + """The status endpoint returns feature-flag state.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + response = client.get("/api/admin/system-reset/status") + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 200 + data = response.json() + assert "enabled" in data + assert "factory_reset_on_startup" in data + + def test_full_reset_success(self, client): + """Full reset succeeds with correct confirmation and feature flag.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + with patch( + "app.utils.system_reset.perform_full_reset", return_value={"database": {}, "filesystem": {}} + ): + response = client.post( + "/api/admin/system-reset/full", + json={"confirmation": "DELETE"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + def test_reimport_success(self, client): + """Reimport succeeds with correct confirmation.""" + from app.api.system_reset import _require_admin + + client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True} + try: + with patch("app.api.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + with patch( + "app.utils.system_reset.perform_reset_and_reimport", + return_value={"database": {}, "filesystem": {}, "reimport": {"files_moved": 3}}, + ): + response = client.post( + "/api/admin/system-reset/reimport", + json={"confirmation": "REIMPORT"}, + ) + finally: + client.app.dependency_overrides.pop(_require_admin, None) + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +# --------------------------------------------------------------------------- +# Integration tests for the view +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +class TestSystemResetView: + """Tests for the /admin/system-reset view.""" + + def test_view_redirects_when_disabled(self, client): + """When ENABLE_FACTORY_RESET=False, accessing the page redirects away.""" + with client: + client.cookies.set("session", "test") + with patch("app.views.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = False + response = client.get("/admin/system-reset", follow_redirects=False) + # Redirect to /settings (302) when disabled, or to login (302/307) when unauthenticated + assert response.status_code in (302, 307) + + def test_view_requires_auth(self, client): + """Unauthenticated users are redirected away from the page.""" + with patch("app.views.system_reset.settings") as mock_s: + mock_s.enable_factory_reset = True + mock_s.factory_reset_on_startup = False + response = client.get("/admin/system-reset", follow_redirects=False) + # Should redirect to login since there's no active session + assert response.status_code in (302, 307)