feat(system-reset): add system reset and factory reset feature
- Add FACTORY_RESET_ON_STARTUP and ENABLE_FACTORY_RESET config settings - Create app/utils/system_reset.py with core reset logic (wipe DB + files, reimport) - Create app/api/system_reset.py with admin-only API endpoints - Create app/views/system_reset.py with admin-only UI view - Create frontend/templates/system_reset.html with confirmation dialogs - Auto-reset on startup when FACTORY_RESET_ON_STARTUP=true - Re-import uses watch folder mechanism for re-ingestion - Register routers in API and views init files - Add i18n keys and SETTING_METADATA entries - Add nav links in base.html (desktop + mobile) Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -43,6 +43,7 @@ from app.api.shared_links import public_router as shared_links_public_router
|
||||
from app.api.shared_links import router as shared_links_router
|
||||
from app.api.similarity import router as similarity_router
|
||||
from app.api.subscriptions import router as subscriptions_router
|
||||
from app.api.system_reset import router as system_reset_router
|
||||
from app.api.translation import router as translation_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
|
||||
@@ -97,4 +98,5 @@ router.include_router(audit_logs_router)
|
||||
router.include_router(i18n_router)
|
||||
router.include_router(mobile_router)
|
||||
router.include_router(compliance_router)
|
||||
router.include_router(system_reset_router)
|
||||
router.include_router(translation_router)
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
System reset API endpoints for DocuElevate.
|
||||
|
||||
Provides admin-only REST endpoints for:
|
||||
- Full system reset (wipe all user data)
|
||||
- Reset with re-import (move originals → reimport folder, wipe, re-ingest)
|
||||
|
||||
Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and
|
||||
admin privileges.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"])
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
def _require_feature_enabled() -> None:
|
||||
"""Raise 404 when the factory-reset feature flag is off."""
|
||||
if not settings.enable_factory_reset:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.",
|
||||
)
|
||||
|
||||
|
||||
class ResetRequest(BaseModel):
|
||||
"""Body for system reset endpoints. Requires explicit confirmation."""
|
||||
|
||||
confirmation: str
|
||||
|
||||
|
||||
@router.post("/full")
|
||||
async def full_reset(
|
||||
body: ResetRequest,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Wipe all user data (database + work-files).
|
||||
|
||||
The caller must send ``{"confirmation": "DELETE"}`` to proceed.
|
||||
"""
|
||||
_require_feature_enabled()
|
||||
|
||||
if body.confirmation != "DELETE":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.',
|
||||
)
|
||||
|
||||
from app.utils.system_reset import perform_full_reset
|
||||
|
||||
try:
|
||||
result = perform_full_reset(db)
|
||||
except Exception as exc:
|
||||
logger.exception("Full system reset failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"System reset failed: {exc}",
|
||||
) from exc
|
||||
|
||||
return {"status": "ok", "result": result}
|
||||
|
||||
|
||||
@router.post("/reimport")
|
||||
async def reset_and_reimport(
|
||||
body: ResetRequest,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, and
|
||||
configure the reimport folder as a watch folder for automatic
|
||||
re-ingestion.
|
||||
|
||||
The caller must send ``{"confirmation": "REIMPORT"}`` to proceed.
|
||||
"""
|
||||
_require_feature_enabled()
|
||||
|
||||
if body.confirmation != "REIMPORT":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.',
|
||||
)
|
||||
|
||||
from app.utils.system_reset import perform_reset_and_reimport
|
||||
|
||||
try:
|
||||
result = perform_reset_and_reimport(db)
|
||||
except Exception as exc:
|
||||
logger.exception("Reset-and-reimport failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Reset and reimport failed: {exc}",
|
||||
) from exc
|
||||
|
||||
return {"status": "ok", "result": result}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def reset_status(_admin: AdminUser) -> dict:
|
||||
"""Return whether the system reset feature is enabled."""
|
||||
return {
|
||||
"enabled": settings.enable_factory_reset,
|
||||
"factory_reset_on_startup": settings.factory_reset_on_startup,
|
||||
}
|
||||
@@ -639,6 +639,25 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
# System reset / factory reset settings
|
||||
factory_reset_on_startup: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When enabled, DocuElevate wipes all user data (database rows and "
|
||||
"work-files on disk) on every startup so the instance always comes "
|
||||
"up in a clean, fresh state. Useful for demo or testing environments. "
|
||||
"Default: False."
|
||||
),
|
||||
)
|
||||
enable_factory_reset: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Show the 'System Reset' page in the admin UI. When enabled, "
|
||||
"administrators can trigger a full data wipe or a wipe-and-reimport "
|
||||
"directly from the web interface. Default: False."
|
||||
),
|
||||
)
|
||||
|
||||
# PDF/A archival conversion settings
|
||||
enable_pdfa_conversion: bool = Field(
|
||||
default=False,
|
||||
|
||||
@@ -170,6 +170,12 @@ async def lifespan(app: FastAPI):
|
||||
# Startup: Initialize database
|
||||
init_db() # Create tables if they don't exist
|
||||
|
||||
# Factory reset on startup — wipe all user data before anything else
|
||||
if settings.factory_reset_on_startup:
|
||||
from app.utils.system_reset import perform_startup_reset
|
||||
|
||||
perform_startup_reset()
|
||||
|
||||
# Load settings from database after DB initialization
|
||||
from app.database import SessionLocal
|
||||
from app.utils.config_loader import load_settings_from_db
|
||||
|
||||
@@ -1900,6 +1900,28 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"factory_reset_on_startup": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Wipe all user data on every startup so the instance always starts fresh. "
|
||||
"Useful for demo/testing environments. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"enable_factory_reset": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Show the System Reset page in the admin UI. Allows administrators to "
|
||||
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Backup / Restore
|
||||
"backup_enabled": {
|
||||
"category": "Backup",
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
System reset utilities for DocuElevate.
|
||||
|
||||
Provides functions to:
|
||||
- Wipe all user data (database rows + work-files on disk) for a fresh start.
|
||||
- Wipe with re-import: move original files to a dedicated folder, wipe
|
||||
everything, then let the watch-folder mechanism re-ingest the files.
|
||||
|
||||
Security: All public functions in this module require admin-level access.
|
||||
They MUST only be invoked from admin-guarded API/view endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Subdirectories inside *workdir* that contain user-generated data.
|
||||
# Everything else (app code, static assets, config) is left untouched.
|
||||
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
|
||||
|
||||
# JSON cache files written by watch-folder / ingest tasks.
|
||||
_CACHE_FILES = (
|
||||
"watch_folder_processed.json",
|
||||
"ftp_ingest_processed.json",
|
||||
"sftp_ingest_processed.json",
|
||||
"dropbox_ingest_processed.json",
|
||||
"gdrive_ingest_processed.json",
|
||||
"onedrive_ingest_processed.json",
|
||||
"nextcloud_ingest_processed.json",
|
||||
"s3_ingest_processed.json",
|
||||
"webdav_ingest_processed.json",
|
||||
"processed_mails.json",
|
||||
"credential_failures.json",
|
||||
)
|
||||
|
||||
# The folder name used for storing files prior to re-import.
|
||||
REIMPORT_FOLDER_NAME = "reimport"
|
||||
|
||||
|
||||
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
|
||||
"""Delete user data subdirectories and cache files inside *workdir*.
|
||||
|
||||
Leaves the workdir directory itself intact so the application can
|
||||
continue to write into it. Also leaves any files that do not belong
|
||||
to the known data subdirectories or caches.
|
||||
|
||||
Returns:
|
||||
A dict with counts of deleted directories and files.
|
||||
"""
|
||||
workdir_path = Path(workdir)
|
||||
deleted_dirs = 0
|
||||
deleted_files = 0
|
||||
|
||||
# Remove data subdirectories
|
||||
for subdir in _USER_DATA_SUBDIRS:
|
||||
target = workdir_path / subdir
|
||||
if target.is_dir():
|
||||
shutil.rmtree(target)
|
||||
logger.info("Deleted data directory: %s", target)
|
||||
deleted_dirs += 1
|
||||
|
||||
# Remove cache / state JSON files
|
||||
for cache_file in _CACHE_FILES:
|
||||
target = workdir_path / cache_file
|
||||
if target.is_file():
|
||||
target.unlink()
|
||||
logger.info("Deleted cache file: %s", target)
|
||||
deleted_files += 1
|
||||
|
||||
# Also remove user_wf_*.json files (per-user watch folder caches)
|
||||
for f in workdir_path.glob("user_wf_*.json"):
|
||||
f.unlink()
|
||||
logger.info("Deleted user watch-folder cache: %s", f)
|
||||
deleted_files += 1
|
||||
|
||||
# Remove loose files in workdir root that are user uploads (uuid-named
|
||||
# files like "a1b2c3d4-…pdf") but NOT application config files.
|
||||
for entry in workdir_path.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() in {
|
||||
".pdf",
|
||||
".png",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".tiff",
|
||||
".tif",
|
||||
".docx",
|
||||
".doc",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".pptx",
|
||||
".heic",
|
||||
".heif",
|
||||
".webp",
|
||||
".bmp",
|
||||
".gif",
|
||||
".txt",
|
||||
".rtf",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".csv",
|
||||
".pages",
|
||||
".numbers",
|
||||
".keynote",
|
||||
}:
|
||||
entry.unlink()
|
||||
logger.info("Deleted loose workdir file: %s", entry)
|
||||
deleted_files += 1
|
||||
|
||||
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
|
||||
|
||||
|
||||
def _wipe_database(db: Session) -> dict[str, int]:
|
||||
"""Delete all user-generated rows from the database.
|
||||
|
||||
Preserves schema (tables, migrations) and system-seeded rows that will
|
||||
be re-created on the next startup (subscription plans, default pipeline,
|
||||
scheduled jobs, compliance templates).
|
||||
|
||||
Returns:
|
||||
A dict mapping table name → number of rows deleted.
|
||||
"""
|
||||
from app.models import (
|
||||
AuditLog,
|
||||
BackupRecord,
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
InAppNotification,
|
||||
ProcessingLog,
|
||||
SavedSearch,
|
||||
SettingsAuditLog,
|
||||
SharedLink,
|
||||
UserImapAccount,
|
||||
UserIntegration,
|
||||
UserNotificationPreference,
|
||||
UserNotificationTarget,
|
||||
)
|
||||
|
||||
# Order matters: delete children before parents to respect FK constraints.
|
||||
tables_to_wipe: list[tuple[str, type]] = [
|
||||
("file_processing_steps", FileProcessingStep),
|
||||
("processing_logs", ProcessingLog),
|
||||
("shared_links", SharedLink),
|
||||
("in_app_notifications", InAppNotification),
|
||||
("user_notification_preferences", UserNotificationPreference),
|
||||
("user_notification_targets", UserNotificationTarget),
|
||||
("user_imap_accounts", UserImapAccount),
|
||||
("user_integrations", UserIntegration),
|
||||
("saved_searches", SavedSearch),
|
||||
("settings_audit_log", SettingsAuditLog),
|
||||
("audit_logs", AuditLog),
|
||||
("backup_records", BackupRecord),
|
||||
("document_metadata", DocumentMetadata),
|
||||
("files", FileRecord),
|
||||
]
|
||||
|
||||
result: dict[str, int] = {}
|
||||
for table_name, model in tables_to_wipe:
|
||||
try:
|
||||
count = db.query(model).delete()
|
||||
result[table_name] = count
|
||||
logger.info("Wiped %d rows from %s", count, table_name)
|
||||
except Exception:
|
||||
logger.exception("Failed to wipe table %s", table_name)
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
db.commit()
|
||||
return result
|
||||
|
||||
|
||||
def perform_full_reset(db: Session) -> dict:
|
||||
"""Perform a complete system reset: wipe database rows + work-files.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database`` and ``filesystem`` sub-dicts.
|
||||
"""
|
||||
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
|
||||
|
||||
db_result = _wipe_database(db)
|
||||
fs_result = _wipe_workdir_data(settings.workdir)
|
||||
|
||||
logger.warning(">>> SYSTEM RESET complete <<<")
|
||||
return {"database": db_result, "filesystem": fs_result}
|
||||
|
||||
|
||||
def perform_reset_and_reimport(db: Session) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, then
|
||||
configure the reimport folder as a watch folder for re-ingestion.
|
||||
|
||||
The watch-folder scanner (``scan_all_watch_folders``) will pick up
|
||||
the files on its next periodic run and process them exactly as if
|
||||
they had been freshly uploaded — respecting the same backoff
|
||||
strategy, size limits, and rate limits.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
|
||||
"""
|
||||
workdir_path = Path(settings.workdir)
|
||||
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
|
||||
original_dir = workdir_path / "original"
|
||||
|
||||
# 1. Collect original files
|
||||
files_moved = 0
|
||||
reimport_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if original_dir.is_dir():
|
||||
for entry in original_dir.iterdir():
|
||||
if entry.is_file():
|
||||
dest = reimport_dir / entry.name
|
||||
# Avoid overwriting: append counter if name clash
|
||||
if dest.exists():
|
||||
stem = dest.stem
|
||||
suffix = dest.suffix
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = reimport_dir / f"{stem}_{counter}{suffix}"
|
||||
counter += 1
|
||||
shutil.copy2(str(entry), str(dest))
|
||||
files_moved += 1
|
||||
|
||||
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
|
||||
|
||||
# 2. Perform the full reset (wipe DB + other workdir data)
|
||||
reset_result = perform_full_reset(db)
|
||||
|
||||
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
|
||||
# and set up watch folder config to point at it.
|
||||
_configure_reimport_watch_folder(str(reimport_dir))
|
||||
|
||||
reset_result["reimport"] = {
|
||||
"files_moved": files_moved,
|
||||
"reimport_folder": str(reimport_dir),
|
||||
}
|
||||
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
|
||||
return reset_result
|
||||
|
||||
|
||||
def _configure_reimport_watch_folder(reimport_path: str) -> None:
|
||||
"""Append *reimport_path* to the application's watch-folder list.
|
||||
|
||||
The watch-folder scanner uses ``settings.watch_folders`` (a
|
||||
comma-separated string). We mutate the runtime setting so the
|
||||
next scan picks up the folder. We also set
|
||||
``watch_folder_delete_after_process = True`` so files are cleaned
|
||||
up after successful processing.
|
||||
"""
|
||||
current = getattr(settings, "watch_folders", None) or ""
|
||||
folders = [f.strip() for f in current.split(",") if f.strip()]
|
||||
|
||||
if reimport_path not in folders:
|
||||
folders.append(reimport_path)
|
||||
|
||||
# Mutate runtime settings (not persisted to .env — ephemeral)
|
||||
object.__setattr__(settings, "watch_folders", ",".join(folders))
|
||||
object.__setattr__(settings, "watch_folder_delete_after_process", True)
|
||||
logger.info("Configured reimport watch folder: %s", reimport_path)
|
||||
|
||||
|
||||
def perform_startup_reset() -> None:
|
||||
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
|
||||
|
||||
Wipes database and filesystem data so the instance starts completely
|
||||
fresh. Uses its own DB session so it runs before the normal lifespan
|
||||
seeding logic.
|
||||
"""
|
||||
from app.database import SessionLocal
|
||||
|
||||
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
perform_full_reset(db)
|
||||
except Exception:
|
||||
logger.exception("Factory reset on startup failed")
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
@@ -34,6 +34,7 @@ from app.views.share import router as share_router
|
||||
from app.views.shared_links import router as shared_links_router
|
||||
from app.views.status import router as status_router
|
||||
from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages
|
||||
from app.views.system_reset import router as system_reset_router # System reset / factory reset
|
||||
from app.views.wizard import router as wizard_router
|
||||
|
||||
# Create a main router that includes all the view routers
|
||||
@@ -67,3 +68,4 @@ router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs
|
||||
router.include_router(audit_logs_router) # Comprehensive audit log viewer
|
||||
router.include_router(help_router) # Built-in help / How-To docs
|
||||
router.include_router(compliance_router) # Compliance templates dashboard
|
||||
router.include_router(system_reset_router) # System reset / factory reset
|
||||
|
||||
@@ -94,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None:
|
||||
"allow_signup",
|
||||
getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False),
|
||||
)
|
||||
ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False))
|
||||
|
||||
req = ctx.get("request")
|
||||
if req is not None:
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
System reset view — admin-only UI page.
|
||||
|
||||
Renders a confirmation-heavy page that allows administrators to:
|
||||
1. **Full Reset** — wipe all user data (DB + disk) for a fresh start.
|
||||
2. **Reset & Re-import** — move originals to a reimport folder, wipe,
|
||||
and let the watch-folder mechanism re-ingest them.
|
||||
|
||||
Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import Depends, Request
|
||||
from fastapi.responses import RedirectResponse, Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.views.base import APIRouter, get_db, require_login, templates
|
||||
from app.views.settings import require_admin_access
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/admin/system-reset")
|
||||
@require_login
|
||||
@require_admin_access
|
||||
async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response:
|
||||
"""Render the system reset administration page."""
|
||||
if not settings.enable_factory_reset:
|
||||
return RedirectResponse(url="/settings", status_code=302)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"system_reset.html",
|
||||
{
|
||||
"request": request,
|
||||
"factory_reset_on_startup": settings.factory_reset_on_startup,
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user