fix: resolve merge conflicts with main branch
Merge origin/main into feature branch, resolving 18 conflicts: - app/api/__init__.py: add classification_rules_router alongside system_reset_router - app/config.py: keep system reset settings from main - app/models.py: keep ClassificationRuleModel alongside main's models - app/utils/settings_service.py: keep factory reset settings from main - app/views/__init__.py: keep system_reset_router from main - tests/conftest.py: add ClassificationRuleModel import - migrations/env.py: add ClassificationRuleModel import - .env.demo, docs/*, frontend/*, mobile/*: keep additions from main - BUILD_DATE, GIT_SHA, RUNTIME_INFO, VERSION, CHANGELOG.md: accept main's version Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -7,6 +7,11 @@ GOTENBERG_URL=http://gotenberg:3000
|
||||
ALLOW_FILE_DELETE=true # Allow deletion of file records
|
||||
COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2)
|
||||
|
||||
# **System Reset / Factory Reset**
|
||||
# FACTORY_RESET_ON_STARTUP=false # Wipe all user data on every startup (demo/testing only)
|
||||
# ENABLE_FACTORY_RESET=false # Show the System Reset page in admin UI
|
||||
|
||||
|
||||
# **Logging**
|
||||
# LOG_LEVEL controls the Python root-logger level.
|
||||
# Accepted values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO).
|
||||
|
||||
@@ -44,6 +44,18 @@ jobs:
|
||||
- run: ruff check app/ tests/
|
||||
- run: ruff format --check app/ tests/
|
||||
|
||||
migration-chain:
|
||||
name: Alembic Migration Chain Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
- name: Validate migration chain
|
||||
run: python scripts/check_alembic_migrations.py
|
||||
|
||||
html-lint:
|
||||
name: HTML Accessibility Lint
|
||||
runs-on: ubuntu-latest
|
||||
@@ -138,7 +150,7 @@ jobs:
|
||||
build:
|
||||
name: Build & Push Docker Image
|
||||
runs-on: ubuntu-latest
|
||||
needs: [run-tests, mypy, dependency-scan, html-lint]
|
||||
needs: [run-tests, mypy, dependency-scan, html-lint, migration-chain]
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
||||
@@ -48,6 +48,16 @@ repos:
|
||||
.env.demo
|
||||
)$
|
||||
|
||||
# Alembic migration chain validation
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: check-alembic-migrations
|
||||
name: Check Alembic migration chain
|
||||
entry: python scripts/check_alembic_migrations.py
|
||||
language: python
|
||||
pass_filenames: false
|
||||
files: ^migrations/versions/.*\.py$
|
||||
|
||||
# Conventional commits validation
|
||||
- repo: https://github.com/compilerla/conventional-pre-commit
|
||||
rev: v3.0.0
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
2026-03-16T21:40:03Z
|
||||
2026-03-17T09:55:43Z
|
||||
|
||||
@@ -10,6 +10,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
<!-- version list -->
|
||||
|
||||
## v0.154.0 (2026-03-17)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **db**: Address code review feedback - fix comment stripping, type hints, test skip, and docs
|
||||
([`a007b4f`](https://github.com/christianlouis/DocuElevate/commit/a007b4fd98ed327a9874082766e0f3b8c32a1a68))
|
||||
|
||||
### Features
|
||||
|
||||
- **db**: Add migration chain CI validation, pre-commit hook, script template, docs, and tests
|
||||
([`aa49fa3`](https://github.com/christianlouis/DocuElevate/commit/aa49fa3ae6ae0bfbb0f9efa6304cf109eca1c402))
|
||||
|
||||
|
||||
## v0.153.1 (2026-03-16)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **mobile**: Add root index.tsx redirect to prevent stale Hello World screen
|
||||
([`1350aa6`](https://github.com/christianlouis/DocuElevate/commit/1350aa6a5ede19d924a9a4c277ef8089b940ea3c))
|
||||
|
||||
|
||||
## v0.153.0 (2026-03-16)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **system-reset**: Address code review feedback
|
||||
([`7dffdc0`](https://github.com/christianlouis/DocuElevate/commit/7dffdc05542cbfe37a91f86b661790038195fef5))
|
||||
|
||||
### Features
|
||||
|
||||
- **system-reset**: Add system reset and factory reset feature
|
||||
([`a88d790`](https://github.com/christianlouis/DocuElevate/commit/a88d790445e5aae9f8ba04e735501d6c93589dec))
|
||||
|
||||
### Testing
|
||||
|
||||
- **system-reset**: Add comprehensive tests and documentation
|
||||
([`96bfba8`](https://github.com/christianlouis/DocuElevate/commit/96bfba8057e506f561d21318b0205e521cf67109))
|
||||
|
||||
|
||||
## v0.152.0 (2026-03-16)
|
||||
|
||||
### Features
|
||||
|
||||
+6
-6
@@ -1,10 +1,10 @@
|
||||
DocuElevate Build Information
|
||||
==============================
|
||||
Version: 0.152.0
|
||||
Build Date: 2026-03-16T21:40:03Z
|
||||
Git Commit: 793b6d3e2562518cb404579f656e82a6dc60da02
|
||||
Git Short SHA: 793b6d3
|
||||
Version: 0.154.0
|
||||
Build Date: 2026-03-17T09:55:43Z
|
||||
Git Commit: 7b7554decf527d2c9f7eee653a1f503249b7d138
|
||||
Git Short SHA: 7b7554d
|
||||
Git Branch: main
|
||||
Commit Date: 2026-03-16T22:39:40+01:00
|
||||
Build Timestamp: 2026-03-16T21:40:03Z
|
||||
Commit Date: 2026-03-17T10:55:10+01:00
|
||||
Build Timestamp: 2026-03-17T09:55:43Z
|
||||
==============================
|
||||
|
||||
@@ -44,6 +44,7 @@ from app.api.shared_links import public_router as shared_links_public_router
|
||||
from app.api.shared_links import router as shared_links_router
|
||||
from app.api.similarity import router as similarity_router
|
||||
from app.api.subscriptions import router as subscriptions_router
|
||||
from app.api.system_reset import router as system_reset_router
|
||||
from app.api.translation import router as translation_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
|
||||
@@ -98,5 +99,6 @@ router.include_router(audit_logs_router)
|
||||
router.include_router(i18n_router)
|
||||
router.include_router(mobile_router)
|
||||
router.include_router(compliance_router)
|
||||
router.include_router(system_reset_router)
|
||||
router.include_router(translation_router)
|
||||
router.include_router(classification_rules_router)
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
System reset API endpoints for DocuElevate.
|
||||
|
||||
Provides admin-only REST endpoints for:
|
||||
- Full system reset (wipe all user data)
|
||||
- Reset with re-import (move originals → reimport folder, wipe, re-ingest)
|
||||
|
||||
Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and
|
||||
admin privileges.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"])
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin. Raises 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
def _require_feature_enabled() -> None:
|
||||
"""Raise 404 when the factory-reset feature flag is off."""
|
||||
if not settings.enable_factory_reset:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.",
|
||||
)
|
||||
|
||||
|
||||
class ResetRequest(BaseModel):
|
||||
"""Body for system reset endpoints. Requires explicit confirmation."""
|
||||
|
||||
confirmation: str
|
||||
|
||||
|
||||
@router.post("/full")
|
||||
async def full_reset(
|
||||
body: ResetRequest,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Wipe all user data (database + work-files).
|
||||
|
||||
The caller must send ``{"confirmation": "DELETE"}`` to proceed.
|
||||
"""
|
||||
_require_feature_enabled()
|
||||
|
||||
if body.confirmation != "DELETE":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.',
|
||||
)
|
||||
|
||||
from app.utils.system_reset import perform_full_reset
|
||||
|
||||
try:
|
||||
result = perform_full_reset(db)
|
||||
except Exception as exc:
|
||||
logger.exception("Full system reset failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"System reset failed: {exc}",
|
||||
) from exc
|
||||
|
||||
return {"status": "ok", "result": result}
|
||||
|
||||
|
||||
@router.post("/reimport")
|
||||
async def reset_and_reimport(
|
||||
body: ResetRequest,
|
||||
_admin: AdminUser,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, and
|
||||
configure the reimport folder as a watch folder for automatic
|
||||
re-ingestion.
|
||||
|
||||
The caller must send ``{"confirmation": "REIMPORT"}`` to proceed.
|
||||
"""
|
||||
_require_feature_enabled()
|
||||
|
||||
if body.confirmation != "REIMPORT":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.',
|
||||
)
|
||||
|
||||
from app.utils.system_reset import perform_reset_and_reimport
|
||||
|
||||
try:
|
||||
result = perform_reset_and_reimport(db)
|
||||
except Exception as exc:
|
||||
logger.exception("Reset-and-reimport failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Reset and reimport failed: {exc}",
|
||||
) from exc
|
||||
|
||||
return {"status": "ok", "result": result}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def reset_status(_admin: AdminUser) -> dict:
|
||||
"""Return whether the system reset feature is enabled."""
|
||||
return {
|
||||
"enabled": settings.enable_factory_reset,
|
||||
"factory_reset_on_startup": settings.factory_reset_on_startup,
|
||||
}
|
||||
@@ -639,6 +639,25 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
# System reset / factory reset settings
|
||||
factory_reset_on_startup: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When enabled, DocuElevate wipes all user data (database rows and "
|
||||
"work-files on disk) on every startup so the instance always comes "
|
||||
"up in a clean, fresh state. Useful for demo or testing environments. "
|
||||
"Default: False."
|
||||
),
|
||||
)
|
||||
enable_factory_reset: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Show the 'System Reset' page in the admin UI. When enabled, "
|
||||
"administrators can trigger a full data wipe or a wipe-and-reimport "
|
||||
"directly from the web interface. Default: False."
|
||||
),
|
||||
)
|
||||
|
||||
# PDF/A archival conversion settings
|
||||
enable_pdfa_conversion: bool = Field(
|
||||
default=False,
|
||||
|
||||
@@ -170,6 +170,12 @@ async def lifespan(app: FastAPI):
|
||||
# Startup: Initialize database
|
||||
init_db() # Create tables if they don't exist
|
||||
|
||||
# Factory reset on startup — wipe all user data before anything else
|
||||
if settings.factory_reset_on_startup:
|
||||
from app.utils.system_reset import perform_startup_reset
|
||||
|
||||
perform_startup_reset()
|
||||
|
||||
# Load settings from database after DB initialization
|
||||
from app.database import SessionLocal
|
||||
from app.utils.config_loader import load_settings_from_db
|
||||
|
||||
@@ -980,6 +980,7 @@ class ClassificationRuleModel(Base):
|
||||
__table_args__ = (UniqueConstraint("owner_id", "name", name="uq_classification_rules_owner_name"),)
|
||||
|
||||
|
||||
|
||||
class MobileDevice(Base):
|
||||
"""Registered mobile device for push notifications.
|
||||
|
||||
|
||||
@@ -1900,6 +1900,28 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"factory_reset_on_startup": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Wipe all user data on every startup so the instance always starts fresh. "
|
||||
"Useful for demo/testing environments. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": True,
|
||||
},
|
||||
"enable_factory_reset": {
|
||||
"category": "Feature Flags",
|
||||
"description": (
|
||||
"Show the System Reset page in the admin UI. Allows administrators to "
|
||||
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Backup / Restore
|
||||
"backup_enabled": {
|
||||
"category": "Backup",
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
"""
|
||||
System reset utilities for DocuElevate.
|
||||
|
||||
Provides functions to:
|
||||
- Wipe all user data (database rows + work-files on disk) for a fresh start.
|
||||
- Wipe with re-import: move original files to a dedicated folder, wipe
|
||||
everything, then let the watch-folder mechanism re-ingest the files.
|
||||
|
||||
Security: All public functions in this module require admin-level access.
|
||||
They MUST only be invoked from admin-guarded API/view endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Subdirectories inside *workdir* that contain user-generated data.
|
||||
# Everything else (app code, static assets, config) is left untouched.
|
||||
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
|
||||
|
||||
# JSON cache files written by watch-folder / ingest tasks.
|
||||
_CACHE_FILES = (
|
||||
"watch_folder_processed.json",
|
||||
"ftp_ingest_processed.json",
|
||||
"sftp_ingest_processed.json",
|
||||
"dropbox_ingest_processed.json",
|
||||
"gdrive_ingest_processed.json",
|
||||
"onedrive_ingest_processed.json",
|
||||
"nextcloud_ingest_processed.json",
|
||||
"s3_ingest_processed.json",
|
||||
"webdav_ingest_processed.json",
|
||||
"processed_mails.json",
|
||||
"credential_failures.json",
|
||||
)
|
||||
|
||||
# The folder name used for storing files prior to re-import.
|
||||
REIMPORT_FOLDER_NAME = "reimport"
|
||||
|
||||
|
||||
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
|
||||
"""Delete user data subdirectories and cache files inside *workdir*.
|
||||
|
||||
Leaves the workdir directory itself intact so the application can
|
||||
continue to write into it. Also leaves any files that do not belong
|
||||
to the known data subdirectories or caches.
|
||||
|
||||
Returns:
|
||||
A dict with counts of deleted directories and files.
|
||||
"""
|
||||
workdir_path = Path(workdir)
|
||||
deleted_dirs = 0
|
||||
deleted_files = 0
|
||||
|
||||
# Remove data subdirectories
|
||||
for subdir in _USER_DATA_SUBDIRS:
|
||||
target = workdir_path / subdir
|
||||
if target.is_dir():
|
||||
shutil.rmtree(target)
|
||||
logger.info("Deleted data directory: %s", target)
|
||||
deleted_dirs += 1
|
||||
|
||||
# Remove cache / state JSON files
|
||||
for cache_file in _CACHE_FILES:
|
||||
target = workdir_path / cache_file
|
||||
if target.is_file():
|
||||
target.unlink()
|
||||
logger.info("Deleted cache file: %s", target)
|
||||
deleted_files += 1
|
||||
|
||||
# Also remove user_wf_*.json files (per-user watch folder caches)
|
||||
for f in workdir_path.glob("user_wf_*.json"):
|
||||
f.unlink()
|
||||
logger.info("Deleted user watch-folder cache: %s", f)
|
||||
deleted_files += 1
|
||||
|
||||
# Remove loose files in workdir root that are user uploads (uuid-named
|
||||
# files like "a1b2c3d4-…pdf") but NOT application config files.
|
||||
for entry in workdir_path.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() in {
|
||||
".pdf",
|
||||
".png",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".tiff",
|
||||
".tif",
|
||||
".docx",
|
||||
".doc",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".pptx",
|
||||
".heic",
|
||||
".heif",
|
||||
".webp",
|
||||
".bmp",
|
||||
".gif",
|
||||
".txt",
|
||||
".rtf",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".csv",
|
||||
".pages",
|
||||
".numbers",
|
||||
".keynote",
|
||||
}:
|
||||
entry.unlink()
|
||||
logger.info("Deleted loose workdir file: %s", entry)
|
||||
deleted_files += 1
|
||||
|
||||
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
|
||||
|
||||
|
||||
def _wipe_database(db: Session) -> dict[str, int]:
|
||||
"""Delete all user-generated rows from the database.
|
||||
|
||||
Preserves schema (tables, migrations) and system-seeded rows that will
|
||||
be re-created on the next startup (subscription plans, default pipeline,
|
||||
scheduled jobs, compliance templates).
|
||||
|
||||
Returns:
|
||||
A dict mapping table name → number of rows deleted.
|
||||
"""
|
||||
from app.models import (
|
||||
AuditLog,
|
||||
BackupRecord,
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
InAppNotification,
|
||||
ProcessingLog,
|
||||
SavedSearch,
|
||||
SettingsAuditLog,
|
||||
SharedLink,
|
||||
UserImapAccount,
|
||||
UserIntegration,
|
||||
UserNotificationPreference,
|
||||
UserNotificationTarget,
|
||||
)
|
||||
|
||||
# Order matters: delete children before parents to respect FK constraints.
|
||||
tables_to_wipe: list[tuple[str, type]] = [
|
||||
("file_processing_steps", FileProcessingStep),
|
||||
("processing_logs", ProcessingLog),
|
||||
("shared_links", SharedLink),
|
||||
("in_app_notifications", InAppNotification),
|
||||
("user_notification_preferences", UserNotificationPreference),
|
||||
("user_notification_targets", UserNotificationTarget),
|
||||
("user_imap_accounts", UserImapAccount),
|
||||
("user_integrations", UserIntegration),
|
||||
("saved_searches", SavedSearch),
|
||||
("settings_audit_log", SettingsAuditLog),
|
||||
("audit_logs", AuditLog),
|
||||
("backup_records", BackupRecord),
|
||||
("document_metadata", DocumentMetadata),
|
||||
("files", FileRecord),
|
||||
]
|
||||
|
||||
result: dict[str, int] = {}
|
||||
for table_name, model in tables_to_wipe:
|
||||
try:
|
||||
count = db.query(model).delete()
|
||||
result[table_name] = count
|
||||
logger.info("Wiped %d rows from %s", count, table_name)
|
||||
except Exception:
|
||||
logger.exception("Failed to wipe table %s during system reset", table_name)
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
db.commit()
|
||||
return result
|
||||
|
||||
|
||||
def perform_full_reset(db: Session) -> dict:
|
||||
"""Perform a complete system reset: wipe database rows + work-files.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database`` and ``filesystem`` sub-dicts.
|
||||
"""
|
||||
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
|
||||
|
||||
db_result = _wipe_database(db)
|
||||
fs_result = _wipe_workdir_data(settings.workdir)
|
||||
|
||||
logger.warning(">>> SYSTEM RESET complete <<<")
|
||||
return {"database": db_result, "filesystem": fs_result}
|
||||
|
||||
|
||||
def perform_reset_and_reimport(db: Session) -> dict:
|
||||
"""Move original files to a reimport folder, wipe everything, then
|
||||
configure the reimport folder as a watch folder for re-ingestion.
|
||||
|
||||
The watch-folder scanner (``scan_all_watch_folders``) will pick up
|
||||
the files on its next periodic run and process them exactly as if
|
||||
they had been freshly uploaded — respecting the same backoff
|
||||
strategy, size limits, and rate limits.
|
||||
|
||||
Args:
|
||||
db: An active SQLAlchemy session.
|
||||
|
||||
Returns:
|
||||
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
|
||||
"""
|
||||
workdir_path = Path(settings.workdir)
|
||||
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
|
||||
original_dir = workdir_path / "original"
|
||||
|
||||
# 1. Collect original files
|
||||
files_moved = 0
|
||||
reimport_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if original_dir.is_dir():
|
||||
for entry in original_dir.iterdir():
|
||||
if entry.is_file():
|
||||
# Validate the resolved path stays within original_dir (path traversal guard)
|
||||
try:
|
||||
entry.resolve().relative_to(original_dir.resolve())
|
||||
except ValueError:
|
||||
logger.warning("Skipping file outside original dir: %s", entry)
|
||||
continue
|
||||
dest = reimport_dir / entry.name
|
||||
# Avoid overwriting: append counter if name clash
|
||||
if dest.exists():
|
||||
stem = dest.stem
|
||||
suffix = dest.suffix
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = reimport_dir / f"{stem}_{counter}{suffix}"
|
||||
counter += 1
|
||||
shutil.copy2(str(entry), str(dest))
|
||||
files_moved += 1
|
||||
|
||||
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
|
||||
|
||||
# 2. Perform the full reset (wipe DB + other workdir data)
|
||||
reset_result = perform_full_reset(db)
|
||||
|
||||
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
|
||||
# and set up watch folder config to point at it.
|
||||
_configure_reimport_watch_folder(str(reimport_dir))
|
||||
|
||||
reset_result["reimport"] = {
|
||||
"files_moved": files_moved,
|
||||
"reimport_folder": str(reimport_dir),
|
||||
}
|
||||
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
|
||||
return reset_result
|
||||
|
||||
|
||||
def _configure_reimport_watch_folder(reimport_path: str) -> None:
|
||||
"""Append *reimport_path* to the application's watch-folder list.
|
||||
|
||||
The watch-folder scanner uses ``settings.watch_folders`` (a
|
||||
comma-separated string). We mutate the runtime setting so the
|
||||
next scan picks up the folder. We also set
|
||||
``watch_folder_delete_after_process = True`` so files are cleaned
|
||||
up after successful processing.
|
||||
"""
|
||||
current = getattr(settings, "watch_folders", None) or ""
|
||||
folders = [f.strip() for f in current.split(",") if f.strip()]
|
||||
|
||||
if reimport_path not in folders:
|
||||
folders.append(reimport_path)
|
||||
|
||||
# Mutate runtime settings (not persisted to .env — ephemeral)
|
||||
object.__setattr__(settings, "watch_folders", ",".join(folders))
|
||||
object.__setattr__(settings, "watch_folder_delete_after_process", True)
|
||||
logger.info("Configured reimport watch folder: %s", reimport_path)
|
||||
|
||||
|
||||
def perform_startup_reset() -> None:
|
||||
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
|
||||
|
||||
Wipes database and filesystem data so the instance starts completely
|
||||
fresh. Uses its own DB session so it runs before the normal lifespan
|
||||
seeding logic.
|
||||
"""
|
||||
from app.database import SessionLocal
|
||||
|
||||
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
perform_full_reset(db)
|
||||
except Exception:
|
||||
logger.exception("Factory reset on startup failed")
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
@@ -34,6 +34,7 @@ from app.views.share import router as share_router
|
||||
from app.views.shared_links import router as shared_links_router
|
||||
from app.views.status import router as status_router
|
||||
from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages
|
||||
from app.views.system_reset import router as system_reset_router # System reset / factory reset
|
||||
from app.views.wizard import router as wizard_router
|
||||
|
||||
# Create a main router that includes all the view routers
|
||||
@@ -67,3 +68,4 @@ router.include_router(scheduled_jobs_router) # Admin scheduled batch jobs
|
||||
router.include_router(audit_logs_router) # Comprehensive audit log viewer
|
||||
router.include_router(help_router) # Built-in help / How-To docs
|
||||
router.include_router(compliance_router) # Compliance templates dashboard
|
||||
router.include_router(system_reset_router) # System reset / factory reset
|
||||
|
||||
@@ -94,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None:
|
||||
"allow_signup",
|
||||
getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False),
|
||||
)
|
||||
ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False))
|
||||
|
||||
req = ctx.get("request")
|
||||
if req is not None:
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
System reset view — admin-only UI page.
|
||||
|
||||
Renders a confirmation-heavy page that allows administrators to:
|
||||
1. **Full Reset** — wipe all user data (DB + disk) for a fresh start.
|
||||
2. **Reset & Re-import** — move originals to a reimport folder, wipe,
|
||||
and let the watch-folder mechanism re-ingest them.
|
||||
|
||||
Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import Depends, Request
|
||||
from fastapi.responses import RedirectResponse, Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.views.base import APIRouter, get_db, require_login, templates
|
||||
from app.views.settings import require_admin_access
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/admin/system-reset")
|
||||
@require_login
|
||||
@require_admin_access
|
||||
async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response:
|
||||
"""Render the system reset administration page."""
|
||||
if not settings.enable_factory_reset:
|
||||
return RedirectResponse(url="/settings", status_code=302)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"system_reset.html",
|
||||
{
|
||||
"request": request,
|
||||
"factory_reset_on_startup": settings.factory_reset_on_startup,
|
||||
},
|
||||
)
|
||||
+61
@@ -2562,3 +2562,64 @@ query GetDocument($id: Int!) {
|
||||
}
|
||||
```
|
||||
Variables: `{ "id": 42 }`
|
||||
|
||||
## System Reset
|
||||
|
||||
Admin-only endpoints for resetting the system to a clean state. Requires `ENABLE_FACTORY_RESET=True`.
|
||||
|
||||
### GET /api/admin/system-reset/status
|
||||
|
||||
Check whether the system reset feature is enabled.
|
||||
|
||||
**Response (200):**
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"factory_reset_on_startup": false
|
||||
}
|
||||
```
|
||||
|
||||
### POST /api/admin/system-reset/full
|
||||
|
||||
Wipe all user data (database + work-files).
|
||||
|
||||
**Request:**
|
||||
```json
|
||||
{
|
||||
"confirmation": "DELETE"
|
||||
}
|
||||
```
|
||||
|
||||
**Response (200):**
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"result": {
|
||||
"database": { "files": 42, "processing_logs": 100 },
|
||||
"filesystem": { "deleted_dirs": 5, "deleted_files": 12 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### POST /api/admin/system-reset/reimport
|
||||
|
||||
Move original files to a reimport folder, wipe everything, and configure the reimport folder as a watch folder for re-ingestion.
|
||||
|
||||
**Request:**
|
||||
```json
|
||||
{
|
||||
"confirmation": "REIMPORT"
|
||||
}
|
||||
```
|
||||
|
||||
**Response (200):**
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"result": {
|
||||
"database": { "files": 42 },
|
||||
"filesystem": { "deleted_dirs": 5, "deleted_files": 12 },
|
||||
"reimport": { "files_moved": 42, "reimport_folder": "/workdir/reimport" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -17,6 +17,8 @@ Configuration is primarily done through environment variables specified in a `.e
|
||||
| `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` |
|
||||
| `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` |
|
||||
| `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` |
|
||||
| `FACTORY_RESET_ON_STARTUP` | Wipe all user data on every startup (demo/testing). | `false` |
|
||||
| `ENABLE_FACTORY_RESET` | Show the System Reset page in the admin UI. | `false` |
|
||||
|
||||
### Batch Processing Settings
|
||||
|
||||
@@ -1856,6 +1858,52 @@ BACKUP_RETAIN_WEEKLY=13
|
||||
|
||||
You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables.
|
||||
|
||||
## System Reset / Factory Reset
|
||||
|
||||
DocuElevate provides two mechanisms for resetting the system to a clean state. Both are **disabled by default** and must be explicitly enabled.
|
||||
|
||||
### Automatic Reset on Startup
|
||||
|
||||
Set `FACTORY_RESET_ON_STARTUP=true` to wipe all user data (database rows and work-files) every time the application starts. This is useful for demo, testing, or ephemeral environments where you always want a fresh instance.
|
||||
|
||||
```dotenv
|
||||
FACTORY_RESET_ON_STARTUP=true
|
||||
```
|
||||
|
||||
> **Warning:** This destroys all documents, processing history, audit logs, and backups on every restart. Application settings and configuration are preserved.
|
||||
|
||||
### Admin UI Reset Page
|
||||
|
||||
Set `ENABLE_FACTORY_RESET=true` to display the **System Reset** page in the admin navigation menu. From this page, administrators can:
|
||||
|
||||
| Action | Confirmation | Description |
|
||||
|--------|-------------|-------------|
|
||||
| **Full Reset** | Type `DELETE` | Wipes all database rows and work-files. The system returns to its initial state. |
|
||||
| **Reset & Re-import** | Type `REIMPORT` | Copies original files to a `reimport/` folder inside the workdir, wipes everything, then configures the reimport folder as a watch folder so files are automatically re-ingested with the same processing pipeline, rate limits, and backoff strategy as regular uploads. |
|
||||
|
||||
```dotenv
|
||||
ENABLE_FACTORY_RESET=true
|
||||
```
|
||||
|
||||
### API Endpoints
|
||||
|
||||
When `ENABLE_FACTORY_RESET=true`, two admin-only API endpoints are available:
|
||||
|
||||
- `POST /api/admin/system-reset/full` — body: `{"confirmation": "DELETE"}`
|
||||
- `POST /api/admin/system-reset/reimport` — body: `{"confirmation": "REIMPORT"}`
|
||||
- `GET /api/admin/system-reset/status` — returns current feature-flag state
|
||||
|
||||
### What Gets Deleted
|
||||
|
||||
| Deleted | Preserved |
|
||||
|---------|-----------|
|
||||
| All document records (`files` table) | Application settings (`application_settings` table) |
|
||||
| Processing logs and steps | User accounts and profiles |
|
||||
| Audit logs | Subscription plans |
|
||||
| Backup records | Pipelines and scheduled jobs |
|
||||
| Original, processed, and temporary files | The workdir directory itself |
|
||||
| Watch-folder caches and ingestion state | OAuth and integration configuration |
|
||||
|
||||
## Configuration File Location
|
||||
|
||||
The `.env` file should be placed at the root of the project directory. When using Docker Compose, you can reference it with the `env_file` directive in your `docker-compose.yml`.
|
||||
|
||||
@@ -287,6 +287,27 @@ alembic revision --autogenerate -m "describe your change"
|
||||
|
||||
Review the generated file in `migrations/versions/` before applying it.
|
||||
|
||||
> **Tip:** For detailed guidance on naming conventions, idempotent patterns, parallel-branch workflows, and resolving merge conflicts, see the [Migration Workflow Guide](MigrationWorkflow.md).
|
||||
|
||||
### Validating the Migration Chain
|
||||
|
||||
A CI check and pre-commit hook validate that the migration chain has no broken
|
||||
references, duplicate revisions, or diverged heads. Run the check locally:
|
||||
|
||||
```bash
|
||||
python scripts/check_alembic_migrations.py
|
||||
python scripts/check_alembic_migrations.py --verbose # extra detail
|
||||
```
|
||||
|
||||
If you see **"Multiple migration heads detected"**, two branches added
|
||||
migrations from the same parent. Create a merge migration:
|
||||
|
||||
```bash
|
||||
alembic merge heads -m "merge_parallel_branches"
|
||||
```
|
||||
|
||||
For a complete walk-through, see the [Migration Workflow Guide](MigrationWorkflow.md).
|
||||
|
||||
### Automating Migrations in Docker Compose
|
||||
|
||||
Add a short-lived `migrate` service that runs before the API and Worker:
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
# Migration Workflow
|
||||
|
||||
This guide explains how to create, test, and merge Alembic database migrations in DocuElevate — especially when **multiple feature branches** add migrations in parallel.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Quick Reference](#quick-reference)
|
||||
- [Creating a New Migration](#creating-a-new-migration)
|
||||
- [Migration Naming Convention](#migration-naming-convention)
|
||||
- [Idempotent Migration Patterns](#idempotent-migration-patterns)
|
||||
- [Parallel Branch Development](#parallel-branch-development)
|
||||
- [Resolving Migration Conflicts](#resolving-migration-conflicts)
|
||||
- [CI Validation](#ci-validation)
|
||||
- [Pre-commit Hook](#pre-commit-hook)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference
|
||||
|
||||
```bash
|
||||
# Create a new migration after editing app/models.py
|
||||
alembic revision --autogenerate -m "add_foobar_column"
|
||||
|
||||
# Apply all pending migrations
|
||||
alembic upgrade head
|
||||
|
||||
# Check current database version
|
||||
alembic current
|
||||
|
||||
# View migration history
|
||||
alembic history --verbose
|
||||
|
||||
# Detect multiple heads (diverged branches)
|
||||
alembic heads
|
||||
|
||||
# Create a merge migration to resolve multiple heads
|
||||
alembic merge heads -m "merge_parallel_branches"
|
||||
|
||||
# Validate migration chain integrity (CI script)
|
||||
python scripts/check_alembic_migrations.py
|
||||
python scripts/check_alembic_migrations.py --verbose
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Creating a New Migration
|
||||
|
||||
1. **Edit `app/models.py`** — add or modify SQLAlchemy model classes.
|
||||
|
||||
2. **Generate the migration** from the repo root. Use `--rev-id` to set the
|
||||
revision identifier directly (avoids renaming afterwards):
|
||||
|
||||
```bash
|
||||
alembic revision --autogenerate --rev-id 037_add_my_new_table -m "add my new table"
|
||||
```
|
||||
|
||||
This creates `migrations/versions/037_add_my_new_table_add_my_new_table.py`
|
||||
with `revision = "037_add_my_new_table"`. Rename the file to match:
|
||||
|
||||
```bash
|
||||
mv migrations/versions/037_add_my_new_table_add_my_new_table.py \
|
||||
migrations/versions/037_add_my_new_table.py
|
||||
```
|
||||
|
||||
Alternatively, generate with the default hash and then rename:
|
||||
|
||||
```bash
|
||||
alembic revision --autogenerate -m "add_my_new_table"
|
||||
# Rename: mv migrations/versions/<hash>_add_my_new_table.py migrations/versions/037_add_my_new_table.py
|
||||
# Update revision inside the file to match the filename stem.
|
||||
```
|
||||
|
||||
Alembic uses the `migrations/script.py.mako` template to generate the file. The template includes inline comments about idempotent patterns — read them.
|
||||
|
||||
3. **Review the generated code** — autogenerate is helpful but not perfect. Check:
|
||||
- Are new tables and columns detected correctly?
|
||||
- Does the `downgrade()` reverse all changes?
|
||||
- Are SQLite-incompatible operations wrapped in `batch_alter_table()`?
|
||||
|
||||
4. **Test the migration** against a fresh database:
|
||||
|
||||
```bash
|
||||
# Apply
|
||||
alembic upgrade head
|
||||
|
||||
# Rollback
|
||||
alembic downgrade -1
|
||||
|
||||
# Re-apply
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
5. **Run the chain validation**:
|
||||
|
||||
```bash
|
||||
python scripts/check_alembic_migrations.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Migration Naming Convention
|
||||
|
||||
All migration files follow a **sequential numeric prefix** scheme:
|
||||
|
||||
```
|
||||
NNN_short_description.py
|
||||
```
|
||||
|
||||
| Component | Rule |
|
||||
|-----------|------|
|
||||
| `NNN` | Three-digit zero-padded number, incrementing from the previous migration |
|
||||
| `short_description` | Lowercase snake_case summary of the change |
|
||||
|
||||
The **`revision`** variable inside the file **must match the filename stem** exactly:
|
||||
|
||||
```python
|
||||
# File: migrations/versions/037_add_classification_rules.py
|
||||
revision: str = "037_add_classification_rules"
|
||||
down_revision: Union[str, None] = "036_add_document_translation_fields"
|
||||
```
|
||||
|
||||
The CI check (`scripts/check_alembic_migrations.py`) enforces this consistency.
|
||||
|
||||
---
|
||||
|
||||
## Idempotent Migration Patterns
|
||||
|
||||
Migrations should be **idempotent** — safe to run even if the change already exists. This is critical for SQLite compatibility and for recovering from partial failures.
|
||||
|
||||
### Add a Column (only if missing)
|
||||
|
||||
```python
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
|
||||
if "my_table" in inspector.get_table_names():
|
||||
existing = {c["name"] for c in inspector.get_columns("my_table")}
|
||||
if "new_col" not in existing:
|
||||
with op.batch_alter_table("my_table") as batch_op:
|
||||
batch_op.add_column(sa.Column("new_col", sa.String(128), nullable=True))
|
||||
```
|
||||
|
||||
### Create a Table (only if missing)
|
||||
|
||||
```python
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
|
||||
if "new_table" not in inspector.get_table_names():
|
||||
op.create_table(
|
||||
"new_table",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
)
|
||||
```
|
||||
|
||||
### Drop a Column (only if present)
|
||||
|
||||
```python
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
|
||||
if "my_table" in inspector.get_table_names():
|
||||
existing = {c["name"] for c in inspector.get_columns("my_table")}
|
||||
if "new_col" in existing:
|
||||
with op.batch_alter_table("my_table") as batch_op:
|
||||
batch_op.drop_column("new_col")
|
||||
```
|
||||
|
||||
### Use `batch_alter_table` for SQLite
|
||||
|
||||
SQLite does not support `ALTER TABLE DROP COLUMN` or `ALTER TABLE RENAME COLUMN` natively. Alembic's `batch_alter_table` context manager works around this by recreating the table:
|
||||
|
||||
```python
|
||||
with op.batch_alter_table("users") as batch_op:
|
||||
batch_op.add_column(sa.Column("phone", sa.String(20), nullable=True))
|
||||
batch_op.drop_column("fax")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Parallel Branch Development
|
||||
|
||||
When two feature branches both add migrations from the same parent, the migration chain **diverges** into multiple heads. This is normal and expected — Alembic supports it — but the heads must be merged before the code reaches `main`.
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
main: 001 → 002 → 003
|
||||
↘ Branch A: 004_add_widgets
|
||||
↘ Branch B: 004_add_gadgets ← two heads!
|
||||
```
|
||||
|
||||
### How to Avoid Conflicts
|
||||
|
||||
1. **Coordinate** — if two developers are both adding migrations, assign different sequence numbers (e.g., `037_` and `038_`). Even if both depend on `036_`, different numbers prevent filename collisions.
|
||||
|
||||
2. **Rebase early** — before opening a PR, rebase your branch onto the latest `main`:
|
||||
|
||||
```bash
|
||||
git fetch origin main
|
||||
git rebase origin/main
|
||||
```
|
||||
|
||||
If `main` now has a new migration `037_*`, renumber yours to `038_*` and update `down_revision` to point at `037_*`.
|
||||
|
||||
3. **Check for multiple heads** locally:
|
||||
|
||||
```bash
|
||||
python scripts/check_alembic_migrations.py
|
||||
# or
|
||||
alembic heads
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Resolving Migration Conflicts
|
||||
|
||||
If your PR's CI check reports **"Multiple migration heads detected"**, follow these steps:
|
||||
|
||||
### Step 1 — Update Your Branch
|
||||
|
||||
```bash
|
||||
git fetch origin main
|
||||
git merge origin/main
|
||||
# or
|
||||
git rebase origin/main
|
||||
```
|
||||
|
||||
### Step 2 — Check Heads
|
||||
|
||||
```bash
|
||||
python scripts/check_alembic_migrations.py --verbose
|
||||
```
|
||||
|
||||
The output lists the conflicting heads.
|
||||
|
||||
### Step 3 — Create a Merge Migration
|
||||
|
||||
```bash
|
||||
alembic merge heads -m "merge_parallel_branches"
|
||||
```
|
||||
|
||||
This generates a new migration with **two parents** (a merge point):
|
||||
|
||||
```python
|
||||
down_revision = ("037_add_widgets", "037_add_gadgets")
|
||||
```
|
||||
|
||||
### Step 4 — Rename and Validate
|
||||
|
||||
Rename the merge migration to the next sequence number:
|
||||
|
||||
```bash
|
||||
mv migrations/versions/<hash>_merge_parallel_branches.py \
|
||||
migrations/versions/038_merge_parallel_branches.py
|
||||
```
|
||||
|
||||
Update the `revision` inside to match, then validate:
|
||||
|
||||
```bash
|
||||
python scripts/check_alembic_migrations.py
|
||||
```
|
||||
|
||||
### Step 5 — Test
|
||||
|
||||
```bash
|
||||
alembic upgrade head
|
||||
alembic downgrade -1
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CI Validation
|
||||
|
||||
The CI pipeline (`.github/workflows/ci.yml`) includes a **migration-chain** job that runs:
|
||||
|
||||
```bash
|
||||
python scripts/check_alembic_migrations.py
|
||||
```
|
||||
|
||||
This script checks for:
|
||||
|
||||
| Check | Description |
|
||||
|-------|-------------|
|
||||
| Multiple heads | Diverged migration chains that need a merge migration |
|
||||
| Broken references | A `down_revision` that points to a non-existent revision |
|
||||
| Duplicate revisions | Two files declaring the same `revision` identifier |
|
||||
| Filename mismatches | The `revision` variable doesn't match the filename stem |
|
||||
|
||||
The job runs in Stage 1 (fast-fail gates) alongside lint checks. If it fails, the build is blocked until the migration chain is fixed.
|
||||
|
||||
---
|
||||
|
||||
## Pre-commit Hook
|
||||
|
||||
A local pre-commit hook is configured in `.pre-commit-config.yaml` that runs the same check whenever you commit a change to `migrations/versions/`:
|
||||
|
||||
```yaml
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: check-alembic-migrations
|
||||
name: Check Alembic migration chain
|
||||
entry: python scripts/check_alembic_migrations.py
|
||||
language: python
|
||||
pass_filenames: false
|
||||
files: ^migrations/versions/.*\.py$
|
||||
```
|
||||
|
||||
Install the hook:
|
||||
|
||||
```bash
|
||||
pip install pre-commit
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Multiple migration heads detected"
|
||||
|
||||
See [Resolving Migration Conflicts](#resolving-migration-conflicts) above.
|
||||
|
||||
### "Broken chain: revision X references down_revision Y which does not exist"
|
||||
|
||||
You removed or renamed a migration that another migration depends on. Either restore the missing file or update the dependent migration's `down_revision`.
|
||||
|
||||
### "Filename mismatch: file declares revision=X but filename stem is Y"
|
||||
|
||||
The `revision` string inside the Python file must match the filename (without `.py`). Rename the file or update the variable.
|
||||
|
||||
### "relation already exists" when running `alembic upgrade head`
|
||||
|
||||
The database has a table that a pending migration tries to create. Stamp the current state:
|
||||
|
||||
```bash
|
||||
alembic stamp head
|
||||
```
|
||||
|
||||
### Autogenerate doesn't detect my changes
|
||||
|
||||
Ensure all models are imported in `migrations/env.py`. The `from app.models import ...` block at the top must include your new model class.
|
||||
|
||||
### SQLite "no such column" after downgrade
|
||||
|
||||
SQLite has limited `ALTER TABLE` support. Always use `op.batch_alter_table()` for column operations on existing tables.
|
||||
|
||||
---
|
||||
|
||||
## Further Reading
|
||||
|
||||
- [Alembic Tutorial](https://alembic.sqlalchemy.org/en/latest/tutorial.html)
|
||||
- [Alembic Branch / Merge](https://alembic.sqlalchemy.org/en/latest/branches.html)
|
||||
- [Database Configuration Guide](DatabaseConfiguration.md)
|
||||
+33
-1
@@ -279,7 +279,19 @@ If you wish to use **direct FCM/APNs** without Expo's relay, replace the `send_e
|
||||
|
||||
```
|
||||
mobile/
|
||||
├── App.tsx # Root component
|
||||
├── App.tsx # Root component (legacy, not used at runtime)
|
||||
├── app/ # Expo Router file-based routes
|
||||
│ ├── _layout.tsx # Root layout (AuthGuard + providers)
|
||||
│ ├── index.tsx # Root redirect → /(auth)/
|
||||
│ ├── (auth)/ # Unauthenticated route group
|
||||
│ │ ├── _layout.tsx # Stack navigator (headerless)
|
||||
│ │ ├── index.tsx # Welcome screen
|
||||
│ │ └── login.tsx # Login screen
|
||||
│ └── (tabs)/ # Authenticated route group
|
||||
│ ├── _layout.tsx # Tab navigator
|
||||
│ ├── index.tsx # Upload screen (default tab)
|
||||
│ ├── files.tsx # Files screen
|
||||
│ └── profile.tsx # Profile screen
|
||||
├── app.json # Expo/EAS configuration
|
||||
├── eas.json # EAS Build profiles
|
||||
├── package.json
|
||||
@@ -301,6 +313,26 @@ mobile/
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### App shows "Hello World" / default Expo page after update
|
||||
|
||||
If the iOS or Android app shows a generic "Hello World – This is the first page of your app" screen instead of the DocuElevate UI, it means a stale default `index.tsx` file (generated by Expo CLI scaffolding) is being picked up in the `mobile/app/` directory.
|
||||
|
||||
**To fix:**
|
||||
|
||||
1. Delete any leftover default `mobile/app/index.tsx` that is **not** the repository version (the repo version contains a `<Redirect>` to `/(auth)/`).
|
||||
2. Clear the Metro bundler cache and rebuild:
|
||||
```bash
|
||||
cd mobile
|
||||
npx expo start --clear
|
||||
```
|
||||
3. For production builds, run a clean EAS build:
|
||||
```bash
|
||||
eas build --platform ios --clear-cache
|
||||
```
|
||||
|
||||
The repository includes a root `app/index.tsx` that immediately redirects to the authentication flow, so this issue should not recur once the correct file is present.
|
||||
|
||||
|
||||
### "Session expired Local session" during iOS build
|
||||
|
||||
EAS stores an Apple ID session locally (in `~/.expo/`) to manage code-signing certificates and provisioning profiles. This session expires after a few weeks.
|
||||
|
||||
@@ -177,6 +177,11 @@
|
||||
<a href="/admin/backup" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-database w-4 mr-2 text-green-600" aria-hidden="true"></i> {{ _("nav.backup_restore") }}
|
||||
</a>
|
||||
{% if enable_factory_reset %}
|
||||
<a href="/admin/system-reset" role="menuitem" class="flex items-center px-4 py-2 text-sm text-red-600 hover:bg-red-50">
|
||||
<i class="fas fa-skull-crossbones w-4 mr-2 text-red-500" aria-hidden="true"></i> {{ _("nav.system_reset") }}
|
||||
</a>
|
||||
{% endif %}
|
||||
<a href="/admin/audit-logs" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-shield-halved w-4 mr-2 text-indigo-500" aria-hidden="true"></i> Audit Logs
|
||||
</a>
|
||||
@@ -445,6 +450,11 @@
|
||||
<a href="/admin/backup" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
|
||||
<i class="fas fa-database mr-2 text-green-500" aria-hidden="true"></i> {{ _("nav.backup_restore") }}
|
||||
</a>
|
||||
{% if enable_factory_reset %}
|
||||
<a href="/admin/system-reset" class="block px-3 py-3 rounded-md text-base font-medium text-red-600 hover:text-red-800 hover:bg-red-50">
|
||||
<i class="fas fa-skull-crossbones mr-2 text-red-500" aria-hidden="true"></i> {{ _("nav.system_reset") }}
|
||||
</a>
|
||||
{% endif %}
|
||||
<a href="/status" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50"
|
||||
{% if request and request.url.path == '/status' %}aria-current="page"{% endif %}>
|
||||
<i class="fas fa-circle-dot mr-2 text-gray-400" aria-hidden="true"></i> {{ _("nav.status") }}
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ _("system_reset.page_title") }} - DocuElevate{% endblock %}
|
||||
|
||||
{% block head_extra %}
|
||||
<style>
|
||||
.reset-card { transition: box-shadow 0.2s ease; }
|
||||
.reset-card:hover { box-shadow: 0 4px 20px rgba(0,0,0,.08); }
|
||||
.confirmation-input { font-family: 'Courier New', monospace; letter-spacing: 0.1em; text-transform: uppercase; }
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container mx-auto px-4 py-8" x-data="systemResetApp()">
|
||||
|
||||
<!-- Header -->
|
||||
<div class="mb-6">
|
||||
<div class="flex items-center gap-3 mb-2">
|
||||
<h1 class="text-3xl font-bold text-gray-900 dark:text-white">
|
||||
<i class="fas fa-skull-crossbones mr-2 text-red-600" aria-hidden="true"></i>
|
||||
{{ _("system_reset.heading") }}
|
||||
</h1>
|
||||
</div>
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">
|
||||
{{ _("system_reset.subtitle") }}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Factory Reset on Startup banner -->
|
||||
{% if factory_reset_on_startup %}
|
||||
<div class="mb-6 rounded-lg border border-yellow-300 bg-yellow-50 dark:bg-yellow-900/20 dark:border-yellow-700 p-4" role="alert">
|
||||
<div class="flex items-start gap-3">
|
||||
<i class="fas fa-exclamation-triangle text-yellow-600 mt-0.5" aria-hidden="true"></i>
|
||||
<div>
|
||||
<p class="font-semibold text-yellow-800 dark:text-yellow-200">{{ _("system_reset.startup_reset_active") }}</p>
|
||||
<p class="text-sm text-yellow-700 dark:text-yellow-300 mt-1">{{ _("system_reset.startup_reset_desc") }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Warning banner -->
|
||||
<div class="mb-8 rounded-lg border-2 border-red-300 bg-red-50 dark:bg-red-900/20 dark:border-red-700 p-6" role="alert">
|
||||
<div class="flex items-start gap-3">
|
||||
<i class="fas fa-radiation text-red-600 text-2xl mt-0.5" aria-hidden="true"></i>
|
||||
<div>
|
||||
<p class="font-bold text-red-800 dark:text-red-200 text-lg">{{ _("system_reset.danger_zone") }}</p>
|
||||
<p class="text-sm text-red-700 dark:text-red-300 mt-1">{{ _("system_reset.danger_desc") }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-8">
|
||||
|
||||
<!-- Full Reset Card -->
|
||||
<div class="reset-card rounded-xl border-2 border-red-200 dark:border-red-800 bg-white dark:bg-gray-800 p-6">
|
||||
<div class="flex items-center gap-3 mb-4">
|
||||
<div class="h-10 w-10 rounded-full bg-red-100 dark:bg-red-900 flex items-center justify-center">
|
||||
<i class="fas fa-trash-alt text-red-600" aria-hidden="true"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h2 class="text-xl font-bold text-gray-900 dark:text-white">{{ _("system_reset.full_reset_title") }}</h2>
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">{{ _("system_reset.full_reset_subtitle") }}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-3 mb-6">
|
||||
<p class="text-sm text-gray-700 dark:text-gray-300">{{ _("system_reset.full_reset_desc") }}</p>
|
||||
<ul class="text-sm text-gray-600 dark:text-gray-400 space-y-1 ml-4 list-disc">
|
||||
<li>{{ _("system_reset.full_reset_item_db") }}</li>
|
||||
<li>{{ _("system_reset.full_reset_item_files") }}</li>
|
||||
<li>{{ _("system_reset.full_reset_item_cache") }}</li>
|
||||
<li>{{ _("system_reset.full_reset_item_settings_kept") }}</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="border-t border-gray-200 dark:border-gray-700 pt-4">
|
||||
<label for="fullResetConfirm" class="block text-sm font-semibold text-red-700 dark:text-red-400 mb-2">
|
||||
{{ _("system_reset.type_delete") }}
|
||||
</label>
|
||||
<input id="fullResetConfirm"
|
||||
type="text"
|
||||
x-model="fullResetInput"
|
||||
class="confirmation-input w-full px-3 py-2 border-2 border-red-300 dark:border-red-700 rounded-lg
|
||||
text-center text-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white
|
||||
focus:outline-none focus:ring-2 focus:ring-red-500 focus:border-red-500"
|
||||
placeholder="DELETE"
|
||||
autocomplete="off"
|
||||
aria-describedby="fullResetHelp" />
|
||||
<p id="fullResetHelp" class="text-xs text-gray-500 dark:text-gray-400 mt-1">{{ _("system_reset.type_delete_help") }}</p>
|
||||
|
||||
<button @click="executeFullReset()"
|
||||
:disabled="fullResetInput !== 'DELETE' || loading"
|
||||
type="button"
|
||||
class="mt-4 w-full inline-flex items-center justify-center px-4 py-3 border border-transparent
|
||||
text-base font-bold rounded-lg text-white
|
||||
bg-red-600 hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-red-500
|
||||
disabled:opacity-40 disabled:cursor-not-allowed transition min-h-[44px]"
|
||||
aria-label="{{ _('system_reset.full_reset_button') }}">
|
||||
<template x-if="loading && activeAction === 'full'">
|
||||
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
|
||||
</template>
|
||||
<i x-show="!(loading && activeAction === 'full')" class="fas fa-trash-alt mr-2" aria-hidden="true"></i>
|
||||
{{ _("system_reset.full_reset_button") }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Reset & Re-import Card -->
|
||||
<div class="reset-card rounded-xl border-2 border-orange-200 dark:border-orange-800 bg-white dark:bg-gray-800 p-6">
|
||||
<div class="flex items-center gap-3 mb-4">
|
||||
<div class="h-10 w-10 rounded-full bg-orange-100 dark:bg-orange-900 flex items-center justify-center">
|
||||
<i class="fas fa-recycle text-orange-600" aria-hidden="true"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h2 class="text-xl font-bold text-gray-900 dark:text-white">{{ _("system_reset.reimport_title") }}</h2>
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">{{ _("system_reset.reimport_subtitle") }}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-3 mb-6">
|
||||
<p class="text-sm text-gray-700 dark:text-gray-300">{{ _("system_reset.reimport_desc") }}</p>
|
||||
<ol class="text-sm text-gray-600 dark:text-gray-400 space-y-1 ml-4 list-decimal">
|
||||
<li>{{ _("system_reset.reimport_step_1") }}</li>
|
||||
<li>{{ _("system_reset.reimport_step_2") }}</li>
|
||||
<li>{{ _("system_reset.reimport_step_3") }}</li>
|
||||
</ol>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400 italic">{{ _("system_reset.reimport_note") }}</p>
|
||||
</div>
|
||||
|
||||
<div class="border-t border-gray-200 dark:border-gray-700 pt-4">
|
||||
<label for="reimportConfirm" class="block text-sm font-semibold text-orange-700 dark:text-orange-400 mb-2">
|
||||
{{ _("system_reset.type_reimport") }}
|
||||
</label>
|
||||
<input id="reimportConfirm"
|
||||
type="text"
|
||||
x-model="reimportInput"
|
||||
class="confirmation-input w-full px-3 py-2 border-2 border-orange-300 dark:border-orange-700 rounded-lg
|
||||
text-center text-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white
|
||||
focus:outline-none focus:ring-2 focus:ring-orange-500 focus:border-orange-500"
|
||||
placeholder="REIMPORT"
|
||||
autocomplete="off"
|
||||
aria-describedby="reimportHelp" />
|
||||
<p id="reimportHelp" class="text-xs text-gray-500 dark:text-gray-400 mt-1">{{ _("system_reset.type_reimport_help") }}</p>
|
||||
|
||||
<button @click="executeReimport()"
|
||||
:disabled="reimportInput !== 'REIMPORT' || loading"
|
||||
type="button"
|
||||
class="mt-4 w-full inline-flex items-center justify-center px-4 py-3 border border-transparent
|
||||
text-base font-bold rounded-lg text-white
|
||||
bg-orange-600 hover:bg-orange-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-orange-500
|
||||
disabled:opacity-40 disabled:cursor-not-allowed transition min-h-[44px]"
|
||||
aria-label="{{ _('system_reset.reimport_button') }}">
|
||||
<template x-if="loading && activeAction === 'reimport'">
|
||||
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
|
||||
</template>
|
||||
<i x-show="!(loading && activeAction === 'reimport')" class="fas fa-recycle mr-2" aria-hidden="true"></i>
|
||||
{{ _("system_reset.reimport_button") }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Result banner (shown after a reset completes) -->
|
||||
<div x-show="resultMessage" x-cloak
|
||||
class="mt-8 rounded-lg p-4"
|
||||
:class="resultSuccess ? 'bg-green-50 dark:bg-green-900/20 border border-green-300 dark:border-green-700' :
|
||||
'bg-red-50 dark:bg-red-900/20 border border-red-300 dark:border-red-700'"
|
||||
:role="resultSuccess ? 'status' : 'alert'" aria-live="polite">
|
||||
<div class="flex items-start gap-3">
|
||||
<i :class="resultSuccess ? 'fas fa-check-circle text-green-600' : 'fas fa-times-circle text-red-600'" aria-hidden="true"></i>
|
||||
<div>
|
||||
<p class="font-semibold" :class="resultSuccess ? 'text-green-800 dark:text-green-200' : 'text-red-800 dark:text-red-200'"
|
||||
x-text="resultMessage"></p>
|
||||
<pre x-show="resultDetail" x-text="resultDetail"
|
||||
class="mt-2 text-xs overflow-x-auto whitespace-pre-wrap"
|
||||
:class="resultSuccess ? 'text-green-700 dark:text-green-300' : 'text-red-700 dark:text-red-300'"></pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function systemResetApp() {
|
||||
const __i18n = {
|
||||
successFull: {{ _("system_reset.js_success_full") | tojson }},
|
||||
successReimport: {{ _("system_reset.js_success_reimport") | tojson }},
|
||||
errorGeneric: {{ _("system_reset.js_error_generic") | tojson }},
|
||||
};
|
||||
|
||||
return {
|
||||
fullResetInput: '',
|
||||
reimportInput: '',
|
||||
loading: false,
|
||||
activeAction: null,
|
||||
resultMessage: null,
|
||||
resultDetail: null,
|
||||
resultSuccess: false,
|
||||
|
||||
async executeFullReset() {
|
||||
if (this.fullResetInput !== 'DELETE') return;
|
||||
this.loading = true;
|
||||
this.activeAction = 'full';
|
||||
this.resultMessage = null;
|
||||
try {
|
||||
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content || '';
|
||||
const resp = await fetch('/api/admin/system-reset/full', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': csrfToken },
|
||||
body: JSON.stringify({ confirmation: 'DELETE' }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (resp.ok) {
|
||||
this.resultSuccess = true;
|
||||
this.resultMessage = __i18n.successFull;
|
||||
this.resultDetail = JSON.stringify(data.result, null, 2);
|
||||
} else {
|
||||
this.resultSuccess = false;
|
||||
this.resultMessage = data.detail || __i18n.errorGeneric;
|
||||
}
|
||||
} catch (err) {
|
||||
this.resultSuccess = false;
|
||||
this.resultMessage = __i18n.errorGeneric;
|
||||
} finally {
|
||||
this.loading = false;
|
||||
this.fullResetInput = '';
|
||||
}
|
||||
},
|
||||
|
||||
async executeReimport() {
|
||||
if (this.reimportInput !== 'REIMPORT') return;
|
||||
this.loading = true;
|
||||
this.activeAction = 'reimport';
|
||||
this.resultMessage = null;
|
||||
try {
|
||||
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content || '';
|
||||
const resp = await fetch('/api/admin/system-reset/reimport', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': csrfToken },
|
||||
body: JSON.stringify({ confirmation: 'REIMPORT' }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (resp.ok) {
|
||||
this.resultSuccess = true;
|
||||
this.resultMessage = __i18n.successReimport;
|
||||
this.resultDetail = JSON.stringify(data.result, null, 2);
|
||||
} else {
|
||||
this.resultSuccess = false;
|
||||
this.resultMessage = data.detail || __i18n.errorGeneric;
|
||||
}
|
||||
} catch (err) {
|
||||
this.resultSuccess = false;
|
||||
this.resultMessage = __i18n.errorGeneric;
|
||||
} finally {
|
||||
this.loading = false;
|
||||
this.reimportInput = '';
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -1184,6 +1184,7 @@
|
||||
"nav.skip_to_content": "Skip to main content",
|
||||
"nav.status": "Status",
|
||||
"nav.subscription": "Subscription",
|
||||
"nav.system_reset": "System Reset",
|
||||
"nav.toggle_dark_mode": "Toggle dark mode",
|
||||
"nav.toggle_nav": "Toggle navigation menu",
|
||||
"nav.upload": "Upload",
|
||||
@@ -1723,6 +1724,36 @@
|
||||
"subscription.upgrade_info": "Upgrades take effect immediately. Downgrades are scheduled for the end of your current billing period.",
|
||||
"subscription.upgrade_to_prefix": "Upgrade to",
|
||||
"subscription.usage_heading": "Usage",
|
||||
"system_reset.danger_desc": "The actions below will permanently destroy data. They cannot be undone. Application settings and configuration are preserved, but all documents, files, processing history, and audit logs will be deleted.",
|
||||
"system_reset.danger_zone": "Danger Zone — Irreversible Actions",
|
||||
"system_reset.full_reset_button": "Wipe All Data",
|
||||
"system_reset.full_reset_desc": "Permanently deletes all user data from the database and removes all work files from disk. The application will be in its initial state after this operation.",
|
||||
"system_reset.full_reset_item_cache": "All watch-folder caches and ingestion state",
|
||||
"system_reset.full_reset_item_db": "All document records, processing logs, and audit history",
|
||||
"system_reset.full_reset_item_files": "All original, processed, and temporary files on disk",
|
||||
"system_reset.full_reset_item_settings_kept": "Application settings and configuration are preserved",
|
||||
"system_reset.full_reset_subtitle": "Wipe everything and start fresh",
|
||||
"system_reset.full_reset_title": "Full System Reset",
|
||||
"system_reset.heading": "System Reset",
|
||||
"system_reset.js_error_generic": "An error occurred. Please check the server logs for details.",
|
||||
"system_reset.js_success_full": "System reset complete. All user data has been wiped.",
|
||||
"system_reset.js_success_reimport": "Reset complete. Original files have been staged for re-import via the watch folder.",
|
||||
"system_reset.page_title": "System Reset",
|
||||
"system_reset.reimport_button": "Reset & Re-import",
|
||||
"system_reset.reimport_desc": "Copies your original files to a special reimport folder, wipes everything, then lets the watch-folder mechanism re-process them as if they were freshly uploaded.",
|
||||
"system_reset.reimport_note": "Re-imported files will go through the full processing pipeline with the same rate limits and backoff strategy as regular uploads.",
|
||||
"system_reset.reimport_step_1": "Original files are copied to a dedicated reimport folder",
|
||||
"system_reset.reimport_step_2": "All data (database + work files) is wiped clean",
|
||||
"system_reset.reimport_step_3": "The reimport folder is configured as a watch folder for automatic re-ingestion",
|
||||
"system_reset.reimport_subtitle": "Wipe and re-process all original files",
|
||||
"system_reset.reimport_title": "Reset & Re-import",
|
||||
"system_reset.startup_reset_active": "Factory Reset on Startup is ACTIVE",
|
||||
"system_reset.startup_reset_desc": "FACTORY_RESET_ON_STARTUP is enabled. All user data is wiped every time the application starts.",
|
||||
"system_reset.subtitle": "Reset DocuElevate to a clean, fresh state. All user data will be permanently deleted.",
|
||||
"system_reset.type_delete": "Type DELETE to confirm",
|
||||
"system_reset.type_delete_help": "You must type the word DELETE in capital letters to enable the reset button.",
|
||||
"system_reset.type_reimport": "Type REIMPORT to confirm",
|
||||
"system_reset.type_reimport_help": "You must type the word REIMPORT in capital letters to enable the button.",
|
||||
"terms.cookie_link": "Cookie Policy",
|
||||
"terms.heading": "Terms of Service",
|
||||
"terms.last_updated": "Last Updated:",
|
||||
|
||||
@@ -24,6 +24,7 @@ from app.models import ( # noqa: F401
|
||||
ApplicationSettings,
|
||||
AuditLog,
|
||||
BackupRecord,
|
||||
ClassificationRuleModel,
|
||||
ComplianceTemplate,
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""${message}."""
|
||||
# Use ``op.batch_alter_table()`` for SQLite compatibility.
|
||||
# Always check whether the table/column already exists before altering
|
||||
# to keep migrations idempotent (safe to re-run).
|
||||
#
|
||||
# Example – add a column only if it is missing:
|
||||
#
|
||||
# conn = op.get_bind()
|
||||
# inspector = sa.inspect(conn)
|
||||
# if "my_table" in inspector.get_table_names():
|
||||
# existing = {c["name"] for c in inspector.get_columns("my_table")}
|
||||
# if "new_col" not in existing:
|
||||
# with op.batch_alter_table("my_table") as batch_op:
|
||||
# batch_op.add_column(sa.Column("new_col", sa.String(128), nullable=True))
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reverse ${message}."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -118,6 +118,7 @@ function AuthGuard() {
|
||||
|
||||
return (
|
||||
<Stack screenOptions={{ headerShown: false }}>
|
||||
<Stack.Screen name="index" />
|
||||
<Stack.Screen name="(auth)" />
|
||||
<Stack.Screen name="(tabs)" />
|
||||
</Stack>
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Root index route – redirects to the auth flow on launch.
|
||||
*
|
||||
* expo-router renders this when the "/" route is matched (i.e. on cold start).
|
||||
* Without this file, a stale default scaffold page ("Hello World") can appear
|
||||
* if one was left behind by a previous build or Expo CLI scaffolding.
|
||||
*
|
||||
* The redirect targets the (auth) group; the AuthGuard in _layout.tsx will
|
||||
* immediately forward authenticated users to (tabs).
|
||||
*/
|
||||
|
||||
import { Redirect } from "expo-router";
|
||||
import React from "react";
|
||||
|
||||
export default function RootIndex() {
|
||||
return <Redirect href="/(auth)/" />;
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Alembic migration chain integrity.
|
||||
|
||||
This script checks the migration files in ``migrations/versions/`` for
|
||||
common problems that arise when multiple feature branches add migrations
|
||||
in parallel and then get merged into *main*.
|
||||
|
||||
Checks performed
|
||||
~~~~~~~~~~~~~~~~
|
||||
1. **Multiple heads** – more than one migration without a child means the
|
||||
chain has diverged and a merge migration is needed.
|
||||
2. **Broken down-revision references** – a migration points to a
|
||||
``down_revision`` that does not exist.
|
||||
3. **Duplicate revision IDs** – two files declare the same ``revision``.
|
||||
4. **Revision / filename mismatch** – the ``revision`` variable inside a
|
||||
file does not match the stem of the filename (minus the numeric
|
||||
prefix).
|
||||
|
||||
Exit codes
|
||||
~~~~~~~~~~
|
||||
* **0** – all checks passed.
|
||||
* **1** – one or more problems detected (details printed to *stderr*).
|
||||
* **2** – unexpected runtime error.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/check_alembic_migrations.py # from repo root
|
||||
python scripts/check_alembic_migrations.py --verbose # extra detail
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_REVISION_RE = re.compile(r'^revision\s*(?::\s*str\s*)?=\s*["\'](.+?)["\']', re.MULTILINE)
|
||||
_DOWN_REV_RE = re.compile(
|
||||
r"^down_revision\s*(?::\s*Union\[str,\s*(?:None|tuple)\]\s*)?=\s*(.+)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_down_revision(raw: str) -> list[str] | None:
|
||||
"""Parse a ``down_revision`` value into a list of parent revisions.
|
||||
|
||||
Returns ``None`` for the root migration (``down_revision = None``).
|
||||
Returns a list with one or more strings otherwise. Tuples are
|
||||
returned for merge migrations (e.g. ``("017_a", "017_b")``).
|
||||
"""
|
||||
# Strip inline comments (e.g. ``None # type: ignore``)
|
||||
raw = raw.strip()
|
||||
if "#" in raw:
|
||||
raw = raw[: raw.index("#")].strip()
|
||||
try:
|
||||
value = ast.literal_eval(raw)
|
||||
except (ValueError, SyntaxError):
|
||||
return [raw.strip("\"' ")]
|
||||
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
return [value]
|
||||
if isinstance(value, (tuple, list)):
|
||||
return [str(v) for v in value]
|
||||
return [str(value)]
|
||||
|
||||
|
||||
def _parse_migration(path: Path) -> dict | None:
|
||||
"""Extract ``revision`` and ``down_revision`` from a migration file."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
|
||||
rev_match = _REVISION_RE.search(text)
|
||||
down_match = _DOWN_REV_RE.search(text)
|
||||
|
||||
if not rev_match:
|
||||
return None # not a valid migration file
|
||||
|
||||
revision = rev_match.group(1)
|
||||
down_revision = _parse_down_revision(down_match.group(1)) if down_match else None
|
||||
|
||||
return {
|
||||
"path": path,
|
||||
"revision": revision,
|
||||
"down_revision": down_revision,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def check_migrations(versions_dir: Path, *, verbose: bool = False) -> list[str]:
|
||||
"""Run all migration-chain checks and return a list of error messages."""
|
||||
errors: list[str] = []
|
||||
|
||||
# Collect all migrations ------------------------------------------------
|
||||
migrations: dict[str, dict] = {}
|
||||
py_files = sorted(versions_dir.glob("*.py"))
|
||||
if not py_files:
|
||||
errors.append(f"No migration files found in {versions_dir}")
|
||||
return errors
|
||||
|
||||
for path in py_files:
|
||||
if path.name == "__init__.py":
|
||||
continue
|
||||
info = _parse_migration(path)
|
||||
if info is None:
|
||||
if verbose:
|
||||
print(f" SKIP {path.name} (no revision found)", file=sys.stderr)
|
||||
continue
|
||||
rev = info["revision"]
|
||||
|
||||
# Check 1 – duplicate revision IDs
|
||||
if rev in migrations:
|
||||
errors.append(f"Duplicate revision '{rev}' in:\n - {migrations[rev]['path'].name}\n - {path.name}")
|
||||
else:
|
||||
migrations[rev] = info
|
||||
|
||||
if verbose:
|
||||
parents = info["down_revision"] or ["(root)"]
|
||||
print(f" {rev} ← {', '.join(parents)}", file=sys.stderr)
|
||||
|
||||
# Build child map -------------------------------------------------------
|
||||
all_revisions = set(migrations.keys())
|
||||
children: dict[str, list[str]] = {rev: [] for rev in all_revisions}
|
||||
|
||||
for rev, info in migrations.items():
|
||||
parents = info["down_revision"]
|
||||
if parents is None:
|
||||
continue
|
||||
for parent in parents:
|
||||
# Check 2 – broken down_revision references
|
||||
if parent not in all_revisions:
|
||||
errors.append(
|
||||
f"Broken chain: '{rev}' ({info['path'].name}) references "
|
||||
f"down_revision '{parent}' which does not exist."
|
||||
)
|
||||
else:
|
||||
children[parent].append(rev)
|
||||
|
||||
# Check 3 – multiple heads (revisions with no children) -----------------
|
||||
heads = [rev for rev, kids in children.items() if not kids]
|
||||
if len(heads) > 1:
|
||||
head_details = "\n".join(f" - {h} ({migrations[h]['path'].name})" for h in sorted(heads))
|
||||
errors.append(
|
||||
f"Multiple migration heads detected ({len(heads)}). "
|
||||
f"Create a merge migration to resolve:\n{head_details}\n\n"
|
||||
f' Fix: alembic merge heads -m "merge_parallel_branches"'
|
||||
)
|
||||
|
||||
# Check 4 – revision / filename consistency -----------------------------
|
||||
for rev, info in migrations.items():
|
||||
stem = info["path"].stem # e.g. "017_add_pipelines"
|
||||
if rev != stem:
|
||||
errors.append(
|
||||
f"Filename mismatch: file '{info['path'].name}' declares "
|
||||
f"revision='{rev}' but filename stem is '{stem}'."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI entry-point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI entry-point. Returns 0 on success, 1 on failure, 2 on error."""
|
||||
parser = argparse.ArgumentParser(description="Check Alembic migration chain integrity.")
|
||||
parser.add_argument(
|
||||
"--versions-dir",
|
||||
type=Path,
|
||||
default=Path("migrations/versions"),
|
||||
help="Path to Alembic versions directory (default: migrations/versions)",
|
||||
)
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Print extra diagnostic info")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.versions_dir.is_dir():
|
||||
print(f"ERROR: versions directory not found: {args.versions_dir}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if args.verbose:
|
||||
print("Scanning migrations…", file=sys.stderr)
|
||||
|
||||
errors = check_migrations(args.versions_dir, verbose=args.verbose)
|
||||
|
||||
if errors:
|
||||
print(f"\n{'=' * 60}", file=sys.stderr)
|
||||
print(f" Migration chain problems found: {len(errors)}", file=sys.stderr)
|
||||
print(f"{'=' * 60}\n", file=sys.stderr)
|
||||
for i, err in enumerate(errors, 1):
|
||||
print(f" [{i}] {err}\n", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("✓ Migration chain is valid.", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Tests for scripts/check_alembic_migrations.py."""
|
||||
|
||||
# The script lives outside of the ``app`` package, so we import it by path.
|
||||
import importlib.util
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "check_alembic_migrations.py"
|
||||
_spec = importlib.util.spec_from_file_location("check_alembic_migrations", _SCRIPT)
|
||||
assert _spec and _spec.loader
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod) # type: ignore[union-attr]
|
||||
|
||||
check_migrations = _mod.check_migrations
|
||||
main = _mod.main
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_migration(
|
||||
directory: Path, filename: str, revision: str, down_revision: str | tuple[str, ...] | None
|
||||
) -> Path:
|
||||
"""Helper to create a minimal migration file."""
|
||||
if down_revision is None:
|
||||
down_rev_str = "None"
|
||||
elif isinstance(down_revision, tuple):
|
||||
down_rev_str = repr(down_revision)
|
||||
else:
|
||||
down_rev_str = f'"{down_revision}"'
|
||||
|
||||
content = textwrap.dedent(f'''\
|
||||
"""Test migration."""
|
||||
from typing import Union
|
||||
revision: str = "{revision}"
|
||||
down_revision: Union[str, None] = {down_rev_str}
|
||||
depends_on: Union[str, None] = None
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
''')
|
||||
path = directory / filename
|
||||
path.write_text(content)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def versions_dir(tmp_path: Path) -> Path:
|
||||
"""Return a temporary versions directory."""
|
||||
d = tmp_path / "versions"
|
||||
d.mkdir()
|
||||
return d
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestCheckMigrations:
|
||||
"""Tests for the check_migrations function."""
|
||||
|
||||
def test_valid_linear_chain(self, versions_dir: Path) -> None:
|
||||
"""A simple linear chain should pass with no errors."""
|
||||
_write_migration(versions_dir, "001_initial.py", "001_initial", None)
|
||||
_write_migration(versions_dir, "002_add_col.py", "002_add_col", "001_initial")
|
||||
_write_migration(versions_dir, "003_add_table.py", "003_add_table", "002_add_col")
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert errors == []
|
||||
|
||||
def test_valid_merge_migration(self, versions_dir: Path) -> None:
|
||||
"""A chain with a merge point should pass."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
_write_migration(versions_dir, "002_a.py", "002_a", "001_base")
|
||||
_write_migration(versions_dir, "002_b.py", "002_b", "001_base")
|
||||
|
||||
# Merge file with tuple down_revision
|
||||
content = textwrap.dedent('''\
|
||||
"""Merge."""
|
||||
from typing import Union
|
||||
revision: str = "003_merge"
|
||||
down_revision: Union[str, tuple] = ("002_a", "002_b")
|
||||
depends_on: Union[str, None] = None
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
''')
|
||||
(versions_dir / "003_merge.py").write_text(content)
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert errors == []
|
||||
|
||||
def test_multiple_heads_detected(self, versions_dir: Path) -> None:
|
||||
"""Two unmerged branches should report multiple heads."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
_write_migration(versions_dir, "002_a.py", "002_a", "001_base")
|
||||
_write_migration(versions_dir, "002_b.py", "002_b", "001_base")
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert len(errors) == 1
|
||||
assert "Multiple migration heads" in errors[0]
|
||||
assert "002_a" in errors[0]
|
||||
assert "002_b" in errors[0]
|
||||
|
||||
def test_broken_down_revision(self, versions_dir: Path) -> None:
|
||||
"""A migration pointing to a non-existent parent should be flagged."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
_write_migration(versions_dir, "002_orphan.py", "002_orphan", "NONEXISTENT")
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert any("Broken chain" in e for e in errors)
|
||||
assert any("NONEXISTENT" in e for e in errors)
|
||||
|
||||
def test_duplicate_revision(self, versions_dir: Path) -> None:
|
||||
"""Two files declaring the same revision should be flagged."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
_write_migration(versions_dir, "002_first.py", "002_dup", "001_base")
|
||||
_write_migration(versions_dir, "002_second.py", "002_dup", "001_base")
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert any("Duplicate revision" in e for e in errors)
|
||||
|
||||
def test_filename_mismatch(self, versions_dir: Path) -> None:
|
||||
"""A file whose revision doesn't match its filename should be flagged."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
# filename stem is "002_wrong_name" but revision says "002_correct_name"
|
||||
_write_migration(versions_dir, "002_wrong_name.py", "002_correct_name", "001_base")
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert any("Filename mismatch" in e for e in errors)
|
||||
|
||||
def test_empty_directory(self, versions_dir: Path) -> None:
|
||||
"""An empty versions directory should report an error."""
|
||||
errors = check_migrations(versions_dir)
|
||||
assert len(errors) == 1
|
||||
assert "No migration files found" in errors[0]
|
||||
|
||||
def test_init_py_is_skipped(self, versions_dir: Path) -> None:
|
||||
"""__init__.py files should be ignored."""
|
||||
(versions_dir / "__init__.py").write_text("")
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert errors == []
|
||||
|
||||
def test_non_migration_file_skipped(self, versions_dir: Path) -> None:
|
||||
"""A .py file without a revision variable should be silently skipped."""
|
||||
(versions_dir / "helper.py").write_text("# just a helper\nx = 1\n")
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
|
||||
errors = check_migrations(versions_dir)
|
||||
assert errors == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMainCLI:
|
||||
"""Tests for the CLI entry-point."""
|
||||
|
||||
def test_success_returns_zero(self, versions_dir: Path) -> None:
|
||||
"""Valid chain should exit 0."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
rc = main(["--versions-dir", str(versions_dir)])
|
||||
assert rc == 0
|
||||
|
||||
def test_failure_returns_one(self, versions_dir: Path) -> None:
|
||||
"""Invalid chain should exit 1."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
_write_migration(versions_dir, "002_a.py", "002_a", "001_base")
|
||||
_write_migration(versions_dir, "002_b.py", "002_b", "001_base")
|
||||
|
||||
rc = main(["--versions-dir", str(versions_dir)])
|
||||
assert rc == 1
|
||||
|
||||
def test_missing_directory_returns_two(self, tmp_path: Path) -> None:
|
||||
"""Non-existent versions directory should exit 2."""
|
||||
rc = main(["--versions-dir", str(tmp_path / "does_not_exist")])
|
||||
assert rc == 2
|
||||
|
||||
def test_verbose_flag(self, versions_dir: Path) -> None:
|
||||
"""The --verbose flag should not crash."""
|
||||
_write_migration(versions_dir, "001_base.py", "001_base", None)
|
||||
rc = main(["--versions-dir", str(versions_dir), "--verbose"])
|
||||
assert rc == 0
|
||||
|
||||
def test_real_migrations(self) -> None:
|
||||
"""Smoke test against the actual project migrations."""
|
||||
real_dir = Path(__file__).resolve().parent.parent / "migrations" / "versions"
|
||||
if not real_dir.is_dir():
|
||||
pytest.skip("migrations/versions directory not found in working tree")
|
||||
rc = main(["--versions-dir", str(real_dir)])
|
||||
assert rc == 0
|
||||
@@ -0,0 +1,393 @@
|
||||
"""Tests for the system reset feature (app/api/system_reset.py, app/utils/system_reset.py, app/views/system_reset.py)."""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.database import Base
|
||||
from app.models import (
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
ProcessingLog,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_workdir():
|
||||
"""Create a temporary workdir populated with sample user data."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create data subdirectories with dummy files
|
||||
for subdir in ("original", "processed", "tmp", "pdfa", "backups"):
|
||||
d = Path(tmpdir) / subdir
|
||||
d.mkdir()
|
||||
(d / "sample.pdf").write_bytes(b"%PDF-1.4 fake")
|
||||
|
||||
# Create cache files
|
||||
for cache in ("watch_folder_processed.json", "ftp_ingest_processed.json"):
|
||||
(Path(tmpdir) / cache).write_text("{}")
|
||||
|
||||
# Create a per-user watch folder cache
|
||||
(Path(tmpdir) / "user_wf_42.json").write_text("{}")
|
||||
|
||||
# Create a loose PDF in workdir root
|
||||
(Path(tmpdir) / "abc123.pdf").write_bytes(b"%PDF-1.4 loose")
|
||||
|
||||
yield tmpdir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_db_session():
|
||||
"""Fresh in-memory database with sample user data rows."""
|
||||
engine = create_engine(
|
||||
"sqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
# Seed with sample data
|
||||
fr = FileRecord(
|
||||
filehash="abc123",
|
||||
original_filename="test.pdf",
|
||||
local_filename="uuid.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
session.add(fr)
|
||||
session.flush()
|
||||
|
||||
session.add(ProcessingLog(file_id=fr.id, task_id="t1", step_name="hash_file", status="success"))
|
||||
session.add(FileProcessingStep(file_id=fr.id, step_name="hash_file", status="success"))
|
||||
session.add(DocumentMetadata(filename="test.pdf", sender="Alice", recipient="Bob"))
|
||||
session.commit()
|
||||
|
||||
yield session
|
||||
|
||||
session.close()
|
||||
Base.metadata.drop_all(bind=engine)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests for app/utils/system_reset.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestWipeWorkdirData:
|
||||
"""Tests for _wipe_workdir_data()."""
|
||||
|
||||
def test_removes_data_subdirs(self, reset_workdir):
|
||||
from app.utils.system_reset import _wipe_workdir_data
|
||||
|
||||
result = _wipe_workdir_data(reset_workdir)
|
||||
|
||||
# All data subdirectories should be gone
|
||||
for subdir in ("original", "processed", "tmp", "pdfa", "backups"):
|
||||
assert not (Path(reset_workdir) / subdir).exists()
|
||||
|
||||
assert result["deleted_dirs"] == 5
|
||||
|
||||
def test_removes_cache_files(self, reset_workdir):
|
||||
from app.utils.system_reset import _wipe_workdir_data
|
||||
|
||||
result = _wipe_workdir_data(reset_workdir)
|
||||
|
||||
assert not (Path(reset_workdir) / "watch_folder_processed.json").exists()
|
||||
assert not (Path(reset_workdir) / "ftp_ingest_processed.json").exists()
|
||||
assert not (Path(reset_workdir) / "user_wf_42.json").exists()
|
||||
assert result["deleted_files"] >= 3
|
||||
|
||||
def test_removes_loose_document_files(self, reset_workdir):
|
||||
from app.utils.system_reset import _wipe_workdir_data
|
||||
|
||||
_wipe_workdir_data(reset_workdir)
|
||||
assert not (Path(reset_workdir) / "abc123.pdf").exists()
|
||||
|
||||
def test_preserves_workdir_directory(self, reset_workdir):
|
||||
from app.utils.system_reset import _wipe_workdir_data
|
||||
|
||||
_wipe_workdir_data(reset_workdir)
|
||||
assert Path(reset_workdir).is_dir()
|
||||
|
||||
def test_handles_empty_workdir(self):
|
||||
"""No errors when workdir has no data dirs or caches."""
|
||||
from app.utils.system_reset import _wipe_workdir_data
|
||||
|
||||
with tempfile.TemporaryDirectory() as empty_dir:
|
||||
result = _wipe_workdir_data(empty_dir)
|
||||
assert result["deleted_dirs"] == 0
|
||||
assert result["deleted_files"] == 0
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestWipeDatabase:
|
||||
"""Tests for _wipe_database()."""
|
||||
|
||||
def test_deletes_all_user_data(self, reset_db_session):
|
||||
from app.utils.system_reset import _wipe_database
|
||||
|
||||
result = _wipe_database(reset_db_session)
|
||||
|
||||
assert result.get("files", 0) >= 1
|
||||
assert result.get("processing_logs", 0) >= 1
|
||||
assert result.get("file_processing_steps", 0) >= 1
|
||||
assert result.get("document_metadata", 0) >= 1
|
||||
|
||||
def test_tables_are_empty_after_wipe(self, reset_db_session):
|
||||
from app.utils.system_reset import _wipe_database
|
||||
|
||||
_wipe_database(reset_db_session)
|
||||
|
||||
assert reset_db_session.query(FileRecord).count() == 0
|
||||
assert reset_db_session.query(ProcessingLog).count() == 0
|
||||
assert reset_db_session.query(FileProcessingStep).count() == 0
|
||||
assert reset_db_session.query(DocumentMetadata).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestPerformFullReset:
|
||||
"""Tests for perform_full_reset()."""
|
||||
|
||||
def test_wipes_db_and_filesystem(self, reset_db_session, reset_workdir):
|
||||
from app.utils.system_reset import perform_full_reset
|
||||
|
||||
with patch("app.utils.system_reset.settings") as mock_settings:
|
||||
mock_settings.workdir = reset_workdir
|
||||
result = perform_full_reset(reset_db_session)
|
||||
|
||||
assert "database" in result
|
||||
assert "filesystem" in result
|
||||
assert reset_db_session.query(FileRecord).count() == 0
|
||||
assert not (Path(reset_workdir) / "original").exists()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestPerformResetAndReimport:
|
||||
"""Tests for perform_reset_and_reimport()."""
|
||||
|
||||
def test_copies_originals_to_reimport_then_wipes(self, reset_db_session, reset_workdir):
|
||||
from app.utils.system_reset import perform_reset_and_reimport
|
||||
|
||||
with patch("app.utils.system_reset.settings") as mock_settings:
|
||||
mock_settings.workdir = reset_workdir
|
||||
mock_settings.watch_folders = ""
|
||||
mock_settings.watch_folder_delete_after_process = False
|
||||
result = perform_reset_and_reimport(reset_db_session)
|
||||
|
||||
reimport_dir = Path(reset_workdir) / "reimport"
|
||||
assert reimport_dir.is_dir()
|
||||
assert result["reimport"]["files_moved"] >= 1
|
||||
|
||||
# DB should be wiped
|
||||
assert reset_db_session.query(FileRecord).count() == 0
|
||||
|
||||
# Reimport folder should contain the original file
|
||||
reimport_files = list(reimport_dir.iterdir())
|
||||
assert len(reimport_files) >= 1
|
||||
|
||||
def test_configures_watch_folder(self, reset_db_session, reset_workdir):
|
||||
from app.utils.system_reset import perform_reset_and_reimport
|
||||
|
||||
with patch("app.utils.system_reset.settings") as mock_settings:
|
||||
mock_settings.workdir = reset_workdir
|
||||
mock_settings.watch_folders = "/some/other/folder"
|
||||
mock_settings.watch_folder_delete_after_process = False
|
||||
perform_reset_and_reimport(reset_db_session)
|
||||
|
||||
reimport_path = str(Path(reset_workdir) / "reimport")
|
||||
# watch_folders should now include the reimport path
|
||||
assert reimport_path in mock_settings.watch_folders
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestStartupReset:
|
||||
"""Tests for perform_startup_reset()."""
|
||||
|
||||
def test_startup_reset_calls_full_reset(self):
|
||||
from app.utils.system_reset import perform_startup_reset
|
||||
|
||||
with patch("app.utils.system_reset.perform_full_reset") as mock_reset:
|
||||
with patch("app.database.SessionLocal") as mock_sl:
|
||||
mock_db = mock_sl.return_value
|
||||
perform_startup_reset()
|
||||
|
||||
mock_reset.assert_called_once_with(mock_db)
|
||||
mock_db.close.assert_called_once()
|
||||
|
||||
def test_startup_reset_handles_errors(self):
|
||||
from app.utils.system_reset import perform_startup_reset
|
||||
|
||||
with patch("app.utils.system_reset.perform_full_reset", side_effect=RuntimeError("boom")):
|
||||
with patch("app.database.SessionLocal") as mock_sl:
|
||||
mock_db = mock_sl.return_value
|
||||
# Should not raise
|
||||
perform_startup_reset()
|
||||
mock_db.rollback.assert_called_once()
|
||||
mock_db.close.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration tests for API endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestSystemResetApi:
|
||||
"""Tests for the /api/admin/system-reset/ endpoints."""
|
||||
|
||||
def test_full_reset_requires_admin(self, client):
|
||||
"""Non-admin users get 403."""
|
||||
response = client.post(
|
||||
"/api/admin/system-reset/full",
|
||||
json={"confirmation": "DELETE"},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
|
||||
def test_full_reset_requires_feature_flag(self, client):
|
||||
"""Returns 404 when ENABLE_FACTORY_RESET is false."""
|
||||
from app.api.system_reset import _require_admin
|
||||
|
||||
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
|
||||
try:
|
||||
with patch("app.api.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = False
|
||||
response = client.post(
|
||||
"/api/admin/system-reset/full",
|
||||
json={"confirmation": "DELETE"},
|
||||
)
|
||||
finally:
|
||||
client.app.dependency_overrides.pop(_require_admin, None)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_full_reset_requires_confirmation(self, client):
|
||||
"""Wrong confirmation string gets 400."""
|
||||
from app.api.system_reset import _require_admin
|
||||
|
||||
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
|
||||
try:
|
||||
with patch("app.api.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = True
|
||||
response = client.post(
|
||||
"/api/admin/system-reset/full",
|
||||
json={"confirmation": "WRONG"},
|
||||
)
|
||||
finally:
|
||||
client.app.dependency_overrides.pop(_require_admin, None)
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_reimport_requires_confirmation(self, client):
|
||||
"""Wrong confirmation string gets 400."""
|
||||
from app.api.system_reset import _require_admin
|
||||
|
||||
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
|
||||
try:
|
||||
with patch("app.api.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = True
|
||||
response = client.post(
|
||||
"/api/admin/system-reset/reimport",
|
||||
json={"confirmation": "WRONG"},
|
||||
)
|
||||
finally:
|
||||
client.app.dependency_overrides.pop(_require_admin, None)
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_status_endpoint(self, client):
|
||||
"""The status endpoint returns feature-flag state."""
|
||||
from app.api.system_reset import _require_admin
|
||||
|
||||
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
|
||||
try:
|
||||
response = client.get("/api/admin/system-reset/status")
|
||||
finally:
|
||||
client.app.dependency_overrides.pop(_require_admin, None)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "enabled" in data
|
||||
assert "factory_reset_on_startup" in data
|
||||
|
||||
def test_full_reset_success(self, client):
|
||||
"""Full reset succeeds with correct confirmation and feature flag."""
|
||||
from app.api.system_reset import _require_admin
|
||||
|
||||
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
|
||||
try:
|
||||
with patch("app.api.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = True
|
||||
with patch(
|
||||
"app.utils.system_reset.perform_full_reset", return_value={"database": {}, "filesystem": {}}
|
||||
):
|
||||
response = client.post(
|
||||
"/api/admin/system-reset/full",
|
||||
json={"confirmation": "DELETE"},
|
||||
)
|
||||
finally:
|
||||
client.app.dependency_overrides.pop(_require_admin, None)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
def test_reimport_success(self, client):
|
||||
"""Reimport succeeds with correct confirmation."""
|
||||
from app.api.system_reset import _require_admin
|
||||
|
||||
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
|
||||
try:
|
||||
with patch("app.api.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = True
|
||||
with patch(
|
||||
"app.utils.system_reset.perform_reset_and_reimport",
|
||||
return_value={"database": {}, "filesystem": {}, "reimport": {"files_moved": 3}},
|
||||
):
|
||||
response = client.post(
|
||||
"/api/admin/system-reset/reimport",
|
||||
json={"confirmation": "REIMPORT"},
|
||||
)
|
||||
finally:
|
||||
client.app.dependency_overrides.pop(_require_admin, None)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration tests for the view
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestSystemResetView:
|
||||
"""Tests for the /admin/system-reset view."""
|
||||
|
||||
def test_view_redirects_when_disabled(self, client):
|
||||
"""When ENABLE_FACTORY_RESET=False, accessing the page redirects away."""
|
||||
with client:
|
||||
client.cookies.set("session", "test")
|
||||
with patch("app.views.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = False
|
||||
response = client.get("/admin/system-reset", follow_redirects=False)
|
||||
# Redirect to /settings (302) when disabled, or to login (302/307) when unauthenticated
|
||||
assert response.status_code in (302, 307)
|
||||
|
||||
def test_view_requires_auth(self, client):
|
||||
"""Unauthenticated users are redirected away from the page."""
|
||||
with patch("app.views.system_reset.settings") as mock_s:
|
||||
mock_s.enable_factory_reset = True
|
||||
mock_s.factory_reset_on_startup = False
|
||||
response = client.get("/admin/system-reset", follow_redirects=False)
|
||||
# Should redirect to login since there's no active session
|
||||
assert response.status_code in (302, 307)
|
||||
Reference in New Issue
Block a user