Merge branch 'main' into copilot/implement-log-off-everywhere-functionality

This commit is contained in:
Christian Krakau-Louis
2026-03-17 12:12:50 +01:00
committed by GitHub
31 changed files with 2271 additions and 11 deletions
+4
View File
@@ -7,6 +7,10 @@ GOTENBERG_URL=http://gotenberg:3000
ALLOW_FILE_DELETE=true # Allow deletion of file records ALLOW_FILE_DELETE=true # Allow deletion of file records
COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2) COMPLIANCE_ENABLED=true # Enable compliance templates dashboard (GDPR, HIPAA, SOC 2)
# **System Reset / Factory Reset**
# FACTORY_RESET_ON_STARTUP=false # Wipe all user data on every startup (demo/testing only)
# ENABLE_FACTORY_RESET=false # Show the System Reset page in admin UI
# **Logging** # **Logging**
# LOG_LEVEL controls the Python root-logger level. # LOG_LEVEL controls the Python root-logger level.
# Accepted values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO). # Accepted values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO).
+13 -1
View File
@@ -44,6 +44,18 @@ jobs:
- run: ruff check app/ tests/ - run: ruff check app/ tests/
- run: ruff format --check app/ tests/ - run: ruff format --check app/ tests/
migration-chain:
name: Alembic Migration Chain Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Validate migration chain
run: python scripts/check_alembic_migrations.py
html-lint: html-lint:
name: HTML Accessibility Lint name: HTML Accessibility Lint
runs-on: ubuntu-latest runs-on: ubuntu-latest
@@ -138,7 +150,7 @@ jobs:
build: build:
name: Build & Push Docker Image name: Build & Push Docker Image
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [run-tests, mypy, dependency-scan, html-lint] needs: [run-tests, mypy, dependency-scan, html-lint, migration-chain]
if: github.event_name == 'push' if: github.event_name == 'push'
steps: steps:
- name: Checkout Code - name: Checkout Code
+10
View File
@@ -48,6 +48,16 @@ repos:
.env.demo .env.demo
)$ )$
# Alembic migration chain validation
- repo: local
hooks:
- id: check-alembic-migrations
name: Check Alembic migration chain
entry: python scripts/check_alembic_migrations.py
language: python
pass_filenames: false
files: ^migrations/versions/.*\.py$
# Conventional commits validation # Conventional commits validation
- repo: https://github.com/compilerla/conventional-pre-commit - repo: https://github.com/compilerla/conventional-pre-commit
rev: v3.0.0 rev: v3.0.0
+1 -1
View File
@@ -1 +1 @@
2026-03-16T21:40:03Z 2026-03-17T09:55:43Z
+39
View File
@@ -10,6 +10,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
<!-- version list --> <!-- version list -->
## v0.154.0 (2026-03-17)
### Bug Fixes
- **db**: Address code review feedback - fix comment stripping, type hints, test skip, and docs
([`a007b4f`](https://github.com/christianlouis/DocuElevate/commit/a007b4fd98ed327a9874082766e0f3b8c32a1a68))
### Features
- **db**: Add migration chain CI validation, pre-commit hook, script template, docs, and tests
([`aa49fa3`](https://github.com/christianlouis/DocuElevate/commit/aa49fa3ae6ae0bfbb0f9efa6304cf109eca1c402))
## v0.153.1 (2026-03-16)
### Bug Fixes
- **mobile**: Add root index.tsx redirect to prevent stale Hello World screen
([`1350aa6`](https://github.com/christianlouis/DocuElevate/commit/1350aa6a5ede19d924a9a4c277ef8089b940ea3c))
## v0.153.0 (2026-03-16)
### Bug Fixes
- **system-reset**: Address code review feedback
([`7dffdc0`](https://github.com/christianlouis/DocuElevate/commit/7dffdc05542cbfe37a91f86b661790038195fef5))
### Features
- **system-reset**: Add system reset and factory reset feature
([`a88d790`](https://github.com/christianlouis/DocuElevate/commit/a88d790445e5aae9f8ba04e735501d6c93589dec))
### Testing
- **system-reset**: Add comprehensive tests and documentation
([`96bfba8`](https://github.com/christianlouis/DocuElevate/commit/96bfba8057e506f561d21318b0205e521cf67109))
## v0.152.0 (2026-03-16) ## v0.152.0 (2026-03-16)
### Features ### Features
+1 -1
View File
@@ -1 +1 @@
793b6d3 7b7554d
+6 -6
View File
@@ -1,10 +1,10 @@
DocuElevate Build Information DocuElevate Build Information
============================== ==============================
Version: 0.152.0 Version: 0.154.0
Build Date: 2026-03-16T21:40:03Z Build Date: 2026-03-17T09:55:43Z
Git Commit: 793b6d3e2562518cb404579f656e82a6dc60da02 Git Commit: 7b7554decf527d2c9f7eee653a1f503249b7d138
Git Short SHA: 793b6d3 Git Short SHA: 7b7554d
Git Branch: main Git Branch: main
Commit Date: 2026-03-16T22:39:40+01:00 Commit Date: 2026-03-17T10:55:10+01:00
Build Timestamp: 2026-03-16T21:40:03Z Build Timestamp: 2026-03-17T09:55:43Z
============================== ==============================
+1 -1
View File
@@ -1 +1 @@
0.152.0 0.154.0
+2
View File
@@ -45,6 +45,7 @@ from app.api.shared_links import public_router as shared_links_public_router
from app.api.shared_links import router as shared_links_router from app.api.shared_links import router as shared_links_router
from app.api.similarity import router as similarity_router from app.api.similarity import router as similarity_router
from app.api.subscriptions import router as subscriptions_router from app.api.subscriptions import router as subscriptions_router
from app.api.system_reset import router as system_reset_router
from app.api.translation import router as translation_router from app.api.translation import router as translation_router
from app.api.url_upload import router as url_upload_router from app.api.url_upload import router as url_upload_router
@@ -101,4 +102,5 @@ router.include_router(mobile_router)
router.include_router(sessions_router) router.include_router(sessions_router)
router.include_router(qr_auth_router) router.include_router(qr_auth_router)
router.include_router(compliance_router) router.include_router(compliance_router)
router.include_router(system_reset_router)
router.include_router(translation_router) router.include_router(translation_router)
+124
View File
@@ -0,0 +1,124 @@
"""
System reset API endpoints for DocuElevate.
Provides admin-only REST endpoints for:
- Full system reset (wipe all user data)
- Reset with re-import (move originals → reimport folder, wipe, re-ingest)
Both operations require the ``ENABLE_FACTORY_RESET=True`` feature flag and
admin privileges.
"""
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.config import settings
from app.database import get_db
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin/system-reset", tags=["system-reset"])
def _require_admin(request: Request) -> dict:
"""Ensure the caller is an admin. Raises 403 otherwise."""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
AdminUser = Annotated[dict, Depends(_require_admin)]
def _require_feature_enabled() -> None:
"""Raise 404 when the factory-reset feature flag is off."""
if not settings.enable_factory_reset:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="System reset is not enabled. Set ENABLE_FACTORY_RESET=True to activate.",
)
class ResetRequest(BaseModel):
"""Body for system reset endpoints. Requires explicit confirmation."""
confirmation: str
@router.post("/full")
async def full_reset(
body: ResetRequest,
_admin: AdminUser,
db: Session = Depends(get_db),
) -> dict:
"""Wipe all user data (database + work-files).
The caller must send ``{"confirmation": "DELETE"}`` to proceed.
"""
_require_feature_enabled()
if body.confirmation != "DELETE":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Confirmation required: send {"confirmation": "DELETE"} to proceed.',
)
from app.utils.system_reset import perform_full_reset
try:
result = perform_full_reset(db)
except Exception as exc:
logger.exception("Full system reset failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"System reset failed: {exc}",
) from exc
return {"status": "ok", "result": result}
@router.post("/reimport")
async def reset_and_reimport(
body: ResetRequest,
_admin: AdminUser,
db: Session = Depends(get_db),
) -> dict:
"""Move original files to a reimport folder, wipe everything, and
configure the reimport folder as a watch folder for automatic
re-ingestion.
The caller must send ``{"confirmation": "REIMPORT"}`` to proceed.
"""
_require_feature_enabled()
if body.confirmation != "REIMPORT":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Confirmation required: send {"confirmation": "REIMPORT"} to proceed.',
)
from app.utils.system_reset import perform_reset_and_reimport
try:
result = perform_reset_and_reimport(db)
except Exception as exc:
logger.exception("Reset-and-reimport failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Reset and reimport failed: {exc}",
) from exc
return {"status": "ok", "result": result}
@router.get("/status")
async def reset_status(_admin: AdminUser) -> dict:
"""Return whether the system reset feature is enabled."""
return {
"enabled": settings.enable_factory_reset,
"factory_reset_on_startup": settings.factory_reset_on_startup,
}
+19
View File
@@ -659,6 +659,25 @@ class Settings(BaseSettings):
), ),
) )
# System reset / factory reset settings
factory_reset_on_startup: bool = Field(
default=False,
description=(
"When enabled, DocuElevate wipes all user data (database rows and "
"work-files on disk) on every startup so the instance always comes "
"up in a clean, fresh state. Useful for demo or testing environments. "
"Default: False."
),
)
enable_factory_reset: bool = Field(
default=False,
description=(
"Show the 'System Reset' page in the admin UI. When enabled, "
"administrators can trigger a full data wipe or a wipe-and-reimport "
"directly from the web interface. Default: False."
),
)
# PDF/A archival conversion settings # PDF/A archival conversion settings
enable_pdfa_conversion: bool = Field( enable_pdfa_conversion: bool = Field(
default=False, default=False,
+6
View File
@@ -170,6 +170,12 @@ async def lifespan(app: FastAPI):
# Startup: Initialize database # Startup: Initialize database
init_db() # Create tables if they don't exist init_db() # Create tables if they don't exist
# Factory reset on startup — wipe all user data before anything else
if settings.factory_reset_on_startup:
from app.utils.system_reset import perform_startup_reset
perform_startup_reset()
# Load settings from database after DB initialization # Load settings from database after DB initialization
from app.database import SessionLocal from app.database import SessionLocal
from app.utils.config_loader import load_settings_from_db from app.utils.config_loader import load_settings_from_db
+22
View File
@@ -1924,6 +1924,28 @@ SETTING_METADATA = {
"required": False, "required": False,
"restart_required": False, "restart_required": False,
}, },
"factory_reset_on_startup": {
"category": "Feature Flags",
"description": (
"Wipe all user data on every startup so the instance always starts fresh. "
"Useful for demo/testing environments. Default: False."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": True,
},
"enable_factory_reset": {
"category": "Feature Flags",
"description": (
"Show the System Reset page in the admin UI. Allows administrators to "
"trigger a full data wipe or a wipe-and-reimport from the web interface. Default: False."
),
"type": "boolean",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Backup / Restore # Backup / Restore
"backup_enabled": { "backup_enabled": {
"category": "Backup", "category": "Backup",
+296
View File
@@ -0,0 +1,296 @@
"""
System reset utilities for DocuElevate.
Provides functions to:
- Wipe all user data (database rows + work-files on disk) for a fresh start.
- Wipe with re-import: move original files to a dedicated folder, wipe
everything, then let the watch-folder mechanism re-ingest the files.
Security: All public functions in this module require admin-level access.
They MUST only be invoked from admin-guarded API/view endpoints.
"""
import logging
import shutil
from pathlib import Path
from sqlalchemy.orm import Session
from app.config import settings
logger = logging.getLogger(__name__)
# Subdirectories inside *workdir* that contain user-generated data.
# Everything else (app code, static assets, config) is left untouched.
_USER_DATA_SUBDIRS = ("original", "processed", "tmp", "pdfa", "backups")
# JSON cache files written by watch-folder / ingest tasks.
_CACHE_FILES = (
"watch_folder_processed.json",
"ftp_ingest_processed.json",
"sftp_ingest_processed.json",
"dropbox_ingest_processed.json",
"gdrive_ingest_processed.json",
"onedrive_ingest_processed.json",
"nextcloud_ingest_processed.json",
"s3_ingest_processed.json",
"webdav_ingest_processed.json",
"processed_mails.json",
"credential_failures.json",
)
# The folder name used for storing files prior to re-import.
REIMPORT_FOLDER_NAME = "reimport"
def _wipe_workdir_data(workdir: str) -> dict[str, int]:
"""Delete user data subdirectories and cache files inside *workdir*.
Leaves the workdir directory itself intact so the application can
continue to write into it. Also leaves any files that do not belong
to the known data subdirectories or caches.
Returns:
A dict with counts of deleted directories and files.
"""
workdir_path = Path(workdir)
deleted_dirs = 0
deleted_files = 0
# Remove data subdirectories
for subdir in _USER_DATA_SUBDIRS:
target = workdir_path / subdir
if target.is_dir():
shutil.rmtree(target)
logger.info("Deleted data directory: %s", target)
deleted_dirs += 1
# Remove cache / state JSON files
for cache_file in _CACHE_FILES:
target = workdir_path / cache_file
if target.is_file():
target.unlink()
logger.info("Deleted cache file: %s", target)
deleted_files += 1
# Also remove user_wf_*.json files (per-user watch folder caches)
for f in workdir_path.glob("user_wf_*.json"):
f.unlink()
logger.info("Deleted user watch-folder cache: %s", f)
deleted_files += 1
# Remove loose files in workdir root that are user uploads (uuid-named
# files like "a1b2c3d4-…pdf") but NOT application config files.
for entry in workdir_path.iterdir():
if entry.is_file() and entry.suffix.lower() in {
".pdf",
".png",
".jpg",
".jpeg",
".tiff",
".tif",
".docx",
".doc",
".xlsx",
".xls",
".pptx",
".heic",
".heif",
".webp",
".bmp",
".gif",
".txt",
".rtf",
".odt",
".ods",
".odp",
".csv",
".pages",
".numbers",
".keynote",
}:
entry.unlink()
logger.info("Deleted loose workdir file: %s", entry)
deleted_files += 1
return {"deleted_dirs": deleted_dirs, "deleted_files": deleted_files}
def _wipe_database(db: Session) -> dict[str, int]:
"""Delete all user-generated rows from the database.
Preserves schema (tables, migrations) and system-seeded rows that will
be re-created on the next startup (subscription plans, default pipeline,
scheduled jobs, compliance templates).
Returns:
A dict mapping table name → number of rows deleted.
"""
from app.models import (
AuditLog,
BackupRecord,
DocumentMetadata,
FileProcessingStep,
FileRecord,
InAppNotification,
ProcessingLog,
SavedSearch,
SettingsAuditLog,
SharedLink,
UserImapAccount,
UserIntegration,
UserNotificationPreference,
UserNotificationTarget,
)
# Order matters: delete children before parents to respect FK constraints.
tables_to_wipe: list[tuple[str, type]] = [
("file_processing_steps", FileProcessingStep),
("processing_logs", ProcessingLog),
("shared_links", SharedLink),
("in_app_notifications", InAppNotification),
("user_notification_preferences", UserNotificationPreference),
("user_notification_targets", UserNotificationTarget),
("user_imap_accounts", UserImapAccount),
("user_integrations", UserIntegration),
("saved_searches", SavedSearch),
("settings_audit_log", SettingsAuditLog),
("audit_logs", AuditLog),
("backup_records", BackupRecord),
("document_metadata", DocumentMetadata),
("files", FileRecord),
]
result: dict[str, int] = {}
for table_name, model in tables_to_wipe:
try:
count = db.query(model).delete()
result[table_name] = count
logger.info("Wiped %d rows from %s", count, table_name)
except Exception:
logger.exception("Failed to wipe table %s during system reset", table_name)
db.rollback()
raise
db.commit()
return result
def perform_full_reset(db: Session) -> dict:
"""Perform a complete system reset: wipe database rows + work-files.
Args:
db: An active SQLAlchemy session.
Returns:
Summary dict with ``database`` and ``filesystem`` sub-dicts.
"""
logger.warning(">>> SYSTEM RESET: wiping all user data <<<")
db_result = _wipe_database(db)
fs_result = _wipe_workdir_data(settings.workdir)
logger.warning(">>> SYSTEM RESET complete <<<")
return {"database": db_result, "filesystem": fs_result}
def perform_reset_and_reimport(db: Session) -> dict:
"""Move original files to a reimport folder, wipe everything, then
configure the reimport folder as a watch folder for re-ingestion.
The watch-folder scanner (``scan_all_watch_folders``) will pick up
the files on its next periodic run and process them exactly as if
they had been freshly uploaded — respecting the same backoff
strategy, size limits, and rate limits.
Args:
db: An active SQLAlchemy session.
Returns:
Summary dict with ``database``, ``filesystem``, and ``reimport`` sub-dicts.
"""
workdir_path = Path(settings.workdir)
reimport_dir = workdir_path / REIMPORT_FOLDER_NAME
original_dir = workdir_path / "original"
# 1. Collect original files
files_moved = 0
reimport_dir.mkdir(parents=True, exist_ok=True)
if original_dir.is_dir():
for entry in original_dir.iterdir():
if entry.is_file():
# Validate the resolved path stays within original_dir (path traversal guard)
try:
entry.resolve().relative_to(original_dir.resolve())
except ValueError:
logger.warning("Skipping file outside original dir: %s", entry)
continue
dest = reimport_dir / entry.name
# Avoid overwriting: append counter if name clash
if dest.exists():
stem = dest.stem
suffix = dest.suffix
counter = 1
while dest.exists():
dest = reimport_dir / f"{stem}_{counter}{suffix}"
counter += 1
shutil.copy2(str(entry), str(dest))
files_moved += 1
logger.info("Copied %d original files to reimport folder: %s", files_moved, reimport_dir)
# 2. Perform the full reset (wipe DB + other workdir data)
reset_result = perform_full_reset(db)
# 3. Ensure the reimport folder survived the wipe (it's not in _USER_DATA_SUBDIRS)
# and set up watch folder config to point at it.
_configure_reimport_watch_folder(str(reimport_dir))
reset_result["reimport"] = {
"files_moved": files_moved,
"reimport_folder": str(reimport_dir),
}
logger.warning(">>> SYSTEM RESET with re-import configured — %d files staged <<<", files_moved)
return reset_result
def _configure_reimport_watch_folder(reimport_path: str) -> None:
"""Append *reimport_path* to the application's watch-folder list.
The watch-folder scanner uses ``settings.watch_folders`` (a
comma-separated string). We mutate the runtime setting so the
next scan picks up the folder. We also set
``watch_folder_delete_after_process = True`` so files are cleaned
up after successful processing.
"""
current = getattr(settings, "watch_folders", None) or ""
folders = [f.strip() for f in current.split(",") if f.strip()]
if reimport_path not in folders:
folders.append(reimport_path)
# Mutate runtime settings (not persisted to .env — ephemeral)
object.__setattr__(settings, "watch_folders", ",".join(folders))
object.__setattr__(settings, "watch_folder_delete_after_process", True)
logger.info("Configured reimport watch folder: %s", reimport_path)
def perform_startup_reset() -> None:
"""Called during application startup when ``FACTORY_RESET_ON_STARTUP=True``.
Wipes database and filesystem data so the instance starts completely
fresh. Uses its own DB session so it runs before the normal lifespan
seeding logic.
"""
from app.database import SessionLocal
logger.warning("FACTORY_RESET_ON_STARTUP is enabled — wiping all data")
db = SessionLocal()
try:
perform_full_reset(db)
except Exception:
logger.exception("Factory reset on startup failed")
db.rollback()
finally:
db.close()
+2
View File
@@ -36,6 +36,7 @@ from app.views.share import router as share_router
from app.views.shared_links import router as shared_links_router from app.views.shared_links import router as shared_links_router
from app.views.status import router as status_router from app.views.status import router as status_router
from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages from app.views.subscriptions import router as subscriptions_router # Pricing + subscription pages
from app.views.system_reset import router as system_reset_router # System reset / factory reset
from app.views.wizard import router as wizard_router from app.views.wizard import router as wizard_router
# Create a main router that includes all the view routers # Create a main router that includes all the view routers
@@ -71,3 +72,4 @@ router.include_router(audit_logs_router) # Comprehensive audit log viewer
router.include_router(help_router) # Built-in help / How-To docs router.include_router(help_router) # Built-in help / How-To docs
router.include_router(compliance_router) # Compliance templates dashboard router.include_router(compliance_router) # Compliance templates dashboard
router.include_router(devices_router) # Mobile devices dashboard router.include_router(devices_router) # Mobile devices dashboard
router.include_router(system_reset_router) # System reset / factory reset
+1
View File
@@ -94,6 +94,7 @@ def _inject_global_context(ctx: dict) -> None:
"allow_signup", "allow_signup",
getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False), getattr(settings, "multi_user_enabled", False) and getattr(settings, "allow_local_signup", False),
) )
ctx.setdefault("enable_factory_reset", getattr(settings, "enable_factory_reset", False))
req = ctx.get("request") req = ctx.get("request")
if req is not None: if req is not None:
+40
View File
@@ -0,0 +1,40 @@
"""
System reset view — admin-only UI page.
Renders a confirmation-heavy page that allows administrators to:
1. **Full Reset** — wipe all user data (DB + disk) for a fresh start.
2. **Reset & Re-import** — move originals to a reimport folder, wipe,
and let the watch-folder mechanism re-ingest them.
Both options are gated behind the ``ENABLE_FACTORY_RESET`` feature flag.
"""
import logging
from fastapi import Depends, Request
from fastapi.responses import RedirectResponse, Response
from sqlalchemy.orm import Session
from app.config import settings
from app.views.base import APIRouter, get_db, require_login, templates
from app.views.settings import require_admin_access
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/admin/system-reset")
@require_login
@require_admin_access
async def system_reset_page(request: Request, db: Session = Depends(get_db)) -> Response:
"""Render the system reset administration page."""
if not settings.enable_factory_reset:
return RedirectResponse(url="/settings", status_code=302)
return templates.TemplateResponse(
"system_reset.html",
{
"request": request,
"factory_reset_on_startup": settings.factory_reset_on_startup,
},
)
+61
View File
@@ -2405,3 +2405,64 @@ query GetDocument($id: Int!) {
} }
``` ```
Variables: `{ "id": 42 }` Variables: `{ "id": 42 }`
## System Reset
Admin-only endpoints for resetting the system to a clean state. Requires `ENABLE_FACTORY_RESET=True`.
### GET /api/admin/system-reset/status
Check whether the system reset feature is enabled.
**Response (200):**
```json
{
"enabled": true,
"factory_reset_on_startup": false
}
```
### POST /api/admin/system-reset/full
Wipe all user data (database + work-files).
**Request:**
```json
{
"confirmation": "DELETE"
}
```
**Response (200):**
```json
{
"status": "ok",
"result": {
"database": { "files": 42, "processing_logs": 100 },
"filesystem": { "deleted_dirs": 5, "deleted_files": 12 }
}
}
```
### POST /api/admin/system-reset/reimport
Move original files to a reimport folder, wipe everything, and configure the reimport folder as a watch folder for re-ingestion.
**Request:**
```json
{
"confirmation": "REIMPORT"
}
```
**Response (200):**
```json
{
"status": "ok",
"result": {
"database": { "files": 42 },
"filesystem": { "deleted_dirs": 5, "deleted_files": 12 },
"reimport": { "files_moved": 42, "reimport_folder": "/workdir/reimport" }
}
}
```
+48
View File
@@ -17,6 +17,8 @@ Configuration is primarily done through environment variables specified in a `.e
| `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` | | `EXTERNAL_HOSTNAME` | The external hostname for the application. | `docuelevate.example.com` |
| `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` | | `ALLOW_FILE_DELETE` | Enable file deletion in the web interface (`true`/`false`). | `true` |
| `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` | | `COMPLIANCE_ENABLED` | Enable the compliance templates dashboard (GDPR, HIPAA, SOC 2). | `true` |
| `FACTORY_RESET_ON_STARTUP` | Wipe all user data on every startup (demo/testing). | `false` |
| `ENABLE_FACTORY_RESET` | Show the System Reset page in the admin UI. | `false` |
### Batch Processing Settings ### Batch Processing Settings
@@ -1859,6 +1861,52 @@ BACKUP_RETAIN_WEEKLY=13
You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables. You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables.
## System Reset / Factory Reset
DocuElevate provides two mechanisms for resetting the system to a clean state. Both are **disabled by default** and must be explicitly enabled.
### Automatic Reset on Startup
Set `FACTORY_RESET_ON_STARTUP=true` to wipe all user data (database rows and work-files) every time the application starts. This is useful for demo, testing, or ephemeral environments where you always want a fresh instance.
```dotenv
FACTORY_RESET_ON_STARTUP=true
```
> **Warning:** This destroys all documents, processing history, audit logs, and backups on every restart. Application settings and configuration are preserved.
### Admin UI Reset Page
Set `ENABLE_FACTORY_RESET=true` to display the **System Reset** page in the admin navigation menu. From this page, administrators can:
| Action | Confirmation | Description |
|--------|-------------|-------------|
| **Full Reset** | Type `DELETE` | Wipes all database rows and work-files. The system returns to its initial state. |
| **Reset & Re-import** | Type `REIMPORT` | Copies original files to a `reimport/` folder inside the workdir, wipes everything, then configures the reimport folder as a watch folder so files are automatically re-ingested with the same processing pipeline, rate limits, and backoff strategy as regular uploads. |
```dotenv
ENABLE_FACTORY_RESET=true
```
### API Endpoints
When `ENABLE_FACTORY_RESET=true`, two admin-only API endpoints are available:
- `POST /api/admin/system-reset/full` — body: `{"confirmation": "DELETE"}`
- `POST /api/admin/system-reset/reimport` — body: `{"confirmation": "REIMPORT"}`
- `GET /api/admin/system-reset/status` — returns current feature-flag state
### What Gets Deleted
| Deleted | Preserved |
|---------|-----------|
| All document records (`files` table) | Application settings (`application_settings` table) |
| Processing logs and steps | User accounts and profiles |
| Audit logs | Subscription plans |
| Backup records | Pipelines and scheduled jobs |
| Original, processed, and temporary files | The workdir directory itself |
| Watch-folder caches and ingestion state | OAuth and integration configuration |
## Configuration File Location ## Configuration File Location
The `.env` file should be placed at the root of the project directory. When using Docker Compose, you can reference it with the `env_file` directive in your `docker-compose.yml`. The `.env` file should be placed at the root of the project directory. When using Docker Compose, you can reference it with the `env_file` directive in your `docker-compose.yml`.
+21
View File
@@ -287,6 +287,27 @@ alembic revision --autogenerate -m "describe your change"
Review the generated file in `migrations/versions/` before applying it. Review the generated file in `migrations/versions/` before applying it.
> **Tip:** For detailed guidance on naming conventions, idempotent patterns, parallel-branch workflows, and resolving merge conflicts, see the [Migration Workflow Guide](MigrationWorkflow.md).
### Validating the Migration Chain
A CI check and pre-commit hook validate that the migration chain has no broken
references, duplicate revisions, or diverged heads. Run the check locally:
```bash
python scripts/check_alembic_migrations.py
python scripts/check_alembic_migrations.py --verbose # extra detail
```
If you see **"Multiple migration heads detected"**, two branches added
migrations from the same parent. Create a merge migration:
```bash
alembic merge heads -m "merge_parallel_branches"
```
For a complete walk-through, see the [Migration Workflow Guide](MigrationWorkflow.md).
### Automating Migrations in Docker Compose ### Automating Migrations in Docker Compose
Add a short-lived `migrate` service that runs before the API and Worker: Add a short-lived `migrate` service that runs before the API and Worker:
+360
View File
@@ -0,0 +1,360 @@
# Migration Workflow
This guide explains how to create, test, and merge Alembic database migrations in DocuElevate — especially when **multiple feature branches** add migrations in parallel.
## Table of Contents
- [Quick Reference](#quick-reference)
- [Creating a New Migration](#creating-a-new-migration)
- [Migration Naming Convention](#migration-naming-convention)
- [Idempotent Migration Patterns](#idempotent-migration-patterns)
- [Parallel Branch Development](#parallel-branch-development)
- [Resolving Migration Conflicts](#resolving-migration-conflicts)
- [CI Validation](#ci-validation)
- [Pre-commit Hook](#pre-commit-hook)
- [Troubleshooting](#troubleshooting)
---
## Quick Reference
```bash
# Create a new migration after editing app/models.py
alembic revision --autogenerate -m "add_foobar_column"
# Apply all pending migrations
alembic upgrade head
# Check current database version
alembic current
# View migration history
alembic history --verbose
# Detect multiple heads (diverged branches)
alembic heads
# Create a merge migration to resolve multiple heads
alembic merge heads -m "merge_parallel_branches"
# Validate migration chain integrity (CI script)
python scripts/check_alembic_migrations.py
python scripts/check_alembic_migrations.py --verbose
```
---
## Creating a New Migration
1. **Edit `app/models.py`** — add or modify SQLAlchemy model classes.
2. **Generate the migration** from the repo root. Use `--rev-id` to set the
revision identifier directly (avoids renaming afterwards):
```bash
alembic revision --autogenerate --rev-id 037_add_my_new_table -m "add my new table"
```
This creates `migrations/versions/037_add_my_new_table_add_my_new_table.py`
with `revision = "037_add_my_new_table"`. Rename the file to match:
```bash
mv migrations/versions/037_add_my_new_table_add_my_new_table.py \
migrations/versions/037_add_my_new_table.py
```
Alternatively, generate with the default hash and then rename:
```bash
alembic revision --autogenerate -m "add_my_new_table"
# Rename: mv migrations/versions/<hash>_add_my_new_table.py migrations/versions/037_add_my_new_table.py
# Update revision inside the file to match the filename stem.
```
Alembic uses the `migrations/script.py.mako` template to generate the file. The template includes inline comments about idempotent patterns — read them.
3. **Review the generated code** — autogenerate is helpful but not perfect. Check:
- Are new tables and columns detected correctly?
- Does the `downgrade()` reverse all changes?
- Are SQLite-incompatible operations wrapped in `batch_alter_table()`?
4. **Test the migration** against a fresh database:
```bash
# Apply
alembic upgrade head
# Rollback
alembic downgrade -1
# Re-apply
alembic upgrade head
```
5. **Run the chain validation**:
```bash
python scripts/check_alembic_migrations.py
```
---
## Migration Naming Convention
All migration files follow a **sequential numeric prefix** scheme:
```
NNN_short_description.py
```
| Component | Rule |
|-----------|------|
| `NNN` | Three-digit zero-padded number, incrementing from the previous migration |
| `short_description` | Lowercase snake_case summary of the change |
The **`revision`** variable inside the file **must match the filename stem** exactly:
```python
# File: migrations/versions/037_add_classification_rules.py
revision: str = "037_add_classification_rules"
down_revision: Union[str, None] = "036_add_document_translation_fields"
```
The CI check (`scripts/check_alembic_migrations.py`) enforces this consistency.
---
## Idempotent Migration Patterns
Migrations should be **idempotent** — safe to run even if the change already exists. This is critical for SQLite compatibility and for recovering from partial failures.
### Add a Column (only if missing)
```python
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if "my_table" in inspector.get_table_names():
existing = {c["name"] for c in inspector.get_columns("my_table")}
if "new_col" not in existing:
with op.batch_alter_table("my_table") as batch_op:
batch_op.add_column(sa.Column("new_col", sa.String(128), nullable=True))
```
### Create a Table (only if missing)
```python
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if "new_table" not in inspector.get_table_names():
op.create_table(
"new_table",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("name", sa.String(255), nullable=False),
)
```
### Drop a Column (only if present)
```python
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if "my_table" in inspector.get_table_names():
existing = {c["name"] for c in inspector.get_columns("my_table")}
if "new_col" in existing:
with op.batch_alter_table("my_table") as batch_op:
batch_op.drop_column("new_col")
```
### Use `batch_alter_table` for SQLite
SQLite does not support `ALTER TABLE DROP COLUMN` or `ALTER TABLE RENAME COLUMN` natively. Alembic's `batch_alter_table` context manager works around this by recreating the table:
```python
with op.batch_alter_table("users") as batch_op:
batch_op.add_column(sa.Column("phone", sa.String(20), nullable=True))
batch_op.drop_column("fax")
```
---
## Parallel Branch Development
When two feature branches both add migrations from the same parent, the migration chain **diverges** into multiple heads. This is normal and expected — Alembic supports it — but the heads must be merged before the code reaches `main`.
### Example
```
main: 001 → 002 → 003
↘ Branch A: 004_add_widgets
↘ Branch B: 004_add_gadgets ← two heads!
```
### How to Avoid Conflicts
1. **Coordinate** — if two developers are both adding migrations, assign different sequence numbers (e.g., `037_` and `038_`). Even if both depend on `036_`, different numbers prevent filename collisions.
2. **Rebase early** — before opening a PR, rebase your branch onto the latest `main`:
```bash
git fetch origin main
git rebase origin/main
```
If `main` now has a new migration `037_*`, renumber yours to `038_*` and update `down_revision` to point at `037_*`.
3. **Check for multiple heads** locally:
```bash
python scripts/check_alembic_migrations.py
# or
alembic heads
```
---
## Resolving Migration Conflicts
If your PR's CI check reports **"Multiple migration heads detected"**, follow these steps:
### Step 1 — Update Your Branch
```bash
git fetch origin main
git merge origin/main
# or
git rebase origin/main
```
### Step 2 — Check Heads
```bash
python scripts/check_alembic_migrations.py --verbose
```
The output lists the conflicting heads.
### Step 3 — Create a Merge Migration
```bash
alembic merge heads -m "merge_parallel_branches"
```
This generates a new migration with **two parents** (a merge point):
```python
down_revision = ("037_add_widgets", "037_add_gadgets")
```
### Step 4 — Rename and Validate
Rename the merge migration to the next sequence number:
```bash
mv migrations/versions/<hash>_merge_parallel_branches.py \
migrations/versions/038_merge_parallel_branches.py
```
Update the `revision` inside to match, then validate:
```bash
python scripts/check_alembic_migrations.py
```
### Step 5 — Test
```bash
alembic upgrade head
alembic downgrade -1
alembic upgrade head
```
---
## CI Validation
The CI pipeline (`.github/workflows/ci.yml`) includes a **migration-chain** job that runs:
```bash
python scripts/check_alembic_migrations.py
```
This script checks for:
| Check | Description |
|-------|-------------|
| Multiple heads | Diverged migration chains that need a merge migration |
| Broken references | A `down_revision` that points to a non-existent revision |
| Duplicate revisions | Two files declaring the same `revision` identifier |
| Filename mismatches | The `revision` variable doesn't match the filename stem |
The job runs in Stage 1 (fast-fail gates) alongside lint checks. If it fails, the build is blocked until the migration chain is fixed.
---
## Pre-commit Hook
A local pre-commit hook is configured in `.pre-commit-config.yaml` that runs the same check whenever you commit a change to `migrations/versions/`:
```yaml
- repo: local
hooks:
- id: check-alembic-migrations
name: Check Alembic migration chain
entry: python scripts/check_alembic_migrations.py
language: python
pass_filenames: false
files: ^migrations/versions/.*\.py$
```
Install the hook:
```bash
pip install pre-commit
pre-commit install
```
---
## Troubleshooting
### "Multiple migration heads detected"
See [Resolving Migration Conflicts](#resolving-migration-conflicts) above.
### "Broken chain: revision X references down_revision Y which does not exist"
You removed or renamed a migration that another migration depends on. Either restore the missing file or update the dependent migration's `down_revision`.
### "Filename mismatch: file declares revision=X but filename stem is Y"
The `revision` string inside the Python file must match the filename (without `.py`). Rename the file or update the variable.
### "relation already exists" when running `alembic upgrade head`
The database has a table that a pending migration tries to create. Stamp the current state:
```bash
alembic stamp head
```
### Autogenerate doesn't detect my changes
Ensure all models are imported in `migrations/env.py`. The `from app.models import ...` block at the top must include your new model class.
### SQLite "no such column" after downgrade
SQLite has limited `ALTER TABLE` support. Always use `op.batch_alter_table()` for column operations on existing tables.
---
## Further Reading
- [Alembic Tutorial](https://alembic.sqlalchemy.org/en/latest/tutorial.html)
- [Alembic Branch / Merge](https://alembic.sqlalchemy.org/en/latest/branches.html)
- [Database Configuration Guide](DatabaseConfiguration.md)
+32 -1
View File
@@ -279,7 +279,19 @@ If you wish to use **direct FCM/APNs** without Expo's relay, replace the `send_e
``` ```
mobile/ mobile/
├── App.tsx # Root component ├── App.tsx # Root component (legacy, not used at runtime)
├── app/ # Expo Router file-based routes
│ ├── _layout.tsx # Root layout (AuthGuard + providers)
│ ├── index.tsx # Root redirect → /(auth)/
│ ├── (auth)/ # Unauthenticated route group
│ │ ├── _layout.tsx # Stack navigator (headerless)
│ │ ├── index.tsx # Welcome screen
│ │ └── login.tsx # Login screen
│ └── (tabs)/ # Authenticated route group
│ ├── _layout.tsx # Tab navigator
│ ├── index.tsx # Upload screen (default tab)
│ ├── files.tsx # Files screen
│ └── profile.tsx # Profile screen
├── app.json # Expo/EAS configuration ├── app.json # Expo/EAS configuration
├── eas.json # EAS Build profiles ├── eas.json # EAS Build profiles
├── package.json ├── package.json
@@ -301,6 +313,25 @@ mobile/
## Troubleshooting ## Troubleshooting
### App shows "Hello World" / default Expo page after update
If the iOS or Android app shows a generic "Hello World This is the first page of your app" screen instead of the DocuElevate UI, it means a stale default `index.tsx` file (generated by Expo CLI scaffolding) is being picked up in the `mobile/app/` directory.
**To fix:**
1. Delete any leftover default `mobile/app/index.tsx` that is **not** the repository version (the repo version contains a `<Redirect>` to `/(auth)/`).
2. Clear the Metro bundler cache and rebuild:
```bash
cd mobile
npx expo start --clear
```
3. For production builds, run a clean EAS build:
```bash
eas build --platform ios --clear-cache
```
The repository includes a root `app/index.tsx` that immediately redirects to the authentication flow, so this issue should not recur once the correct file is present.
### "Session expired Local session" during iOS build ### "Session expired Local session" during iOS build
EAS stores an Apple ID session locally (in `~/.expo/`) to manage code-signing certificates and provisioning profiles. This session expires after a few weeks. EAS stores an Apple ID session locally (in `~/.expo/`) to manage code-signing certificates and provisioning profiles. This session expires after a few weeks.
+10
View File
@@ -177,6 +177,11 @@
<a href="/admin/backup" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100"> <a href="/admin/backup" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
<i class="fas fa-database w-4 mr-2 text-green-600" aria-hidden="true"></i> {{ _("nav.backup_restore") }} <i class="fas fa-database w-4 mr-2 text-green-600" aria-hidden="true"></i> {{ _("nav.backup_restore") }}
</a> </a>
{% if enable_factory_reset %}
<a href="/admin/system-reset" role="menuitem" class="flex items-center px-4 py-2 text-sm text-red-600 hover:bg-red-50">
<i class="fas fa-skull-crossbones w-4 mr-2 text-red-500" aria-hidden="true"></i> {{ _("nav.system_reset") }}
</a>
{% endif %}
<a href="/admin/audit-logs" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100"> <a href="/admin/audit-logs" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
<i class="fas fa-shield-halved w-4 mr-2 text-indigo-500" aria-hidden="true"></i> Audit Logs <i class="fas fa-shield-halved w-4 mr-2 text-indigo-500" aria-hidden="true"></i> Audit Logs
</a> </a>
@@ -445,6 +450,11 @@
<a href="/admin/backup" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50"> <a href="/admin/backup" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
<i class="fas fa-database mr-2 text-green-500" aria-hidden="true"></i> {{ _("nav.backup_restore") }} <i class="fas fa-database mr-2 text-green-500" aria-hidden="true"></i> {{ _("nav.backup_restore") }}
</a> </a>
{% if enable_factory_reset %}
<a href="/admin/system-reset" class="block px-3 py-3 rounded-md text-base font-medium text-red-600 hover:text-red-800 hover:bg-red-50">
<i class="fas fa-skull-crossbones mr-2 text-red-500" aria-hidden="true"></i> {{ _("nav.system_reset") }}
</a>
{% endif %}
<a href="/status" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50" <a href="/status" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50"
{% if request and request.url.path == '/status' %}aria-current="page"{% endif %}> {% if request and request.url.path == '/status' %}aria-current="page"{% endif %}>
<i class="fas fa-circle-dot mr-2 text-gray-400" aria-hidden="true"></i> {{ _("nav.status") }} <i class="fas fa-circle-dot mr-2 text-gray-400" aria-hidden="true"></i> {{ _("nav.status") }}
+261
View File
@@ -0,0 +1,261 @@
{% extends "base.html" %}
{% block title %}{{ _("system_reset.page_title") }} - DocuElevate{% endblock %}
{% block head_extra %}
<style>
.reset-card { transition: box-shadow 0.2s ease; }
.reset-card:hover { box-shadow: 0 4px 20px rgba(0,0,0,.08); }
.confirmation-input { font-family: 'Courier New', monospace; letter-spacing: 0.1em; text-transform: uppercase; }
</style>
{% endblock %}
{% block content %}
<div class="container mx-auto px-4 py-8" x-data="systemResetApp()">
<!-- Header -->
<div class="mb-6">
<div class="flex items-center gap-3 mb-2">
<h1 class="text-3xl font-bold text-gray-900 dark:text-white">
<i class="fas fa-skull-crossbones mr-2 text-red-600" aria-hidden="true"></i>
{{ _("system_reset.heading") }}
</h1>
</div>
<p class="text-sm text-gray-500 dark:text-gray-400">
{{ _("system_reset.subtitle") }}
</p>
</div>
<!-- Factory Reset on Startup banner -->
{% if factory_reset_on_startup %}
<div class="mb-6 rounded-lg border border-yellow-300 bg-yellow-50 dark:bg-yellow-900/20 dark:border-yellow-700 p-4" role="alert">
<div class="flex items-start gap-3">
<i class="fas fa-exclamation-triangle text-yellow-600 mt-0.5" aria-hidden="true"></i>
<div>
<p class="font-semibold text-yellow-800 dark:text-yellow-200">{{ _("system_reset.startup_reset_active") }}</p>
<p class="text-sm text-yellow-700 dark:text-yellow-300 mt-1">{{ _("system_reset.startup_reset_desc") }}</p>
</div>
</div>
</div>
{% endif %}
<!-- Warning banner -->
<div class="mb-8 rounded-lg border-2 border-red-300 bg-red-50 dark:bg-red-900/20 dark:border-red-700 p-6" role="alert">
<div class="flex items-start gap-3">
<i class="fas fa-radiation text-red-600 text-2xl mt-0.5" aria-hidden="true"></i>
<div>
<p class="font-bold text-red-800 dark:text-red-200 text-lg">{{ _("system_reset.danger_zone") }}</p>
<p class="text-sm text-red-700 dark:text-red-300 mt-1">{{ _("system_reset.danger_desc") }}</p>
</div>
</div>
</div>
<div class="grid grid-cols-1 lg:grid-cols-2 gap-8">
<!-- Full Reset Card -->
<div class="reset-card rounded-xl border-2 border-red-200 dark:border-red-800 bg-white dark:bg-gray-800 p-6">
<div class="flex items-center gap-3 mb-4">
<div class="h-10 w-10 rounded-full bg-red-100 dark:bg-red-900 flex items-center justify-center">
<i class="fas fa-trash-alt text-red-600" aria-hidden="true"></i>
</div>
<div>
<h2 class="text-xl font-bold text-gray-900 dark:text-white">{{ _("system_reset.full_reset_title") }}</h2>
<p class="text-sm text-gray-500 dark:text-gray-400">{{ _("system_reset.full_reset_subtitle") }}</p>
</div>
</div>
<div class="space-y-3 mb-6">
<p class="text-sm text-gray-700 dark:text-gray-300">{{ _("system_reset.full_reset_desc") }}</p>
<ul class="text-sm text-gray-600 dark:text-gray-400 space-y-1 ml-4 list-disc">
<li>{{ _("system_reset.full_reset_item_db") }}</li>
<li>{{ _("system_reset.full_reset_item_files") }}</li>
<li>{{ _("system_reset.full_reset_item_cache") }}</li>
<li>{{ _("system_reset.full_reset_item_settings_kept") }}</li>
</ul>
</div>
<div class="border-t border-gray-200 dark:border-gray-700 pt-4">
<label for="fullResetConfirm" class="block text-sm font-semibold text-red-700 dark:text-red-400 mb-2">
{{ _("system_reset.type_delete") }}
</label>
<input id="fullResetConfirm"
type="text"
x-model="fullResetInput"
class="confirmation-input w-full px-3 py-2 border-2 border-red-300 dark:border-red-700 rounded-lg
text-center text-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white
focus:outline-none focus:ring-2 focus:ring-red-500 focus:border-red-500"
placeholder="DELETE"
autocomplete="off"
aria-describedby="fullResetHelp" />
<p id="fullResetHelp" class="text-xs text-gray-500 dark:text-gray-400 mt-1">{{ _("system_reset.type_delete_help") }}</p>
<button @click="executeFullReset()"
:disabled="fullResetInput !== 'DELETE' || loading"
type="button"
class="mt-4 w-full inline-flex items-center justify-center px-4 py-3 border border-transparent
text-base font-bold rounded-lg text-white
bg-red-600 hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-red-500
disabled:opacity-40 disabled:cursor-not-allowed transition min-h-[44px]"
aria-label="{{ _('system_reset.full_reset_button') }}">
<template x-if="loading && activeAction === 'full'">
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
</template>
<i x-show="!(loading && activeAction === 'full')" class="fas fa-trash-alt mr-2" aria-hidden="true"></i>
{{ _("system_reset.full_reset_button") }}
</button>
</div>
</div>
<!-- Reset & Re-import Card -->
<div class="reset-card rounded-xl border-2 border-orange-200 dark:border-orange-800 bg-white dark:bg-gray-800 p-6">
<div class="flex items-center gap-3 mb-4">
<div class="h-10 w-10 rounded-full bg-orange-100 dark:bg-orange-900 flex items-center justify-center">
<i class="fas fa-recycle text-orange-600" aria-hidden="true"></i>
</div>
<div>
<h2 class="text-xl font-bold text-gray-900 dark:text-white">{{ _("system_reset.reimport_title") }}</h2>
<p class="text-sm text-gray-500 dark:text-gray-400">{{ _("system_reset.reimport_subtitle") }}</p>
</div>
</div>
<div class="space-y-3 mb-6">
<p class="text-sm text-gray-700 dark:text-gray-300">{{ _("system_reset.reimport_desc") }}</p>
<ol class="text-sm text-gray-600 dark:text-gray-400 space-y-1 ml-4 list-decimal">
<li>{{ _("system_reset.reimport_step_1") }}</li>
<li>{{ _("system_reset.reimport_step_2") }}</li>
<li>{{ _("system_reset.reimport_step_3") }}</li>
</ol>
<p class="text-xs text-gray-500 dark:text-gray-400 italic">{{ _("system_reset.reimport_note") }}</p>
</div>
<div class="border-t border-gray-200 dark:border-gray-700 pt-4">
<label for="reimportConfirm" class="block text-sm font-semibold text-orange-700 dark:text-orange-400 mb-2">
{{ _("system_reset.type_reimport") }}
</label>
<input id="reimportConfirm"
type="text"
x-model="reimportInput"
class="confirmation-input w-full px-3 py-2 border-2 border-orange-300 dark:border-orange-700 rounded-lg
text-center text-lg bg-white dark:bg-gray-900 text-gray-900 dark:text-white
focus:outline-none focus:ring-2 focus:ring-orange-500 focus:border-orange-500"
placeholder="REIMPORT"
autocomplete="off"
aria-describedby="reimportHelp" />
<p id="reimportHelp" class="text-xs text-gray-500 dark:text-gray-400 mt-1">{{ _("system_reset.type_reimport_help") }}</p>
<button @click="executeReimport()"
:disabled="reimportInput !== 'REIMPORT' || loading"
type="button"
class="mt-4 w-full inline-flex items-center justify-center px-4 py-3 border border-transparent
text-base font-bold rounded-lg text-white
bg-orange-600 hover:bg-orange-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-orange-500
disabled:opacity-40 disabled:cursor-not-allowed transition min-h-[44px]"
aria-label="{{ _('system_reset.reimport_button') }}">
<template x-if="loading && activeAction === 'reimport'">
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
</template>
<i x-show="!(loading && activeAction === 'reimport')" class="fas fa-recycle mr-2" aria-hidden="true"></i>
{{ _("system_reset.reimport_button") }}
</button>
</div>
</div>
</div>
<!-- Result banner (shown after a reset completes) -->
<div x-show="resultMessage" x-cloak
class="mt-8 rounded-lg p-4"
:class="resultSuccess ? 'bg-green-50 dark:bg-green-900/20 border border-green-300 dark:border-green-700' :
'bg-red-50 dark:bg-red-900/20 border border-red-300 dark:border-red-700'"
:role="resultSuccess ? 'status' : 'alert'" aria-live="polite">
<div class="flex items-start gap-3">
<i :class="resultSuccess ? 'fas fa-check-circle text-green-600' : 'fas fa-times-circle text-red-600'" aria-hidden="true"></i>
<div>
<p class="font-semibold" :class="resultSuccess ? 'text-green-800 dark:text-green-200' : 'text-red-800 dark:text-red-200'"
x-text="resultMessage"></p>
<pre x-show="resultDetail" x-text="resultDetail"
class="mt-2 text-xs overflow-x-auto whitespace-pre-wrap"
:class="resultSuccess ? 'text-green-700 dark:text-green-300' : 'text-red-700 dark:text-red-300'"></pre>
</div>
</div>
</div>
</div>
<script>
function systemResetApp() {
const __i18n = {
successFull: {{ _("system_reset.js_success_full") | tojson }},
successReimport: {{ _("system_reset.js_success_reimport") | tojson }},
errorGeneric: {{ _("system_reset.js_error_generic") | tojson }},
};
return {
fullResetInput: '',
reimportInput: '',
loading: false,
activeAction: null,
resultMessage: null,
resultDetail: null,
resultSuccess: false,
async executeFullReset() {
if (this.fullResetInput !== 'DELETE') return;
this.loading = true;
this.activeAction = 'full';
this.resultMessage = null;
try {
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content || '';
const resp = await fetch('/api/admin/system-reset/full', {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': csrfToken },
body: JSON.stringify({ confirmation: 'DELETE' }),
});
const data = await resp.json();
if (resp.ok) {
this.resultSuccess = true;
this.resultMessage = __i18n.successFull;
this.resultDetail = JSON.stringify(data.result, null, 2);
} else {
this.resultSuccess = false;
this.resultMessage = data.detail || __i18n.errorGeneric;
}
} catch (err) {
this.resultSuccess = false;
this.resultMessage = __i18n.errorGeneric;
} finally {
this.loading = false;
this.fullResetInput = '';
}
},
async executeReimport() {
if (this.reimportInput !== 'REIMPORT') return;
this.loading = true;
this.activeAction = 'reimport';
this.resultMessage = null;
try {
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content || '';
const resp = await fetch('/api/admin/system-reset/reimport', {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': csrfToken },
body: JSON.stringify({ confirmation: 'REIMPORT' }),
});
const data = await resp.json();
if (resp.ok) {
this.resultSuccess = true;
this.resultMessage = __i18n.successReimport;
this.resultDetail = JSON.stringify(data.result, null, 2);
} else {
this.resultSuccess = false;
this.resultMessage = data.detail || __i18n.errorGeneric;
}
} catch (err) {
this.resultSuccess = false;
this.resultMessage = __i18n.errorGeneric;
} finally {
this.loading = false;
this.reimportInput = '';
}
},
};
}
</script>
{% endblock %}
+31
View File
@@ -1216,6 +1216,7 @@
"nav.skip_to_content": "Skip to main content", "nav.skip_to_content": "Skip to main content",
"nav.status": "Status", "nav.status": "Status",
"nav.subscription": "Subscription", "nav.subscription": "Subscription",
"nav.system_reset": "System Reset",
"nav.toggle_dark_mode": "Toggle dark mode", "nav.toggle_dark_mode": "Toggle dark mode",
"nav.toggle_nav": "Toggle navigation menu", "nav.toggle_nav": "Toggle navigation menu",
"nav.upload": "Upload", "nav.upload": "Upload",
@@ -1788,6 +1789,36 @@
"subscription.upgrade_info": "Upgrades take effect immediately. Downgrades are scheduled for the end of your current billing period.", "subscription.upgrade_info": "Upgrades take effect immediately. Downgrades are scheduled for the end of your current billing period.",
"subscription.upgrade_to_prefix": "Upgrade to", "subscription.upgrade_to_prefix": "Upgrade to",
"subscription.usage_heading": "Usage", "subscription.usage_heading": "Usage",
"system_reset.danger_desc": "The actions below will permanently destroy data. They cannot be undone. Application settings and configuration are preserved, but all documents, files, processing history, and audit logs will be deleted.",
"system_reset.danger_zone": "Danger Zone — Irreversible Actions",
"system_reset.full_reset_button": "Wipe All Data",
"system_reset.full_reset_desc": "Permanently deletes all user data from the database and removes all work files from disk. The application will be in its initial state after this operation.",
"system_reset.full_reset_item_cache": "All watch-folder caches and ingestion state",
"system_reset.full_reset_item_db": "All document records, processing logs, and audit history",
"system_reset.full_reset_item_files": "All original, processed, and temporary files on disk",
"system_reset.full_reset_item_settings_kept": "Application settings and configuration are preserved",
"system_reset.full_reset_subtitle": "Wipe everything and start fresh",
"system_reset.full_reset_title": "Full System Reset",
"system_reset.heading": "System Reset",
"system_reset.js_error_generic": "An error occurred. Please check the server logs for details.",
"system_reset.js_success_full": "System reset complete. All user data has been wiped.",
"system_reset.js_success_reimport": "Reset complete. Original files have been staged for re-import via the watch folder.",
"system_reset.page_title": "System Reset",
"system_reset.reimport_button": "Reset & Re-import",
"system_reset.reimport_desc": "Copies your original files to a special reimport folder, wipes everything, then lets the watch-folder mechanism re-process them as if they were freshly uploaded.",
"system_reset.reimport_note": "Re-imported files will go through the full processing pipeline with the same rate limits and backoff strategy as regular uploads.",
"system_reset.reimport_step_1": "Original files are copied to a dedicated reimport folder",
"system_reset.reimport_step_2": "All data (database + work files) is wiped clean",
"system_reset.reimport_step_3": "The reimport folder is configured as a watch folder for automatic re-ingestion",
"system_reset.reimport_subtitle": "Wipe and re-process all original files",
"system_reset.reimport_title": "Reset & Re-import",
"system_reset.startup_reset_active": "Factory Reset on Startup is ACTIVE",
"system_reset.startup_reset_desc": "FACTORY_RESET_ON_STARTUP is enabled. All user data is wiped every time the application starts.",
"system_reset.subtitle": "Reset DocuElevate to a clean, fresh state. All user data will be permanently deleted.",
"system_reset.type_delete": "Type DELETE to confirm",
"system_reset.type_delete_help": "You must type the word DELETE in capital letters to enable the reset button.",
"system_reset.type_reimport": "Type REIMPORT to confirm",
"system_reset.type_reimport_help": "You must type the word REIMPORT in capital letters to enable the button.",
"terms.cookie_link": "Cookie Policy", "terms.cookie_link": "Cookie Policy",
"terms.heading": "Terms of Service", "terms.heading": "Terms of Service",
"terms.last_updated": "Last Updated:", "terms.last_updated": "Last Updated:",
+40
View File
@@ -0,0 +1,40 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
depends_on: Union[str, None] = None
def upgrade() -> None:
"""${message}."""
# Use ``op.batch_alter_table()`` for SQLite compatibility.
# Always check whether the table/column already exists before altering
# to keep migrations idempotent (safe to re-run).
#
# Example add a column only if it is missing:
#
# conn = op.get_bind()
# inspector = sa.inspect(conn)
# if "my_table" in inspector.get_table_names():
# existing = {c["name"] for c in inspector.get_columns("my_table")}
# if "new_col" not in existing:
# with op.batch_alter_table("my_table") as batch_op:
# batch_op.add_column(sa.Column("new_col", sa.String(128), nullable=True))
${upgrades if upgrades else "pass"}
def downgrade() -> None:
"""Reverse ${message}."""
${downgrades if downgrades else "pass"}
+1
View File
@@ -118,6 +118,7 @@ function AuthGuard() {
return ( return (
<Stack screenOptions={{ headerShown: false }}> <Stack screenOptions={{ headerShown: false }}>
<Stack.Screen name="index" />
<Stack.Screen name="(auth)" /> <Stack.Screen name="(auth)" />
<Stack.Screen name="(tabs)" /> <Stack.Screen name="(tabs)" />
</Stack> </Stack>
+17
View File
@@ -0,0 +1,17 @@
/**
* Root index route redirects to the auth flow on launch.
*
* expo-router renders this when the "/" route is matched (i.e. on cold start).
* Without this file, a stale default scaffold page ("Hello World") can appear
* if one was left behind by a previous build or Expo CLI scaffolding.
*
* The redirect targets the (auth) group; the AuthGuard in _layout.tsx will
* immediately forward authenticated users to (tabs).
*/
import { Redirect } from "expo-router";
import React from "react";
export default function RootIndex() {
return <Redirect href="/(auth)/" />;
}
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Validate Alembic migration chain integrity.
This script checks the migration files in ``migrations/versions/`` for
common problems that arise when multiple feature branches add migrations
in parallel and then get merged into *main*.
Checks performed
~~~~~~~~~~~~~~~~
1. **Multiple heads** more than one migration without a child means the
chain has diverged and a merge migration is needed.
2. **Broken down-revision references** a migration points to a
``down_revision`` that does not exist.
3. **Duplicate revision IDs** two files declare the same ``revision``.
4. **Revision / filename mismatch** the ``revision`` variable inside a
file does not match the stem of the filename (minus the numeric
prefix).
Exit codes
~~~~~~~~~~
* **0** all checks passed.
* **1** one or more problems detected (details printed to *stderr*).
* **2** unexpected runtime error.
Usage::
python scripts/check_alembic_migrations.py # from repo root
python scripts/check_alembic_migrations.py --verbose # extra detail
"""
from __future__ import annotations
import argparse
import ast
import re
import sys
from pathlib import Path
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_REVISION_RE = re.compile(r'^revision\s*(?::\s*str\s*)?=\s*["\'](.+?)["\']', re.MULTILINE)
_DOWN_REV_RE = re.compile(
r"^down_revision\s*(?::\s*Union\[str,\s*(?:None|tuple)\]\s*)?=\s*(.+)",
re.MULTILINE,
)
def _parse_down_revision(raw: str) -> list[str] | None:
"""Parse a ``down_revision`` value into a list of parent revisions.
Returns ``None`` for the root migration (``down_revision = None``).
Returns a list with one or more strings otherwise. Tuples are
returned for merge migrations (e.g. ``("017_a", "017_b")``).
"""
# Strip inline comments (e.g. ``None # type: ignore``)
raw = raw.strip()
if "#" in raw:
raw = raw[: raw.index("#")].strip()
try:
value = ast.literal_eval(raw)
except (ValueError, SyntaxError):
return [raw.strip("\"' ")]
if value is None:
return None
if isinstance(value, str):
return [value]
if isinstance(value, (tuple, list)):
return [str(v) for v in value]
return [str(value)]
def _parse_migration(path: Path) -> dict | None:
"""Extract ``revision`` and ``down_revision`` from a migration file."""
text = path.read_text(encoding="utf-8")
rev_match = _REVISION_RE.search(text)
down_match = _DOWN_REV_RE.search(text)
if not rev_match:
return None # not a valid migration file
revision = rev_match.group(1)
down_revision = _parse_down_revision(down_match.group(1)) if down_match else None
return {
"path": path,
"revision": revision,
"down_revision": down_revision,
}
# ---------------------------------------------------------------------------
# Checks
# ---------------------------------------------------------------------------
def check_migrations(versions_dir: Path, *, verbose: bool = False) -> list[str]:
"""Run all migration-chain checks and return a list of error messages."""
errors: list[str] = []
# Collect all migrations ------------------------------------------------
migrations: dict[str, dict] = {}
py_files = sorted(versions_dir.glob("*.py"))
if not py_files:
errors.append(f"No migration files found in {versions_dir}")
return errors
for path in py_files:
if path.name == "__init__.py":
continue
info = _parse_migration(path)
if info is None:
if verbose:
print(f" SKIP {path.name} (no revision found)", file=sys.stderr)
continue
rev = info["revision"]
# Check 1 duplicate revision IDs
if rev in migrations:
errors.append(f"Duplicate revision '{rev}' in:\n - {migrations[rev]['path'].name}\n - {path.name}")
else:
migrations[rev] = info
if verbose:
parents = info["down_revision"] or ["(root)"]
print(f" {rev}{', '.join(parents)}", file=sys.stderr)
# Build child map -------------------------------------------------------
all_revisions = set(migrations.keys())
children: dict[str, list[str]] = {rev: [] for rev in all_revisions}
for rev, info in migrations.items():
parents = info["down_revision"]
if parents is None:
continue
for parent in parents:
# Check 2 broken down_revision references
if parent not in all_revisions:
errors.append(
f"Broken chain: '{rev}' ({info['path'].name}) references "
f"down_revision '{parent}' which does not exist."
)
else:
children[parent].append(rev)
# Check 3 multiple heads (revisions with no children) -----------------
heads = [rev for rev, kids in children.items() if not kids]
if len(heads) > 1:
head_details = "\n".join(f" - {h} ({migrations[h]['path'].name})" for h in sorted(heads))
errors.append(
f"Multiple migration heads detected ({len(heads)}). "
f"Create a merge migration to resolve:\n{head_details}\n\n"
f' Fix: alembic merge heads -m "merge_parallel_branches"'
)
# Check 4 revision / filename consistency -----------------------------
for rev, info in migrations.items():
stem = info["path"].stem # e.g. "017_add_pipelines"
if rev != stem:
errors.append(
f"Filename mismatch: file '{info['path'].name}' declares "
f"revision='{rev}' but filename stem is '{stem}'."
)
return errors
# ---------------------------------------------------------------------------
# CLI entry-point
# ---------------------------------------------------------------------------
def main(argv: list[str] | None = None) -> int:
"""CLI entry-point. Returns 0 on success, 1 on failure, 2 on error."""
parser = argparse.ArgumentParser(description="Check Alembic migration chain integrity.")
parser.add_argument(
"--versions-dir",
type=Path,
default=Path("migrations/versions"),
help="Path to Alembic versions directory (default: migrations/versions)",
)
parser.add_argument("--verbose", "-v", action="store_true", help="Print extra diagnostic info")
args = parser.parse_args(argv)
if not args.versions_dir.is_dir():
print(f"ERROR: versions directory not found: {args.versions_dir}", file=sys.stderr)
return 2
if args.verbose:
print("Scanning migrations…", file=sys.stderr)
errors = check_migrations(args.versions_dir, verbose=args.verbose)
if errors:
print(f"\n{'=' * 60}", file=sys.stderr)
print(f" Migration chain problems found: {len(errors)}", file=sys.stderr)
print(f"{'=' * 60}\n", file=sys.stderr)
for i, err in enumerate(errors, 1):
print(f" [{i}] {err}\n", file=sys.stderr)
return 1
print("✓ Migration chain is valid.", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main())
+199
View File
@@ -0,0 +1,199 @@
"""Tests for scripts/check_alembic_migrations.py."""
# The script lives outside of the ``app`` package, so we import it by path.
import importlib.util
import textwrap
from pathlib import Path
import pytest
_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "check_alembic_migrations.py"
_spec = importlib.util.spec_from_file_location("check_alembic_migrations", _SCRIPT)
assert _spec and _spec.loader
_mod = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_mod) # type: ignore[union-attr]
check_migrations = _mod.check_migrations
main = _mod.main
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
def _write_migration(
directory: Path, filename: str, revision: str, down_revision: str | tuple[str, ...] | None
) -> Path:
"""Helper to create a minimal migration file."""
if down_revision is None:
down_rev_str = "None"
elif isinstance(down_revision, tuple):
down_rev_str = repr(down_revision)
else:
down_rev_str = f'"{down_revision}"'
content = textwrap.dedent(f'''\
"""Test migration."""
from typing import Union
revision: str = "{revision}"
down_revision: Union[str, None] = {down_rev_str}
depends_on: Union[str, None] = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
''')
path = directory / filename
path.write_text(content)
return path
@pytest.fixture
def versions_dir(tmp_path: Path) -> Path:
"""Return a temporary versions directory."""
d = tmp_path / "versions"
d.mkdir()
return d
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@pytest.mark.unit
class TestCheckMigrations:
"""Tests for the check_migrations function."""
def test_valid_linear_chain(self, versions_dir: Path) -> None:
"""A simple linear chain should pass with no errors."""
_write_migration(versions_dir, "001_initial.py", "001_initial", None)
_write_migration(versions_dir, "002_add_col.py", "002_add_col", "001_initial")
_write_migration(versions_dir, "003_add_table.py", "003_add_table", "002_add_col")
errors = check_migrations(versions_dir)
assert errors == []
def test_valid_merge_migration(self, versions_dir: Path) -> None:
"""A chain with a merge point should pass."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
_write_migration(versions_dir, "002_a.py", "002_a", "001_base")
_write_migration(versions_dir, "002_b.py", "002_b", "001_base")
# Merge file with tuple down_revision
content = textwrap.dedent('''\
"""Merge."""
from typing import Union
revision: str = "003_merge"
down_revision: Union[str, tuple] = ("002_a", "002_b")
depends_on: Union[str, None] = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
''')
(versions_dir / "003_merge.py").write_text(content)
errors = check_migrations(versions_dir)
assert errors == []
def test_multiple_heads_detected(self, versions_dir: Path) -> None:
"""Two unmerged branches should report multiple heads."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
_write_migration(versions_dir, "002_a.py", "002_a", "001_base")
_write_migration(versions_dir, "002_b.py", "002_b", "001_base")
errors = check_migrations(versions_dir)
assert len(errors) == 1
assert "Multiple migration heads" in errors[0]
assert "002_a" in errors[0]
assert "002_b" in errors[0]
def test_broken_down_revision(self, versions_dir: Path) -> None:
"""A migration pointing to a non-existent parent should be flagged."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
_write_migration(versions_dir, "002_orphan.py", "002_orphan", "NONEXISTENT")
errors = check_migrations(versions_dir)
assert any("Broken chain" in e for e in errors)
assert any("NONEXISTENT" in e for e in errors)
def test_duplicate_revision(self, versions_dir: Path) -> None:
"""Two files declaring the same revision should be flagged."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
_write_migration(versions_dir, "002_first.py", "002_dup", "001_base")
_write_migration(versions_dir, "002_second.py", "002_dup", "001_base")
errors = check_migrations(versions_dir)
assert any("Duplicate revision" in e for e in errors)
def test_filename_mismatch(self, versions_dir: Path) -> None:
"""A file whose revision doesn't match its filename should be flagged."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
# filename stem is "002_wrong_name" but revision says "002_correct_name"
_write_migration(versions_dir, "002_wrong_name.py", "002_correct_name", "001_base")
errors = check_migrations(versions_dir)
assert any("Filename mismatch" in e for e in errors)
def test_empty_directory(self, versions_dir: Path) -> None:
"""An empty versions directory should report an error."""
errors = check_migrations(versions_dir)
assert len(errors) == 1
assert "No migration files found" in errors[0]
def test_init_py_is_skipped(self, versions_dir: Path) -> None:
"""__init__.py files should be ignored."""
(versions_dir / "__init__.py").write_text("")
_write_migration(versions_dir, "001_base.py", "001_base", None)
errors = check_migrations(versions_dir)
assert errors == []
def test_non_migration_file_skipped(self, versions_dir: Path) -> None:
"""A .py file without a revision variable should be silently skipped."""
(versions_dir / "helper.py").write_text("# just a helper\nx = 1\n")
_write_migration(versions_dir, "001_base.py", "001_base", None)
errors = check_migrations(versions_dir)
assert errors == []
@pytest.mark.unit
class TestMainCLI:
"""Tests for the CLI entry-point."""
def test_success_returns_zero(self, versions_dir: Path) -> None:
"""Valid chain should exit 0."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
rc = main(["--versions-dir", str(versions_dir)])
assert rc == 0
def test_failure_returns_one(self, versions_dir: Path) -> None:
"""Invalid chain should exit 1."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
_write_migration(versions_dir, "002_a.py", "002_a", "001_base")
_write_migration(versions_dir, "002_b.py", "002_b", "001_base")
rc = main(["--versions-dir", str(versions_dir)])
assert rc == 1
def test_missing_directory_returns_two(self, tmp_path: Path) -> None:
"""Non-existent versions directory should exit 2."""
rc = main(["--versions-dir", str(tmp_path / "does_not_exist")])
assert rc == 2
def test_verbose_flag(self, versions_dir: Path) -> None:
"""The --verbose flag should not crash."""
_write_migration(versions_dir, "001_base.py", "001_base", None)
rc = main(["--versions-dir", str(versions_dir), "--verbose"])
assert rc == 0
def test_real_migrations(self) -> None:
"""Smoke test against the actual project migrations."""
real_dir = Path(__file__).resolve().parent.parent / "migrations" / "versions"
if not real_dir.is_dir():
pytest.skip("migrations/versions directory not found in working tree")
rc = main(["--versions-dir", str(real_dir)])
assert rc == 0
+393
View File
@@ -0,0 +1,393 @@
"""Tests for the system reset feature (app/api/system_reset.py, app/utils/system_reset.py, app/views/system_reset.py)."""
import tempfile
from pathlib import Path
from unittest.mock import patch
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
from app.database import Base
from app.models import (
DocumentMetadata,
FileProcessingStep,
FileRecord,
ProcessingLog,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def reset_workdir():
"""Create a temporary workdir populated with sample user data."""
with tempfile.TemporaryDirectory() as tmpdir:
# Create data subdirectories with dummy files
for subdir in ("original", "processed", "tmp", "pdfa", "backups"):
d = Path(tmpdir) / subdir
d.mkdir()
(d / "sample.pdf").write_bytes(b"%PDF-1.4 fake")
# Create cache files
for cache in ("watch_folder_processed.json", "ftp_ingest_processed.json"):
(Path(tmpdir) / cache).write_text("{}")
# Create a per-user watch folder cache
(Path(tmpdir) / "user_wf_42.json").write_text("{}")
# Create a loose PDF in workdir root
(Path(tmpdir) / "abc123.pdf").write_bytes(b"%PDF-1.4 loose")
yield tmpdir
@pytest.fixture
def reset_db_session():
"""Fresh in-memory database with sample user data rows."""
engine = create_engine(
"sqlite:///:memory:",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session = Session()
# Seed with sample data
fr = FileRecord(
filehash="abc123",
original_filename="test.pdf",
local_filename="uuid.pdf",
file_size=1024,
mime_type="application/pdf",
)
session.add(fr)
session.flush()
session.add(ProcessingLog(file_id=fr.id, task_id="t1", step_name="hash_file", status="success"))
session.add(FileProcessingStep(file_id=fr.id, step_name="hash_file", status="success"))
session.add(DocumentMetadata(filename="test.pdf", sender="Alice", recipient="Bob"))
session.commit()
yield session
session.close()
Base.metadata.drop_all(bind=engine)
# ---------------------------------------------------------------------------
# Unit tests for app/utils/system_reset.py
# ---------------------------------------------------------------------------
@pytest.mark.unit
class TestWipeWorkdirData:
"""Tests for _wipe_workdir_data()."""
def test_removes_data_subdirs(self, reset_workdir):
from app.utils.system_reset import _wipe_workdir_data
result = _wipe_workdir_data(reset_workdir)
# All data subdirectories should be gone
for subdir in ("original", "processed", "tmp", "pdfa", "backups"):
assert not (Path(reset_workdir) / subdir).exists()
assert result["deleted_dirs"] == 5
def test_removes_cache_files(self, reset_workdir):
from app.utils.system_reset import _wipe_workdir_data
result = _wipe_workdir_data(reset_workdir)
assert not (Path(reset_workdir) / "watch_folder_processed.json").exists()
assert not (Path(reset_workdir) / "ftp_ingest_processed.json").exists()
assert not (Path(reset_workdir) / "user_wf_42.json").exists()
assert result["deleted_files"] >= 3
def test_removes_loose_document_files(self, reset_workdir):
from app.utils.system_reset import _wipe_workdir_data
_wipe_workdir_data(reset_workdir)
assert not (Path(reset_workdir) / "abc123.pdf").exists()
def test_preserves_workdir_directory(self, reset_workdir):
from app.utils.system_reset import _wipe_workdir_data
_wipe_workdir_data(reset_workdir)
assert Path(reset_workdir).is_dir()
def test_handles_empty_workdir(self):
"""No errors when workdir has no data dirs or caches."""
from app.utils.system_reset import _wipe_workdir_data
with tempfile.TemporaryDirectory() as empty_dir:
result = _wipe_workdir_data(empty_dir)
assert result["deleted_dirs"] == 0
assert result["deleted_files"] == 0
@pytest.mark.unit
class TestWipeDatabase:
"""Tests for _wipe_database()."""
def test_deletes_all_user_data(self, reset_db_session):
from app.utils.system_reset import _wipe_database
result = _wipe_database(reset_db_session)
assert result.get("files", 0) >= 1
assert result.get("processing_logs", 0) >= 1
assert result.get("file_processing_steps", 0) >= 1
assert result.get("document_metadata", 0) >= 1
def test_tables_are_empty_after_wipe(self, reset_db_session):
from app.utils.system_reset import _wipe_database
_wipe_database(reset_db_session)
assert reset_db_session.query(FileRecord).count() == 0
assert reset_db_session.query(ProcessingLog).count() == 0
assert reset_db_session.query(FileProcessingStep).count() == 0
assert reset_db_session.query(DocumentMetadata).count() == 0
@pytest.mark.unit
class TestPerformFullReset:
"""Tests for perform_full_reset()."""
def test_wipes_db_and_filesystem(self, reset_db_session, reset_workdir):
from app.utils.system_reset import perform_full_reset
with patch("app.utils.system_reset.settings") as mock_settings:
mock_settings.workdir = reset_workdir
result = perform_full_reset(reset_db_session)
assert "database" in result
assert "filesystem" in result
assert reset_db_session.query(FileRecord).count() == 0
assert not (Path(reset_workdir) / "original").exists()
@pytest.mark.unit
class TestPerformResetAndReimport:
"""Tests for perform_reset_and_reimport()."""
def test_copies_originals_to_reimport_then_wipes(self, reset_db_session, reset_workdir):
from app.utils.system_reset import perform_reset_and_reimport
with patch("app.utils.system_reset.settings") as mock_settings:
mock_settings.workdir = reset_workdir
mock_settings.watch_folders = ""
mock_settings.watch_folder_delete_after_process = False
result = perform_reset_and_reimport(reset_db_session)
reimport_dir = Path(reset_workdir) / "reimport"
assert reimport_dir.is_dir()
assert result["reimport"]["files_moved"] >= 1
# DB should be wiped
assert reset_db_session.query(FileRecord).count() == 0
# Reimport folder should contain the original file
reimport_files = list(reimport_dir.iterdir())
assert len(reimport_files) >= 1
def test_configures_watch_folder(self, reset_db_session, reset_workdir):
from app.utils.system_reset import perform_reset_and_reimport
with patch("app.utils.system_reset.settings") as mock_settings:
mock_settings.workdir = reset_workdir
mock_settings.watch_folders = "/some/other/folder"
mock_settings.watch_folder_delete_after_process = False
perform_reset_and_reimport(reset_db_session)
reimport_path = str(Path(reset_workdir) / "reimport")
# watch_folders should now include the reimport path
assert reimport_path in mock_settings.watch_folders
@pytest.mark.unit
class TestStartupReset:
"""Tests for perform_startup_reset()."""
def test_startup_reset_calls_full_reset(self):
from app.utils.system_reset import perform_startup_reset
with patch("app.utils.system_reset.perform_full_reset") as mock_reset:
with patch("app.database.SessionLocal") as mock_sl:
mock_db = mock_sl.return_value
perform_startup_reset()
mock_reset.assert_called_once_with(mock_db)
mock_db.close.assert_called_once()
def test_startup_reset_handles_errors(self):
from app.utils.system_reset import perform_startup_reset
with patch("app.utils.system_reset.perform_full_reset", side_effect=RuntimeError("boom")):
with patch("app.database.SessionLocal") as mock_sl:
mock_db = mock_sl.return_value
# Should not raise
perform_startup_reset()
mock_db.rollback.assert_called_once()
mock_db.close.assert_called_once()
# ---------------------------------------------------------------------------
# Integration tests for API endpoints
# ---------------------------------------------------------------------------
@pytest.mark.integration
class TestSystemResetApi:
"""Tests for the /api/admin/system-reset/ endpoints."""
def test_full_reset_requires_admin(self, client):
"""Non-admin users get 403."""
response = client.post(
"/api/admin/system-reset/full",
json={"confirmation": "DELETE"},
)
assert response.status_code == 403
def test_full_reset_requires_feature_flag(self, client):
"""Returns 404 when ENABLE_FACTORY_RESET is false."""
from app.api.system_reset import _require_admin
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
try:
with patch("app.api.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = False
response = client.post(
"/api/admin/system-reset/full",
json={"confirmation": "DELETE"},
)
finally:
client.app.dependency_overrides.pop(_require_admin, None)
assert response.status_code == 404
def test_full_reset_requires_confirmation(self, client):
"""Wrong confirmation string gets 400."""
from app.api.system_reset import _require_admin
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
try:
with patch("app.api.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = True
response = client.post(
"/api/admin/system-reset/full",
json={"confirmation": "WRONG"},
)
finally:
client.app.dependency_overrides.pop(_require_admin, None)
assert response.status_code == 400
def test_reimport_requires_confirmation(self, client):
"""Wrong confirmation string gets 400."""
from app.api.system_reset import _require_admin
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
try:
with patch("app.api.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = True
response = client.post(
"/api/admin/system-reset/reimport",
json={"confirmation": "WRONG"},
)
finally:
client.app.dependency_overrides.pop(_require_admin, None)
assert response.status_code == 400
def test_status_endpoint(self, client):
"""The status endpoint returns feature-flag state."""
from app.api.system_reset import _require_admin
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
try:
response = client.get("/api/admin/system-reset/status")
finally:
client.app.dependency_overrides.pop(_require_admin, None)
assert response.status_code == 200
data = response.json()
assert "enabled" in data
assert "factory_reset_on_startup" in data
def test_full_reset_success(self, client):
"""Full reset succeeds with correct confirmation and feature flag."""
from app.api.system_reset import _require_admin
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
try:
with patch("app.api.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = True
with patch(
"app.utils.system_reset.perform_full_reset", return_value={"database": {}, "filesystem": {}}
):
response = client.post(
"/api/admin/system-reset/full",
json={"confirmation": "DELETE"},
)
finally:
client.app.dependency_overrides.pop(_require_admin, None)
assert response.status_code == 200
assert response.json()["status"] == "ok"
def test_reimport_success(self, client):
"""Reimport succeeds with correct confirmation."""
from app.api.system_reset import _require_admin
client.app.dependency_overrides[_require_admin] = lambda: {"is_admin": True}
try:
with patch("app.api.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = True
with patch(
"app.utils.system_reset.perform_reset_and_reimport",
return_value={"database": {}, "filesystem": {}, "reimport": {"files_moved": 3}},
):
response = client.post(
"/api/admin/system-reset/reimport",
json={"confirmation": "REIMPORT"},
)
finally:
client.app.dependency_overrides.pop(_require_admin, None)
assert response.status_code == 200
assert response.json()["status"] == "ok"
# ---------------------------------------------------------------------------
# Integration tests for the view
# ---------------------------------------------------------------------------
@pytest.mark.integration
class TestSystemResetView:
"""Tests for the /admin/system-reset view."""
def test_view_redirects_when_disabled(self, client):
"""When ENABLE_FACTORY_RESET=False, accessing the page redirects away."""
with client:
client.cookies.set("session", "test")
with patch("app.views.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = False
response = client.get("/admin/system-reset", follow_redirects=False)
# Redirect to /settings (302) when disabled, or to login (302/307) when unauthenticated
assert response.status_code in (302, 307)
def test_view_requires_auth(self, client):
"""Unauthenticated users are redirected away from the page."""
with patch("app.views.system_reset.settings") as mock_s:
mock_s.enable_factory_reset = True
mock_s.factory_reset_on_startup = False
response = client.get("/admin/system-reset", follow_redirects=False)
# Should redirect to login since there's no active session
assert response.status_code in (302, 307)