diff --git a/app/api/backup.py b/app/api/backup.py index 2c83058c..8262814c 100644 --- a/app/api/backup.py +++ b/app/api/backup.py @@ -117,88 +117,104 @@ async def restore_backup( """Restore the database from an uploaded gzip-compressed SQL dump. **Warning**: This overwrites the current database contents. - Only SQLite databases are supported. - The uploaded file must be a ``.db.gz`` file produced by the DocuElevate - backup task (a gzip-compressed SQLite ``.dump()`` SQL script). + Supported formats (must match the currently configured database backend): + + - ``*.db.gz`` – gzip-compressed SQLite ``.dump()`` SQL script (SQLite backend) + - ``*.pgsql.gz`` – gzip-compressed ``pg_dump --format=plain`` output (PostgreSQL backend) + - ``*.mysql.gz`` – gzip-compressed ``mysqldump`` output (MySQL / MariaDB backend) """ - from app.tasks.backup_tasks import _db_path - - db_path = _db_path() - if db_path is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Restore is only supported for SQLite databases.", - ) - - if not file.filename or not file.filename.endswith(".db.gz"): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Uploaded file must be a .db.gz backup archive.", - ) - - import gzip - import sqlite3 import tempfile from pathlib import Path - # Write the upload to a temp file first so we can validate it - with tempfile.NamedTemporaryFile(suffix=".db.gz", delete=False) as tmp: + from sqlalchemy.engine.url import make_url + + from app.config import settings as app_settings + from app.tasks.backup_tasks import ( + _archive_ext_for_backend, + _db_path, + _restore_mysql, + _restore_postgresql, + _restore_sqlite, + ) + + url = make_url(app_settings.database_url) + backend = url.get_backend_name() + expected_ext = _archive_ext_for_backend(backend) + + if not file.filename or not file.filename.endswith(expected_ext): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Uploaded file must be a '{expected_ext}' backup archive for the current database backend ({backend})." + ), + ) + + # Write upload to a temp file + with tempfile.NamedTemporaryFile(suffix=expected_ext, delete=False) as tmp: tmp_path = Path(tmp.name) content = await file.read() tmp.write(content) try: - # Decompress and read SQL statements - with gzip.open(str(tmp_path), "rt", encoding="utf-8") as gz: - sql_script = gz.read() - except Exception as exc: - tmp_path.unlink(missing_ok=True) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to decompress backup file: {exc}", - ) from exc + if backend == "sqlite": + db_path = _db_path() + if db_path is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Restore is only supported for file-based SQLite databases.", + ) + # Close the application DB session before replacing the file + db.close() + try: + _restore_sqlite(db_path, tmp_path) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(exc), + ) from exc - # Create a fresh in-memory DB from the script to validate it - try: - mem_conn = sqlite3.connect(":memory:") - mem_conn.executescript(sql_script) - mem_conn.close() - except sqlite3.Error as exc: - tmp_path.unlink(missing_ok=True) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Backup file contains invalid SQL: {exc}", - ) from exc + elif backend == "postgresql": + db.close() + try: + _restore_postgresql(app_settings.database_url, tmp_path) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"psql binary not found – is PostgreSQL client installed? ({exc})", + ) from exc + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"PostgreSQL restore failed: {exc}", + ) from exc - # Close the application DB session before replacing the file - db.close() + elif backend == "mysql": + db.close() + try: + _restore_mysql(app_settings.database_url, tmp_path) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"mysql binary not found – is MySQL client installed? ({exc})", + ) from exc + except RuntimeError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"MySQL restore failed: {exc}", + ) from exc - # Preserve the current DB before overwriting - import shutil + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Database backend '{backend}' does not support restore.", + ) - bak = str(db_path) + ".pre_restore" - try: - shutil.copy2(str(db_path), bak) - except OSError as exc: - logger.warning(f"Could not create pre-restore backup at {bak}: {exc}") - - try: - # Write the restored database - restore_conn = sqlite3.connect(str(db_path)) - restore_conn.executescript(sql_script) - restore_conn.close() - except sqlite3.Error as exc: - # Attempt rollback - try: - if os.path.exists(bak): - shutil.copy2(bak, str(db_path)) - except OSError as rollback_exc: - logger.error(f"Rollback failed; database may be corrupted: {rollback_exc}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Restore failed: {exc}", - ) from exc finally: tmp_path.unlink(missing_ok=True) diff --git a/app/tasks/backup_tasks.py b/app/tasks/backup_tasks.py index c94ca1eb..f40014f7 100644 --- a/app/tasks/backup_tasks.py +++ b/app/tasks/backup_tasks.py @@ -16,12 +16,19 @@ Three separate Celery-beat entries call ``create_backup`` with the appropriate After each backup is created ``_apply_retention`` prunes old local backups for that tier. Remote copies are pruned by ``_prune_remote_backups`` which mirrors the same retention limits. + +Supported database backends +---------------------------- +- **SQLite** – dumped via Python's built-in ``sqlite3.iterdump()``; archive extension ``.db.gz`` +- **PostgreSQL** – dumped via ``pg_dump --format=plain``; archive extension ``.pgsql.gz`` +- **MySQL / MariaDB** – dumped via ``mysqldump --single-transaction``; archive extension ``.mysql.gz`` """ import gzip import hashlib import logging import os +import subprocess from datetime import datetime, timezone from pathlib import Path @@ -42,6 +49,13 @@ _BACKUP_TYPE_RETAIN: dict[str, str] = { "weekly": "backup_retain_weekly", } +#: Map of backend name → archive file extension. +_BACKEND_EXTENSIONS: dict[str, str] = { + "sqlite": ".db.gz", + "postgresql": ".pgsql.gz", + "mysql": ".mysql.gz", +} + def _backup_dir() -> Path: """Return (and create) the local backup directory.""" @@ -51,6 +65,14 @@ def _backup_dir() -> Path: return path +def _db_backend() -> str: + """Return the database backend name (e.g. ``'sqlite'``, ``'postgresql'``, ``'mysql'``).""" + from sqlalchemy.engine.url import make_url + + url = make_url(settings.database_url) + return url.get_backend_name() + + def _db_path() -> Path | None: """Return the SQLite database file path, or None for non-SQLite databases.""" from sqlalchemy.engine.url import make_url @@ -64,6 +86,20 @@ def _db_path() -> Path | None: return Path(db) +def _archive_ext_for_backend(backend: str) -> str: + """Return the archive file extension for the given database backend. + + Args: + backend: Backend name as returned by + ``sqlalchemy.engine.url.URL.get_backend_name()`` (e.g. ``'sqlite'``). + + Returns: + File extension string including the leading dot, e.g. ``'.db.gz'``. + Falls back to ``'.sql.gz'`` for unknown backends. + """ + return _BACKEND_EXTENSIONS.get(backend, ".sql.gz") + + def _sha256(path: Path) -> str: """Return the SHA-256 hex digest of *path*.""" h = hashlib.sha256() @@ -86,6 +122,277 @@ def _dump_sqlite(db_path: Path, dest: Path) -> None: conn.close() +def _dump_postgresql(db_url: str, dest: Path) -> None: + """Write a gzip-compressed ``pg_dump`` of the PostgreSQL database to *dest*. + + Uses ``PGPASSWORD`` environment variable so the password is never exposed on + the process command line. + + Args: + db_url: Full SQLAlchemy database URL (e.g. ``postgresql://user:pass@host/db``). + dest: Destination path for the ``.pgsql.gz`` archive. + + Raises: + RuntimeError: If ``pg_dump`` exits with a non-zero return code. + FileNotFoundError: If the ``pg_dump`` binary is not found. + """ + from sqlalchemy.engine.url import make_url + + url = make_url(db_url) + env = os.environ.copy() + if url.password: + env["PGPASSWORD"] = str(url.password) + + # Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL), + # not from user-controlled input. shell=False (the default when passing a list) is used + # so there is no shell interpretation of the argument values. + cmd: list[str] = ["pg_dump", "--format=plain", "--no-password"] + if url.host: + cmd.extend(["-h", url.host]) + if url.port: + cmd.extend(["-p", str(url.port)]) + if url.username: + cmd.extend(["-U", url.username]) + if url.database: + cmd.append(url.database) + + with gzip.open(str(dest), "wb") as gz: + proc = subprocess.Popen( # noqa: S603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + stdout = proc.stdout + if stdout is None: # pragma: no cover – guaranteed by stdout=PIPE + raise RuntimeError("pg_dump produced no stdout pipe") + try: + while True: + chunk = stdout.read(65536) + if not chunk: + break + gz.write(chunk) + finally: + stdout.close() + stderr_bytes = proc.stderr.read() if proc.stderr else b"" + proc.wait() + + if proc.returncode != 0: + dest.unlink(missing_ok=True) + raise RuntimeError( + f"pg_dump exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}" + ) + + +def _dump_mysql(db_url: str, dest: Path) -> None: + """Write a gzip-compressed ``mysqldump`` of the MySQL database to *dest*. + + Uses the ``MYSQL_PWD`` environment variable so the password is never exposed + on the process command line. + + Args: + db_url: Full SQLAlchemy database URL + (e.g. ``mysql+pymysql://user:pass@host/db``). + dest: Destination path for the ``.mysql.gz`` archive. + + Raises: + RuntimeError: If ``mysqldump`` exits with a non-zero return code. + FileNotFoundError: If the ``mysqldump`` binary is not found. + """ + from sqlalchemy.engine.url import make_url + + url = make_url(db_url) + env = os.environ.copy() + if url.password: + env["MYSQL_PWD"] = str(url.password) + + # Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL). + # shell=False (list form) prevents shell interpretation of argument values. + cmd: list[str] = ["mysqldump", "--single-transaction", "--routines", "--triggers"] + if url.host: + cmd.extend(["-h", url.host]) + if url.port: + cmd.extend(["-P", str(url.port)]) + if url.username: + cmd.extend(["-u", url.username]) + if url.database: + cmd.append(url.database) + + with gzip.open(str(dest), "wb") as gz: + proc = subprocess.Popen( # noqa: S603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + stdout = proc.stdout + if stdout is None: # pragma: no cover – guaranteed by stdout=PIPE + raise RuntimeError("mysqldump produced no stdout pipe") + try: + while True: + chunk = stdout.read(65536) + if not chunk: + break + gz.write(chunk) + finally: + stdout.close() + stderr_bytes = proc.stderr.read() if proc.stderr else b"" + proc.wait() + + if proc.returncode != 0: + dest.unlink(missing_ok=True) + raise RuntimeError( + f"mysqldump exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}" + ) + + +def _restore_sqlite(db_path: Path, archive_path: Path) -> None: + """Restore a SQLite database from a gzip-compressed SQL dump archive. + + Validates the SQL by replaying it on an in-memory database before touching + the live file. Saves a ``.pre_restore`` rollback copy first. + + Args: + db_path: Path to the live SQLite database file to overwrite. + archive_path: Path to the ``.db.gz`` gzip-compressed SQL dump. + + Raises: + ValueError: If the archive cannot be decompressed or contains invalid SQL. + RuntimeError: If writing the restored database fails. + """ + import shutil + import sqlite3 + + # Decompress and read SQL statements + try: + with gzip.open(str(archive_path), "rt", encoding="utf-8") as gz: + sql_script = gz.read() + except Exception as exc: + raise ValueError(f"Failed to decompress backup file: {exc}") from exc + + # Validate by replaying on an in-memory database + try: + mem_conn = sqlite3.connect(":memory:") + mem_conn.executescript(sql_script) + mem_conn.close() + except sqlite3.Error as exc: + raise ValueError(f"Backup file contains invalid SQL: {exc}") from exc + + # Preserve the current DB before overwriting + bak = str(db_path) + ".pre_restore" + try: + shutil.copy2(str(db_path), bak) + except OSError as exc: + logger.warning(f"Could not create pre-restore backup at {bak}: {exc}") + + try: + restore_conn = sqlite3.connect(str(db_path)) + restore_conn.executescript(sql_script) + restore_conn.close() + except sqlite3.Error as exc: + # Attempt rollback to the pre-restore copy + try: + if os.path.exists(bak): + shutil.copy2(bak, str(db_path)) + except OSError as rollback_exc: + logger.error(f"Rollback failed; database may be corrupted: {rollback_exc}") + raise RuntimeError(f"SQLite restore failed: {exc}") from exc + + +def _restore_postgresql(db_url: str, archive_path: Path) -> None: + """Restore a PostgreSQL database from a gzip-compressed SQL dump archive. + + Pipes the decompressed dump to ``psql``. Uses ``PGPASSWORD`` so the + password is never exposed on the process command line. + + Args: + db_url: Full SQLAlchemy database URL. + archive_path: Path to the ``.pgsql.gz`` gzip-compressed ``pg_dump`` archive. + + Raises: + RuntimeError: If ``psql`` exits with a non-zero return code. + FileNotFoundError: If the ``psql`` binary is not found. + """ + from sqlalchemy.engine.url import make_url + + url = make_url(db_url) + env = os.environ.copy() + if url.password: + env["PGPASSWORD"] = str(url.password) + + # Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL). + # shell=False (list form) prevents shell interpretation of argument values. + cmd: list[str] = ["psql", "--no-password"] + if url.host: + cmd.extend(["-h", url.host]) + if url.port: + cmd.extend(["-p", str(url.port)]) + if url.username: + cmd.extend(["-U", url.username]) + if url.database: + cmd.append(url.database) + + with gzip.open(str(archive_path), "rb") as gz: + proc = subprocess.Popen( # noqa: S603 + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + _, stderr_bytes = proc.communicate(input=gz.read()) + + if proc.returncode != 0: + raise RuntimeError(f"psql exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}") + + +def _restore_mysql(db_url: str, archive_path: Path) -> None: + """Restore a MySQL database from a gzip-compressed SQL dump archive. + + Pipes the decompressed dump to ``mysql``. Uses the ``MYSQL_PWD`` + environment variable so the password is never exposed on the command line. + + Args: + db_url: Full SQLAlchemy database URL. + archive_path: Path to the ``.mysql.gz`` gzip-compressed ``mysqldump`` archive. + + Raises: + RuntimeError: If ``mysql`` exits with a non-zero return code. + FileNotFoundError: If the ``mysql`` binary is not found. + """ + from sqlalchemy.engine.url import make_url + + url = make_url(db_url) + env = os.environ.copy() + if url.password: + env["MYSQL_PWD"] = str(url.password) + + # Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL). + # shell=False (list form) prevents shell interpretation of argument values. + cmd: list[str] = ["mysql"] + if url.host: + cmd.extend(["-h", url.host]) + if url.port: + cmd.extend(["-P", str(url.port)]) + if url.username: + cmd.extend(["-u", url.username]) + if url.database: + cmd.append(url.database) + + with gzip.open(str(archive_path), "rb") as gz: + proc = subprocess.Popen( # noqa: S603 + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + _, stderr_bytes = proc.communicate(input=gz.read()) + + if proc.returncode != 0: + raise RuntimeError(f"mysql exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}") + + def _apply_retention(backup_type: str, db: object) -> None: """Delete local backups beyond the retention limit for *backup_type*. @@ -314,6 +621,11 @@ def _email_backup(archive_path: Path, filename: str) -> None: def create_backup(self, backup_type: str = "hourly") -> dict: """Create a database backup archive and apply retention. + Supports SQLite (``.db.gz``), PostgreSQL (``.pgsql.gz``), and + MySQL / MariaDB (``.mysql.gz``) databases. The native dump tool for the + configured backend (``sqlite3``, ``pg_dump``, or ``mysqldump``) must be + available on the worker's ``PATH``. + Args: backup_type: ``"hourly"``, ``"daily"``, or ``"weekly"``. @@ -327,19 +639,27 @@ def create_backup(self, backup_type: str = "hourly") -> dict: logger.debug("Backup is disabled; skipping create_backup task.") return {"status": "disabled"} + backend = _db_backend() + ext = _archive_ext_for_backend(backend) + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S") - filename = f"backup_{backup_type}_{ts}.db.gz" + filename = f"backup_{backup_type}_{ts}{ext}" archive_path = _backup_dir() / filename - db_path = _db_path() - if db_path is None: - logger.warning("Backup task skipped: non-SQLite databases are not supported for file-based backups.") + # SQLite: verify the database file exists before attempting to dump it + db_path: Path | None = None + if backend == "sqlite": + db_path = _db_path() + if db_path is None: + logger.warning("Backup task skipped: in-memory SQLite databases are not supported.") + return {"status": "unsupported_db"} + if not db_path.exists(): + logger.error(f"Database file not found: {db_path}") + return {"status": "error", "detail": f"DB file missing: {db_path}"} + elif backend not in ("postgresql", "mysql"): + logger.warning(f"Backup task skipped: unsupported database backend '{backend}'.") return {"status": "unsupported_db"} - if not db_path.exists(): - logger.error(f"Database file not found: {db_path}") - return {"status": "error", "detail": f"DB file missing: {db_path}"} - status = "ok" checksum: str | None = None size_bytes = 0 @@ -347,7 +667,15 @@ def create_backup(self, backup_type: str = "hourly") -> dict: remote_path: str | None = None try: - _dump_sqlite(db_path, archive_path) + if backend == "sqlite": + # db_path is guaranteed non-None: we returned early if it were None + if db_path is None: # pragma: no cover + return {"status": "error", "detail": "db_path unexpectedly None"} + _dump_sqlite(db_path, archive_path) + elif backend == "postgresql": + _dump_postgresql(settings.database_url, archive_path) + elif backend == "mysql": + _dump_mysql(settings.database_url, archive_path) size_bytes = archive_path.stat().st_size checksum = _sha256(archive_path) logger.info(f"Created {backup_type} backup: {archive_path} ({size_bytes:,} bytes)") diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index a48c4d97..ef52229a 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -1056,9 +1056,13 @@ Webhook URLs, secrets, and subscribed events are configured per-webhook via the ### Backup & Restore -DocuElevate can automatically back up the SQLite database on a scheduled basis. +DocuElevate automatically backs up the database on a scheduled basis. Backups are managed from the **Admin → Backup & Restore** dashboard. +Supported database backends: **SQLite** (`.db.gz`), **PostgreSQL** (`.pgsql.gz`), **MySQL / MariaDB** (`.mysql.gz`). +For PostgreSQL and MySQL backups the respective CLI client (`pg_dump` / `psql` or `mysqldump` / `mysql`) must be installed on the Celery worker host. +See the [Database Configuration Guide](DatabaseConfiguration.md#backup-procedures) for setup details. + | **Variable** | **Description** | **Default** | |--------------------------------|-----------------------------------------------------------------------------------------------|---------------------| | `BACKUP_ENABLED` | Enable or disable automatic scheduled backups (`True`/`False`). | `True` | diff --git a/docs/DatabaseConfiguration.md b/docs/DatabaseConfiguration.md index 0e5811aa..9eb1fece 100644 --- a/docs/DatabaseConfiguration.md +++ b/docs/DatabaseConfiguration.md @@ -341,29 +341,91 @@ Disable `prepared_statements` when using PgBouncer in transaction mode. ## Backup Procedures -### PostgreSQL +DocuElevate's built-in **Backup & Restore** feature (Admin → Backup & Restore) supports all three +database backends natively, using the native dump tools of each database. -**Manual backup:** +| Backend | Backup tool | Archive extension | Restore tool | +|----------------|--------------|-------------------|--------------| +| SQLite | `sqlite3` (built-in Python) | `.db.gz` | `sqlite3` (built-in Python) | +| PostgreSQL | `pg_dump` | `.pgsql.gz` | `psql` | +| MySQL/MariaDB | `mysqldump` | `.mysql.gz` | `mysql` | + +Passwords are passed via the `PGPASSWORD` (PostgreSQL) and `MYSQL_PWD` (MySQL) environment +variables so they are never exposed on the process command line. + +### Prerequisites + +For PostgreSQL and MySQL backups the corresponding CLI client must be installed on the +worker host (the container / server that runs Celery workers): + +```bash +# PostgreSQL clients (Debian/Ubuntu) +apt-get install -y postgresql-client + +# MySQL clients (Debian/Ubuntu) +apt-get install -y default-mysql-client +``` + +The binaries required are: + +- **PostgreSQL**: `pg_dump` (backup) and `psql` (restore) +- **MySQL / MariaDB**: `mysqldump` (backup) and `mysql` (restore) + +### Using the Admin Dashboard + +Navigate to **Admin → Backup & Restore** to: + +- Trigger manual backups (hourly / daily / weekly) +- Download backup archives +- Upload and restore a backup archive +- Configure retention and remote storage destinations + +### PostgreSQL – manual backup/restore + +**Manual backup using DocuElevate's archive format (for use with the UI restore):** + +```bash +pg_dump --format=plain --no-password \ + -h localhost -U docuelevate docuelevate \ + | gzip > docuelevate_$(date +%Y%m%d_%H%M).pgsql.gz +``` + +**Restore via the DocuElevate UI:** upload the `.pgsql.gz` file on the Backup & Restore page. + +**Manual restore using native tools (custom format):** ```bash pg_dump -h localhost -U docuelevate -F c docuelevate > docuelevate_$(date +%Y%m%d_%H%M).dump -``` - -**Restore:** - -```bash pg_restore -h localhost -U docuelevate -d docuelevate docuelevate_20240101_1200.dump ``` **Automated daily backup (cron example):** ```cron -0 2 * * * pg_dump -h localhost -U docuelevate -F c docuelevate | gzip > /backups/docuelevate_$(date +\%Y\%m\%d).dump.gz +0 2 * * * pg_dump --format=plain -h localhost -U docuelevate docuelevate | gzip > /backups/docuelevate_$(date +\%Y\%m\%d).pgsql.gz ``` Use your cloud provider's automated backup feature when available (e.g., RDS automated snapshots, Cloud SQL backups). -### SQLite +### MySQL / MariaDB – manual backup/restore + +**Manual backup using DocuElevate's archive format (for use with the UI restore):** + +```bash +MYSQL_PWD=yourpassword mysqldump --single-transaction --routines --triggers \ + -h localhost -u docuelevate docuelevate \ + | gzip > docuelevate_$(date +%Y%m%d_%H%M).mysql.gz +``` + +**Restore via the DocuElevate UI:** upload the `.mysql.gz` file on the Backup & Restore page. + +**Manual restore using native tools:** + +```bash +gunzip -c docuelevate_20240101_1200.mysql.gz | mysql -h localhost -u docuelevate -p docuelevate +``` + +### SQLite – manual backup/restore ```bash # Stop the application first, or use SQLite's online backup API diff --git a/tests/test_backup.py b/tests/test_backup.py index d4859493..34dd115b 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -236,6 +236,254 @@ class TestBackupTaskHelpers: result = _db_path() assert result is None + def test_db_backend_sqlite(self): + """_db_backend() returns 'sqlite' for SQLite URLs.""" + from app.tasks.backup_tasks import _db_backend + + with patch("app.tasks.backup_tasks.settings") as mock_settings: + mock_settings.database_url = "sqlite:////tmp/test.db" + assert _db_backend() == "sqlite" + + def test_db_backend_postgresql(self): + """_db_backend() returns 'postgresql' for PostgreSQL URLs.""" + from app.tasks.backup_tasks import _db_backend + + with patch("app.tasks.backup_tasks.settings") as mock_settings: + mock_settings.database_url = "postgresql://user:pass@localhost/db" + assert _db_backend() == "postgresql" + + def test_db_backend_mysql(self): + """_db_backend() returns 'mysql' for MySQL URLs.""" + from app.tasks.backup_tasks import _db_backend + + with patch("app.tasks.backup_tasks.settings") as mock_settings: + mock_settings.database_url = "mysql+pymysql://user:pass@localhost/db" + assert _db_backend() == "mysql" + + def test_archive_ext_sqlite(self): + """_archive_ext_for_backend() returns '.db.gz' for sqlite.""" + from app.tasks.backup_tasks import _archive_ext_for_backend + + assert _archive_ext_for_backend("sqlite") == ".db.gz" + + def test_archive_ext_postgresql(self): + """_archive_ext_for_backend() returns '.pgsql.gz' for postgresql.""" + from app.tasks.backup_tasks import _archive_ext_for_backend + + assert _archive_ext_for_backend("postgresql") == ".pgsql.gz" + + def test_archive_ext_mysql(self): + """_archive_ext_for_backend() returns '.mysql.gz' for mysql.""" + from app.tasks.backup_tasks import _archive_ext_for_backend + + assert _archive_ext_for_backend("mysql") == ".mysql.gz" + + def test_archive_ext_unknown(self): + """_archive_ext_for_backend() falls back to '.sql.gz' for unknown backends.""" + from app.tasks.backup_tasks import _archive_ext_for_backend + + assert _archive_ext_for_backend("mssql") == ".sql.gz" + + def test_dump_postgresql_success(self, tmp_path): + """_dump_postgresql() streams pg_dump output into a gzip archive.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _dump_postgresql + + dest = tmp_path / "dump.pgsql.gz" + fake_sql = b"-- PostgreSQL database dump\nSELECT 1;\n" + + mock_proc = MagicMock() + mock_proc.stdout.read.side_effect = [fake_sql, b""] + mock_proc.stderr.read.return_value = b"" + mock_proc.returncode = 0 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + _dump_postgresql("postgresql://user:pass@localhost/testdb", dest) + + assert dest.exists() + with gzip.open(str(dest), "rb") as gz: + assert gz.read() == fake_sql + + def test_dump_postgresql_failure(self, tmp_path): + """_dump_postgresql() raises RuntimeError when pg_dump fails.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _dump_postgresql + + dest = tmp_path / "dump.pgsql.gz" + + mock_proc = MagicMock() + mock_proc.stdout.read.side_effect = [b""] + mock_proc.stderr.read.return_value = b"FATAL: connection refused" + mock_proc.returncode = 1 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + with pytest.raises(RuntimeError, match="pg_dump exited with code 1"): + _dump_postgresql("postgresql://user:pass@localhost/testdb", dest) + + def test_dump_mysql_success(self, tmp_path): + """_dump_mysql() streams mysqldump output into a gzip archive.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _dump_mysql + + dest = tmp_path / "dump.mysql.gz" + fake_sql = b"-- MySQL dump\nCREATE TABLE t (id INT);\n" + + mock_proc = MagicMock() + mock_proc.stdout.read.side_effect = [fake_sql, b""] + mock_proc.stderr.read.return_value = b"" + mock_proc.returncode = 0 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + _dump_mysql("mysql+pymysql://user:pass@localhost/testdb", dest) + + assert dest.exists() + with gzip.open(str(dest), "rb") as gz: + assert gz.read() == fake_sql + + def test_dump_mysql_failure(self, tmp_path): + """_dump_mysql() raises RuntimeError when mysqldump fails.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _dump_mysql + + dest = tmp_path / "dump.mysql.gz" + + mock_proc = MagicMock() + mock_proc.stdout.read.side_effect = [b""] + mock_proc.stderr.read.return_value = b"ERROR: Access denied" + mock_proc.returncode = 1 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + with pytest.raises(RuntimeError, match="mysqldump exited with code 1"): + _dump_mysql("mysql+pymysql://user:pass@localhost/testdb", dest) + + def test_restore_sqlite_success(self, tmp_path): + """_restore_sqlite() applies a valid SQL dump to a SQLite file.""" + from app.tasks.backup_tasks import _restore_sqlite + + db_file = tmp_path / "test.db" + conn = sqlite3.connect(str(db_file)) + conn.execute("CREATE TABLE old (id INTEGER)") + conn.commit() + conn.close() + + # Create a valid dump archive + sql = "BEGIN TRANSACTION;\nCREATE TABLE new_tbl (x TEXT);\nCOMMIT;\n" + archive = tmp_path / "dump.db.gz" + with gzip.open(str(archive), "wt") as gz: + gz.write(sql) + + _restore_sqlite(db_file, archive) + + conn2 = sqlite3.connect(str(db_file)) + tables = [r[0] for r in conn2.execute("SELECT name FROM sqlite_master WHERE type='table'")] + conn2.close() + assert "new_tbl" in tables + + def test_restore_sqlite_invalid_gz(self, tmp_path): + """_restore_sqlite() raises ValueError for corrupt gzip content.""" + from app.tasks.backup_tasks import _restore_sqlite + + db_file = tmp_path / "test.db" + db_file.write_bytes(b"") + archive = tmp_path / "bad.db.gz" + archive.write_bytes(b"not gzip data") + + with pytest.raises(ValueError, match="Failed to decompress"): + _restore_sqlite(db_file, archive) + + def test_restore_sqlite_invalid_sql(self, tmp_path): + """_restore_sqlite() raises ValueError for invalid SQL content.""" + from app.tasks.backup_tasks import _restore_sqlite + + db_file = tmp_path / "test.db" + db_file.write_bytes(b"") + archive = tmp_path / "bad.db.gz" + with gzip.open(str(archive), "wt") as gz: + gz.write("THIS IS NOT VALID SQL!!!;\n") + + with pytest.raises(ValueError, match="invalid SQL"): + _restore_sqlite(db_file, archive) + + def test_restore_postgresql_success(self, tmp_path): + """_restore_postgresql() pipes the archive to psql.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _restore_postgresql + + fake_sql = b"-- PostgreSQL dump\nSELECT 1;\n" + archive = tmp_path / "dump.pgsql.gz" + with gzip.open(str(archive), "wb") as gz: + gz.write(fake_sql) + + mock_proc = MagicMock() + mock_proc.communicate.return_value = (b"", b"") + mock_proc.returncode = 0 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + _restore_postgresql("postgresql://user:pass@localhost/testdb", archive) + + mock_proc.communicate.assert_called_once_with(input=fake_sql) + + def test_restore_postgresql_failure(self, tmp_path): + """_restore_postgresql() raises RuntimeError when psql fails.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _restore_postgresql + + archive = tmp_path / "dump.pgsql.gz" + with gzip.open(str(archive), "wb") as gz: + gz.write(b"SELECT 1;") + + mock_proc = MagicMock() + mock_proc.communicate.return_value = (b"", b"ERROR: invalid input") + mock_proc.returncode = 1 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + with pytest.raises(RuntimeError, match="psql exited with code 1"): + _restore_postgresql("postgresql://user:pass@localhost/testdb", archive) + + def test_restore_mysql_success(self, tmp_path): + """_restore_mysql() pipes the archive to mysql.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _restore_mysql + + fake_sql = b"-- MySQL dump\nSELECT 1;\n" + archive = tmp_path / "dump.mysql.gz" + with gzip.open(str(archive), "wb") as gz: + gz.write(fake_sql) + + mock_proc = MagicMock() + mock_proc.communicate.return_value = (b"", b"") + mock_proc.returncode = 0 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + _restore_mysql("mysql+pymysql://user:pass@localhost/testdb", archive) + + mock_proc.communicate.assert_called_once_with(input=fake_sql) + + def test_restore_mysql_failure(self, tmp_path): + """_restore_mysql() raises RuntimeError when mysql fails.""" + from unittest.mock import MagicMock + + from app.tasks.backup_tasks import _restore_mysql + + archive = tmp_path / "dump.mysql.gz" + with gzip.open(str(archive), "wb") as gz: + gz.write(b"SELECT 1;") + + mock_proc = MagicMock() + mock_proc.communicate.return_value = (b"", b"ERROR: Access denied") + mock_proc.returncode = 1 + + with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc): + with pytest.raises(RuntimeError, match="mysql exited with code 1"): + _restore_mysql("mysql+pymysql://user:pass@localhost/testdb", archive) + def test_apply_retention_prunes_old(self, tmp_path, db_session): """_apply_retention() deletes backups beyond the retention limit.""" from app.tasks.backup_tasks import _apply_retention @@ -352,20 +600,28 @@ class TestCreateBackupTask: result = create_backup("hourly") assert result["status"] == "disabled" - def test_non_sqlite_db(self): - """create_backup returns unsupported_db for non-SQLite databases.""" + def test_unsupported_db_backend(self): + """create_backup returns unsupported_db for backends other than sqlite/postgresql/mysql.""" from app.tasks.backup_tasks import create_backup - with ( - patch("app.tasks.backup_tasks.settings") as mock_settings, - patch("app.tasks.backup_tasks._db_path", return_value=None), - ): + with patch("app.tasks.backup_tasks.settings") as mock_settings: mock_settings.backup_enabled = True + mock_settings.database_url = "mssql+pyodbc://user:pass@server/db" + result = create_backup("hourly") + assert result["status"] == "unsupported_db" + + def test_in_memory_sqlite_unsupported(self): + """create_backup returns unsupported_db for in-memory SQLite.""" + from app.tasks.backup_tasks import create_backup + + with patch("app.tasks.backup_tasks.settings") as mock_settings: + mock_settings.backup_enabled = True + mock_settings.database_url = "sqlite:///:memory:" result = create_backup("hourly") assert result["status"] == "unsupported_db" def test_missing_db_file(self, tmp_path): - """create_backup returns error when the DB file does not exist.""" + """create_backup returns error when the SQLite DB file does not exist.""" from app.tasks.backup_tasks import create_backup missing = tmp_path / "does_not_exist.db" @@ -375,6 +631,7 @@ class TestCreateBackupTask: patch("app.tasks.backup_tasks._db_path", return_value=missing), ): mock_settings.backup_enabled = True + mock_settings.database_url = f"sqlite:///{missing}" result = create_backup("hourly") assert result["status"] == "error" @@ -400,6 +657,7 @@ class TestCreateBackupTask: ): backup_dir.mkdir(parents=True, exist_ok=True) mock_settings.backup_enabled = True + mock_settings.database_url = f"sqlite:///{db_file}" mock_db = MagicMock() mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db) mock_sl.return_value.__exit__ = MagicMock(return_value=False) @@ -408,6 +666,69 @@ class TestCreateBackupTask: assert result["status"] == "ok" assert "filename" in result assert result["filename"].startswith("backup_hourly_") + assert result["filename"].endswith(".db.gz") + + def test_successful_backup_postgresql(self, tmp_path): + """create_backup creates a .pgsql.gz archive for PostgreSQL databases.""" + from app.tasks.backup_tasks import create_backup + + backup_dir = tmp_path / "backups" + backup_dir.mkdir() + + def fake_pg_dump(db_url: str, dest: Path) -> None: + with gzip.open(str(dest), "wb") as gz: + gz.write(b"-- PostgreSQL dump\n") + + with ( + patch("app.tasks.backup_tasks.settings") as mock_settings, + patch("app.tasks.backup_tasks._backup_dir", return_value=backup_dir), + patch("app.tasks.backup_tasks._dump_postgresql", side_effect=fake_pg_dump), + patch("app.tasks.backup_tasks._upload_remote", return_value=None), + patch("app.tasks.backup_tasks._apply_retention"), + patch("app.tasks.backup_tasks._prune_remote_backups"), + patch("app.tasks.backup_tasks.SessionLocal") as mock_sl, + ): + mock_settings.backup_enabled = True + mock_settings.database_url = "postgresql://user:pass@localhost/testdb" + mock_db = MagicMock() + mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_sl.return_value.__exit__ = MagicMock(return_value=False) + result = create_backup("daily") + + assert result["status"] == "ok" + assert result["filename"].endswith(".pgsql.gz") + assert "daily" in result["filename"] + + def test_successful_backup_mysql(self, tmp_path): + """create_backup creates a .mysql.gz archive for MySQL databases.""" + from app.tasks.backup_tasks import create_backup + + backup_dir = tmp_path / "backups" + backup_dir.mkdir() + + def fake_mysql_dump(db_url: str, dest: Path) -> None: + with gzip.open(str(dest), "wb") as gz: + gz.write(b"-- MySQL dump\n") + + with ( + patch("app.tasks.backup_tasks.settings") as mock_settings, + patch("app.tasks.backup_tasks._backup_dir", return_value=backup_dir), + patch("app.tasks.backup_tasks._dump_mysql", side_effect=fake_mysql_dump), + patch("app.tasks.backup_tasks._upload_remote", return_value=None), + patch("app.tasks.backup_tasks._apply_retention"), + patch("app.tasks.backup_tasks._prune_remote_backups"), + patch("app.tasks.backup_tasks.SessionLocal") as mock_sl, + ): + mock_settings.backup_enabled = True + mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb" + mock_db = MagicMock() + mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db) + mock_sl.return_value.__exit__ = MagicMock(return_value=False) + result = create_backup("weekly") + + assert result["status"] == "ok" + assert result["filename"].endswith(".mysql.gz") + assert "weekly" in result["filename"] def test_invalid_backup_type_defaults_to_hourly(self, tmp_path): """create_backup normalises unknown backup_type to 'hourly'.""" @@ -431,6 +752,7 @@ class TestCreateBackupTask: patch("app.tasks.backup_tasks.SessionLocal") as mock_sl, ): mock_settings.backup_enabled = True + mock_settings.database_url = f"sqlite:///{db_file}" mock_db = MagicMock() mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db) mock_sl.return_value.__exit__ = MagicMock(return_value=False) @@ -549,19 +871,24 @@ class TestBackupAPIEndpoints: assert resp.status_code == 403 def test_restore_wrong_extension(self, admin_client): - """POST /api/admin/backup/restore rejects non-.db.gz files.""" + """POST /api/admin/backup/restore rejects files with wrong extension for current backend.""" + # Default test env uses sqlite:///:memory: → expects .db.gz resp = admin_client.post( "/api/admin/backup/restore", files={"file": ("backup.zip", b"data", "application/zip")}, ) assert resp.status_code == 400 - def test_restore_invalid_gz_content(self, admin_client): + def test_restore_invalid_gz_content(self, admin_client, tmp_path): """POST /api/admin/backup/restore rejects corrupt gzip data.""" - resp = admin_client.post( - "/api/admin/backup/restore", - files={"file": ("backup.db.gz", b"not gzip data at all", "application/gzip")}, - ) + db_file = tmp_path / "test.db" + db_file.write_bytes(b"") + + with patch("app.tasks.backup_tasks._db_path", return_value=db_file): + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.db.gz", b"not gzip data at all", "application/gzip")}, + ) assert resp.status_code == 400 def test_restore_valid_archive(self, admin_client, tmp_path): @@ -581,11 +908,11 @@ class TestBackupAPIEndpoints: assert resp.status_code == 200 assert resp.json()["status"] == "restored" - def test_restore_non_sqlite_db(self, admin_client): - """POST /api/admin/backup/restore returns 400 for non-SQLite database.""" - sql = "BEGIN TRANSACTION;\nCOMMIT;\n" - gz_data = gzip.compress(sql.encode()) + def test_restore_in_memory_sqlite(self, admin_client): + """POST /api/admin/backup/restore returns 400 for in-memory SQLite (no file to restore to).""" + gz_data = gzip.compress(b"BEGIN TRANSACTION;\nCOMMIT;\n") + # _db_path() returns None for :memory: URLs → 400 with patch("app.tasks.backup_tasks._db_path", return_value=None): resp = admin_client.post( "/api/admin/backup/restore", @@ -593,6 +920,106 @@ class TestBackupAPIEndpoints: ) assert resp.status_code == 400 + def test_restore_wrong_extension_for_postgresql(self, admin_client): + """POST /api/admin/backup/restore returns 400 when uploading .db.gz for PostgreSQL backend.""" + gz_data = gzip.compress(b"-- PostgreSQL dump") + + with patch("app.config.settings") as mock_settings: + mock_settings.database_url = "postgresql://user:pass@localhost/testdb" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.db.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 400 + + def test_restore_wrong_extension_for_mysql(self, admin_client): + """POST /api/admin/backup/restore returns 400 when uploading .db.gz for MySQL backend.""" + gz_data = gzip.compress(b"-- MySQL dump") + + with patch("app.config.settings") as mock_settings: + mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.db.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 400 + + def test_restore_postgresql_success(self, admin_client): + """POST /api/admin/backup/restore succeeds for PostgreSQL database.""" + gz_data = gzip.compress(b"-- PostgreSQL dump\n") + + with ( + patch("app.config.settings") as mock_settings, + patch("app.tasks.backup_tasks._restore_postgresql") as mock_restore, + ): + mock_settings.database_url = "postgresql://user:pass@localhost/testdb" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "restored" + mock_restore.assert_called_once() + + def test_restore_mysql_success(self, admin_client): + """POST /api/admin/backup/restore succeeds for MySQL database.""" + gz_data = gzip.compress(b"-- MySQL dump\n") + + with ( + patch("app.config.settings") as mock_settings, + patch("app.tasks.backup_tasks._restore_mysql") as mock_restore, + ): + mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.mysql.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "restored" + mock_restore.assert_called_once() + + def test_restore_postgresql_runtime_error(self, admin_client): + """POST /api/admin/backup/restore returns 500 when psql command fails.""" + gz_data = gzip.compress(b"-- PostgreSQL dump\n") + + with ( + patch("app.config.settings") as mock_settings, + patch("app.tasks.backup_tasks._restore_postgresql", side_effect=RuntimeError("psql failed")), + ): + mock_settings.database_url = "postgresql://user:pass@localhost/testdb" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 500 + + def test_restore_postgresql_missing_binary(self, admin_client): + """POST /api/admin/backup/restore returns 500 when psql binary is missing.""" + gz_data = gzip.compress(b"-- PostgreSQL dump\n") + + with ( + patch("app.config.settings") as mock_settings, + patch("app.tasks.backup_tasks._restore_postgresql", side_effect=FileNotFoundError("psql not found")), + ): + mock_settings.database_url = "postgresql://user:pass@localhost/testdb" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 500 + + def test_restore_unsupported_backend(self, admin_client): + """POST /api/admin/backup/restore returns 400 for an unsupported database backend.""" + gz_data = gzip.compress(b"-- some dump\n") + + with patch("app.config.settings") as mock_settings: + mock_settings.database_url = "mssql+pyodbc://user:pass@server/db" + resp = admin_client.post( + "/api/admin/backup/restore", + files={"file": ("backup.sql.gz", gz_data, "application/gzip")}, + ) + assert resp.status_code == 400 + # --------------------------------------------------------------------------- # View tests