feat(backup): extend backup and restore to PostgreSQL and MySQL/MariaDB
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
+84
-68
@@ -117,88 +117,104 @@ async def restore_backup(
|
||||
"""Restore the database from an uploaded gzip-compressed SQL dump.
|
||||
|
||||
**Warning**: This overwrites the current database contents.
|
||||
Only SQLite databases are supported.
|
||||
|
||||
The uploaded file must be a ``.db.gz`` file produced by the DocuElevate
|
||||
backup task (a gzip-compressed SQLite ``.dump()`` SQL script).
|
||||
Supported formats (must match the currently configured database backend):
|
||||
|
||||
- ``*.db.gz`` – gzip-compressed SQLite ``.dump()`` SQL script (SQLite backend)
|
||||
- ``*.pgsql.gz`` – gzip-compressed ``pg_dump --format=plain`` output (PostgreSQL backend)
|
||||
- ``*.mysql.gz`` – gzip-compressed ``mysqldump`` output (MySQL / MariaDB backend)
|
||||
"""
|
||||
from app.tasks.backup_tasks import _db_path
|
||||
|
||||
db_path = _db_path()
|
||||
if db_path is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Restore is only supported for SQLite databases.",
|
||||
)
|
||||
|
||||
if not file.filename or not file.filename.endswith(".db.gz"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Uploaded file must be a .db.gz backup archive.",
|
||||
)
|
||||
|
||||
import gzip
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
# Write the upload to a temp file first so we can validate it
|
||||
with tempfile.NamedTemporaryFile(suffix=".db.gz", delete=False) as tmp:
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
from app.config import settings as app_settings
|
||||
from app.tasks.backup_tasks import (
|
||||
_archive_ext_for_backend,
|
||||
_db_path,
|
||||
_restore_mysql,
|
||||
_restore_postgresql,
|
||||
_restore_sqlite,
|
||||
)
|
||||
|
||||
url = make_url(app_settings.database_url)
|
||||
backend = url.get_backend_name()
|
||||
expected_ext = _archive_ext_for_backend(backend)
|
||||
|
||||
if not file.filename or not file.filename.endswith(expected_ext):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
f"Uploaded file must be a '{expected_ext}' backup archive for the current database backend ({backend})."
|
||||
),
|
||||
)
|
||||
|
||||
# Write upload to a temp file
|
||||
with tempfile.NamedTemporaryFile(suffix=expected_ext, delete=False) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
content = await file.read()
|
||||
tmp.write(content)
|
||||
|
||||
try:
|
||||
# Decompress and read SQL statements
|
||||
with gzip.open(str(tmp_path), "rt", encoding="utf-8") as gz:
|
||||
sql_script = gz.read()
|
||||
except Exception as exc:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Failed to decompress backup file: {exc}",
|
||||
) from exc
|
||||
if backend == "sqlite":
|
||||
db_path = _db_path()
|
||||
if db_path is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Restore is only supported for file-based SQLite databases.",
|
||||
)
|
||||
# Close the application DB session before replacing the file
|
||||
db.close()
|
||||
try:
|
||||
_restore_sqlite(db_path, tmp_path)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
|
||||
# Create a fresh in-memory DB from the script to validate it
|
||||
try:
|
||||
mem_conn = sqlite3.connect(":memory:")
|
||||
mem_conn.executescript(sql_script)
|
||||
mem_conn.close()
|
||||
except sqlite3.Error as exc:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Backup file contains invalid SQL: {exc}",
|
||||
) from exc
|
||||
elif backend == "postgresql":
|
||||
db.close()
|
||||
try:
|
||||
_restore_postgresql(app_settings.database_url, tmp_path)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"psql binary not found – is PostgreSQL client installed? ({exc})",
|
||||
) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"PostgreSQL restore failed: {exc}",
|
||||
) from exc
|
||||
|
||||
# Close the application DB session before replacing the file
|
||||
db.close()
|
||||
elif backend == "mysql":
|
||||
db.close()
|
||||
try:
|
||||
_restore_mysql(app_settings.database_url, tmp_path)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"mysql binary not found – is MySQL client installed? ({exc})",
|
||||
) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"MySQL restore failed: {exc}",
|
||||
) from exc
|
||||
|
||||
# Preserve the current DB before overwriting
|
||||
import shutil
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Database backend '{backend}' does not support restore.",
|
||||
)
|
||||
|
||||
bak = str(db_path) + ".pre_restore"
|
||||
try:
|
||||
shutil.copy2(str(db_path), bak)
|
||||
except OSError as exc:
|
||||
logger.warning(f"Could not create pre-restore backup at {bak}: {exc}")
|
||||
|
||||
try:
|
||||
# Write the restored database
|
||||
restore_conn = sqlite3.connect(str(db_path))
|
||||
restore_conn.executescript(sql_script)
|
||||
restore_conn.close()
|
||||
except sqlite3.Error as exc:
|
||||
# Attempt rollback
|
||||
try:
|
||||
if os.path.exists(bak):
|
||||
shutil.copy2(bak, str(db_path))
|
||||
except OSError as rollback_exc:
|
||||
logger.error(f"Rollback failed; database may be corrupted: {rollback_exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Restore failed: {exc}",
|
||||
) from exc
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
+337
-9
@@ -16,12 +16,19 @@ Three separate Celery-beat entries call ``create_backup`` with the appropriate
|
||||
After each backup is created ``_apply_retention`` prunes old local backups for
|
||||
that tier. Remote copies are pruned by ``_prune_remote_backups`` which mirrors
|
||||
the same retention limits.
|
||||
|
||||
Supported database backends
|
||||
----------------------------
|
||||
- **SQLite** – dumped via Python's built-in ``sqlite3.iterdump()``; archive extension ``.db.gz``
|
||||
- **PostgreSQL** – dumped via ``pg_dump --format=plain``; archive extension ``.pgsql.gz``
|
||||
- **MySQL / MariaDB** – dumped via ``mysqldump --single-transaction``; archive extension ``.mysql.gz``
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
@@ -42,6 +49,13 @@ _BACKUP_TYPE_RETAIN: dict[str, str] = {
|
||||
"weekly": "backup_retain_weekly",
|
||||
}
|
||||
|
||||
#: Map of backend name → archive file extension.
|
||||
_BACKEND_EXTENSIONS: dict[str, str] = {
|
||||
"sqlite": ".db.gz",
|
||||
"postgresql": ".pgsql.gz",
|
||||
"mysql": ".mysql.gz",
|
||||
}
|
||||
|
||||
|
||||
def _backup_dir() -> Path:
|
||||
"""Return (and create) the local backup directory."""
|
||||
@@ -51,6 +65,14 @@ def _backup_dir() -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def _db_backend() -> str:
|
||||
"""Return the database backend name (e.g. ``'sqlite'``, ``'postgresql'``, ``'mysql'``)."""
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
url = make_url(settings.database_url)
|
||||
return url.get_backend_name()
|
||||
|
||||
|
||||
def _db_path() -> Path | None:
|
||||
"""Return the SQLite database file path, or None for non-SQLite databases."""
|
||||
from sqlalchemy.engine.url import make_url
|
||||
@@ -64,6 +86,20 @@ def _db_path() -> Path | None:
|
||||
return Path(db)
|
||||
|
||||
|
||||
def _archive_ext_for_backend(backend: str) -> str:
|
||||
"""Return the archive file extension for the given database backend.
|
||||
|
||||
Args:
|
||||
backend: Backend name as returned by
|
||||
``sqlalchemy.engine.url.URL.get_backend_name()`` (e.g. ``'sqlite'``).
|
||||
|
||||
Returns:
|
||||
File extension string including the leading dot, e.g. ``'.db.gz'``.
|
||||
Falls back to ``'.sql.gz'`` for unknown backends.
|
||||
"""
|
||||
return _BACKEND_EXTENSIONS.get(backend, ".sql.gz")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
"""Return the SHA-256 hex digest of *path*."""
|
||||
h = hashlib.sha256()
|
||||
@@ -86,6 +122,277 @@ def _dump_sqlite(db_path: Path, dest: Path) -> None:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _dump_postgresql(db_url: str, dest: Path) -> None:
|
||||
"""Write a gzip-compressed ``pg_dump`` of the PostgreSQL database to *dest*.
|
||||
|
||||
Uses ``PGPASSWORD`` environment variable so the password is never exposed on
|
||||
the process command line.
|
||||
|
||||
Args:
|
||||
db_url: Full SQLAlchemy database URL (e.g. ``postgresql://user:pass@host/db``).
|
||||
dest: Destination path for the ``.pgsql.gz`` archive.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``pg_dump`` exits with a non-zero return code.
|
||||
FileNotFoundError: If the ``pg_dump`` binary is not found.
|
||||
"""
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
url = make_url(db_url)
|
||||
env = os.environ.copy()
|
||||
if url.password:
|
||||
env["PGPASSWORD"] = str(url.password)
|
||||
|
||||
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL),
|
||||
# not from user-controlled input. shell=False (the default when passing a list) is used
|
||||
# so there is no shell interpretation of the argument values.
|
||||
cmd: list[str] = ["pg_dump", "--format=plain", "--no-password"]
|
||||
if url.host:
|
||||
cmd.extend(["-h", url.host])
|
||||
if url.port:
|
||||
cmd.extend(["-p", str(url.port)])
|
||||
if url.username:
|
||||
cmd.extend(["-U", url.username])
|
||||
if url.database:
|
||||
cmd.append(url.database)
|
||||
|
||||
with gzip.open(str(dest), "wb") as gz:
|
||||
proc = subprocess.Popen( # noqa: S603
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
stdout = proc.stdout
|
||||
if stdout is None: # pragma: no cover – guaranteed by stdout=PIPE
|
||||
raise RuntimeError("pg_dump produced no stdout pipe")
|
||||
try:
|
||||
while True:
|
||||
chunk = stdout.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
gz.write(chunk)
|
||||
finally:
|
||||
stdout.close()
|
||||
stderr_bytes = proc.stderr.read() if proc.stderr else b""
|
||||
proc.wait()
|
||||
|
||||
if proc.returncode != 0:
|
||||
dest.unlink(missing_ok=True)
|
||||
raise RuntimeError(
|
||||
f"pg_dump exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}"
|
||||
)
|
||||
|
||||
|
||||
def _dump_mysql(db_url: str, dest: Path) -> None:
|
||||
"""Write a gzip-compressed ``mysqldump`` of the MySQL database to *dest*.
|
||||
|
||||
Uses the ``MYSQL_PWD`` environment variable so the password is never exposed
|
||||
on the process command line.
|
||||
|
||||
Args:
|
||||
db_url: Full SQLAlchemy database URL
|
||||
(e.g. ``mysql+pymysql://user:pass@host/db``).
|
||||
dest: Destination path for the ``.mysql.gz`` archive.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``mysqldump`` exits with a non-zero return code.
|
||||
FileNotFoundError: If the ``mysqldump`` binary is not found.
|
||||
"""
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
url = make_url(db_url)
|
||||
env = os.environ.copy()
|
||||
if url.password:
|
||||
env["MYSQL_PWD"] = str(url.password)
|
||||
|
||||
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL).
|
||||
# shell=False (list form) prevents shell interpretation of argument values.
|
||||
cmd: list[str] = ["mysqldump", "--single-transaction", "--routines", "--triggers"]
|
||||
if url.host:
|
||||
cmd.extend(["-h", url.host])
|
||||
if url.port:
|
||||
cmd.extend(["-P", str(url.port)])
|
||||
if url.username:
|
||||
cmd.extend(["-u", url.username])
|
||||
if url.database:
|
||||
cmd.append(url.database)
|
||||
|
||||
with gzip.open(str(dest), "wb") as gz:
|
||||
proc = subprocess.Popen( # noqa: S603
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
stdout = proc.stdout
|
||||
if stdout is None: # pragma: no cover – guaranteed by stdout=PIPE
|
||||
raise RuntimeError("mysqldump produced no stdout pipe")
|
||||
try:
|
||||
while True:
|
||||
chunk = stdout.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
gz.write(chunk)
|
||||
finally:
|
||||
stdout.close()
|
||||
stderr_bytes = proc.stderr.read() if proc.stderr else b""
|
||||
proc.wait()
|
||||
|
||||
if proc.returncode != 0:
|
||||
dest.unlink(missing_ok=True)
|
||||
raise RuntimeError(
|
||||
f"mysqldump exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}"
|
||||
)
|
||||
|
||||
|
||||
def _restore_sqlite(db_path: Path, archive_path: Path) -> None:
|
||||
"""Restore a SQLite database from a gzip-compressed SQL dump archive.
|
||||
|
||||
Validates the SQL by replaying it on an in-memory database before touching
|
||||
the live file. Saves a ``<db_path>.pre_restore`` rollback copy first.
|
||||
|
||||
Args:
|
||||
db_path: Path to the live SQLite database file to overwrite.
|
||||
archive_path: Path to the ``.db.gz`` gzip-compressed SQL dump.
|
||||
|
||||
Raises:
|
||||
ValueError: If the archive cannot be decompressed or contains invalid SQL.
|
||||
RuntimeError: If writing the restored database fails.
|
||||
"""
|
||||
import shutil
|
||||
import sqlite3
|
||||
|
||||
# Decompress and read SQL statements
|
||||
try:
|
||||
with gzip.open(str(archive_path), "rt", encoding="utf-8") as gz:
|
||||
sql_script = gz.read()
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Failed to decompress backup file: {exc}") from exc
|
||||
|
||||
# Validate by replaying on an in-memory database
|
||||
try:
|
||||
mem_conn = sqlite3.connect(":memory:")
|
||||
mem_conn.executescript(sql_script)
|
||||
mem_conn.close()
|
||||
except sqlite3.Error as exc:
|
||||
raise ValueError(f"Backup file contains invalid SQL: {exc}") from exc
|
||||
|
||||
# Preserve the current DB before overwriting
|
||||
bak = str(db_path) + ".pre_restore"
|
||||
try:
|
||||
shutil.copy2(str(db_path), bak)
|
||||
except OSError as exc:
|
||||
logger.warning(f"Could not create pre-restore backup at {bak}: {exc}")
|
||||
|
||||
try:
|
||||
restore_conn = sqlite3.connect(str(db_path))
|
||||
restore_conn.executescript(sql_script)
|
||||
restore_conn.close()
|
||||
except sqlite3.Error as exc:
|
||||
# Attempt rollback to the pre-restore copy
|
||||
try:
|
||||
if os.path.exists(bak):
|
||||
shutil.copy2(bak, str(db_path))
|
||||
except OSError as rollback_exc:
|
||||
logger.error(f"Rollback failed; database may be corrupted: {rollback_exc}")
|
||||
raise RuntimeError(f"SQLite restore failed: {exc}") from exc
|
||||
|
||||
|
||||
def _restore_postgresql(db_url: str, archive_path: Path) -> None:
|
||||
"""Restore a PostgreSQL database from a gzip-compressed SQL dump archive.
|
||||
|
||||
Pipes the decompressed dump to ``psql``. Uses ``PGPASSWORD`` so the
|
||||
password is never exposed on the process command line.
|
||||
|
||||
Args:
|
||||
db_url: Full SQLAlchemy database URL.
|
||||
archive_path: Path to the ``.pgsql.gz`` gzip-compressed ``pg_dump`` archive.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``psql`` exits with a non-zero return code.
|
||||
FileNotFoundError: If the ``psql`` binary is not found.
|
||||
"""
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
url = make_url(db_url)
|
||||
env = os.environ.copy()
|
||||
if url.password:
|
||||
env["PGPASSWORD"] = str(url.password)
|
||||
|
||||
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL).
|
||||
# shell=False (list form) prevents shell interpretation of argument values.
|
||||
cmd: list[str] = ["psql", "--no-password"]
|
||||
if url.host:
|
||||
cmd.extend(["-h", url.host])
|
||||
if url.port:
|
||||
cmd.extend(["-p", str(url.port)])
|
||||
if url.username:
|
||||
cmd.extend(["-U", url.username])
|
||||
if url.database:
|
||||
cmd.append(url.database)
|
||||
|
||||
with gzip.open(str(archive_path), "rb") as gz:
|
||||
proc = subprocess.Popen( # noqa: S603
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
_, stderr_bytes = proc.communicate(input=gz.read())
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"psql exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}")
|
||||
|
||||
|
||||
def _restore_mysql(db_url: str, archive_path: Path) -> None:
|
||||
"""Restore a MySQL database from a gzip-compressed SQL dump archive.
|
||||
|
||||
Pipes the decompressed dump to ``mysql``. Uses the ``MYSQL_PWD``
|
||||
environment variable so the password is never exposed on the command line.
|
||||
|
||||
Args:
|
||||
db_url: Full SQLAlchemy database URL.
|
||||
archive_path: Path to the ``.mysql.gz`` gzip-compressed ``mysqldump`` archive.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``mysql`` exits with a non-zero return code.
|
||||
FileNotFoundError: If the ``mysql`` binary is not found.
|
||||
"""
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
url = make_url(db_url)
|
||||
env = os.environ.copy()
|
||||
if url.password:
|
||||
env["MYSQL_PWD"] = str(url.password)
|
||||
|
||||
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL).
|
||||
# shell=False (list form) prevents shell interpretation of argument values.
|
||||
cmd: list[str] = ["mysql"]
|
||||
if url.host:
|
||||
cmd.extend(["-h", url.host])
|
||||
if url.port:
|
||||
cmd.extend(["-P", str(url.port)])
|
||||
if url.username:
|
||||
cmd.extend(["-u", url.username])
|
||||
if url.database:
|
||||
cmd.append(url.database)
|
||||
|
||||
with gzip.open(str(archive_path), "rb") as gz:
|
||||
proc = subprocess.Popen( # noqa: S603
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
_, stderr_bytes = proc.communicate(input=gz.read())
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"mysql exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}")
|
||||
|
||||
|
||||
def _apply_retention(backup_type: str, db: object) -> None:
|
||||
"""Delete local backups beyond the retention limit for *backup_type*.
|
||||
|
||||
@@ -314,6 +621,11 @@ def _email_backup(archive_path: Path, filename: str) -> None:
|
||||
def create_backup(self, backup_type: str = "hourly") -> dict:
|
||||
"""Create a database backup archive and apply retention.
|
||||
|
||||
Supports SQLite (``.db.gz``), PostgreSQL (``.pgsql.gz``), and
|
||||
MySQL / MariaDB (``.mysql.gz``) databases. The native dump tool for the
|
||||
configured backend (``sqlite3``, ``pg_dump``, or ``mysqldump``) must be
|
||||
available on the worker's ``PATH``.
|
||||
|
||||
Args:
|
||||
backup_type: ``"hourly"``, ``"daily"``, or ``"weekly"``.
|
||||
|
||||
@@ -327,19 +639,27 @@ def create_backup(self, backup_type: str = "hourly") -> dict:
|
||||
logger.debug("Backup is disabled; skipping create_backup task.")
|
||||
return {"status": "disabled"}
|
||||
|
||||
backend = _db_backend()
|
||||
ext = _archive_ext_for_backend(backend)
|
||||
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")
|
||||
filename = f"backup_{backup_type}_{ts}.db.gz"
|
||||
filename = f"backup_{backup_type}_{ts}{ext}"
|
||||
archive_path = _backup_dir() / filename
|
||||
|
||||
db_path = _db_path()
|
||||
if db_path is None:
|
||||
logger.warning("Backup task skipped: non-SQLite databases are not supported for file-based backups.")
|
||||
# SQLite: verify the database file exists before attempting to dump it
|
||||
db_path: Path | None = None
|
||||
if backend == "sqlite":
|
||||
db_path = _db_path()
|
||||
if db_path is None:
|
||||
logger.warning("Backup task skipped: in-memory SQLite databases are not supported.")
|
||||
return {"status": "unsupported_db"}
|
||||
if not db_path.exists():
|
||||
logger.error(f"Database file not found: {db_path}")
|
||||
return {"status": "error", "detail": f"DB file missing: {db_path}"}
|
||||
elif backend not in ("postgresql", "mysql"):
|
||||
logger.warning(f"Backup task skipped: unsupported database backend '{backend}'.")
|
||||
return {"status": "unsupported_db"}
|
||||
|
||||
if not db_path.exists():
|
||||
logger.error(f"Database file not found: {db_path}")
|
||||
return {"status": "error", "detail": f"DB file missing: {db_path}"}
|
||||
|
||||
status = "ok"
|
||||
checksum: str | None = None
|
||||
size_bytes = 0
|
||||
@@ -347,7 +667,15 @@ def create_backup(self, backup_type: str = "hourly") -> dict:
|
||||
remote_path: str | None = None
|
||||
|
||||
try:
|
||||
_dump_sqlite(db_path, archive_path)
|
||||
if backend == "sqlite":
|
||||
# db_path is guaranteed non-None: we returned early if it were None
|
||||
if db_path is None: # pragma: no cover
|
||||
return {"status": "error", "detail": "db_path unexpectedly None"}
|
||||
_dump_sqlite(db_path, archive_path)
|
||||
elif backend == "postgresql":
|
||||
_dump_postgresql(settings.database_url, archive_path)
|
||||
elif backend == "mysql":
|
||||
_dump_mysql(settings.database_url, archive_path)
|
||||
size_bytes = archive_path.stat().st_size
|
||||
checksum = _sha256(archive_path)
|
||||
logger.info(f"Created {backup_type} backup: {archive_path} ({size_bytes:,} bytes)")
|
||||
|
||||
@@ -1037,9 +1037,13 @@ Webhook URLs, secrets, and subscribed events are configured per-webhook via the
|
||||
|
||||
### Backup & Restore
|
||||
|
||||
DocuElevate can automatically back up the SQLite database on a scheduled basis.
|
||||
DocuElevate automatically backs up the database on a scheduled basis.
|
||||
Backups are managed from the **Admin → Backup & Restore** dashboard.
|
||||
|
||||
Supported database backends: **SQLite** (`.db.gz`), **PostgreSQL** (`.pgsql.gz`), **MySQL / MariaDB** (`.mysql.gz`).
|
||||
For PostgreSQL and MySQL backups the respective CLI client (`pg_dump` / `psql` or `mysqldump` / `mysql`) must be installed on the Celery worker host.
|
||||
See the [Database Configuration Guide](DatabaseConfiguration.md#backup-procedures) for setup details.
|
||||
|
||||
| **Variable** | **Description** | **Default** |
|
||||
|--------------------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
||||
| `BACKUP_ENABLED` | Enable or disable automatic scheduled backups (`True`/`False`). | `True` |
|
||||
|
||||
@@ -341,29 +341,91 @@ Disable `prepared_statements` when using PgBouncer in transaction mode.
|
||||
|
||||
## Backup Procedures
|
||||
|
||||
### PostgreSQL
|
||||
DocuElevate's built-in **Backup & Restore** feature (Admin → Backup & Restore) supports all three
|
||||
database backends natively, using the native dump tools of each database.
|
||||
|
||||
**Manual backup:**
|
||||
| Backend | Backup tool | Archive extension | Restore tool |
|
||||
|----------------|--------------|-------------------|--------------|
|
||||
| SQLite | `sqlite3` (built-in Python) | `.db.gz` | `sqlite3` (built-in Python) |
|
||||
| PostgreSQL | `pg_dump` | `.pgsql.gz` | `psql` |
|
||||
| MySQL/MariaDB | `mysqldump` | `.mysql.gz` | `mysql` |
|
||||
|
||||
Passwords are passed via the `PGPASSWORD` (PostgreSQL) and `MYSQL_PWD` (MySQL) environment
|
||||
variables so they are never exposed on the process command line.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
For PostgreSQL and MySQL backups the corresponding CLI client must be installed on the
|
||||
worker host (the container / server that runs Celery workers):
|
||||
|
||||
```bash
|
||||
# PostgreSQL clients (Debian/Ubuntu)
|
||||
apt-get install -y postgresql-client
|
||||
|
||||
# MySQL clients (Debian/Ubuntu)
|
||||
apt-get install -y default-mysql-client
|
||||
```
|
||||
|
||||
The binaries required are:
|
||||
|
||||
- **PostgreSQL**: `pg_dump` (backup) and `psql` (restore)
|
||||
- **MySQL / MariaDB**: `mysqldump` (backup) and `mysql` (restore)
|
||||
|
||||
### Using the Admin Dashboard
|
||||
|
||||
Navigate to **Admin → Backup & Restore** to:
|
||||
|
||||
- Trigger manual backups (hourly / daily / weekly)
|
||||
- Download backup archives
|
||||
- Upload and restore a backup archive
|
||||
- Configure retention and remote storage destinations
|
||||
|
||||
### PostgreSQL – manual backup/restore
|
||||
|
||||
**Manual backup using DocuElevate's archive format (for use with the UI restore):**
|
||||
|
||||
```bash
|
||||
pg_dump --format=plain --no-password \
|
||||
-h localhost -U docuelevate docuelevate \
|
||||
| gzip > docuelevate_$(date +%Y%m%d_%H%M).pgsql.gz
|
||||
```
|
||||
|
||||
**Restore via the DocuElevate UI:** upload the `.pgsql.gz` file on the Backup & Restore page.
|
||||
|
||||
**Manual restore using native tools (custom format):**
|
||||
|
||||
```bash
|
||||
pg_dump -h localhost -U docuelevate -F c docuelevate > docuelevate_$(date +%Y%m%d_%H%M).dump
|
||||
```
|
||||
|
||||
**Restore:**
|
||||
|
||||
```bash
|
||||
pg_restore -h localhost -U docuelevate -d docuelevate docuelevate_20240101_1200.dump
|
||||
```
|
||||
|
||||
**Automated daily backup (cron example):**
|
||||
|
||||
```cron
|
||||
0 2 * * * pg_dump -h localhost -U docuelevate -F c docuelevate | gzip > /backups/docuelevate_$(date +\%Y\%m\%d).dump.gz
|
||||
0 2 * * * pg_dump --format=plain -h localhost -U docuelevate docuelevate | gzip > /backups/docuelevate_$(date +\%Y\%m\%d).pgsql.gz
|
||||
```
|
||||
|
||||
Use your cloud provider's automated backup feature when available (e.g., RDS automated snapshots, Cloud SQL backups).
|
||||
|
||||
### SQLite
|
||||
### MySQL / MariaDB – manual backup/restore
|
||||
|
||||
**Manual backup using DocuElevate's archive format (for use with the UI restore):**
|
||||
|
||||
```bash
|
||||
MYSQL_PWD=yourpassword mysqldump --single-transaction --routines --triggers \
|
||||
-h localhost -u docuelevate docuelevate \
|
||||
| gzip > docuelevate_$(date +%Y%m%d_%H%M).mysql.gz
|
||||
```
|
||||
|
||||
**Restore via the DocuElevate UI:** upload the `.mysql.gz` file on the Backup & Restore page.
|
||||
|
||||
**Manual restore using native tools:**
|
||||
|
||||
```bash
|
||||
gunzip -c docuelevate_20240101_1200.mysql.gz | mysql -h localhost -u docuelevate -p docuelevate
|
||||
```
|
||||
|
||||
### SQLite – manual backup/restore
|
||||
|
||||
```bash
|
||||
# Stop the application first, or use SQLite's online backup API
|
||||
|
||||
+444
-17
@@ -236,6 +236,254 @@ class TestBackupTaskHelpers:
|
||||
result = _db_path()
|
||||
assert result is None
|
||||
|
||||
def test_db_backend_sqlite(self):
|
||||
"""_db_backend() returns 'sqlite' for SQLite URLs."""
|
||||
from app.tasks.backup_tasks import _db_backend
|
||||
|
||||
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||
mock_settings.database_url = "sqlite:////tmp/test.db"
|
||||
assert _db_backend() == "sqlite"
|
||||
|
||||
def test_db_backend_postgresql(self):
|
||||
"""_db_backend() returns 'postgresql' for PostgreSQL URLs."""
|
||||
from app.tasks.backup_tasks import _db_backend
|
||||
|
||||
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||
mock_settings.database_url = "postgresql://user:pass@localhost/db"
|
||||
assert _db_backend() == "postgresql"
|
||||
|
||||
def test_db_backend_mysql(self):
|
||||
"""_db_backend() returns 'mysql' for MySQL URLs."""
|
||||
from app.tasks.backup_tasks import _db_backend
|
||||
|
||||
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/db"
|
||||
assert _db_backend() == "mysql"
|
||||
|
||||
def test_archive_ext_sqlite(self):
|
||||
"""_archive_ext_for_backend() returns '.db.gz' for sqlite."""
|
||||
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||
|
||||
assert _archive_ext_for_backend("sqlite") == ".db.gz"
|
||||
|
||||
def test_archive_ext_postgresql(self):
|
||||
"""_archive_ext_for_backend() returns '.pgsql.gz' for postgresql."""
|
||||
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||
|
||||
assert _archive_ext_for_backend("postgresql") == ".pgsql.gz"
|
||||
|
||||
def test_archive_ext_mysql(self):
|
||||
"""_archive_ext_for_backend() returns '.mysql.gz' for mysql."""
|
||||
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||
|
||||
assert _archive_ext_for_backend("mysql") == ".mysql.gz"
|
||||
|
||||
def test_archive_ext_unknown(self):
|
||||
"""_archive_ext_for_backend() falls back to '.sql.gz' for unknown backends."""
|
||||
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||
|
||||
assert _archive_ext_for_backend("mssql") == ".sql.gz"
|
||||
|
||||
def test_dump_postgresql_success(self, tmp_path):
|
||||
"""_dump_postgresql() streams pg_dump output into a gzip archive."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _dump_postgresql
|
||||
|
||||
dest = tmp_path / "dump.pgsql.gz"
|
||||
fake_sql = b"-- PostgreSQL database dump\nSELECT 1;\n"
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.stdout.read.side_effect = [fake_sql, b""]
|
||||
mock_proc.stderr.read.return_value = b""
|
||||
mock_proc.returncode = 0
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
_dump_postgresql("postgresql://user:pass@localhost/testdb", dest)
|
||||
|
||||
assert dest.exists()
|
||||
with gzip.open(str(dest), "rb") as gz:
|
||||
assert gz.read() == fake_sql
|
||||
|
||||
def test_dump_postgresql_failure(self, tmp_path):
|
||||
"""_dump_postgresql() raises RuntimeError when pg_dump fails."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _dump_postgresql
|
||||
|
||||
dest = tmp_path / "dump.pgsql.gz"
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.stdout.read.side_effect = [b""]
|
||||
mock_proc.stderr.read.return_value = b"FATAL: connection refused"
|
||||
mock_proc.returncode = 1
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
with pytest.raises(RuntimeError, match="pg_dump exited with code 1"):
|
||||
_dump_postgresql("postgresql://user:pass@localhost/testdb", dest)
|
||||
|
||||
def test_dump_mysql_success(self, tmp_path):
|
||||
"""_dump_mysql() streams mysqldump output into a gzip archive."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _dump_mysql
|
||||
|
||||
dest = tmp_path / "dump.mysql.gz"
|
||||
fake_sql = b"-- MySQL dump\nCREATE TABLE t (id INT);\n"
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.stdout.read.side_effect = [fake_sql, b""]
|
||||
mock_proc.stderr.read.return_value = b""
|
||||
mock_proc.returncode = 0
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
_dump_mysql("mysql+pymysql://user:pass@localhost/testdb", dest)
|
||||
|
||||
assert dest.exists()
|
||||
with gzip.open(str(dest), "rb") as gz:
|
||||
assert gz.read() == fake_sql
|
||||
|
||||
def test_dump_mysql_failure(self, tmp_path):
|
||||
"""_dump_mysql() raises RuntimeError when mysqldump fails."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _dump_mysql
|
||||
|
||||
dest = tmp_path / "dump.mysql.gz"
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.stdout.read.side_effect = [b""]
|
||||
mock_proc.stderr.read.return_value = b"ERROR: Access denied"
|
||||
mock_proc.returncode = 1
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
with pytest.raises(RuntimeError, match="mysqldump exited with code 1"):
|
||||
_dump_mysql("mysql+pymysql://user:pass@localhost/testdb", dest)
|
||||
|
||||
def test_restore_sqlite_success(self, tmp_path):
|
||||
"""_restore_sqlite() applies a valid SQL dump to a SQLite file."""
|
||||
from app.tasks.backup_tasks import _restore_sqlite
|
||||
|
||||
db_file = tmp_path / "test.db"
|
||||
conn = sqlite3.connect(str(db_file))
|
||||
conn.execute("CREATE TABLE old (id INTEGER)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Create a valid dump archive
|
||||
sql = "BEGIN TRANSACTION;\nCREATE TABLE new_tbl (x TEXT);\nCOMMIT;\n"
|
||||
archive = tmp_path / "dump.db.gz"
|
||||
with gzip.open(str(archive), "wt") as gz:
|
||||
gz.write(sql)
|
||||
|
||||
_restore_sqlite(db_file, archive)
|
||||
|
||||
conn2 = sqlite3.connect(str(db_file))
|
||||
tables = [r[0] for r in conn2.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||||
conn2.close()
|
||||
assert "new_tbl" in tables
|
||||
|
||||
def test_restore_sqlite_invalid_gz(self, tmp_path):
|
||||
"""_restore_sqlite() raises ValueError for corrupt gzip content."""
|
||||
from app.tasks.backup_tasks import _restore_sqlite
|
||||
|
||||
db_file = tmp_path / "test.db"
|
||||
db_file.write_bytes(b"")
|
||||
archive = tmp_path / "bad.db.gz"
|
||||
archive.write_bytes(b"not gzip data")
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to decompress"):
|
||||
_restore_sqlite(db_file, archive)
|
||||
|
||||
def test_restore_sqlite_invalid_sql(self, tmp_path):
|
||||
"""_restore_sqlite() raises ValueError for invalid SQL content."""
|
||||
from app.tasks.backup_tasks import _restore_sqlite
|
||||
|
||||
db_file = tmp_path / "test.db"
|
||||
db_file.write_bytes(b"")
|
||||
archive = tmp_path / "bad.db.gz"
|
||||
with gzip.open(str(archive), "wt") as gz:
|
||||
gz.write("THIS IS NOT VALID SQL!!!;\n")
|
||||
|
||||
with pytest.raises(ValueError, match="invalid SQL"):
|
||||
_restore_sqlite(db_file, archive)
|
||||
|
||||
def test_restore_postgresql_success(self, tmp_path):
|
||||
"""_restore_postgresql() pipes the archive to psql."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _restore_postgresql
|
||||
|
||||
fake_sql = b"-- PostgreSQL dump\nSELECT 1;\n"
|
||||
archive = tmp_path / "dump.pgsql.gz"
|
||||
with gzip.open(str(archive), "wb") as gz:
|
||||
gz.write(fake_sql)
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.communicate.return_value = (b"", b"")
|
||||
mock_proc.returncode = 0
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
_restore_postgresql("postgresql://user:pass@localhost/testdb", archive)
|
||||
|
||||
mock_proc.communicate.assert_called_once_with(input=fake_sql)
|
||||
|
||||
def test_restore_postgresql_failure(self, tmp_path):
|
||||
"""_restore_postgresql() raises RuntimeError when psql fails."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _restore_postgresql
|
||||
|
||||
archive = tmp_path / "dump.pgsql.gz"
|
||||
with gzip.open(str(archive), "wb") as gz:
|
||||
gz.write(b"SELECT 1;")
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.communicate.return_value = (b"", b"ERROR: invalid input")
|
||||
mock_proc.returncode = 1
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
with pytest.raises(RuntimeError, match="psql exited with code 1"):
|
||||
_restore_postgresql("postgresql://user:pass@localhost/testdb", archive)
|
||||
|
||||
def test_restore_mysql_success(self, tmp_path):
|
||||
"""_restore_mysql() pipes the archive to mysql."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _restore_mysql
|
||||
|
||||
fake_sql = b"-- MySQL dump\nSELECT 1;\n"
|
||||
archive = tmp_path / "dump.mysql.gz"
|
||||
with gzip.open(str(archive), "wb") as gz:
|
||||
gz.write(fake_sql)
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.communicate.return_value = (b"", b"")
|
||||
mock_proc.returncode = 0
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
_restore_mysql("mysql+pymysql://user:pass@localhost/testdb", archive)
|
||||
|
||||
mock_proc.communicate.assert_called_once_with(input=fake_sql)
|
||||
|
||||
def test_restore_mysql_failure(self, tmp_path):
|
||||
"""_restore_mysql() raises RuntimeError when mysql fails."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.tasks.backup_tasks import _restore_mysql
|
||||
|
||||
archive = tmp_path / "dump.mysql.gz"
|
||||
with gzip.open(str(archive), "wb") as gz:
|
||||
gz.write(b"SELECT 1;")
|
||||
|
||||
mock_proc = MagicMock()
|
||||
mock_proc.communicate.return_value = (b"", b"ERROR: Access denied")
|
||||
mock_proc.returncode = 1
|
||||
|
||||
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||
with pytest.raises(RuntimeError, match="mysql exited with code 1"):
|
||||
_restore_mysql("mysql+pymysql://user:pass@localhost/testdb", archive)
|
||||
|
||||
def test_apply_retention_prunes_old(self, tmp_path, db_session):
|
||||
"""_apply_retention() deletes backups beyond the retention limit."""
|
||||
from app.tasks.backup_tasks import _apply_retention
|
||||
@@ -352,20 +600,28 @@ class TestCreateBackupTask:
|
||||
result = create_backup("hourly")
|
||||
assert result["status"] == "disabled"
|
||||
|
||||
def test_non_sqlite_db(self):
|
||||
"""create_backup returns unsupported_db for non-SQLite databases."""
|
||||
def test_unsupported_db_backend(self):
|
||||
"""create_backup returns unsupported_db for backends other than sqlite/postgresql/mysql."""
|
||||
from app.tasks.backup_tasks import create_backup
|
||||
|
||||
with (
|
||||
patch("app.tasks.backup_tasks.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._db_path", return_value=None),
|
||||
):
|
||||
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = "mssql+pyodbc://user:pass@server/db"
|
||||
result = create_backup("hourly")
|
||||
assert result["status"] == "unsupported_db"
|
||||
|
||||
def test_in_memory_sqlite_unsupported(self):
|
||||
"""create_backup returns unsupported_db for in-memory SQLite."""
|
||||
from app.tasks.backup_tasks import create_backup
|
||||
|
||||
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = "sqlite:///:memory:"
|
||||
result = create_backup("hourly")
|
||||
assert result["status"] == "unsupported_db"
|
||||
|
||||
def test_missing_db_file(self, tmp_path):
|
||||
"""create_backup returns error when the DB file does not exist."""
|
||||
"""create_backup returns error when the SQLite DB file does not exist."""
|
||||
from app.tasks.backup_tasks import create_backup
|
||||
|
||||
missing = tmp_path / "does_not_exist.db"
|
||||
@@ -375,6 +631,7 @@ class TestCreateBackupTask:
|
||||
patch("app.tasks.backup_tasks._db_path", return_value=missing),
|
||||
):
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = f"sqlite:///{missing}"
|
||||
result = create_backup("hourly")
|
||||
assert result["status"] == "error"
|
||||
|
||||
@@ -400,6 +657,7 @@ class TestCreateBackupTask:
|
||||
):
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = f"sqlite:///{db_file}"
|
||||
mock_db = MagicMock()
|
||||
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||
@@ -408,6 +666,69 @@ class TestCreateBackupTask:
|
||||
assert result["status"] == "ok"
|
||||
assert "filename" in result
|
||||
assert result["filename"].startswith("backup_hourly_")
|
||||
assert result["filename"].endswith(".db.gz")
|
||||
|
||||
def test_successful_backup_postgresql(self, tmp_path):
|
||||
"""create_backup creates a .pgsql.gz archive for PostgreSQL databases."""
|
||||
from app.tasks.backup_tasks import create_backup
|
||||
|
||||
backup_dir = tmp_path / "backups"
|
||||
backup_dir.mkdir()
|
||||
|
||||
def fake_pg_dump(db_url: str, dest: Path) -> None:
|
||||
with gzip.open(str(dest), "wb") as gz:
|
||||
gz.write(b"-- PostgreSQL dump\n")
|
||||
|
||||
with (
|
||||
patch("app.tasks.backup_tasks.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._backup_dir", return_value=backup_dir),
|
||||
patch("app.tasks.backup_tasks._dump_postgresql", side_effect=fake_pg_dump),
|
||||
patch("app.tasks.backup_tasks._upload_remote", return_value=None),
|
||||
patch("app.tasks.backup_tasks._apply_retention"),
|
||||
patch("app.tasks.backup_tasks._prune_remote_backups"),
|
||||
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
||||
):
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||
mock_db = MagicMock()
|
||||
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||
result = create_backup("daily")
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["filename"].endswith(".pgsql.gz")
|
||||
assert "daily" in result["filename"]
|
||||
|
||||
def test_successful_backup_mysql(self, tmp_path):
|
||||
"""create_backup creates a .mysql.gz archive for MySQL databases."""
|
||||
from app.tasks.backup_tasks import create_backup
|
||||
|
||||
backup_dir = tmp_path / "backups"
|
||||
backup_dir.mkdir()
|
||||
|
||||
def fake_mysql_dump(db_url: str, dest: Path) -> None:
|
||||
with gzip.open(str(dest), "wb") as gz:
|
||||
gz.write(b"-- MySQL dump\n")
|
||||
|
||||
with (
|
||||
patch("app.tasks.backup_tasks.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._backup_dir", return_value=backup_dir),
|
||||
patch("app.tasks.backup_tasks._dump_mysql", side_effect=fake_mysql_dump),
|
||||
patch("app.tasks.backup_tasks._upload_remote", return_value=None),
|
||||
patch("app.tasks.backup_tasks._apply_retention"),
|
||||
patch("app.tasks.backup_tasks._prune_remote_backups"),
|
||||
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
||||
):
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb"
|
||||
mock_db = MagicMock()
|
||||
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||
result = create_backup("weekly")
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["filename"].endswith(".mysql.gz")
|
||||
assert "weekly" in result["filename"]
|
||||
|
||||
def test_invalid_backup_type_defaults_to_hourly(self, tmp_path):
|
||||
"""create_backup normalises unknown backup_type to 'hourly'."""
|
||||
@@ -431,6 +752,7 @@ class TestCreateBackupTask:
|
||||
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
||||
):
|
||||
mock_settings.backup_enabled = True
|
||||
mock_settings.database_url = f"sqlite:///{db_file}"
|
||||
mock_db = MagicMock()
|
||||
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||
@@ -549,19 +871,24 @@ class TestBackupAPIEndpoints:
|
||||
assert resp.status_code == 403
|
||||
|
||||
def test_restore_wrong_extension(self, admin_client):
|
||||
"""POST /api/admin/backup/restore rejects non-.db.gz files."""
|
||||
"""POST /api/admin/backup/restore rejects files with wrong extension for current backend."""
|
||||
# Default test env uses sqlite:///:memory: → expects .db.gz
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.zip", b"data", "application/zip")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_restore_invalid_gz_content(self, admin_client):
|
||||
def test_restore_invalid_gz_content(self, admin_client, tmp_path):
|
||||
"""POST /api/admin/backup/restore rejects corrupt gzip data."""
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.db.gz", b"not gzip data at all", "application/gzip")},
|
||||
)
|
||||
db_file = tmp_path / "test.db"
|
||||
db_file.write_bytes(b"")
|
||||
|
||||
with patch("app.tasks.backup_tasks._db_path", return_value=db_file):
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.db.gz", b"not gzip data at all", "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_restore_valid_archive(self, admin_client, tmp_path):
|
||||
@@ -581,11 +908,11 @@ class TestBackupAPIEndpoints:
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "restored"
|
||||
|
||||
def test_restore_non_sqlite_db(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 400 for non-SQLite database."""
|
||||
sql = "BEGIN TRANSACTION;\nCOMMIT;\n"
|
||||
gz_data = gzip.compress(sql.encode())
|
||||
def test_restore_in_memory_sqlite(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 400 for in-memory SQLite (no file to restore to)."""
|
||||
gz_data = gzip.compress(b"BEGIN TRANSACTION;\nCOMMIT;\n")
|
||||
|
||||
# _db_path() returns None for :memory: URLs → 400
|
||||
with patch("app.tasks.backup_tasks._db_path", return_value=None):
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
@@ -593,6 +920,106 @@ class TestBackupAPIEndpoints:
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_restore_wrong_extension_for_postgresql(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 400 when uploading .db.gz for PostgreSQL backend."""
|
||||
gz_data = gzip.compress(b"-- PostgreSQL dump")
|
||||
|
||||
with patch("app.config.settings") as mock_settings:
|
||||
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.db.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_restore_wrong_extension_for_mysql(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 400 when uploading .db.gz for MySQL backend."""
|
||||
gz_data = gzip.compress(b"-- MySQL dump")
|
||||
|
||||
with patch("app.config.settings") as mock_settings:
|
||||
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.db.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_restore_postgresql_success(self, admin_client):
|
||||
"""POST /api/admin/backup/restore succeeds for PostgreSQL database."""
|
||||
gz_data = gzip.compress(b"-- PostgreSQL dump\n")
|
||||
|
||||
with (
|
||||
patch("app.config.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._restore_postgresql") as mock_restore,
|
||||
):
|
||||
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "restored"
|
||||
mock_restore.assert_called_once()
|
||||
|
||||
def test_restore_mysql_success(self, admin_client):
|
||||
"""POST /api/admin/backup/restore succeeds for MySQL database."""
|
||||
gz_data = gzip.compress(b"-- MySQL dump\n")
|
||||
|
||||
with (
|
||||
patch("app.config.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._restore_mysql") as mock_restore,
|
||||
):
|
||||
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.mysql.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "restored"
|
||||
mock_restore.assert_called_once()
|
||||
|
||||
def test_restore_postgresql_runtime_error(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 500 when psql command fails."""
|
||||
gz_data = gzip.compress(b"-- PostgreSQL dump\n")
|
||||
|
||||
with (
|
||||
patch("app.config.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._restore_postgresql", side_effect=RuntimeError("psql failed")),
|
||||
):
|
||||
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 500
|
||||
|
||||
def test_restore_postgresql_missing_binary(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 500 when psql binary is missing."""
|
||||
gz_data = gzip.compress(b"-- PostgreSQL dump\n")
|
||||
|
||||
with (
|
||||
patch("app.config.settings") as mock_settings,
|
||||
patch("app.tasks.backup_tasks._restore_postgresql", side_effect=FileNotFoundError("psql not found")),
|
||||
):
|
||||
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 500
|
||||
|
||||
def test_restore_unsupported_backend(self, admin_client):
|
||||
"""POST /api/admin/backup/restore returns 400 for an unsupported database backend."""
|
||||
gz_data = gzip.compress(b"-- some dump\n")
|
||||
|
||||
with patch("app.config.settings") as mock_settings:
|
||||
mock_settings.database_url = "mssql+pyodbc://user:pass@server/db"
|
||||
resp = admin_client.post(
|
||||
"/api/admin/backup/restore",
|
||||
files={"file": ("backup.sql.gz", gz_data, "application/gzip")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# View tests
|
||||
|
||||
Reference in New Issue
Block a user