feat(backup): extend backup and restore to PostgreSQL and MySQL/MariaDB
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
+84
-68
@@ -117,88 +117,104 @@ async def restore_backup(
|
|||||||
"""Restore the database from an uploaded gzip-compressed SQL dump.
|
"""Restore the database from an uploaded gzip-compressed SQL dump.
|
||||||
|
|
||||||
**Warning**: This overwrites the current database contents.
|
**Warning**: This overwrites the current database contents.
|
||||||
Only SQLite databases are supported.
|
|
||||||
|
|
||||||
The uploaded file must be a ``.db.gz`` file produced by the DocuElevate
|
Supported formats (must match the currently configured database backend):
|
||||||
backup task (a gzip-compressed SQLite ``.dump()`` SQL script).
|
|
||||||
|
- ``*.db.gz`` – gzip-compressed SQLite ``.dump()`` SQL script (SQLite backend)
|
||||||
|
- ``*.pgsql.gz`` – gzip-compressed ``pg_dump --format=plain`` output (PostgreSQL backend)
|
||||||
|
- ``*.mysql.gz`` – gzip-compressed ``mysqldump`` output (MySQL / MariaDB backend)
|
||||||
"""
|
"""
|
||||||
from app.tasks.backup_tasks import _db_path
|
|
||||||
|
|
||||||
db_path = _db_path()
|
|
||||||
if db_path is None:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
|
||||||
detail="Restore is only supported for SQLite databases.",
|
|
||||||
)
|
|
||||||
|
|
||||||
if not file.filename or not file.filename.endswith(".db.gz"):
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
|
||||||
detail="Uploaded file must be a .db.gz backup archive.",
|
|
||||||
)
|
|
||||||
|
|
||||||
import gzip
|
|
||||||
import sqlite3
|
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Write the upload to a temp file first so we can validate it
|
from sqlalchemy.engine.url import make_url
|
||||||
with tempfile.NamedTemporaryFile(suffix=".db.gz", delete=False) as tmp:
|
|
||||||
|
from app.config import settings as app_settings
|
||||||
|
from app.tasks.backup_tasks import (
|
||||||
|
_archive_ext_for_backend,
|
||||||
|
_db_path,
|
||||||
|
_restore_mysql,
|
||||||
|
_restore_postgresql,
|
||||||
|
_restore_sqlite,
|
||||||
|
)
|
||||||
|
|
||||||
|
url = make_url(app_settings.database_url)
|
||||||
|
backend = url.get_backend_name()
|
||||||
|
expected_ext = _archive_ext_for_backend(backend)
|
||||||
|
|
||||||
|
if not file.filename or not file.filename.endswith(expected_ext):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=(
|
||||||
|
f"Uploaded file must be a '{expected_ext}' backup archive for the current database backend ({backend})."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Write upload to a temp file
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=expected_ext, delete=False) as tmp:
|
||||||
tmp_path = Path(tmp.name)
|
tmp_path = Path(tmp.name)
|
||||||
content = await file.read()
|
content = await file.read()
|
||||||
tmp.write(content)
|
tmp.write(content)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Decompress and read SQL statements
|
if backend == "sqlite":
|
||||||
with gzip.open(str(tmp_path), "rt", encoding="utf-8") as gz:
|
db_path = _db_path()
|
||||||
sql_script = gz.read()
|
if db_path is None:
|
||||||
except Exception as exc:
|
raise HTTPException(
|
||||||
tmp_path.unlink(missing_ok=True)
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
raise HTTPException(
|
detail="Restore is only supported for file-based SQLite databases.",
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
)
|
||||||
detail=f"Failed to decompress backup file: {exc}",
|
# Close the application DB session before replacing the file
|
||||||
) from exc
|
db.close()
|
||||||
|
try:
|
||||||
|
_restore_sqlite(db_path, tmp_path)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
|
except RuntimeError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
|
|
||||||
# Create a fresh in-memory DB from the script to validate it
|
elif backend == "postgresql":
|
||||||
try:
|
db.close()
|
||||||
mem_conn = sqlite3.connect(":memory:")
|
try:
|
||||||
mem_conn.executescript(sql_script)
|
_restore_postgresql(app_settings.database_url, tmp_path)
|
||||||
mem_conn.close()
|
except FileNotFoundError as exc:
|
||||||
except sqlite3.Error as exc:
|
raise HTTPException(
|
||||||
tmp_path.unlink(missing_ok=True)
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
raise HTTPException(
|
detail=f"psql binary not found – is PostgreSQL client installed? ({exc})",
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
) from exc
|
||||||
detail=f"Backup file contains invalid SQL: {exc}",
|
except RuntimeError as exc:
|
||||||
) from exc
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"PostgreSQL restore failed: {exc}",
|
||||||
|
) from exc
|
||||||
|
|
||||||
# Close the application DB session before replacing the file
|
elif backend == "mysql":
|
||||||
db.close()
|
db.close()
|
||||||
|
try:
|
||||||
|
_restore_mysql(app_settings.database_url, tmp_path)
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"mysql binary not found – is MySQL client installed? ({exc})",
|
||||||
|
) from exc
|
||||||
|
except RuntimeError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"MySQL restore failed: {exc}",
|
||||||
|
) from exc
|
||||||
|
|
||||||
# Preserve the current DB before overwriting
|
else:
|
||||||
import shutil
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Database backend '{backend}' does not support restore.",
|
||||||
|
)
|
||||||
|
|
||||||
bak = str(db_path) + ".pre_restore"
|
|
||||||
try:
|
|
||||||
shutil.copy2(str(db_path), bak)
|
|
||||||
except OSError as exc:
|
|
||||||
logger.warning(f"Could not create pre-restore backup at {bak}: {exc}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Write the restored database
|
|
||||||
restore_conn = sqlite3.connect(str(db_path))
|
|
||||||
restore_conn.executescript(sql_script)
|
|
||||||
restore_conn.close()
|
|
||||||
except sqlite3.Error as exc:
|
|
||||||
# Attempt rollback
|
|
||||||
try:
|
|
||||||
if os.path.exists(bak):
|
|
||||||
shutil.copy2(bak, str(db_path))
|
|
||||||
except OSError as rollback_exc:
|
|
||||||
logger.error(f"Rollback failed; database may be corrupted: {rollback_exc}")
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
||||||
detail=f"Restore failed: {exc}",
|
|
||||||
) from exc
|
|
||||||
finally:
|
finally:
|
||||||
tmp_path.unlink(missing_ok=True)
|
tmp_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|||||||
+337
-9
@@ -16,12 +16,19 @@ Three separate Celery-beat entries call ``create_backup`` with the appropriate
|
|||||||
After each backup is created ``_apply_retention`` prunes old local backups for
|
After each backup is created ``_apply_retention`` prunes old local backups for
|
||||||
that tier. Remote copies are pruned by ``_prune_remote_backups`` which mirrors
|
that tier. Remote copies are pruned by ``_prune_remote_backups`` which mirrors
|
||||||
the same retention limits.
|
the same retention limits.
|
||||||
|
|
||||||
|
Supported database backends
|
||||||
|
----------------------------
|
||||||
|
- **SQLite** – dumped via Python's built-in ``sqlite3.iterdump()``; archive extension ``.db.gz``
|
||||||
|
- **PostgreSQL** – dumped via ``pg_dump --format=plain``; archive extension ``.pgsql.gz``
|
||||||
|
- **MySQL / MariaDB** – dumped via ``mysqldump --single-transaction``; archive extension ``.mysql.gz``
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import gzip
|
import gzip
|
||||||
import hashlib
|
import hashlib
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import subprocess
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -42,6 +49,13 @@ _BACKUP_TYPE_RETAIN: dict[str, str] = {
|
|||||||
"weekly": "backup_retain_weekly",
|
"weekly": "backup_retain_weekly",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#: Map of backend name → archive file extension.
|
||||||
|
_BACKEND_EXTENSIONS: dict[str, str] = {
|
||||||
|
"sqlite": ".db.gz",
|
||||||
|
"postgresql": ".pgsql.gz",
|
||||||
|
"mysql": ".mysql.gz",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _backup_dir() -> Path:
|
def _backup_dir() -> Path:
|
||||||
"""Return (and create) the local backup directory."""
|
"""Return (and create) the local backup directory."""
|
||||||
@@ -51,6 +65,14 @@ def _backup_dir() -> Path:
|
|||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _db_backend() -> str:
|
||||||
|
"""Return the database backend name (e.g. ``'sqlite'``, ``'postgresql'``, ``'mysql'``)."""
|
||||||
|
from sqlalchemy.engine.url import make_url
|
||||||
|
|
||||||
|
url = make_url(settings.database_url)
|
||||||
|
return url.get_backend_name()
|
||||||
|
|
||||||
|
|
||||||
def _db_path() -> Path | None:
|
def _db_path() -> Path | None:
|
||||||
"""Return the SQLite database file path, or None for non-SQLite databases."""
|
"""Return the SQLite database file path, or None for non-SQLite databases."""
|
||||||
from sqlalchemy.engine.url import make_url
|
from sqlalchemy.engine.url import make_url
|
||||||
@@ -64,6 +86,20 @@ def _db_path() -> Path | None:
|
|||||||
return Path(db)
|
return Path(db)
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_ext_for_backend(backend: str) -> str:
|
||||||
|
"""Return the archive file extension for the given database backend.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
backend: Backend name as returned by
|
||||||
|
``sqlalchemy.engine.url.URL.get_backend_name()`` (e.g. ``'sqlite'``).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
File extension string including the leading dot, e.g. ``'.db.gz'``.
|
||||||
|
Falls back to ``'.sql.gz'`` for unknown backends.
|
||||||
|
"""
|
||||||
|
return _BACKEND_EXTENSIONS.get(backend, ".sql.gz")
|
||||||
|
|
||||||
|
|
||||||
def _sha256(path: Path) -> str:
|
def _sha256(path: Path) -> str:
|
||||||
"""Return the SHA-256 hex digest of *path*."""
|
"""Return the SHA-256 hex digest of *path*."""
|
||||||
h = hashlib.sha256()
|
h = hashlib.sha256()
|
||||||
@@ -86,6 +122,277 @@ def _dump_sqlite(db_path: Path, dest: Path) -> None:
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_postgresql(db_url: str, dest: Path) -> None:
|
||||||
|
"""Write a gzip-compressed ``pg_dump`` of the PostgreSQL database to *dest*.
|
||||||
|
|
||||||
|
Uses ``PGPASSWORD`` environment variable so the password is never exposed on
|
||||||
|
the process command line.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db_url: Full SQLAlchemy database URL (e.g. ``postgresql://user:pass@host/db``).
|
||||||
|
dest: Destination path for the ``.pgsql.gz`` archive.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: If ``pg_dump`` exits with a non-zero return code.
|
||||||
|
FileNotFoundError: If the ``pg_dump`` binary is not found.
|
||||||
|
"""
|
||||||
|
from sqlalchemy.engine.url import make_url
|
||||||
|
|
||||||
|
url = make_url(db_url)
|
||||||
|
env = os.environ.copy()
|
||||||
|
if url.password:
|
||||||
|
env["PGPASSWORD"] = str(url.password)
|
||||||
|
|
||||||
|
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL),
|
||||||
|
# not from user-controlled input. shell=False (the default when passing a list) is used
|
||||||
|
# so there is no shell interpretation of the argument values.
|
||||||
|
cmd: list[str] = ["pg_dump", "--format=plain", "--no-password"]
|
||||||
|
if url.host:
|
||||||
|
cmd.extend(["-h", url.host])
|
||||||
|
if url.port:
|
||||||
|
cmd.extend(["-p", str(url.port)])
|
||||||
|
if url.username:
|
||||||
|
cmd.extend(["-U", url.username])
|
||||||
|
if url.database:
|
||||||
|
cmd.append(url.database)
|
||||||
|
|
||||||
|
with gzip.open(str(dest), "wb") as gz:
|
||||||
|
proc = subprocess.Popen( # noqa: S603
|
||||||
|
cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
stdout = proc.stdout
|
||||||
|
if stdout is None: # pragma: no cover – guaranteed by stdout=PIPE
|
||||||
|
raise RuntimeError("pg_dump produced no stdout pipe")
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = stdout.read(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
gz.write(chunk)
|
||||||
|
finally:
|
||||||
|
stdout.close()
|
||||||
|
stderr_bytes = proc.stderr.read() if proc.stderr else b""
|
||||||
|
proc.wait()
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
|
raise RuntimeError(
|
||||||
|
f"pg_dump exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_mysql(db_url: str, dest: Path) -> None:
|
||||||
|
"""Write a gzip-compressed ``mysqldump`` of the MySQL database to *dest*.
|
||||||
|
|
||||||
|
Uses the ``MYSQL_PWD`` environment variable so the password is never exposed
|
||||||
|
on the process command line.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db_url: Full SQLAlchemy database URL
|
||||||
|
(e.g. ``mysql+pymysql://user:pass@host/db``).
|
||||||
|
dest: Destination path for the ``.mysql.gz`` archive.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: If ``mysqldump`` exits with a non-zero return code.
|
||||||
|
FileNotFoundError: If the ``mysqldump`` binary is not found.
|
||||||
|
"""
|
||||||
|
from sqlalchemy.engine.url import make_url
|
||||||
|
|
||||||
|
url = make_url(db_url)
|
||||||
|
env = os.environ.copy()
|
||||||
|
if url.password:
|
||||||
|
env["MYSQL_PWD"] = str(url.password)
|
||||||
|
|
||||||
|
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL).
|
||||||
|
# shell=False (list form) prevents shell interpretation of argument values.
|
||||||
|
cmd: list[str] = ["mysqldump", "--single-transaction", "--routines", "--triggers"]
|
||||||
|
if url.host:
|
||||||
|
cmd.extend(["-h", url.host])
|
||||||
|
if url.port:
|
||||||
|
cmd.extend(["-P", str(url.port)])
|
||||||
|
if url.username:
|
||||||
|
cmd.extend(["-u", url.username])
|
||||||
|
if url.database:
|
||||||
|
cmd.append(url.database)
|
||||||
|
|
||||||
|
with gzip.open(str(dest), "wb") as gz:
|
||||||
|
proc = subprocess.Popen( # noqa: S603
|
||||||
|
cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
stdout = proc.stdout
|
||||||
|
if stdout is None: # pragma: no cover – guaranteed by stdout=PIPE
|
||||||
|
raise RuntimeError("mysqldump produced no stdout pipe")
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = stdout.read(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
gz.write(chunk)
|
||||||
|
finally:
|
||||||
|
stdout.close()
|
||||||
|
stderr_bytes = proc.stderr.read() if proc.stderr else b""
|
||||||
|
proc.wait()
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
dest.unlink(missing_ok=True)
|
||||||
|
raise RuntimeError(
|
||||||
|
f"mysqldump exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_sqlite(db_path: Path, archive_path: Path) -> None:
|
||||||
|
"""Restore a SQLite database from a gzip-compressed SQL dump archive.
|
||||||
|
|
||||||
|
Validates the SQL by replaying it on an in-memory database before touching
|
||||||
|
the live file. Saves a ``<db_path>.pre_restore`` rollback copy first.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db_path: Path to the live SQLite database file to overwrite.
|
||||||
|
archive_path: Path to the ``.db.gz`` gzip-compressed SQL dump.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If the archive cannot be decompressed or contains invalid SQL.
|
||||||
|
RuntimeError: If writing the restored database fails.
|
||||||
|
"""
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
# Decompress and read SQL statements
|
||||||
|
try:
|
||||||
|
with gzip.open(str(archive_path), "rt", encoding="utf-8") as gz:
|
||||||
|
sql_script = gz.read()
|
||||||
|
except Exception as exc:
|
||||||
|
raise ValueError(f"Failed to decompress backup file: {exc}") from exc
|
||||||
|
|
||||||
|
# Validate by replaying on an in-memory database
|
||||||
|
try:
|
||||||
|
mem_conn = sqlite3.connect(":memory:")
|
||||||
|
mem_conn.executescript(sql_script)
|
||||||
|
mem_conn.close()
|
||||||
|
except sqlite3.Error as exc:
|
||||||
|
raise ValueError(f"Backup file contains invalid SQL: {exc}") from exc
|
||||||
|
|
||||||
|
# Preserve the current DB before overwriting
|
||||||
|
bak = str(db_path) + ".pre_restore"
|
||||||
|
try:
|
||||||
|
shutil.copy2(str(db_path), bak)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.warning(f"Could not create pre-restore backup at {bak}: {exc}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
restore_conn = sqlite3.connect(str(db_path))
|
||||||
|
restore_conn.executescript(sql_script)
|
||||||
|
restore_conn.close()
|
||||||
|
except sqlite3.Error as exc:
|
||||||
|
# Attempt rollback to the pre-restore copy
|
||||||
|
try:
|
||||||
|
if os.path.exists(bak):
|
||||||
|
shutil.copy2(bak, str(db_path))
|
||||||
|
except OSError as rollback_exc:
|
||||||
|
logger.error(f"Rollback failed; database may be corrupted: {rollback_exc}")
|
||||||
|
raise RuntimeError(f"SQLite restore failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_postgresql(db_url: str, archive_path: Path) -> None:
|
||||||
|
"""Restore a PostgreSQL database from a gzip-compressed SQL dump archive.
|
||||||
|
|
||||||
|
Pipes the decompressed dump to ``psql``. Uses ``PGPASSWORD`` so the
|
||||||
|
password is never exposed on the process command line.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db_url: Full SQLAlchemy database URL.
|
||||||
|
archive_path: Path to the ``.pgsql.gz`` gzip-compressed ``pg_dump`` archive.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: If ``psql`` exits with a non-zero return code.
|
||||||
|
FileNotFoundError: If the ``psql`` binary is not found.
|
||||||
|
"""
|
||||||
|
from sqlalchemy.engine.url import make_url
|
||||||
|
|
||||||
|
url = make_url(db_url)
|
||||||
|
env = os.environ.copy()
|
||||||
|
if url.password:
|
||||||
|
env["PGPASSWORD"] = str(url.password)
|
||||||
|
|
||||||
|
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL).
|
||||||
|
# shell=False (list form) prevents shell interpretation of argument values.
|
||||||
|
cmd: list[str] = ["psql", "--no-password"]
|
||||||
|
if url.host:
|
||||||
|
cmd.extend(["-h", url.host])
|
||||||
|
if url.port:
|
||||||
|
cmd.extend(["-p", str(url.port)])
|
||||||
|
if url.username:
|
||||||
|
cmd.extend(["-U", url.username])
|
||||||
|
if url.database:
|
||||||
|
cmd.append(url.database)
|
||||||
|
|
||||||
|
with gzip.open(str(archive_path), "rb") as gz:
|
||||||
|
proc = subprocess.Popen( # noqa: S603
|
||||||
|
cmd,
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
_, stderr_bytes = proc.communicate(input=gz.read())
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise RuntimeError(f"psql exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}")
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_mysql(db_url: str, archive_path: Path) -> None:
|
||||||
|
"""Restore a MySQL database from a gzip-compressed SQL dump archive.
|
||||||
|
|
||||||
|
Pipes the decompressed dump to ``mysql``. Uses the ``MYSQL_PWD``
|
||||||
|
environment variable so the password is never exposed on the command line.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db_url: Full SQLAlchemy database URL.
|
||||||
|
archive_path: Path to the ``.mysql.gz`` gzip-compressed ``mysqldump`` archive.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: If ``mysql`` exits with a non-zero return code.
|
||||||
|
FileNotFoundError: If the ``mysql`` binary is not found.
|
||||||
|
"""
|
||||||
|
from sqlalchemy.engine.url import make_url
|
||||||
|
|
||||||
|
url = make_url(db_url)
|
||||||
|
env = os.environ.copy()
|
||||||
|
if url.password:
|
||||||
|
env["MYSQL_PWD"] = str(url.password)
|
||||||
|
|
||||||
|
# Command arguments are built from the SQLAlchemy URL (admin-configured DATABASE_URL).
|
||||||
|
# shell=False (list form) prevents shell interpretation of argument values.
|
||||||
|
cmd: list[str] = ["mysql"]
|
||||||
|
if url.host:
|
||||||
|
cmd.extend(["-h", url.host])
|
||||||
|
if url.port:
|
||||||
|
cmd.extend(["-P", str(url.port)])
|
||||||
|
if url.username:
|
||||||
|
cmd.extend(["-u", url.username])
|
||||||
|
if url.database:
|
||||||
|
cmd.append(url.database)
|
||||||
|
|
||||||
|
with gzip.open(str(archive_path), "rb") as gz:
|
||||||
|
proc = subprocess.Popen( # noqa: S603
|
||||||
|
cmd,
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
_, stderr_bytes = proc.communicate(input=gz.read())
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise RuntimeError(f"mysql exited with code {proc.returncode}: {stderr_bytes.decode(errors='replace').strip()}")
|
||||||
|
|
||||||
|
|
||||||
def _apply_retention(backup_type: str, db: object) -> None:
|
def _apply_retention(backup_type: str, db: object) -> None:
|
||||||
"""Delete local backups beyond the retention limit for *backup_type*.
|
"""Delete local backups beyond the retention limit for *backup_type*.
|
||||||
|
|
||||||
@@ -314,6 +621,11 @@ def _email_backup(archive_path: Path, filename: str) -> None:
|
|||||||
def create_backup(self, backup_type: str = "hourly") -> dict:
|
def create_backup(self, backup_type: str = "hourly") -> dict:
|
||||||
"""Create a database backup archive and apply retention.
|
"""Create a database backup archive and apply retention.
|
||||||
|
|
||||||
|
Supports SQLite (``.db.gz``), PostgreSQL (``.pgsql.gz``), and
|
||||||
|
MySQL / MariaDB (``.mysql.gz``) databases. The native dump tool for the
|
||||||
|
configured backend (``sqlite3``, ``pg_dump``, or ``mysqldump``) must be
|
||||||
|
available on the worker's ``PATH``.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
backup_type: ``"hourly"``, ``"daily"``, or ``"weekly"``.
|
backup_type: ``"hourly"``, ``"daily"``, or ``"weekly"``.
|
||||||
|
|
||||||
@@ -327,19 +639,27 @@ def create_backup(self, backup_type: str = "hourly") -> dict:
|
|||||||
logger.debug("Backup is disabled; skipping create_backup task.")
|
logger.debug("Backup is disabled; skipping create_backup task.")
|
||||||
return {"status": "disabled"}
|
return {"status": "disabled"}
|
||||||
|
|
||||||
|
backend = _db_backend()
|
||||||
|
ext = _archive_ext_for_backend(backend)
|
||||||
|
|
||||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")
|
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")
|
||||||
filename = f"backup_{backup_type}_{ts}.db.gz"
|
filename = f"backup_{backup_type}_{ts}{ext}"
|
||||||
archive_path = _backup_dir() / filename
|
archive_path = _backup_dir() / filename
|
||||||
|
|
||||||
db_path = _db_path()
|
# SQLite: verify the database file exists before attempting to dump it
|
||||||
if db_path is None:
|
db_path: Path | None = None
|
||||||
logger.warning("Backup task skipped: non-SQLite databases are not supported for file-based backups.")
|
if backend == "sqlite":
|
||||||
|
db_path = _db_path()
|
||||||
|
if db_path is None:
|
||||||
|
logger.warning("Backup task skipped: in-memory SQLite databases are not supported.")
|
||||||
|
return {"status": "unsupported_db"}
|
||||||
|
if not db_path.exists():
|
||||||
|
logger.error(f"Database file not found: {db_path}")
|
||||||
|
return {"status": "error", "detail": f"DB file missing: {db_path}"}
|
||||||
|
elif backend not in ("postgresql", "mysql"):
|
||||||
|
logger.warning(f"Backup task skipped: unsupported database backend '{backend}'.")
|
||||||
return {"status": "unsupported_db"}
|
return {"status": "unsupported_db"}
|
||||||
|
|
||||||
if not db_path.exists():
|
|
||||||
logger.error(f"Database file not found: {db_path}")
|
|
||||||
return {"status": "error", "detail": f"DB file missing: {db_path}"}
|
|
||||||
|
|
||||||
status = "ok"
|
status = "ok"
|
||||||
checksum: str | None = None
|
checksum: str | None = None
|
||||||
size_bytes = 0
|
size_bytes = 0
|
||||||
@@ -347,7 +667,15 @@ def create_backup(self, backup_type: str = "hourly") -> dict:
|
|||||||
remote_path: str | None = None
|
remote_path: str | None = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
_dump_sqlite(db_path, archive_path)
|
if backend == "sqlite":
|
||||||
|
# db_path is guaranteed non-None: we returned early if it were None
|
||||||
|
if db_path is None: # pragma: no cover
|
||||||
|
return {"status": "error", "detail": "db_path unexpectedly None"}
|
||||||
|
_dump_sqlite(db_path, archive_path)
|
||||||
|
elif backend == "postgresql":
|
||||||
|
_dump_postgresql(settings.database_url, archive_path)
|
||||||
|
elif backend == "mysql":
|
||||||
|
_dump_mysql(settings.database_url, archive_path)
|
||||||
size_bytes = archive_path.stat().st_size
|
size_bytes = archive_path.stat().st_size
|
||||||
checksum = _sha256(archive_path)
|
checksum = _sha256(archive_path)
|
||||||
logger.info(f"Created {backup_type} backup: {archive_path} ({size_bytes:,} bytes)")
|
logger.info(f"Created {backup_type} backup: {archive_path} ({size_bytes:,} bytes)")
|
||||||
|
|||||||
@@ -1037,9 +1037,13 @@ Webhook URLs, secrets, and subscribed events are configured per-webhook via the
|
|||||||
|
|
||||||
### Backup & Restore
|
### Backup & Restore
|
||||||
|
|
||||||
DocuElevate can automatically back up the SQLite database on a scheduled basis.
|
DocuElevate automatically backs up the database on a scheduled basis.
|
||||||
Backups are managed from the **Admin → Backup & Restore** dashboard.
|
Backups are managed from the **Admin → Backup & Restore** dashboard.
|
||||||
|
|
||||||
|
Supported database backends: **SQLite** (`.db.gz`), **PostgreSQL** (`.pgsql.gz`), **MySQL / MariaDB** (`.mysql.gz`).
|
||||||
|
For PostgreSQL and MySQL backups the respective CLI client (`pg_dump` / `psql` or `mysqldump` / `mysql`) must be installed on the Celery worker host.
|
||||||
|
See the [Database Configuration Guide](DatabaseConfiguration.md#backup-procedures) for setup details.
|
||||||
|
|
||||||
| **Variable** | **Description** | **Default** |
|
| **Variable** | **Description** | **Default** |
|
||||||
|--------------------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
|--------------------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
||||||
| `BACKUP_ENABLED` | Enable or disable automatic scheduled backups (`True`/`False`). | `True` |
|
| `BACKUP_ENABLED` | Enable or disable automatic scheduled backups (`True`/`False`). | `True` |
|
||||||
|
|||||||
@@ -341,29 +341,91 @@ Disable `prepared_statements` when using PgBouncer in transaction mode.
|
|||||||
|
|
||||||
## Backup Procedures
|
## Backup Procedures
|
||||||
|
|
||||||
### PostgreSQL
|
DocuElevate's built-in **Backup & Restore** feature (Admin → Backup & Restore) supports all three
|
||||||
|
database backends natively, using the native dump tools of each database.
|
||||||
|
|
||||||
**Manual backup:**
|
| Backend | Backup tool | Archive extension | Restore tool |
|
||||||
|
|----------------|--------------|-------------------|--------------|
|
||||||
|
| SQLite | `sqlite3` (built-in Python) | `.db.gz` | `sqlite3` (built-in Python) |
|
||||||
|
| PostgreSQL | `pg_dump` | `.pgsql.gz` | `psql` |
|
||||||
|
| MySQL/MariaDB | `mysqldump` | `.mysql.gz` | `mysql` |
|
||||||
|
|
||||||
|
Passwords are passed via the `PGPASSWORD` (PostgreSQL) and `MYSQL_PWD` (MySQL) environment
|
||||||
|
variables so they are never exposed on the process command line.
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
For PostgreSQL and MySQL backups the corresponding CLI client must be installed on the
|
||||||
|
worker host (the container / server that runs Celery workers):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# PostgreSQL clients (Debian/Ubuntu)
|
||||||
|
apt-get install -y postgresql-client
|
||||||
|
|
||||||
|
# MySQL clients (Debian/Ubuntu)
|
||||||
|
apt-get install -y default-mysql-client
|
||||||
|
```
|
||||||
|
|
||||||
|
The binaries required are:
|
||||||
|
|
||||||
|
- **PostgreSQL**: `pg_dump` (backup) and `psql` (restore)
|
||||||
|
- **MySQL / MariaDB**: `mysqldump` (backup) and `mysql` (restore)
|
||||||
|
|
||||||
|
### Using the Admin Dashboard
|
||||||
|
|
||||||
|
Navigate to **Admin → Backup & Restore** to:
|
||||||
|
|
||||||
|
- Trigger manual backups (hourly / daily / weekly)
|
||||||
|
- Download backup archives
|
||||||
|
- Upload and restore a backup archive
|
||||||
|
- Configure retention and remote storage destinations
|
||||||
|
|
||||||
|
### PostgreSQL – manual backup/restore
|
||||||
|
|
||||||
|
**Manual backup using DocuElevate's archive format (for use with the UI restore):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pg_dump --format=plain --no-password \
|
||||||
|
-h localhost -U docuelevate docuelevate \
|
||||||
|
| gzip > docuelevate_$(date +%Y%m%d_%H%M).pgsql.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
**Restore via the DocuElevate UI:** upload the `.pgsql.gz` file on the Backup & Restore page.
|
||||||
|
|
||||||
|
**Manual restore using native tools (custom format):**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pg_dump -h localhost -U docuelevate -F c docuelevate > docuelevate_$(date +%Y%m%d_%H%M).dump
|
pg_dump -h localhost -U docuelevate -F c docuelevate > docuelevate_$(date +%Y%m%d_%H%M).dump
|
||||||
```
|
|
||||||
|
|
||||||
**Restore:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pg_restore -h localhost -U docuelevate -d docuelevate docuelevate_20240101_1200.dump
|
pg_restore -h localhost -U docuelevate -d docuelevate docuelevate_20240101_1200.dump
|
||||||
```
|
```
|
||||||
|
|
||||||
**Automated daily backup (cron example):**
|
**Automated daily backup (cron example):**
|
||||||
|
|
||||||
```cron
|
```cron
|
||||||
0 2 * * * pg_dump -h localhost -U docuelevate -F c docuelevate | gzip > /backups/docuelevate_$(date +\%Y\%m\%d).dump.gz
|
0 2 * * * pg_dump --format=plain -h localhost -U docuelevate docuelevate | gzip > /backups/docuelevate_$(date +\%Y\%m\%d).pgsql.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
Use your cloud provider's automated backup feature when available (e.g., RDS automated snapshots, Cloud SQL backups).
|
Use your cloud provider's automated backup feature when available (e.g., RDS automated snapshots, Cloud SQL backups).
|
||||||
|
|
||||||
### SQLite
|
### MySQL / MariaDB – manual backup/restore
|
||||||
|
|
||||||
|
**Manual backup using DocuElevate's archive format (for use with the UI restore):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
MYSQL_PWD=yourpassword mysqldump --single-transaction --routines --triggers \
|
||||||
|
-h localhost -u docuelevate docuelevate \
|
||||||
|
| gzip > docuelevate_$(date +%Y%m%d_%H%M).mysql.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
**Restore via the DocuElevate UI:** upload the `.mysql.gz` file on the Backup & Restore page.
|
||||||
|
|
||||||
|
**Manual restore using native tools:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
gunzip -c docuelevate_20240101_1200.mysql.gz | mysql -h localhost -u docuelevate -p docuelevate
|
||||||
|
```
|
||||||
|
|
||||||
|
### SQLite – manual backup/restore
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Stop the application first, or use SQLite's online backup API
|
# Stop the application first, or use SQLite's online backup API
|
||||||
|
|||||||
+444
-17
@@ -236,6 +236,254 @@ class TestBackupTaskHelpers:
|
|||||||
result = _db_path()
|
result = _db_path()
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
def test_db_backend_sqlite(self):
|
||||||
|
"""_db_backend() returns 'sqlite' for SQLite URLs."""
|
||||||
|
from app.tasks.backup_tasks import _db_backend
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||||
|
mock_settings.database_url = "sqlite:////tmp/test.db"
|
||||||
|
assert _db_backend() == "sqlite"
|
||||||
|
|
||||||
|
def test_db_backend_postgresql(self):
|
||||||
|
"""_db_backend() returns 'postgresql' for PostgreSQL URLs."""
|
||||||
|
from app.tasks.backup_tasks import _db_backend
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||||
|
mock_settings.database_url = "postgresql://user:pass@localhost/db"
|
||||||
|
assert _db_backend() == "postgresql"
|
||||||
|
|
||||||
|
def test_db_backend_mysql(self):
|
||||||
|
"""_db_backend() returns 'mysql' for MySQL URLs."""
|
||||||
|
from app.tasks.backup_tasks import _db_backend
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||||
|
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/db"
|
||||||
|
assert _db_backend() == "mysql"
|
||||||
|
|
||||||
|
def test_archive_ext_sqlite(self):
|
||||||
|
"""_archive_ext_for_backend() returns '.db.gz' for sqlite."""
|
||||||
|
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||||
|
|
||||||
|
assert _archive_ext_for_backend("sqlite") == ".db.gz"
|
||||||
|
|
||||||
|
def test_archive_ext_postgresql(self):
|
||||||
|
"""_archive_ext_for_backend() returns '.pgsql.gz' for postgresql."""
|
||||||
|
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||||
|
|
||||||
|
assert _archive_ext_for_backend("postgresql") == ".pgsql.gz"
|
||||||
|
|
||||||
|
def test_archive_ext_mysql(self):
|
||||||
|
"""_archive_ext_for_backend() returns '.mysql.gz' for mysql."""
|
||||||
|
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||||
|
|
||||||
|
assert _archive_ext_for_backend("mysql") == ".mysql.gz"
|
||||||
|
|
||||||
|
def test_archive_ext_unknown(self):
|
||||||
|
"""_archive_ext_for_backend() falls back to '.sql.gz' for unknown backends."""
|
||||||
|
from app.tasks.backup_tasks import _archive_ext_for_backend
|
||||||
|
|
||||||
|
assert _archive_ext_for_backend("mssql") == ".sql.gz"
|
||||||
|
|
||||||
|
def test_dump_postgresql_success(self, tmp_path):
|
||||||
|
"""_dump_postgresql() streams pg_dump output into a gzip archive."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _dump_postgresql
|
||||||
|
|
||||||
|
dest = tmp_path / "dump.pgsql.gz"
|
||||||
|
fake_sql = b"-- PostgreSQL database dump\nSELECT 1;\n"
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.stdout.read.side_effect = [fake_sql, b""]
|
||||||
|
mock_proc.stderr.read.return_value = b""
|
||||||
|
mock_proc.returncode = 0
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
_dump_postgresql("postgresql://user:pass@localhost/testdb", dest)
|
||||||
|
|
||||||
|
assert dest.exists()
|
||||||
|
with gzip.open(str(dest), "rb") as gz:
|
||||||
|
assert gz.read() == fake_sql
|
||||||
|
|
||||||
|
def test_dump_postgresql_failure(self, tmp_path):
|
||||||
|
"""_dump_postgresql() raises RuntimeError when pg_dump fails."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _dump_postgresql
|
||||||
|
|
||||||
|
dest = tmp_path / "dump.pgsql.gz"
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.stdout.read.side_effect = [b""]
|
||||||
|
mock_proc.stderr.read.return_value = b"FATAL: connection refused"
|
||||||
|
mock_proc.returncode = 1
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
with pytest.raises(RuntimeError, match="pg_dump exited with code 1"):
|
||||||
|
_dump_postgresql("postgresql://user:pass@localhost/testdb", dest)
|
||||||
|
|
||||||
|
def test_dump_mysql_success(self, tmp_path):
|
||||||
|
"""_dump_mysql() streams mysqldump output into a gzip archive."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _dump_mysql
|
||||||
|
|
||||||
|
dest = tmp_path / "dump.mysql.gz"
|
||||||
|
fake_sql = b"-- MySQL dump\nCREATE TABLE t (id INT);\n"
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.stdout.read.side_effect = [fake_sql, b""]
|
||||||
|
mock_proc.stderr.read.return_value = b""
|
||||||
|
mock_proc.returncode = 0
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
_dump_mysql("mysql+pymysql://user:pass@localhost/testdb", dest)
|
||||||
|
|
||||||
|
assert dest.exists()
|
||||||
|
with gzip.open(str(dest), "rb") as gz:
|
||||||
|
assert gz.read() == fake_sql
|
||||||
|
|
||||||
|
def test_dump_mysql_failure(self, tmp_path):
|
||||||
|
"""_dump_mysql() raises RuntimeError when mysqldump fails."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _dump_mysql
|
||||||
|
|
||||||
|
dest = tmp_path / "dump.mysql.gz"
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.stdout.read.side_effect = [b""]
|
||||||
|
mock_proc.stderr.read.return_value = b"ERROR: Access denied"
|
||||||
|
mock_proc.returncode = 1
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
with pytest.raises(RuntimeError, match="mysqldump exited with code 1"):
|
||||||
|
_dump_mysql("mysql+pymysql://user:pass@localhost/testdb", dest)
|
||||||
|
|
||||||
|
def test_restore_sqlite_success(self, tmp_path):
|
||||||
|
"""_restore_sqlite() applies a valid SQL dump to a SQLite file."""
|
||||||
|
from app.tasks.backup_tasks import _restore_sqlite
|
||||||
|
|
||||||
|
db_file = tmp_path / "test.db"
|
||||||
|
conn = sqlite3.connect(str(db_file))
|
||||||
|
conn.execute("CREATE TABLE old (id INTEGER)")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
# Create a valid dump archive
|
||||||
|
sql = "BEGIN TRANSACTION;\nCREATE TABLE new_tbl (x TEXT);\nCOMMIT;\n"
|
||||||
|
archive = tmp_path / "dump.db.gz"
|
||||||
|
with gzip.open(str(archive), "wt") as gz:
|
||||||
|
gz.write(sql)
|
||||||
|
|
||||||
|
_restore_sqlite(db_file, archive)
|
||||||
|
|
||||||
|
conn2 = sqlite3.connect(str(db_file))
|
||||||
|
tables = [r[0] for r in conn2.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||||||
|
conn2.close()
|
||||||
|
assert "new_tbl" in tables
|
||||||
|
|
||||||
|
def test_restore_sqlite_invalid_gz(self, tmp_path):
|
||||||
|
"""_restore_sqlite() raises ValueError for corrupt gzip content."""
|
||||||
|
from app.tasks.backup_tasks import _restore_sqlite
|
||||||
|
|
||||||
|
db_file = tmp_path / "test.db"
|
||||||
|
db_file.write_bytes(b"")
|
||||||
|
archive = tmp_path / "bad.db.gz"
|
||||||
|
archive.write_bytes(b"not gzip data")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Failed to decompress"):
|
||||||
|
_restore_sqlite(db_file, archive)
|
||||||
|
|
||||||
|
def test_restore_sqlite_invalid_sql(self, tmp_path):
|
||||||
|
"""_restore_sqlite() raises ValueError for invalid SQL content."""
|
||||||
|
from app.tasks.backup_tasks import _restore_sqlite
|
||||||
|
|
||||||
|
db_file = tmp_path / "test.db"
|
||||||
|
db_file.write_bytes(b"")
|
||||||
|
archive = tmp_path / "bad.db.gz"
|
||||||
|
with gzip.open(str(archive), "wt") as gz:
|
||||||
|
gz.write("THIS IS NOT VALID SQL!!!;\n")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="invalid SQL"):
|
||||||
|
_restore_sqlite(db_file, archive)
|
||||||
|
|
||||||
|
def test_restore_postgresql_success(self, tmp_path):
|
||||||
|
"""_restore_postgresql() pipes the archive to psql."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _restore_postgresql
|
||||||
|
|
||||||
|
fake_sql = b"-- PostgreSQL dump\nSELECT 1;\n"
|
||||||
|
archive = tmp_path / "dump.pgsql.gz"
|
||||||
|
with gzip.open(str(archive), "wb") as gz:
|
||||||
|
gz.write(fake_sql)
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.communicate.return_value = (b"", b"")
|
||||||
|
mock_proc.returncode = 0
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
_restore_postgresql("postgresql://user:pass@localhost/testdb", archive)
|
||||||
|
|
||||||
|
mock_proc.communicate.assert_called_once_with(input=fake_sql)
|
||||||
|
|
||||||
|
def test_restore_postgresql_failure(self, tmp_path):
|
||||||
|
"""_restore_postgresql() raises RuntimeError when psql fails."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _restore_postgresql
|
||||||
|
|
||||||
|
archive = tmp_path / "dump.pgsql.gz"
|
||||||
|
with gzip.open(str(archive), "wb") as gz:
|
||||||
|
gz.write(b"SELECT 1;")
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.communicate.return_value = (b"", b"ERROR: invalid input")
|
||||||
|
mock_proc.returncode = 1
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
with pytest.raises(RuntimeError, match="psql exited with code 1"):
|
||||||
|
_restore_postgresql("postgresql://user:pass@localhost/testdb", archive)
|
||||||
|
|
||||||
|
def test_restore_mysql_success(self, tmp_path):
|
||||||
|
"""_restore_mysql() pipes the archive to mysql."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _restore_mysql
|
||||||
|
|
||||||
|
fake_sql = b"-- MySQL dump\nSELECT 1;\n"
|
||||||
|
archive = tmp_path / "dump.mysql.gz"
|
||||||
|
with gzip.open(str(archive), "wb") as gz:
|
||||||
|
gz.write(fake_sql)
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.communicate.return_value = (b"", b"")
|
||||||
|
mock_proc.returncode = 0
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
_restore_mysql("mysql+pymysql://user:pass@localhost/testdb", archive)
|
||||||
|
|
||||||
|
mock_proc.communicate.assert_called_once_with(input=fake_sql)
|
||||||
|
|
||||||
|
def test_restore_mysql_failure(self, tmp_path):
|
||||||
|
"""_restore_mysql() raises RuntimeError when mysql fails."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.tasks.backup_tasks import _restore_mysql
|
||||||
|
|
||||||
|
archive = tmp_path / "dump.mysql.gz"
|
||||||
|
with gzip.open(str(archive), "wb") as gz:
|
||||||
|
gz.write(b"SELECT 1;")
|
||||||
|
|
||||||
|
mock_proc = MagicMock()
|
||||||
|
mock_proc.communicate.return_value = (b"", b"ERROR: Access denied")
|
||||||
|
mock_proc.returncode = 1
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.subprocess.Popen", return_value=mock_proc):
|
||||||
|
with pytest.raises(RuntimeError, match="mysql exited with code 1"):
|
||||||
|
_restore_mysql("mysql+pymysql://user:pass@localhost/testdb", archive)
|
||||||
|
|
||||||
def test_apply_retention_prunes_old(self, tmp_path, db_session):
|
def test_apply_retention_prunes_old(self, tmp_path, db_session):
|
||||||
"""_apply_retention() deletes backups beyond the retention limit."""
|
"""_apply_retention() deletes backups beyond the retention limit."""
|
||||||
from app.tasks.backup_tasks import _apply_retention
|
from app.tasks.backup_tasks import _apply_retention
|
||||||
@@ -352,20 +600,28 @@ class TestCreateBackupTask:
|
|||||||
result = create_backup("hourly")
|
result = create_backup("hourly")
|
||||||
assert result["status"] == "disabled"
|
assert result["status"] == "disabled"
|
||||||
|
|
||||||
def test_non_sqlite_db(self):
|
def test_unsupported_db_backend(self):
|
||||||
"""create_backup returns unsupported_db for non-SQLite databases."""
|
"""create_backup returns unsupported_db for backends other than sqlite/postgresql/mysql."""
|
||||||
from app.tasks.backup_tasks import create_backup
|
from app.tasks.backup_tasks import create_backup
|
||||||
|
|
||||||
with (
|
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||||
patch("app.tasks.backup_tasks.settings") as mock_settings,
|
|
||||||
patch("app.tasks.backup_tasks._db_path", return_value=None),
|
|
||||||
):
|
|
||||||
mock_settings.backup_enabled = True
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = "mssql+pyodbc://user:pass@server/db"
|
||||||
|
result = create_backup("hourly")
|
||||||
|
assert result["status"] == "unsupported_db"
|
||||||
|
|
||||||
|
def test_in_memory_sqlite_unsupported(self):
|
||||||
|
"""create_backup returns unsupported_db for in-memory SQLite."""
|
||||||
|
from app.tasks.backup_tasks import create_backup
|
||||||
|
|
||||||
|
with patch("app.tasks.backup_tasks.settings") as mock_settings:
|
||||||
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = "sqlite:///:memory:"
|
||||||
result = create_backup("hourly")
|
result = create_backup("hourly")
|
||||||
assert result["status"] == "unsupported_db"
|
assert result["status"] == "unsupported_db"
|
||||||
|
|
||||||
def test_missing_db_file(self, tmp_path):
|
def test_missing_db_file(self, tmp_path):
|
||||||
"""create_backup returns error when the DB file does not exist."""
|
"""create_backup returns error when the SQLite DB file does not exist."""
|
||||||
from app.tasks.backup_tasks import create_backup
|
from app.tasks.backup_tasks import create_backup
|
||||||
|
|
||||||
missing = tmp_path / "does_not_exist.db"
|
missing = tmp_path / "does_not_exist.db"
|
||||||
@@ -375,6 +631,7 @@ class TestCreateBackupTask:
|
|||||||
patch("app.tasks.backup_tasks._db_path", return_value=missing),
|
patch("app.tasks.backup_tasks._db_path", return_value=missing),
|
||||||
):
|
):
|
||||||
mock_settings.backup_enabled = True
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = f"sqlite:///{missing}"
|
||||||
result = create_backup("hourly")
|
result = create_backup("hourly")
|
||||||
assert result["status"] == "error"
|
assert result["status"] == "error"
|
||||||
|
|
||||||
@@ -400,6 +657,7 @@ class TestCreateBackupTask:
|
|||||||
):
|
):
|
||||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||||
mock_settings.backup_enabled = True
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = f"sqlite:///{db_file}"
|
||||||
mock_db = MagicMock()
|
mock_db = MagicMock()
|
||||||
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||||
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||||
@@ -408,6 +666,69 @@ class TestCreateBackupTask:
|
|||||||
assert result["status"] == "ok"
|
assert result["status"] == "ok"
|
||||||
assert "filename" in result
|
assert "filename" in result
|
||||||
assert result["filename"].startswith("backup_hourly_")
|
assert result["filename"].startswith("backup_hourly_")
|
||||||
|
assert result["filename"].endswith(".db.gz")
|
||||||
|
|
||||||
|
def test_successful_backup_postgresql(self, tmp_path):
|
||||||
|
"""create_backup creates a .pgsql.gz archive for PostgreSQL databases."""
|
||||||
|
from app.tasks.backup_tasks import create_backup
|
||||||
|
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
def fake_pg_dump(db_url: str, dest: Path) -> None:
|
||||||
|
with gzip.open(str(dest), "wb") as gz:
|
||||||
|
gz.write(b"-- PostgreSQL dump\n")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.tasks.backup_tasks.settings") as mock_settings,
|
||||||
|
patch("app.tasks.backup_tasks._backup_dir", return_value=backup_dir),
|
||||||
|
patch("app.tasks.backup_tasks._dump_postgresql", side_effect=fake_pg_dump),
|
||||||
|
patch("app.tasks.backup_tasks._upload_remote", return_value=None),
|
||||||
|
patch("app.tasks.backup_tasks._apply_retention"),
|
||||||
|
patch("app.tasks.backup_tasks._prune_remote_backups"),
|
||||||
|
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
||||||
|
):
|
||||||
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||||
|
mock_db = MagicMock()
|
||||||
|
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||||
|
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||||
|
result = create_backup("daily")
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert result["filename"].endswith(".pgsql.gz")
|
||||||
|
assert "daily" in result["filename"]
|
||||||
|
|
||||||
|
def test_successful_backup_mysql(self, tmp_path):
|
||||||
|
"""create_backup creates a .mysql.gz archive for MySQL databases."""
|
||||||
|
from app.tasks.backup_tasks import create_backup
|
||||||
|
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
def fake_mysql_dump(db_url: str, dest: Path) -> None:
|
||||||
|
with gzip.open(str(dest), "wb") as gz:
|
||||||
|
gz.write(b"-- MySQL dump\n")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.tasks.backup_tasks.settings") as mock_settings,
|
||||||
|
patch("app.tasks.backup_tasks._backup_dir", return_value=backup_dir),
|
||||||
|
patch("app.tasks.backup_tasks._dump_mysql", side_effect=fake_mysql_dump),
|
||||||
|
patch("app.tasks.backup_tasks._upload_remote", return_value=None),
|
||||||
|
patch("app.tasks.backup_tasks._apply_retention"),
|
||||||
|
patch("app.tasks.backup_tasks._prune_remote_backups"),
|
||||||
|
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
||||||
|
):
|
||||||
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb"
|
||||||
|
mock_db = MagicMock()
|
||||||
|
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||||
|
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||||
|
result = create_backup("weekly")
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert result["filename"].endswith(".mysql.gz")
|
||||||
|
assert "weekly" in result["filename"]
|
||||||
|
|
||||||
def test_invalid_backup_type_defaults_to_hourly(self, tmp_path):
|
def test_invalid_backup_type_defaults_to_hourly(self, tmp_path):
|
||||||
"""create_backup normalises unknown backup_type to 'hourly'."""
|
"""create_backup normalises unknown backup_type to 'hourly'."""
|
||||||
@@ -431,6 +752,7 @@ class TestCreateBackupTask:
|
|||||||
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
patch("app.tasks.backup_tasks.SessionLocal") as mock_sl,
|
||||||
):
|
):
|
||||||
mock_settings.backup_enabled = True
|
mock_settings.backup_enabled = True
|
||||||
|
mock_settings.database_url = f"sqlite:///{db_file}"
|
||||||
mock_db = MagicMock()
|
mock_db = MagicMock()
|
||||||
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
mock_sl.return_value.__enter__ = MagicMock(return_value=mock_db)
|
||||||
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
mock_sl.return_value.__exit__ = MagicMock(return_value=False)
|
||||||
@@ -549,19 +871,24 @@ class TestBackupAPIEndpoints:
|
|||||||
assert resp.status_code == 403
|
assert resp.status_code == 403
|
||||||
|
|
||||||
def test_restore_wrong_extension(self, admin_client):
|
def test_restore_wrong_extension(self, admin_client):
|
||||||
"""POST /api/admin/backup/restore rejects non-.db.gz files."""
|
"""POST /api/admin/backup/restore rejects files with wrong extension for current backend."""
|
||||||
|
# Default test env uses sqlite:///:memory: → expects .db.gz
|
||||||
resp = admin_client.post(
|
resp = admin_client.post(
|
||||||
"/api/admin/backup/restore",
|
"/api/admin/backup/restore",
|
||||||
files={"file": ("backup.zip", b"data", "application/zip")},
|
files={"file": ("backup.zip", b"data", "application/zip")},
|
||||||
)
|
)
|
||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
def test_restore_invalid_gz_content(self, admin_client):
|
def test_restore_invalid_gz_content(self, admin_client, tmp_path):
|
||||||
"""POST /api/admin/backup/restore rejects corrupt gzip data."""
|
"""POST /api/admin/backup/restore rejects corrupt gzip data."""
|
||||||
resp = admin_client.post(
|
db_file = tmp_path / "test.db"
|
||||||
"/api/admin/backup/restore",
|
db_file.write_bytes(b"")
|
||||||
files={"file": ("backup.db.gz", b"not gzip data at all", "application/gzip")},
|
|
||||||
)
|
with patch("app.tasks.backup_tasks._db_path", return_value=db_file):
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.db.gz", b"not gzip data at all", "application/gzip")},
|
||||||
|
)
|
||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
def test_restore_valid_archive(self, admin_client, tmp_path):
|
def test_restore_valid_archive(self, admin_client, tmp_path):
|
||||||
@@ -581,11 +908,11 @@ class TestBackupAPIEndpoints:
|
|||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
assert resp.json()["status"] == "restored"
|
assert resp.json()["status"] == "restored"
|
||||||
|
|
||||||
def test_restore_non_sqlite_db(self, admin_client):
|
def test_restore_in_memory_sqlite(self, admin_client):
|
||||||
"""POST /api/admin/backup/restore returns 400 for non-SQLite database."""
|
"""POST /api/admin/backup/restore returns 400 for in-memory SQLite (no file to restore to)."""
|
||||||
sql = "BEGIN TRANSACTION;\nCOMMIT;\n"
|
gz_data = gzip.compress(b"BEGIN TRANSACTION;\nCOMMIT;\n")
|
||||||
gz_data = gzip.compress(sql.encode())
|
|
||||||
|
|
||||||
|
# _db_path() returns None for :memory: URLs → 400
|
||||||
with patch("app.tasks.backup_tasks._db_path", return_value=None):
|
with patch("app.tasks.backup_tasks._db_path", return_value=None):
|
||||||
resp = admin_client.post(
|
resp = admin_client.post(
|
||||||
"/api/admin/backup/restore",
|
"/api/admin/backup/restore",
|
||||||
@@ -593,6 +920,106 @@ class TestBackupAPIEndpoints:
|
|||||||
)
|
)
|
||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
def test_restore_wrong_extension_for_postgresql(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore returns 400 when uploading .db.gz for PostgreSQL backend."""
|
||||||
|
gz_data = gzip.compress(b"-- PostgreSQL dump")
|
||||||
|
|
||||||
|
with patch("app.config.settings") as mock_settings:
|
||||||
|
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.db.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
def test_restore_wrong_extension_for_mysql(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore returns 400 when uploading .db.gz for MySQL backend."""
|
||||||
|
gz_data = gzip.compress(b"-- MySQL dump")
|
||||||
|
|
||||||
|
with patch("app.config.settings") as mock_settings:
|
||||||
|
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.db.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
def test_restore_postgresql_success(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore succeeds for PostgreSQL database."""
|
||||||
|
gz_data = gzip.compress(b"-- PostgreSQL dump\n")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.config.settings") as mock_settings,
|
||||||
|
patch("app.tasks.backup_tasks._restore_postgresql") as mock_restore,
|
||||||
|
):
|
||||||
|
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json()["status"] == "restored"
|
||||||
|
mock_restore.assert_called_once()
|
||||||
|
|
||||||
|
def test_restore_mysql_success(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore succeeds for MySQL database."""
|
||||||
|
gz_data = gzip.compress(b"-- MySQL dump\n")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.config.settings") as mock_settings,
|
||||||
|
patch("app.tasks.backup_tasks._restore_mysql") as mock_restore,
|
||||||
|
):
|
||||||
|
mock_settings.database_url = "mysql+pymysql://user:pass@localhost/testdb"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.mysql.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json()["status"] == "restored"
|
||||||
|
mock_restore.assert_called_once()
|
||||||
|
|
||||||
|
def test_restore_postgresql_runtime_error(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore returns 500 when psql command fails."""
|
||||||
|
gz_data = gzip.compress(b"-- PostgreSQL dump\n")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.config.settings") as mock_settings,
|
||||||
|
patch("app.tasks.backup_tasks._restore_postgresql", side_effect=RuntimeError("psql failed")),
|
||||||
|
):
|
||||||
|
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 500
|
||||||
|
|
||||||
|
def test_restore_postgresql_missing_binary(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore returns 500 when psql binary is missing."""
|
||||||
|
gz_data = gzip.compress(b"-- PostgreSQL dump\n")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.config.settings") as mock_settings,
|
||||||
|
patch("app.tasks.backup_tasks._restore_postgresql", side_effect=FileNotFoundError("psql not found")),
|
||||||
|
):
|
||||||
|
mock_settings.database_url = "postgresql://user:pass@localhost/testdb"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.pgsql.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 500
|
||||||
|
|
||||||
|
def test_restore_unsupported_backend(self, admin_client):
|
||||||
|
"""POST /api/admin/backup/restore returns 400 for an unsupported database backend."""
|
||||||
|
gz_data = gzip.compress(b"-- some dump\n")
|
||||||
|
|
||||||
|
with patch("app.config.settings") as mock_settings:
|
||||||
|
mock_settings.database_url = "mssql+pyodbc://user:pass@server/db"
|
||||||
|
resp = admin_client.post(
|
||||||
|
"/api/admin/backup/restore",
|
||||||
|
files={"file": ("backup.sql.gz", gz_data, "application/gzip")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# View tests
|
# View tests
|
||||||
|
|||||||
Reference in New Issue
Block a user