style: fix code formatting with black, isort, and flake8

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-12 03:21:41 +00:00
parent f79bd2cb0c
commit ff9a3ff49f
87 changed files with 874 additions and 729 deletions
+2 -6
View File
@@ -425,16 +425,12 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}") logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}")
else: else:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400, detail="Neither original nor local file found on disk. Cannot reprocess."
detail="Neither original nor local file found on disk. Cannot reprocess."
) )
# Queue the file for processing with force_cloud_ocr=True # Queue the file for processing with force_cloud_ocr=True
task = process_document.delay( task = process_document.delay(
source_file, source_file, original_filename=file_record.original_filename, file_id=file_record.id, force_cloud_ocr=True
original_filename=file_record.original_filename,
file_id=file_record.id,
force_cloud_ocr=True
) )
logger.info( logger.info(
+1 -1
View File
@@ -277,7 +277,7 @@ async def process_url(request: Request, url_request: URLUploadRequest):
return { return {
"task_id": task.id, "task_id": task.id,
"status": "queued", "status": "queued",
"message": f"File downloaded from URL and queued for processing", "message": "File downloaded from URL and queued for processing",
"filename": safe_filename, "filename": safe_filename,
"size": downloaded_size, "size": downloaded_size,
} }
+1 -1
View File
@@ -13,6 +13,7 @@ from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401 from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401 from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401 from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
# **Ensure all tasks are imported before Celery starts** # **Ensure all tasks are imported before Celery starts**
from app.tasks.process_document import process_document # noqa: F401 from app.tasks.process_document import process_document # noqa: F401
@@ -33,7 +34,6 @@ from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401 from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401 from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401 from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
celery.conf.task_routes = { celery.conf.task_routes = {
"app.tasks.*": {"queue": "default"}, "app.tasks.*": {"queue": "default"},
+22 -6
View File
@@ -33,7 +33,8 @@ class Settings(BaseSettings):
paperless_host: Optional[str] = None paperless_host: Optional[str] = None
paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
# JSON mapping of metadata field names to Paperless custom field names # JSON mapping of metadata field names to Paperless custom field names
# Example: {"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"} # Example: {"absender": "Sender", "empfaenger": "Recipient",
# "language": "Language", "correspondent": "Correspondent"}
paperless_custom_fields_mapping: Optional[str] = None paperless_custom_fields_mapping: Optional[str] = None
azure_ai_key: str azure_ai_key: str
@@ -176,23 +177,35 @@ class Settings(BaseSettings):
) )
max_single_file_size: Optional[int] = Field( max_single_file_size: Optional[int] = Field(
default=None, default=None,
description="Maximum size for a single file chunk in bytes. If set and file exceeds this, it will be split into smaller chunks for processing. Default: None (no splitting).", description=(
"Maximum size for a single file chunk in bytes. If set and file exceeds this,"
" it will be split into smaller chunks for processing. Default: None (no splitting)."
),
) )
# Deduplication settings - prevents processing of duplicate files # Deduplication settings - prevents processing of duplicate files
enable_deduplication: bool = Field( enable_deduplication: bool = Field(
default=True, default=True,
description="Enable deduplication check before processing. If enabled, files with the same SHA-256 hash as previously processed files will not be processed again. Default: True (enabled).", description=(
"Enable deduplication check before processing. If enabled, files with the same SHA-256 hash"
" as previously processed files will not be processed again. Default: True (enabled)."
),
) )
show_deduplication_step: bool = Field( show_deduplication_step: bool = Field(
default=True, default=True,
description="Show the 'Check for Duplicates' step in processing history. If False, the check is still performed but not displayed. Default: True.", description=(
"Show the 'Check for Duplicates' step in processing history."
" If False, the check is still performed but not displayed. Default: True."
),
) )
# Processing step timeout - prevents files from getting stuck in "in_progress" state # Processing step timeout - prevents files from getting stuck in "in_progress" state
step_timeout: int = Field( step_timeout: int = Field(
default=600, default=600,
description="Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this, it will be marked as failed. Default: 600 seconds (10 minutes).", description=(
"Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this,"
" it will be marked as failed. Default: 600 seconds (10 minutes)."
),
) )
# Security Headers Configuration (see SECURITY_AUDIT.md and docs/DeploymentGuide.md) # Security Headers Configuration (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
@@ -215,7 +228,10 @@ class Settings(BaseSettings):
# Content-Security-Policy (CSP) - Controls resource loading # Content-Security-Policy (CSP) - Controls resource loading
security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.") security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.")
security_header_csp_value: str = Field( security_header_csp_value: str = Field(
default="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;", default=(
"default-src 'self'; script-src 'self' 'unsafe-inline';"
" style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
),
description="CSP header value. Customize based on your application's resource loading needs.", description="CSP header value. Customize based on your application's resource loading needs.",
) )
+1 -1
View File
@@ -8,6 +8,7 @@ from fastapi import FastAPI, HTTPException, Request, status
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates from fastapi.templating import Jinja2Templates
from slowapi.errors import RateLimitExceeded
from starlette.config import Config from starlette.config import Config
from starlette.middleware.sessions import SessionMiddleware from starlette.middleware.sessions import SessionMiddleware
from starlette.middleware.trustedhost import TrustedHostMiddleware from starlette.middleware.trustedhost import TrustedHostMiddleware
@@ -21,7 +22,6 @@ from app.middleware.rate_limit import create_limiter, get_rate_limit_exceeded_ha
from app.middleware.security_headers import SecurityHeadersMiddleware from app.middleware.security_headers import SecurityHeadersMiddleware
from app.utils.config_validator import check_all_configs from app.utils.config_validator import check_all_configs
from app.utils.notification import init_apprise, notify_shutdown, notify_startup from app.utils.notification import init_apprise, notify_shutdown, notify_startup
from slowapi.errors import RateLimitExceeded
# Import the routers - now using views directly instead of frontend # Import the routers - now using views directly instead of frontend
from app.views import router as frontend_router from app.views import router as frontend_router
-1
View File
@@ -21,7 +21,6 @@ from typing import Callable
from fastapi import Request from fastapi import Request
from slowapi import Limiter, _rate_limit_exceeded_handler from slowapi import Limiter, _rate_limit_exceeded_handler
from slowapi.errors import RateLimitExceeded
from slowapi.util import get_remote_address from slowapi.util import get_remote_address
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
-4
View File
@@ -7,10 +7,6 @@ This module provides convenient decorators to apply rate limits to specific endp
Import the limiter from main.py state and use these decorators to protect endpoints. Import the limiter from main.py state and use these decorators to protect endpoints.
""" """
from functools import wraps
from fastapi import Request
# Import will happen at runtime to avoid circular dependencies # Import will happen at runtime to avoid circular dependencies
_limiter = None _limiter = None
+1 -1
View File
@@ -1,6 +1,6 @@
# app/models.py # app/models.py
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func, Boolean from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
from app.database import Base from app.database import Base
+20 -8
View File
@@ -102,7 +102,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
else: else:
logger.error(f"[{task_id}] Local file {local_file_path} not found, cannot embed metadata.") logger.error(f"[{task_id}] Local file {local_file_path} not found, cannot embed metadata.")
log_task_progress( log_task_progress(
task_id, "embed_metadata_into_pdf", "failure", "File not found", file_id=file_id, task_id,
"embed_metadata_into_pdf",
"failure",
"File not found",
file_id=file_id,
detail=( detail=(
f"Local file not found, cannot embed metadata.\n" f"Local file not found, cannot embed metadata.\n"
f"Tried path: {local_file_path}\n" f"Tried path: {local_file_path}\n"
@@ -199,10 +203,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
logger.info(f"[{task_id}] Persisting metadata to JSON") logger.info(f"[{task_id}] Persisting metadata to JSON")
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id) log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
json_path = persist_metadata( json_path = persist_metadata(
metadata, metadata, final_file_path, original_file_path=original_file_path, processed_file_path=final_file_path
final_file_path,
original_file_path=original_file_path,
processed_file_path=final_file_path
) )
logger.info(f"[{task_id}] Metadata persisted to {json_path}") logger.info(f"[{task_id}] Metadata persisted to {json_path}")
log_task_progress( log_task_progress(
@@ -212,7 +213,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
# Trigger the next step: final storage. # Trigger the next step: final storage.
logger.info(f"[{task_id}] Queueing final storage task") logger.info(f"[{task_id}] Queueing final storage task")
log_task_progress( log_task_progress(
task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id, task_id,
"embed_metadata_into_pdf",
"success",
"Metadata embedded, queuing finalization",
file_id=file_id,
detail=( detail=(
f"Metadata embedded into PDF successfully.\n" f"Metadata embedded into PDF successfully.\n"
f"Original file: {original_file}\n" f"Original file: {original_file}\n"
@@ -245,8 +250,15 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as e: except Exception as e:
logger.exception(f"[{task_id}] Failed to embed metadata into {processed_file}: {e}") logger.exception(f"[{task_id}] Failed to embed metadata into {processed_file}: {e}")
log_task_progress( log_task_progress(
task_id, "embed_metadata_into_pdf", "failure", f"Exception: {str(e)}", file_id=file_id, task_id,
detail=f"Failed to embed metadata into {processed_file}.\nOriginal file: {original_file}\nException: {str(e)}", "embed_metadata_into_pdf",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=(
f"Failed to embed metadata into {processed_file}.\n"
f"Original file: {original_file}\nException: {str(e)}"
),
) )
# Clean up temporary file in case of error # Clean up temporary file in case of error
if os.path.exists(processed_file): if os.path.exists(processed_file):
+22 -8
View File
@@ -112,7 +112,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
content = completion.choices[0].message.content content = completion.choices[0].message.content
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...") logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
log_task_progress( log_task_progress(
task_id, "call_openai", "success", "Received OpenAI response", file_id=file_id, task_id,
"call_openai",
"success",
"Received OpenAI response",
file_id=file_id,
detail=f"Raw classification response:\n{content}", detail=f"Raw classification response:\n{content}",
) )
@@ -120,7 +124,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
if not json_text: if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.") logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress( log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id, task_id,
"extract_metadata_with_gpt",
"failure",
"Invalid JSON in response",
file_id=file_id,
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}", detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
) )
return {} return {}
@@ -138,16 +146,18 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
# 1. Potential locale-specific \w behavior # 1. Potential locale-specific \w behavior
# 2. Files literally named ".." which are valid but problematic # 2. Files literally named ".." which are valid but problematic
# 3. Future code changes that might relax the regex # 3. Future code changes that might relax the regex
if not re.match(r'^[\w\-\. ]+$', suggested_filename) or ".." in suggested_filename: if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
logger.warning( logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback"
)
# Reset to empty to trigger fallback to original filename # Reset to empty to trigger fallback to original filename
metadata["filename"] = "" metadata["filename"] = ""
logger.info(f"[{task_id}] Extracted metadata: {metadata}") logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress( log_task_progress(
task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id, task_id,
"parse_metadata",
"success",
f"Parsed metadata: {list(metadata.keys())}",
file_id=file_id,
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}", detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
) )
@@ -164,7 +174,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
except Exception as e: except Exception as e:
logger.exception(f"[{task_id}] OpenAI classification failed for {filename}: {e}") logger.exception(f"[{task_id}] OpenAI classification failed for {filename}: {e}")
log_task_progress( log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", f"Exception: {str(e)}", file_id=file_id, task_id,
"extract_metadata_with_gpt",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=f"OpenAI classification failed for {filename}.\nException: {str(e)}", detail=f"OpenAI classification failed for {filename}.\nException: {str(e)}",
) )
return {} return {}
+1 -4
View File
@@ -42,10 +42,7 @@ def monitor_stalled_steps():
f"Marked as failed due to timeout." f"Marked as failed due to timeout."
) )
else: else:
logger.debug( logger.debug(f"[{datetime.utcnow().isoformat()}] " f"No stalled steps found.")
f"[{datetime.utcnow().isoformat()}] "
f"No stalled steps found."
)
return {"recovered": stalled_count} return {"recovered": stalled_count}
+14 -7
View File
@@ -24,7 +24,9 @@ logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True) @celery.task(base=BaseTaskWithRetry, bind=True)
def process_document(self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False): def process_document(
self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False
):
""" """
Process a document file and trigger appropriate text extraction. Process a document file and trigger appropriate text extraction.
@@ -58,7 +60,10 @@ def process_document(self, original_local_file: str, original_filename: str = No
if not os.path.exists(original_local_file): if not os.path.exists(original_local_file):
logger.error(f"[{task_id}] File {original_local_file} not found.") logger.error(f"[{task_id}] File {original_local_file} not found.")
log_task_progress( log_task_progress(
task_id, "process_document", "failure", "File not found", task_id,
"process_document",
"failure",
"File not found",
detail=f"File not found on disk: {original_local_file}", detail=f"File not found on disk: {original_local_file}",
) )
return {"error": "File not found"} return {"error": "File not found"}
@@ -113,13 +118,15 @@ def process_document(self, original_local_file: str, original_filename: str = No
else: else:
# Check for duplicate only if this is a new file (not reprocessing) # Check for duplicate only if this is a new file (not reprocessing)
# IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file # IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file
existing = db.query(FileRecord).filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)).order_by(FileRecord.created_at.asc()).first() existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.created_at.asc())
.first()
)
if existing is None: if existing is None:
existing = ( existing = (
db.query(FileRecord) db.query(FileRecord).filter(FileRecord.filehash == filehash).order_by(FileRecord.id.asc()).first()
.filter(FileRecord.filehash == filehash)
.order_by(FileRecord.id.asc())
.first()
) )
# A file is only a duplicate if it matches a different file's hash # A file is only a duplicate if it matches a different file's hash
@@ -101,8 +101,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
""" """
task_id = self.request.id task_id = self.request.id
log_task_progress( log_task_progress(
task_id, "process_with_azure_document_intelligence", "in_progress", task_id,
f"Starting OCR for {filename}", file_id=file_id, "process_with_azure_document_intelligence",
"in_progress",
f"Starting OCR for {filename}",
file_id=file_id,
) )
try: try:
tmp_file_path = os.path.join(settings.workdir, "tmp", filename) tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
@@ -117,8 +120,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
) )
logger.error(f"[{task_id}] {error_msg}") logger.error(f"[{task_id}] {error_msg}")
log_task_progress( log_task_progress(
task_id, "validate_file", "failure", task_id,
f"File too large: {filename}", file_id=file_id, detail=error_msg, "validate_file",
"failure",
f"File too large: {filename}",
file_id=file_id,
detail=error_msg,
) )
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"} return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
@@ -130,8 +137,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages" error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
logger.error(f"[{task_id}] {error_msg}") logger.error(f"[{task_id}] {error_msg}")
log_task_progress( log_task_progress(
task_id, "validate_file", "failure", task_id,
f"Too many pages: {filename}", file_id=file_id, detail=error_msg, "validate_file",
"failure",
f"Too many pages: {filename}",
file_id=file_id,
detail=error_msg,
) )
return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"} return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
if page_count is None: if page_count is None:
@@ -140,14 +151,20 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
) )
log_task_progress( log_task_progress(
task_id, "validate_file", "success", task_id,
f"File validation passed for {filename}", file_id=file_id, "validate_file",
"success",
f"File validation passed for {filename}",
file_id=file_id,
) )
logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.") logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.")
log_task_progress( log_task_progress(
task_id, "call_azure_ocr", "in_progress", task_id,
f"Sending {filename} to Azure Document Intelligence", file_id=file_id, "call_azure_ocr",
"in_progress",
f"Sending {filename} to Azure Document Intelligence",
file_id=file_id,
) )
# Open and send the document for processing # Open and send the document for processing
@@ -173,8 +190,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters") logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters")
log_task_progress( log_task_progress(
task_id, "call_azure_ocr", "success", task_id,
f"Azure OCR completed for {filename}", file_id=file_id, "call_azure_ocr",
"success",
f"Azure OCR completed for {filename}",
file_id=file_id,
detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected", detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected",
) )
@@ -182,8 +202,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id) rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id)
log_task_progress( log_task_progress(
task_id, "process_with_azure_document_intelligence", "success", task_id,
f"OCR processing complete for {filename}", file_id=file_id, "process_with_azure_document_intelligence",
"success",
f"OCR processing complete for {filename}",
file_id=file_id,
detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted", detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted",
) )
@@ -191,7 +214,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
except Exception as e: except Exception as e:
logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}") logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}")
log_task_progress( log_task_progress(
task_id, "process_with_azure_document_intelligence", "failure", task_id,
f"OCR failed for {filename}", file_id=file_id, detail=str(e), "process_with_azure_document_intelligence",
"failure",
f"OCR failed for {filename}",
file_id=file_id,
detail=str(e),
) )
raise raise
+28 -15
View File
@@ -63,8 +63,11 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
try: try:
task_id = self.request.id task_id = self.request.id
log_task_progress( log_task_progress(
task_id, "rotate_pdf_pages", "in_progress", task_id,
f"Checking page rotation for {filename}", file_id=file_id, "rotate_pdf_pages",
"in_progress",
f"Checking page rotation for {filename}",
file_id=file_id,
) )
pdf_path = os.path.join(settings.workdir, "tmp", filename) pdf_path = os.path.join(settings.workdir, "tmp", filename)
if not os.path.exists(pdf_path): if not os.path.exists(pdf_path):
@@ -72,12 +75,13 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
# Skip rotation if no rotation data provided # Skip rotation if no rotation data provided
if not rotation_data: if not rotation_data:
logger.info( logger.info(f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction")
f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction"
)
log_task_progress( log_task_progress(
task_id, "rotate_pdf_pages", "success", task_id,
"No rotation needed, proceeding to metadata extraction", file_id=file_id, "rotate_pdf_pages",
"success",
"No rotation needed, proceeding to metadata extraction",
file_id=file_id,
) )
extract_metadata_with_gpt.delay(filename, extracted_text, file_id) extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"} return {"file": filename, "status": "no_rotation_needed"}
@@ -95,16 +99,22 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
f"[{task_id}] No significant rotations detected in {filename}, proceeding with metadata extraction" f"[{task_id}] No significant rotations detected in {filename}, proceeding with metadata extraction"
) )
log_task_progress( log_task_progress(
task_id, "rotate_pdf_pages", "success", task_id,
"No rotation needed, proceeding to metadata extraction", file_id=file_id, "rotate_pdf_pages",
"success",
"No rotation needed, proceeding to metadata extraction",
file_id=file_id,
) )
extract_metadata_with_gpt.delay(filename, extracted_text, file_id) extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"} return {"file": filename, "status": "no_rotation_needed"}
logger.info(f"[{task_id}] Rotating {len(normalized_rotation_data)} pages in {filename}") logger.info(f"[{task_id}] Rotating {len(normalized_rotation_data)} pages in {filename}")
log_task_progress( log_task_progress(
task_id, "apply_rotation", "in_progress", task_id,
f"Rotating {len(normalized_rotation_data)} pages", file_id=file_id, "apply_rotation",
"in_progress",
f"Rotating {len(normalized_rotation_data)} pages",
file_id=file_id,
) )
applied_rotations = {} applied_rotations = {}
@@ -144,8 +154,7 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
if applied_rotations: if applied_rotations:
logger.info( logger.info(
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: " f"[{task_id}] Successfully rotated PDF: {filename} with rotations: " f"{json.dumps(applied_rotations)}"
f"{json.dumps(applied_rotations)}"
) )
else: else:
logger.info( logger.info(
@@ -157,7 +166,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
extract_metadata_with_gpt.delay(filename, extracted_text, file_id) extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
log_task_progress( log_task_progress(
task_id, "rotate_pdf_pages", "success", task_id,
"rotate_pdf_pages",
"success",
f"Rotation complete for {filename}", f"Rotation complete for {filename}",
file_id=file_id, file_id=file_id,
detail={"applied_rotations": applied_rotations}, detail={"applied_rotations": applied_rotations},
@@ -173,7 +184,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
except Exception as e: except Exception as e:
logger.error(f"[{task_id}] Error rotating PDF {filename}: {e}") logger.error(f"[{task_id}] Error rotating PDF {filename}: {e}")
log_task_progress( log_task_progress(
task_id, "rotate_pdf_pages", "failure", task_id,
"rotate_pdf_pages",
"failure",
f"Rotation failed: {str(e)}", f"Rotation failed: {str(e)}",
file_id=file_id, file_id=file_id,
detail={"error": str(e), "filename": filename}, detail={"error": str(e), "filename": filename},
+11 -4
View File
@@ -274,8 +274,15 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
response_text, response_text,
) )
log_task_progress( log_task_progress(
task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id, task_id,
detail=f"Failed to upload document to Paperless.\nFile: {file_path}\nError: {exc}\nResponse: {response_text}", "upload_to_paperless",
"failure",
error_msg,
file_id=file_id,
detail=(
f"Failed to upload document to Paperless.\n"
f"File: {file_path}\nError: {exc}\nResponse: {response_text}"
),
) )
raise raise
@@ -322,9 +329,9 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
# Map each metadata field to its corresponding Paperless custom field # Map each metadata field to its corresponding Paperless custom field
for metadata_field, paperless_field in field_mapping.items(): for metadata_field, paperless_field in field_mapping.items():
if metadata_field in metadata and metadata[metadata_field]: if metadata_field in metadata and metadata[metadata_field]:
# Convert to string to ensure consistent comparison with UNKNOWN_VALUE # Convert to string to ensure consistent comparison
value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else "" value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else ""
if value and value != UNKNOWN_VALUE: if value and value != METADATA_UNKNOWN_PLACEHOLDER:
custom_fields_to_set[paperless_field] = value custom_fields_to_set[paperless_field] = value
logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'") logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'")
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
+1 -3
View File
@@ -103,9 +103,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
except (OSError, ValueError) as e: except (OSError, ValueError) as e:
error_msg = f"Error uploading {filename} to {destination}: {str(e)}" error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
logger.error(f"[{task_id}] {error_msg}") logger.error(f"[{task_id}] {error_msg}")
log_task_progress( log_task_progress(task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg)
task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg
)
raise RuntimeError(error_msg) from e raise RuntimeError(error_msg) from e
+5 -7
View File
@@ -10,7 +10,7 @@ from typing import Optional
from sqlalchemy import or_ from sqlalchemy import or_
from sqlalchemy.orm import Query, Session from sqlalchemy.orm import Query, Session
from app.models import FileRecord, FileProcessingStep from app.models import FileProcessingStep, FileRecord
def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Query: def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Query:
@@ -81,10 +81,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
REAL_STEPS.add("check_for_duplicates") REAL_STEPS.add("check_for_duplicates")
# Filter to only real steps # Filter to only real steps
real_steps_subq = ( real_steps_subq = db.query(FileProcessingStep).filter(FileProcessingStep.step_name.in_(REAL_STEPS))
db.query(FileProcessingStep)
.filter(FileProcessingStep.step_name.in_(REAL_STEPS))
)
if status == "pending": if status == "pending":
# Files with no real steps (never started processing) # Files with no real steps (never started processing)
@@ -108,8 +105,9 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
# Get files with failures or in_progress on real steps # Get files with failures or in_progress on real steps
files_with_issues = ( files_with_issues = (
real_steps_subq real_steps_subq.filter(
.filter(or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress")) or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress")
)
.distinct() .distinct()
.subquery() .subquery()
) )
+1 -1
View File
@@ -7,7 +7,7 @@ from typing import Dict, List
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.models import FileProcessingStep, FileRecord, ProcessingLog from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.utils.step_manager import get_file_overall_status, get_step_summary from app.utils.step_manager import get_file_overall_status
def get_file_processing_status(db: Session, file_id: int) -> Dict: def get_file_processing_status(db: Session, file_id: int) -> Dict:
+1 -4
View File
@@ -6,14 +6,11 @@ for files that were processed before the status tracking table was created.
""" """
import logging import logging
from datetime import datetime
from typing import Dict, List from typing import Dict, List
from sqlalchemy import func
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.models import FileProcessingStep, FileRecord, ProcessingLog from app.models import FileProcessingStep, ProcessingLog
from app.utils.step_manager import MAIN_PROCESSING_STEPS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
+3 -5
View File
@@ -220,10 +220,9 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
# Filter to only real steps # Filter to only real steps
steps = [ steps = [
s for s in all_steps s
if s.step_name in REAL_MAIN_STEPS for s in all_steps
or s.step_name.startswith("queue_") if s.step_name in REAL_MAIN_STEPS or s.step_name.startswith("queue_") or s.step_name.startswith("upload_to_")
or s.step_name.startswith("upload_to_")
] ]
if not steps: if not steps:
@@ -341,4 +340,3 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"total_main_steps": main_steps_count, "total_main_steps": main_steps_count,
"total_upload_tasks": upload_steps_count, "total_upload_tasks": upload_steps_count,
} }
+2 -5
View File
@@ -32,9 +32,7 @@ def get_step_timeout() -> int:
def mark_stalled_steps_as_failed( def mark_stalled_steps_as_failed(
db: Session, db: Session, timeout_seconds: Optional[int] = None, file_id: Optional[int] = None
timeout_seconds: Optional[int] = None,
file_id: Optional[int] = None
) -> int: ) -> int:
""" """
Find and mark any in-progress steps that have exceeded the timeout as failed. Find and mark any in-progress steps that have exceeded the timeout as failed.
@@ -75,8 +73,7 @@ def mark_stalled_steps_as_failed(
return 0 return 0
logger.warning( logger.warning(
f"Found {len(stalled_steps)} stalled step(s) that exceeded " f"Found {len(stalled_steps)} stalled step(s) that exceeded " f"{timeout_seconds}s timeout. Marking as failed."
f"{timeout_seconds}s timeout. Marking as failed."
) )
count = 0 count = 0
+15 -14
View File
@@ -2,13 +2,11 @@
File management views for displaying and managing files. File management views for displaying and managing files.
""" """
import os
from typing import Optional from typing import Optional
from fastapi import Depends, Query, Request from fastapi import Depends, Query, Request
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.config import settings
from app.utils.file_queries import apply_status_filter from app.utils.file_queries import apply_status_filter
from app.utils.file_status import get_files_processing_status from app.utils.file_status import get_files_processing_status
from app.views.base import APIRouter, get_db, logger, require_login, templates from app.views.base import APIRouter, get_db, logger, require_login, templates
@@ -34,9 +32,9 @@ def files_page(
""" """
try: try:
# Import the model here to avoid circular imports # Import the model here to avoid circular imports
from sqlalchemy import asc, desc, or_ from sqlalchemy import asc, desc
from app.models import FileRecord, ProcessingLog from app.models import FileRecord
# Start with base query # Start with base query
query = db.query(FileRecord) query = db.query(FileRecord)
@@ -236,6 +234,7 @@ def _compute_processing_flow(logs):
# Filter out deduplication step if not enabled or if not showing it # Filter out deduplication step if not enabled or if not showing it
from app.config import settings from app.config import settings
if not settings.enable_deduplication or not settings.show_deduplication_step: if not settings.enable_deduplication or not settings.show_deduplication_step:
stages.pop("check_for_duplicates", None) stages.pop("check_for_duplicates", None)
# Update the next pointer for create_file_record # Update the next pointer for create_file_record
@@ -359,16 +358,18 @@ def _compute_step_summary(logs):
main_steps = [] main_steps = []
if settings.enable_deduplication and settings.show_deduplication_step: if settings.enable_deduplication and settings.show_deduplication_step:
main_steps.append("check_for_duplicates") main_steps.append("check_for_duplicates")
main_steps.extend([ main_steps.extend(
"create_file_record", [
"check_text", "create_file_record",
"extract_text", "check_text",
"process_with_azure_document_intelligence", "extract_text",
"extract_metadata_with_gpt", "process_with_azure_document_intelligence",
"embed_metadata_into_pdf", "extract_metadata_with_gpt",
"finalize_document_storage", "embed_metadata_into_pdf",
"send_to_all_destinations", "finalize_document_storage",
]) "send_to_all_destinations",
]
)
upload_prefixes = ["upload_to_", "queue_"] upload_prefixes = ["upload_to_", "queue_"]
+5 -3
View File
@@ -14,16 +14,17 @@ These tests exercise the full application stack end-to-end.
import os import os
import time import time
import pytest
from typing import Generator
from pathlib import Path from pathlib import Path
from typing import Generator
import pytest
# Import testcontainers # Import testcontainers
pytest.importorskip("testcontainers", reason="testcontainers not installed") pytest.importorskip("testcontainers", reason="testcontainers not installed")
from testcontainers.core.container import DockerContainer from testcontainers.core.container import DockerContainer
from testcontainers.minio import MinioContainer
from testcontainers.postgres import PostgresContainer from testcontainers.postgres import PostgresContainer
from testcontainers.redis import RedisContainer from testcontainers.redis import RedisContainer
from testcontainers.minio import MinioContainer
_TEST_CREDENTIAL = "testpass" # noqa: S105 _TEST_CREDENTIAL = "testpass" # noqa: S105
@@ -300,6 +301,7 @@ def db_session_real(postgres_container):
""" """
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from app.database import Base from app.database import Base
# Create engine using PostgreSQL container # Create engine using PostgreSQL container
+3 -1
View File
@@ -1,8 +1,10 @@
"""Tests for app/api/common.py module.""" """Tests for app/api/common.py module."""
import os import os
import pytest
from unittest.mock import patch from unittest.mock import patch
import pytest
from app.api.common import resolve_file_path from app.api.common import resolve_file_path
+3 -1
View File
@@ -2,10 +2,12 @@
Tests for API error handling - ensuring JSON responses for API routes. Tests for API error handling - ensuring JSON responses for API routes.
""" """
from unittest.mock import patch
import pytest import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from app.models import FileRecord from app.models import FileRecord
from unittest.mock import patch
@pytest.mark.integration @pytest.mark.integration
+3 -1
View File
@@ -1,7 +1,9 @@
"""Tests for app/api/logs.py module.""" """Tests for app/api/logs.py module."""
import pytest
from datetime import datetime from datetime import datetime
import pytest
from app.models import FileRecord, ProcessingLog from app.models import FileRecord, ProcessingLog
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/api/openai.py module.""" """Tests for app/api/openai.py module."""
import pytest import pytest
+2
View File
@@ -1,4 +1,5 @@
"""Tests for app/api/process.py module.""" """Tests for app/api/process.py module."""
import pytest import pytest
@@ -44,6 +45,7 @@ class TestProcessEndpoints:
def test_processall_endpoint(self, client, tmp_path): def test_processall_endpoint(self, client, tmp_path):
"""Test POST /api/processall with no PDF files in workdir.""" """Test POST /api/processall with no PDF files in workdir."""
from unittest.mock import patch from unittest.mock import patch
with patch("app.api.process.settings") as mock_settings: with patch("app.api.process.settings") as mock_settings:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
response = client.post("/api/processall") response = client.post("/api/processall")
+3 -1
View File
@@ -1,6 +1,8 @@
"""Tests for app/api/settings.py module.""" """Tests for app/api/settings.py module."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from fastapi import HTTPException from fastapi import HTTPException
from app.api.settings import require_admin from app.api.settings import require_admin
+3 -1
View File
@@ -1,8 +1,10 @@
"""Tests for app/api/user.py module.""" """Tests for app/api/user.py module."""
import pytest
from hashlib import md5 from hashlib import md5
from unittest.mock import MagicMock from unittest.mock import MagicMock
import pytest
from app.api.user import whoami_handler from app.api.user import whoami_handler
+11 -7
View File
@@ -1,9 +1,11 @@
"""Integration tests for auth.py with AUTH_ENABLED=True scenarios.""" """Integration tests for auth.py with AUTH_ENABLED=True scenarios."""
import pytest
from unittest.mock import patch, MagicMock, AsyncMock
import hashlib
from app.auth import get_gravatar_url, get_current_user, require_login import hashlib
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.auth import get_current_user, get_gravatar_url, require_login
@pytest.mark.unit @pytest.mark.unit
@@ -20,8 +22,9 @@ class TestRequireLoginWithAuth:
return {"success": True} return {"success": True}
# Manually create the decorator behavior # Manually create the decorator behavior
from functools import wraps
import inspect import inspect
from functools import wraps
from fastapi import status from fastapi import status
from starlette.responses import RedirectResponse from starlette.responses import RedirectResponse
@@ -45,10 +48,11 @@ class TestRequireLoginWithAuth:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_require_login_allows_authenticated_user(self): async def test_require_login_allows_authenticated_user(self):
"""Test that require_login allows through when user exists in session.""" """Test that require_login allows through when user exists in session."""
from functools import wraps
import inspect import inspect
from starlette.responses import RedirectResponse from functools import wraps
from fastapi import status from fastapi import status
from starlette.responses import RedirectResponse
async def my_route(request): async def my_route(request):
return {"success": True} return {"success": True}
+6 -1
View File
@@ -1,8 +1,10 @@
"""Tests for app/tasks/check_credentials.py module.""" """Tests for app/tasks/check_credentials.py module."""
import json import json
import os import os
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from app.tasks.check_credentials import ( from app.tasks.check_credentials import (
MockRequest, MockRequest,
@@ -95,6 +97,7 @@ class TestUnwrapDecoratedFunction:
def test_returns_same_function_if_not_decorated(self): def test_returns_same_function_if_not_decorated(self):
"""Test returns same function if not decorated.""" """Test returns same function if not decorated."""
def my_func(): def my_func():
return "hello" return "hello"
@@ -103,6 +106,7 @@ class TestUnwrapDecoratedFunction:
def test_unwraps_decorated_function(self): def test_unwraps_decorated_function(self):
"""Test unwraps decorated function.""" """Test unwraps decorated function."""
def inner(): def inner():
return "hello" return "hello"
@@ -116,6 +120,7 @@ class TestUnwrapDecoratedFunction:
def test_unwraps_multiple_levels(self): def test_unwraps_multiple_levels(self):
"""Test unwraps multiple levels of decoration.""" """Test unwraps multiple levels of decoration."""
def original(): def original():
return "hello" return "hello"
+4 -2
View File
@@ -1,13 +1,15 @@
"""Extended tests for app/tasks/check_credentials.py module.""" """Extended tests for app/tasks/check_credentials.py module."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from app.tasks.check_credentials import ( from app.tasks.check_credentials import (
sync_test_openai_connection,
sync_test_azure_connection, sync_test_azure_connection,
sync_test_dropbox_token, sync_test_dropbox_token,
sync_test_google_drive_token, sync_test_google_drive_token,
sync_test_onedrive_token, sync_test_onedrive_token,
sync_test_openai_connection,
) )
+23 -20
View File
@@ -1,9 +1,12 @@
""" """
Unit tests for configuration and security validation. Unit tests for configuration and security validation.
""" """
import pytest
import os import os
import pytest
from pydantic import ValidationError from pydantic import ValidationError
from app.config import Settings from app.config import Settings
@@ -24,7 +27,7 @@ class TestConfigurationValidation:
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=True, auth_enabled=True,
session_secret=None session_secret=None,
) )
assert "SESSION_SECRET must be set" in str(exc_info.value) assert "SESSION_SECRET must be set" in str(exc_info.value)
@@ -41,7 +44,7 @@ class TestConfigurationValidation:
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=True, auth_enabled=True,
session_secret="short" session_secret="short",
) )
assert "at least 32 characters" in str(exc_info.value) assert "at least 32 characters" in str(exc_info.value)
@@ -57,7 +60,7 @@ class TestConfigurationValidation:
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=True, auth_enabled=True,
session_secret="a" * 32 # 32 character secret session_secret="a" * 32, # 32 character secret
) )
assert config.auth_enabled is True assert config.auth_enabled is True
assert len(config.session_secret) == 32 assert len(config.session_secret) == 32
@@ -74,14 +77,12 @@ class TestConfigurationValidation:
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False, auth_enabled=False,
session_secret=None session_secret=None,
) )
assert config.auth_enabled is False assert config.auth_enabled is False
assert config.session_secret is None assert config.session_secret is None
@pytest.mark.unit @pytest.mark.unit
class TestBuildMetadataConfiguration: class TestBuildMetadataConfiguration:
"""Tests for build metadata configuration.""" """Tests for build metadata configuration."""
@@ -98,7 +99,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
assert config.version == "1.2.3-test" assert config.version == "1.2.3-test"
@@ -114,7 +115,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
assert config.build_date == "2026-01-15" assert config.build_date == "2026-01-15"
@@ -130,7 +131,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
assert config.build_date == "2026-01-15T10:30:00Z" assert config.build_date == "2026-01-15T10:30:00Z"
@@ -146,7 +147,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
assert config.git_sha == "abc1234" assert config.git_sha == "abc1234"
@@ -154,7 +155,8 @@ class TestBuildMetadataConfiguration:
"""Test that git_sha defaults to 'unknown' when not set.""" """Test that git_sha defaults to 'unknown' when not set."""
# Mock the file system to ensure no GIT_SHA file exists # Mock the file system to ensure no GIT_SHA file exists
import app.config import app.config
monkeypatch.setattr(app.config.os.path, 'dirname', lambda x: str(tmp_path))
monkeypatch.setattr(app.config.os.path, "dirname", lambda x: str(tmp_path))
config = Settings( config = Settings(
database_url="sqlite:///test.db", database_url="sqlite:///test.db",
@@ -165,7 +167,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
# When no file or env var exists, should return "unknown" # When no file or env var exists, should return "unknown"
assert config.git_sha == "unknown" assert config.git_sha == "unknown"
@@ -173,7 +175,8 @@ class TestBuildMetadataConfiguration:
def test_version_default_when_no_file_or_env(self, monkeypatch, tmp_path): def test_version_default_when_no_file_or_env(self, monkeypatch, tmp_path):
"""Test that version defaults to 'unknown' when no VERSION file or env var exists.""" """Test that version defaults to 'unknown' when no VERSION file or env var exists."""
import app.config import app.config
monkeypatch.setattr(app.config.os.path, 'dirname', lambda x: str(tmp_path))
monkeypatch.setattr(app.config.os.path, "dirname", lambda x: str(tmp_path))
config = Settings( config = Settings(
database_url="sqlite:///test.db", database_url="sqlite:///test.db",
@@ -184,7 +187,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
# When no file or env var exists, should return "unknown" # When no file or env var exists, should return "unknown"
assert config.version == "unknown" assert config.version == "unknown"
@@ -200,7 +203,7 @@ class TestBuildMetadataConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
runtime_info = config.runtime_info runtime_info = config.runtime_info
# Should contain version, build_date, and git_sha in some form # Should contain version, build_date, and git_sha in some form
@@ -223,7 +226,7 @@ class TestNotificationConfiguration:
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False, auth_enabled=False,
notification_urls="discord://webhook1,telegram://webhook2" notification_urls="discord://webhook1,telegram://webhook2",
) )
assert len(config.notification_urls) == 2 assert len(config.notification_urls) == 2
assert "discord://webhook1" in config.notification_urls assert "discord://webhook1" in config.notification_urls
@@ -241,7 +244,7 @@ class TestNotificationConfiguration:
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False, auth_enabled=False,
notification_urls=["discord://webhook1", "telegram://webhook2"] notification_urls=["discord://webhook1", "telegram://webhook2"],
) )
assert len(config.notification_urls) == 2 assert len(config.notification_urls) == 2
@@ -263,7 +266,7 @@ class TestSecurityConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
# Ensure optional credentials are actually optional (None) # Ensure optional credentials are actually optional (None)
@@ -283,7 +286,7 @@ class TestSecurityConfiguration:
azure_endpoint="https://test.example.com", azure_endpoint="https://test.example.com",
gotenberg_url="http://localhost:3000", gotenberg_url="http://localhost:3000",
workdir="/tmp", workdir="/tmp",
auth_enabled=False auth_enabled=False,
) )
# Should not raise an error # Should not raise an error
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/utils/config_loader.py module.""" """Tests for app/utils/config_loader.py module."""
import pytest import pytest
from app.utils.config_loader import convert_setting_value from app.utils.config_loader import convert_setting_value
+5 -3
View File
@@ -1,12 +1,14 @@
"""Tests for app/utils/config_validator/validators.py module.""" """Tests for app/utils/config_validator/validators.py module."""
import pytest
from unittest.mock import patch from unittest.mock import patch
import pytest
from app.utils.config_validator.validators import ( from app.utils.config_validator.validators import (
validate_storage_configs, check_all_configs,
validate_email_config, validate_email_config,
validate_notification_config, validate_notification_config,
check_all_configs, validate_storage_configs,
) )
+6 -1
View File
@@ -1,7 +1,9 @@
"""Tests for app/tasks/convert_to_pdf.py module.""" """Tests for app/tasks/convert_to_pdf.py module."""
import os import os
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -11,6 +13,7 @@ class TestConvertToPdfMimeTypes:
def test_office_extensions_set(self): def test_office_extensions_set(self):
"""Test that the task module is importable.""" """Test that the task module is importable."""
from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.convert_to_pdf import convert_to_pdf
assert callable(convert_to_pdf) assert callable(convert_to_pdf)
@patch("app.tasks.convert_to_pdf.requests") @patch("app.tasks.convert_to_pdf.requests")
@@ -25,6 +28,7 @@ class TestConvertToPdfMimeTypes:
mock_self.request.id = "test-task-id" mock_self.request.id = "test-task-id"
from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.convert_to_pdf import convert_to_pdf
result = convert_to_pdf.__wrapped__(mock_self, "/tmp/test.docx") result = convert_to_pdf.__wrapped__(mock_self, "/tmp/test.docx")
assert result is None assert result is None
@@ -46,5 +50,6 @@ class TestConvertToPdfMimeTypes:
mock_self.request.id = "test-task-id" mock_self.request.id = "test-task-id"
from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.convert_to_pdf import convert_to_pdf
result = convert_to_pdf.__wrapped__(mock_self, str(test_file)) result = convert_to_pdf.__wrapped__(mock_self, str(test_file))
assert result is None assert result is None
+24 -1
View File
@@ -1,6 +1,8 @@
"""Tests to boost coverage for various small modules.""" """Tests to boost coverage for various small modules."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -10,11 +12,13 @@ class TestUtilsCompat:
def test_imports_hash_file(self): def test_imports_hash_file(self):
"""Test that hash_file can be imported from utils.""" """Test that hash_file can be imported from utils."""
from app.utils import hash_file from app.utils import hash_file
assert callable(hash_file) assert callable(hash_file)
def test_imports_log_task_progress(self): def test_imports_log_task_progress(self):
"""Test that log_task_progress can be imported from utils.""" """Test that log_task_progress can be imported from utils."""
from app.utils import log_task_progress from app.utils import log_task_progress
assert callable(log_task_progress) assert callable(log_task_progress)
@@ -25,31 +29,37 @@ class TestConfigValidatorCompat:
def test_imports_validate_email_config(self): def test_imports_validate_email_config(self):
"""Test backward compatible import.""" """Test backward compatible import."""
from app.utils.config_validator import validate_email_config from app.utils.config_validator import validate_email_config
assert callable(validate_email_config) assert callable(validate_email_config)
def test_imports_validate_storage_configs(self): def test_imports_validate_storage_configs(self):
"""Test backward compatible import.""" """Test backward compatible import."""
from app.utils.config_validator import validate_storage_configs from app.utils.config_validator import validate_storage_configs
assert callable(validate_storage_configs) assert callable(validate_storage_configs)
def test_imports_mask_sensitive_value(self): def test_imports_mask_sensitive_value(self):
"""Test backward compatible import.""" """Test backward compatible import."""
from app.utils.config_validator import mask_sensitive_value from app.utils.config_validator import mask_sensitive_value
assert callable(mask_sensitive_value) assert callable(mask_sensitive_value)
def test_imports_get_provider_status(self): def test_imports_get_provider_status(self):
"""Test backward compatible import.""" """Test backward compatible import."""
from app.utils.config_validator import get_provider_status from app.utils.config_validator import get_provider_status
assert callable(get_provider_status) assert callable(get_provider_status)
def test_imports_dump_all_settings(self): def test_imports_dump_all_settings(self):
"""Test backward compatible import.""" """Test backward compatible import."""
from app.utils.config_validator import dump_all_settings from app.utils.config_validator import dump_all_settings
assert callable(dump_all_settings) assert callable(dump_all_settings)
def test_imports_check_all_configs(self): def test_imports_check_all_configs(self):
"""Test backward compatible import.""" """Test backward compatible import."""
from app.utils.config_validator import check_all_configs from app.utils.config_validator import check_all_configs
assert callable(check_all_configs) assert callable(check_all_configs)
@@ -60,6 +70,7 @@ class TestCeleryWorkerImport:
def test_celery_worker_module_exists(self): def test_celery_worker_module_exists(self):
"""Test that celery_worker module can be found.""" """Test that celery_worker module can be found."""
import importlib import importlib
spec = importlib.util.find_spec("app.celery_worker") spec = importlib.util.find_spec("app.celery_worker")
assert spec is not None assert spec is not None
@@ -71,22 +82,26 @@ class TestSettingsDisplayMasking:
def test_dump_all_settings_masks_passwords(self): def test_dump_all_settings_masks_passwords(self):
"""Test that passwords are masked in settings dump.""" """Test that passwords are masked in settings dump."""
from app.utils.config_validator.settings_display import dump_all_settings from app.utils.config_validator.settings_display import dump_all_settings
# Should not raise # Should not raise
dump_all_settings() dump_all_settings()
def test_dump_all_settings_masks_tokens(self): def test_dump_all_settings_masks_tokens(self):
"""Test that tokens are masked in settings dump.""" """Test that tokens are masked in settings dump."""
from app.utils.config_validator.settings_display import dump_all_settings from app.utils.config_validator.settings_display import dump_all_settings
dump_all_settings() dump_all_settings()
def test_dump_all_settings_masks_keys(self): def test_dump_all_settings_masks_keys(self):
"""Test that API keys are masked in settings dump.""" """Test that API keys are masked in settings dump."""
from app.utils.config_validator.settings_display import dump_all_settings from app.utils.config_validator.settings_display import dump_all_settings
dump_all_settings() dump_all_settings()
def test_get_settings_for_display_categories(self): def test_get_settings_for_display_categories(self):
"""Test that all expected categories are returned.""" """Test that all expected categories are returned."""
from app.utils.config_validator.settings_display import get_settings_for_display from app.utils.config_validator.settings_display import get_settings_for_display
result = get_settings_for_display(show_values=True) result = get_settings_for_display(show_values=True)
# Should have multiple categories # Should have multiple categories
assert len(result) > 3 assert len(result) > 3
@@ -102,6 +117,7 @@ class TestNotificationInit:
"""Test init_apprise when no URLs configured.""" """Test init_apprise when no URLs configured."""
mock_settings.notification_urls = [] mock_settings.notification_urls = []
from app.utils.notification import init_apprise from app.utils.notification import init_apprise
result = init_apprise() result = init_apprise()
assert result is not None assert result is not None
@@ -111,6 +127,7 @@ class TestNotificationInit:
"""Test init_apprise with URLs configured.""" """Test init_apprise with URLs configured."""
mock_settings.notification_urls = ["json://localhost"] mock_settings.notification_urls = ["json://localhost"]
from app.utils.notification import init_apprise from app.utils.notification import init_apprise
result = init_apprise() result = init_apprise()
assert result is not None assert result is not None
@@ -127,6 +144,7 @@ class TestNotificationFileProcessed:
mock_send.return_value = True mock_send.return_value = True
from app.utils.notification import notify_file_processed from app.utils.notification import notify_file_processed
result = notify_file_processed( result = notify_file_processed(
filename="test.pdf", filename="test.pdf",
file_size=1048576, file_size=1048576,
@@ -144,6 +162,7 @@ class TestNotificationFileProcessed:
mock_send.return_value = True mock_send.return_value = True
from app.utils.notification import notify_file_processed from app.utils.notification import notify_file_processed
result = notify_file_processed( result = notify_file_processed(
filename="small.pdf", filename="small.pdf",
file_size=512, # Less than 1KB file_size=512, # Less than 1KB
@@ -165,6 +184,7 @@ class TestNotificationCeleryFailure:
mock_send.return_value = True mock_send.return_value = True
from app.utils.notification import notify_celery_failure from app.utils.notification import notify_celery_failure
result = notify_celery_failure( result = notify_celery_failure(
task_name="process_document", task_name="process_document",
task_id="task-123", task_id="task-123",
@@ -188,6 +208,7 @@ class TestNotificationCredentialFailure:
mock_send.return_value = True mock_send.return_value = True
from app.utils.notification import notify_credential_failure from app.utils.notification import notify_credential_failure
result = notify_credential_failure( result = notify_credential_failure(
service_name="OpenAI", service_name="OpenAI",
error="Invalid API key", error="Invalid API key",
@@ -208,6 +229,7 @@ class TestNotificationStartupShutdown:
mock_send.return_value = True mock_send.return_value = True
from app.utils.notification import notify_startup from app.utils.notification import notify_startup
result = notify_startup() result = notify_startup()
assert result is True assert result is True
@@ -220,5 +242,6 @@ class TestNotificationStartupShutdown:
mock_send.return_value = True mock_send.return_value = True
from app.utils.notification import notify_shutdown from app.utils.notification import notify_shutdown
result = notify_shutdown() result = notify_shutdown()
assert result is True assert result is True
+20 -1
View File
@@ -1,6 +1,8 @@
"""Final tests to push coverage over 60%.""" """Final tests to push coverage over 60%."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -10,6 +12,7 @@ class TestCheckCredentialsFunctions:
def test_sync_test_openai_connection(self): def test_sync_test_openai_connection(self):
"""Test sync_test_openai_connection.""" """Test sync_test_openai_connection."""
from app.tasks.check_credentials import sync_test_openai_connection from app.tasks.check_credentials import sync_test_openai_connection
result = sync_test_openai_connection() result = sync_test_openai_connection()
assert isinstance(result, dict) assert isinstance(result, dict)
assert "status" in result assert "status" in result
@@ -17,6 +20,7 @@ class TestCheckCredentialsFunctions:
def test_sync_test_azure_connection(self): def test_sync_test_azure_connection(self):
"""Test sync_test_azure_connection.""" """Test sync_test_azure_connection."""
from app.tasks.check_credentials import sync_test_azure_connection from app.tasks.check_credentials import sync_test_azure_connection
result = sync_test_azure_connection() result = sync_test_azure_connection()
assert isinstance(result, dict) assert isinstance(result, dict)
assert "status" in result assert "status" in result
@@ -24,6 +28,7 @@ class TestCheckCredentialsFunctions:
def test_sync_test_dropbox_token(self): def test_sync_test_dropbox_token(self):
"""Test sync_test_dropbox_token.""" """Test sync_test_dropbox_token."""
from app.tasks.check_credentials import sync_test_dropbox_token from app.tasks.check_credentials import sync_test_dropbox_token
result = sync_test_dropbox_token() result = sync_test_dropbox_token()
assert isinstance(result, dict) assert isinstance(result, dict)
assert "status" in result assert "status" in result
@@ -31,6 +36,7 @@ class TestCheckCredentialsFunctions:
def test_sync_test_google_drive_token(self): def test_sync_test_google_drive_token(self):
"""Test sync_test_google_drive_token.""" """Test sync_test_google_drive_token."""
from app.tasks.check_credentials import sync_test_google_drive_token from app.tasks.check_credentials import sync_test_google_drive_token
result = sync_test_google_drive_token() result = sync_test_google_drive_token()
assert isinstance(result, dict) assert isinstance(result, dict)
assert "status" in result assert "status" in result
@@ -38,6 +44,7 @@ class TestCheckCredentialsFunctions:
def test_sync_test_onedrive_token(self): def test_sync_test_onedrive_token(self):
"""Test sync_test_onedrive_token.""" """Test sync_test_onedrive_token."""
from app.tasks.check_credentials import sync_test_onedrive_token from app.tasks.check_credentials import sync_test_onedrive_token
result = sync_test_onedrive_token() result = sync_test_onedrive_token()
assert isinstance(result, dict) assert isinstance(result, dict)
assert "status" in result assert "status" in result
@@ -45,29 +52,34 @@ class TestCheckCredentialsFunctions:
def test_sync_test_nextcloud_credentials(self): def test_sync_test_nextcloud_credentials(self):
"""Test that check_credentials module has check_credentials task.""" """Test that check_credentials module has check_credentials task."""
from app.tasks.check_credentials import check_credentials from app.tasks.check_credentials import check_credentials
assert callable(check_credentials) assert callable(check_credentials)
def test_sync_test_sftp_credentials(self): def test_sync_test_sftp_credentials(self):
"""Test MockRequest scope attribute.""" """Test MockRequest scope attribute."""
from app.tasks.check_credentials import MockRequest from app.tasks.check_credentials import MockRequest
req = MockRequest() req = MockRequest()
assert hasattr(req, "session") assert hasattr(req, "session")
def test_sync_test_email_credentials(self): def test_sync_test_email_credentials(self):
"""Test MockRequest path_params attribute.""" """Test MockRequest path_params attribute."""
from app.tasks.check_credentials import MockRequest from app.tasks.check_credentials import MockRequest
req = MockRequest() req = MockRequest()
assert isinstance(req.query_params, dict) assert isinstance(req.query_params, dict)
def test_sync_test_ftp_credentials(self): def test_sync_test_ftp_credentials(self):
"""Test MockRequest headers attribute.""" """Test MockRequest headers attribute."""
from app.tasks.check_credentials import MockRequest from app.tasks.check_credentials import MockRequest
req = MockRequest() req = MockRequest()
assert isinstance(req.headers, dict) assert isinstance(req.headers, dict)
def test_sync_test_paperless_credentials(self): def test_sync_test_paperless_credentials(self):
"""Test get_failure_state returns dict.""" """Test get_failure_state returns dict."""
from app.tasks.check_credentials import get_failure_state from app.tasks.check_credentials import get_failure_state
result = get_failure_state() result = get_failure_state()
assert isinstance(result, dict) assert isinstance(result, dict)
@@ -75,7 +87,9 @@ class TestCheckCredentialsFunctions:
"""Test save_failure_state accepts dict.""" """Test save_failure_state accepts dict."""
import os import os
from unittest.mock import patch from unittest.mock import patch
from app.tasks.check_credentials import save_failure_state from app.tasks.check_credentials import save_failure_state
with patch("app.tasks.check_credentials.FAILURE_STATE_FILE", "/tmp/test_failure_state_final.json"): with patch("app.tasks.check_credentials.FAILURE_STATE_FILE", "/tmp/test_failure_state_final.json"):
save_failure_state({"test": "value"}) save_failure_state({"test": "value"})
if os.path.exists("/tmp/test_failure_state_final.json"): if os.path.exists("/tmp/test_failure_state_final.json"):
@@ -89,6 +103,7 @@ class TestImapPullInboxes:
def test_pull_all_inboxes_is_callable(self): def test_pull_all_inboxes_is_callable(self):
"""Test that pull_all_inboxes is callable.""" """Test that pull_all_inboxes is callable."""
from app.tasks.imap_tasks import pull_all_inboxes from app.tasks.imap_tasks import pull_all_inboxes
assert callable(pull_all_inboxes) assert callable(pull_all_inboxes)
@@ -99,6 +114,7 @@ class TestViewsProviderStatus:
def test_get_provider_status_returns_dict(self): def test_get_provider_status_returns_dict(self):
"""Test get_provider_status.""" """Test get_provider_status."""
from app.utils.config_validator.providers import get_provider_status from app.utils.config_validator.providers import get_provider_status
result = get_provider_status() result = get_provider_status()
assert isinstance(result, dict) assert isinstance(result, dict)
assert len(result) > 0 assert len(result) > 0
@@ -111,6 +127,7 @@ class TestSettingsService:
def test_get_settings_by_category(self): def test_get_settings_by_category(self):
"""Test get_settings_by_category.""" """Test get_settings_by_category."""
from app.utils.settings_service import get_settings_by_category from app.utils.settings_service import get_settings_by_category
result = get_settings_by_category() result = get_settings_by_category()
assert isinstance(result, dict) assert isinstance(result, dict)
assert len(result) > 0 assert len(result) > 0
@@ -118,12 +135,14 @@ class TestSettingsService:
def test_get_setting_metadata(self): def test_get_setting_metadata(self):
"""Test get_setting_metadata for a known key.""" """Test get_setting_metadata for a known key."""
from app.utils.settings_service import get_setting_metadata from app.utils.settings_service import get_setting_metadata
result = get_setting_metadata("openai_api_key") result = get_setting_metadata("openai_api_key")
assert isinstance(result, dict) assert isinstance(result, dict)
def test_validate_setting_value(self): def test_validate_setting_value(self):
"""Test validate_setting_value.""" """Test validate_setting_value."""
from app.utils.settings_service import validate_setting_value from app.utils.settings_service import validate_setting_value
# Should return tuple of (is_valid, error_message or None) # Should return tuple of (is_valid, error_message or None)
result = validate_setting_value("openai_api_key", "test-key") result = validate_setting_value("openai_api_key", "test-key")
assert isinstance(result, tuple) assert isinstance(result, tuple)
+6 -4
View File
@@ -1,9 +1,11 @@
"""Tests for app/database.py module.""" """Tests for app/database.py module."""
import os
import pytest
from unittest.mock import patch, MagicMock
from app.database import init_db, get_db import os
from unittest.mock import MagicMock, patch
import pytest
from app.database import get_db, init_db
@pytest.mark.unit @pytest.mark.unit
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/api/diagnostic.py module.""" """Tests for app/api/diagnostic.py module."""
import pytest import pytest
+11 -9
View File
@@ -7,9 +7,10 @@ and test the complete application workflow from API request to file upload.
import os import os
import time import time
from unittest.mock import patch
import pytest import pytest
import requests import requests
from unittest.mock import patch
# Import testcontainers requirement # Import testcontainers requirement
pytest.importorskip("testcontainers", reason="testcontainers not installed") pytest.importorskip("testcontainers", reason="testcontainers not installed")
@@ -22,16 +23,16 @@ except ModuleNotFoundError:
_has_psycopg2 = False _has_psycopg2 = False
from tests.fixtures_integration import ( from tests.fixtures_integration import (
postgres_container,
redis_container,
gotenberg_container,
webdav_container,
sftp_container,
minio_container,
full_infrastructure,
celery_app, celery_app,
celery_worker, celery_worker,
db_session_real, db_session_real,
full_infrastructure,
gotenberg_container,
minio_container,
postgres_container,
redis_container,
sftp_container,
webdav_container,
) )
_TEST_CREDENTIAL = "pass" # noqa: S105 _TEST_CREDENTIAL = "pass" # noqa: S105
@@ -117,9 +118,10 @@ class TestEndToEndWithRedis:
This verifies the Redis broker is working correctly. This verifies the Redis broker is working correctly.
""" """
from app.tasks.upload_to_webdav import upload_to_webdav
import redis import redis
from app.tasks.upload_to_webdav import upload_to_webdav
# Connect to Redis directly # Connect to Redis directly
r = redis.from_url(redis_container["url"]) r = redis.from_url(redis_container["url"])
+3 -1
View File
@@ -1,7 +1,9 @@
"""Tests for app/tasks/embed_metadata_into_pdf.py module.""" """Tests for app/tasks/embed_metadata_into_pdf.py module."""
import os import os
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from app.tasks.embed_metadata_into_pdf import persist_metadata from app.tasks.embed_metadata_into_pdf import persist_metadata
from app.utils.filename_utils import get_unique_filepath_with_counter from app.utils.filename_utils import get_unique_filepath_with_counter
+4 -9
View File
@@ -5,9 +5,10 @@ This test module serves as a regression prevention mechanism to ensure
that endpoints remain accessible after code refactoring or reorganization. that endpoints remain accessible after code refactoring or reorganization.
""" """
import pytest
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
import pytest
# Test constants # Test constants
TEST_URL = "https://example.com/test.pdf" TEST_URL = "https://example.com/test.pdf"
@@ -34,10 +35,7 @@ class TestEndpointRegistration:
mock_process_document.delay.return_value = mock_task mock_process_document.delay.return_value = mock_task
# Make a request to the endpoint - it should not return 404 # Make a request to the endpoint - it should not return 404
response = client.post( response = client.post("/api/process-url", json={"url": TEST_URL})
"/api/process-url",
json={"url": TEST_URL}
)
# The endpoint exists if we don't get a 404 # The endpoint exists if we don't get a 404
# We may get other errors (401, 400, 500, etc.) due to validation or missing mocks, # We may get other errors (401, 400, 500, etc.) due to validation or missing mocks,
@@ -66,10 +64,7 @@ class TestEndpointRegistration:
mock_process_document.delay.return_value = mock_task mock_process_document.delay.return_value = mock_task
# Try POST request # Try POST request
response = client.post( response = client.post("/api/process-url", json={"url": TEST_URL})
"/api/process-url",
json={"url": TEST_URL}
)
# Should not return 405 (Method Not Allowed) # Should not return 405 (Method Not Allowed)
assert response.status_code != 405, ( assert response.status_code != 405, (
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/tasks/extract_metadata_with_gpt.py module.""" """Tests for app/tasks/extract_metadata_with_gpt.py module."""
import pytest import pytest
from app.tasks.extract_metadata_with_gpt import extract_json_from_text from app.tasks.extract_metadata_with_gpt import extract_json_from_text
+19 -12
View File
@@ -1,10 +1,13 @@
""" """
Tests for file listing, pagination, filtering, and detail endpoints. Tests for file listing, pagination, filtering, and detail endpoints.
""" """
from datetime import datetime
import pytest import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from app.models import FileProcessingStep, FileRecord, ProcessingLog from app.models import FileProcessingStep, FileRecord, ProcessingLog
from datetime import datetime
@pytest.mark.integration @pytest.mark.integration
@@ -32,7 +35,7 @@ class TestFileListingPagination:
original_filename=f"test{i}.pdf", original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf", local_filename=f"/tmp/test{i}.pdf",
file_size=1024 * (i + 1), file_size=1024 * (i + 1),
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -53,7 +56,7 @@ class TestFileListingPagination:
original_filename=f"test{i}.pdf", original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf", local_filename=f"/tmp/test{i}.pdf",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -82,7 +85,7 @@ class TestFileListingPagination:
original_filename=name, original_filename=name,
local_filename=f"/tmp/{name}", local_filename=f"/tmp/{name}",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -110,7 +113,7 @@ class TestFileListingPagination:
original_filename=f"file{i}.pdf", original_filename=f"file{i}.pdf",
local_filename=f"/tmp/file{i}.pdf", local_filename=f"/tmp/file{i}.pdf",
file_size=size, file_size=size,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -131,7 +134,7 @@ class TestFileListingPagination:
original_filename=name, original_filename=name,
local_filename=f"/tmp/{name}", local_filename=f"/tmp/{name}",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -160,7 +163,7 @@ class TestFileListingPagination:
original_filename=filename, original_filename=filename,
local_filename=f"/tmp/{filename}", local_filename=f"/tmp/{filename}",
file_size=1024, file_size=1024,
mime_type=mime_type mime_type=mime_type,
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -181,7 +184,7 @@ class TestFileListingPagination:
original_filename="test.pdf", original_filename="test.pdf",
local_filename="/tmp/test.pdf", local_filename="/tmp/test.pdf",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -216,13 +219,17 @@ class TestFileDetailEndpoint:
original_filename="test.pdf", original_filename="test.pdf",
local_filename="/tmp/test.pdf", local_filename="/tmp/test.pdf",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
# Add processing steps (used for status determination) # Add processing steps (used for status determination)
for step_name, status in [("extract_text", "success"), ("extract_metadata_with_gpt", "in_progress"), ("embed_metadata_into_pdf", "success")]: for step_name, status in [
("extract_text", "success"),
("extract_metadata_with_gpt", "in_progress"),
("embed_metadata_into_pdf", "success"),
]:
step = FileProcessingStep( step = FileProcessingStep(
file_id=file_record.id, file_id=file_record.id,
step_name=step_name, step_name=step_name,
@@ -237,7 +244,7 @@ class TestFileDetailEndpoint:
task_id=f"task_{i}", task_id=f"task_{i}",
step_name=f"step_{i}", step_name=f"step_{i}",
status=status, status=status,
message=f"Message {i}" message=f"Message {i}",
) )
db_session.add(log) db_session.add(log)
db_session.commit() db_session.commit()
@@ -272,7 +279,7 @@ class TestFileDetailEndpoint:
original_filename="test.pdf", original_filename="test.pdf",
local_filename="/tmp/test.pdf", local_filename="/tmp/test.pdf",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
+1 -1
View File
@@ -9,7 +9,7 @@ from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from app.database import Base from app.database import Base
from app.models import FileRecord, FileProcessingStep from app.models import FileProcessingStep, FileRecord
from app.utils.file_queries import apply_status_filter from app.utils.file_queries import apply_status_filter
+11 -25
View File
@@ -14,7 +14,7 @@ from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from app.database import Base from app.database import Base
from app.models import FileRecord, FileProcessingStep from app.models import FileProcessingStep, FileRecord
from app.utils.step_manager import get_file_overall_status, get_step_summary, initialize_file_steps, update_step_status from app.utils.step_manager import get_file_overall_status, get_step_summary, initialize_file_steps, update_step_status
@@ -40,10 +40,7 @@ class TestFileStatusCalculation:
""" """
# Create file and initialize steps # Create file and initialize steps
file_record = FileRecord( file_record = FileRecord(
filehash="test1", filehash="test1", original_filename="test.pdf", local_filename="/tmp/test.pdf", file_size=1024
original_filename="test.pdf",
local_filename="/tmp/test.pdf",
file_size=1024
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -52,6 +49,7 @@ class TestFileStatusCalculation:
# Mark all steps as success # Mark all steps as success
from app.utils.step_manager import MAIN_PROCESSING_STEPS from app.utils.step_manager import MAIN_PROCESSING_STEPS
for step_name in MAIN_PROCESSING_STEPS: for step_name in MAIN_PROCESSING_STEPS:
update_step_status(db_session, file_record.id, step_name, "success") update_step_status(db_session, file_record.id, step_name, "success")
@@ -65,10 +63,7 @@ class TestFileStatusCalculation:
Test that status shows "processing" when there are in_progress steps. Test that status shows "processing" when there are in_progress steps.
""" """
file_record = FileRecord( file_record = FileRecord(
filehash="test2", filehash="test2", original_filename="test2.pdf", local_filename="/tmp/test2.pdf", file_size=2048
original_filename="test2.pdf",
local_filename="/tmp/test2.pdf",
file_size=2048
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -89,10 +84,7 @@ class TestFileStatusCalculation:
Test that status shows "failed" when any step has failure status. Test that status shows "failed" when any step has failure status.
""" """
file_record = FileRecord( file_record = FileRecord(
filehash="test3", filehash="test3", original_filename="test3.pdf", local_filename="/tmp/test3.pdf", file_size=3072
original_filename="test3.pdf",
local_filename="/tmp/test3.pdf",
file_size=3072
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -119,10 +111,7 @@ class TestMetricsCounting:
Test that main processing steps are counted correctly. Test that main processing steps are counted correctly.
""" """
file_record = FileRecord( file_record = FileRecord(
filehash="test4", filehash="test4", original_filename="test4.pdf", local_filename="/tmp/test4.pdf", file_size=4096
original_filename="test4.pdf",
local_filename="/tmp/test4.pdf",
file_size=4096
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -138,6 +127,7 @@ class TestMetricsCounting:
# Should count each step once # Should count each step once
from app.utils.step_manager import MAIN_PROCESSING_STEPS from app.utils.step_manager import MAIN_PROCESSING_STEPS
assert summary["total_main_steps"] == len(MAIN_PROCESSING_STEPS) assert summary["total_main_steps"] == len(MAIN_PROCESSING_STEPS)
assert summary["main"]["success"] == 2 assert summary["main"]["success"] == 2
assert summary["main"]["in_progress"] == 1 assert summary["main"]["in_progress"] == 1
@@ -147,16 +137,14 @@ class TestMetricsCounting:
Test that upload tasks are counted correctly. Test that upload tasks are counted correctly.
""" """
file_record = FileRecord( file_record = FileRecord(
filehash="test5", filehash="test5", original_filename="test5.pdf", local_filename="/tmp/test5.pdf", file_size=5120
original_filename="test5.pdf",
local_filename="/tmp/test5.pdf",
file_size=5120
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
initialize_file_steps(db_session, file_record.id) initialize_file_steps(db_session, file_record.id)
from app.utils.step_manager import add_upload_steps from app.utils.step_manager import add_upload_steps
add_upload_steps(db_session, file_record.id, ["dropbox", "s3", "nextcloud"]) add_upload_steps(db_session, file_record.id, ["dropbox", "s3", "nextcloud"])
# Mark upload steps with different statuses # Mark upload steps with different statuses
@@ -177,16 +165,14 @@ class TestMetricsCounting:
Test metrics for a file with multiple successful uploads. Test metrics for a file with multiple successful uploads.
""" """
file_record = FileRecord( file_record = FileRecord(
filehash="test6", filehash="test6", original_filename="test6.pdf", local_filename="/tmp/test6.pdf", file_size=6144
original_filename="test6.pdf",
local_filename="/tmp/test6.pdf",
file_size=6144
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
initialize_file_steps(db_session, file_record.id) initialize_file_steps(db_session, file_record.id)
from app.utils.step_manager import add_upload_steps from app.utils.step_manager import add_upload_steps
services = ["dropbox", "s3", "nextcloud", "google_drive", "onedrive", "webdav"] services = ["dropbox", "s3", "nextcloud", "google_drive", "onedrive", "webdav"]
add_upload_steps(db_session, file_record.id, services) add_upload_steps(db_session, file_record.id, services)
+3 -3
View File
@@ -4,11 +4,12 @@ Tests for app/utils/filename_utils.py
Tests filename sanitization and manipulation functions. Tests filename sanitization and manipulation functions.
""" """
import pytest
import os import os
from datetime import datetime
from pathlib import Path from pathlib import Path
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
from datetime import datetime
import pytest
@pytest.mark.unit @pytest.mark.unit
@@ -259,7 +260,6 @@ class TestFilenameUtilsEdgeCases:
assert "?" not in result assert "?" not in result
@pytest.mark.unit @pytest.mark.unit
class TestUniqueFilepathWithCounter: class TestUniqueFilepathWithCounter:
"""Test unique filepath generation with numeric counter suffix""" """Test unique filepath generation with numeric counter suffix"""
+4 -3
View File
@@ -1,8 +1,10 @@
""" """
Test the /files view UI endpoint to ensure template rendering works correctly. Test the /files view UI endpoint to ensure template rendering works correctly.
""" """
import pytest import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from app.models import FileRecord from app.models import FileRecord
@@ -26,7 +28,7 @@ class TestFilesView:
original_filename=f"test{i}.pdf", original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf", local_filename=f"/tmp/test{i}.pdf",
file_size=1024 * (i + 1), file_size=1024 * (i + 1),
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -54,7 +56,7 @@ class TestFilesView:
original_filename=f"test{i}.pdf", original_filename=f"test{i}.pdf",
local_filename=f"/tmp/test{i}.pdf", local_filename=f"/tmp/test{i}.pdf",
file_size=1024, file_size=1024,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -89,4 +91,3 @@ class TestFilesView:
# Check for drag-and-drop event handlers # Check for drag-and-drop event handlers
assert "dragenter" in content or "drag" in content, "Drag event handlers should be present" assert "dragenter" in content or "drag" in content, "Drag event handlers should be present"
assert "Drop files anywhere to upload" in content, "Drop message should be present" assert "Drop files anywhere to upload" in content, "Drop message should be present"
+4 -1
View File
@@ -1,7 +1,9 @@
"""Tests for app/tasks/finalize_document_storage.py module.""" """Tests for app/tasks/finalize_document_storage.py module."""
import os import os
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -18,4 +20,5 @@ class TestFinalizeDocumentStorageHelpers:
def test_module_imports(self): def test_module_imports(self):
"""Test that the module can be imported without errors.""" """Test that the module can be imported without errors."""
from app.tasks.finalize_document_storage import finalize_document_storage from app.tasks.finalize_document_storage import finalize_document_storage
assert callable(finalize_document_storage) assert callable(finalize_document_storage)
+8 -8
View File
@@ -1,16 +1,18 @@
"""Extended tests for app/tasks/imap_tasks.py module.""" """Extended tests for app/tasks/imap_tasks.py module."""
import os
import json import json
import pytest import os
from datetime import datetime, timezone from datetime import datetime, timezone
from unittest.mock import patch, MagicMock
from email.message import EmailMessage from email.message import EmailMessage
from unittest.mock import MagicMock, patch
import pytest
from app.tasks.imap_tasks import ( from app.tasks.imap_tasks import (
save_processed_emails,
load_processed_emails,
fetch_attachments_and_enqueue, fetch_attachments_and_enqueue,
find_all_mail_xlist, find_all_mail_xlist,
load_processed_emails,
save_processed_emails,
) )
@@ -41,9 +43,7 @@ class TestFetchAttachmentsExtended:
msg = EmailMessage() msg = EmailMessage()
msg["Subject"] = "Test" msg["Subject"] = "Test"
# Create attachment with wrong MIME type but .pdf extension # Create attachment with wrong MIME type but .pdf extension
msg.add_attachment( msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="octet-stream", filename="invoice.pdf")
b"%PDF-1.4", maintype="application", subtype="octet-stream", filename="invoice.pdf"
)
with patch("app.tasks.imap_tasks.settings") as mock_settings: with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
+10 -9
View File
@@ -1,23 +1,24 @@
"""Tests for app/tasks/imap_tasks.py module.""" """Tests for app/tasks/imap_tasks.py module."""
import os
import json import json
import pytest import os
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from unittest.mock import patch, MagicMock
from email.message import EmailMessage from email.message import EmailMessage
from unittest.mock import MagicMock, patch
import pytest
from app.tasks.imap_tasks import ( from app.tasks.imap_tasks import (
load_processed_emails,
save_processed_emails,
cleanup_old_entries,
check_and_pull_mailbox, check_and_pull_mailbox,
fetch_attachments_and_enqueue, cleanup_old_entries,
email_already_has_label, email_already_has_label,
mark_as_processed_with_star, fetch_attachments_and_enqueue,
mark_as_processed_with_label,
find_all_mail_folder, find_all_mail_folder,
get_capabilities, get_capabilities,
load_processed_emails,
mark_as_processed_with_label,
mark_as_processed_with_star,
save_processed_emails,
) )
_TEST_CREDENTIAL = "pass" # noqa: S105 _TEST_CREDENTIAL = "pass" # noqa: S105
+3 -7
View File
@@ -5,9 +5,9 @@ Tests task progress logging functionality.
""" """
import logging import logging
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from unittest.mock import Mock, patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -64,9 +64,7 @@ class TestTaskLogging:
mock_processing_log.return_value = mock_log_entry mock_processing_log.return_value = mock_log_entry
# Call without message # Call without message
log_task_progress( log_task_progress(task_id="task-456", step_name="upload", status="completed", message=None, file_id=None)
task_id="task-456", step_name="upload", status="completed", message=None, file_id=None
)
# Verify called with None for optional parameters # Verify called with None for optional parameters
mock_processing_log.assert_called_once_with( mock_processing_log.assert_called_once_with(
@@ -89,9 +87,7 @@ class TestTaskLogging:
mock_processing_log.return_value = mock_log_entry mock_processing_log.return_value = mock_log_entry
# Call without file_id # Call without file_id
log_task_progress( log_task_progress(task_id="task-789", step_name="metadata", status="running", message="Extracting metadata")
task_id="task-789", step_name="metadata", status="running", message="Extracting metadata"
)
# file_id should default to None # file_id should default to None
mock_processing_log.assert_called_once() mock_processing_log.assert_called_once()
+6 -5
View File
@@ -4,8 +4,9 @@ Tests for app/utils/notification.py
Tests notification utilities and URL masking. Tests notification utilities and URL masking.
""" """
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from unittest.mock import Mock, patch, MagicMock
_TEST_CREDENTIAL_URL = "https://user:password@example.com/notify" # noqa: S105 _TEST_CREDENTIAL_URL = "https://user:password@example.com/notify" # noqa: S105
_TEST_QUERY_URL = "https://example.com/api?key=secret1&password=secret2&public=visible" # noqa: S105 _TEST_QUERY_URL = "https://example.com/api?key=secret1&password=secret2&public=visible" # noqa: S105
@@ -133,8 +134,8 @@ class TestAppriseInitialization:
@patch("app.utils.notification.settings") @patch("app.utils.notification.settings")
def test_init_apprise_with_configured_urls(self, mock_settings, mock_apprise_class): def test_init_apprise_with_configured_urls(self, mock_settings, mock_apprise_class):
"""Test Apprise initialization with configured URLs""" """Test Apprise initialization with configured URLs"""
from app.utils.notification import init_apprise
import app.utils.notification import app.utils.notification
from app.utils.notification import init_apprise
# Reset global # Reset global
app.utils.notification._apprise = None app.utils.notification._apprise = None
@@ -162,8 +163,8 @@ class TestAppriseInitialization:
@patch("app.utils.notification.settings") @patch("app.utils.notification.settings")
def test_init_apprise_no_urls_configured(self, mock_settings, mock_apprise_class): def test_init_apprise_no_urls_configured(self, mock_settings, mock_apprise_class):
"""Test Apprise initialization without configured URLs""" """Test Apprise initialization without configured URLs"""
from app.utils.notification import init_apprise
import app.utils.notification import app.utils.notification
from app.utils.notification import init_apprise
# Reset global # Reset global
app.utils.notification._apprise = None app.utils.notification._apprise = None
@@ -188,8 +189,8 @@ class TestAppriseInitialization:
@patch("app.utils.notification.settings") @patch("app.utils.notification.settings")
def test_init_apprise_caches_instance(self, mock_settings, mock_apprise_class): def test_init_apprise_caches_instance(self, mock_settings, mock_apprise_class):
"""Test that Apprise instance is cached""" """Test that Apprise instance is cached"""
from app.utils.notification import init_apprise
import app.utils.notification import app.utils.notification
from app.utils.notification import init_apprise
# Reset global # Reset global
app.utils.notification._apprise = None app.utils.notification._apprise = None
@@ -214,8 +215,8 @@ class TestAppriseInitialization:
@patch("app.utils.notification.settings") @patch("app.utils.notification.settings")
def test_init_apprise_handles_add_failure(self, mock_settings, mock_apprise_class): def test_init_apprise_handles_add_failure(self, mock_settings, mock_apprise_class):
"""Test handling when adding notification URL fails""" """Test handling when adding notification URL fails"""
from app.utils.notification import init_apprise
import app.utils.notification import app.utils.notification
from app.utils.notification import init_apprise
# Reset global # Reset global
app.utils.notification._apprise = None app.utils.notification._apprise = None
+7 -5
View File
@@ -1,16 +1,18 @@
"""Tests for app/utils/notification.py module.""" """Tests for app/utils/notification.py module."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from app.utils.notification import ( from app.utils.notification import (
_mask_sensitive_url, _mask_sensitive_url,
send_notification, init_apprise,
notify_celery_failure, notify_celery_failure,
notify_credential_failure, notify_credential_failure,
notify_startup,
notify_shutdown,
notify_file_processed, notify_file_processed,
init_apprise, notify_shutdown,
notify_startup,
send_notification,
) )
+5 -10
View File
@@ -4,8 +4,9 @@ Tests for app/utils/oauth_helper.py
Tests OAuth token exchange helper functions. Tests OAuth token exchange helper functions.
""" """
import pytest
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
import pytest
import requests import requests
from fastapi import HTTPException from fastapi import HTTPException
@@ -51,9 +52,7 @@ class TestOAuthTokenExchange:
assert result["expires_in"] == 3600 assert result["expires_in"] == 3600
# Verify request was made correctly # Verify request was made correctly
mock_post.assert_called_once_with( mock_post.assert_called_once_with("https://oauth.example.com/token", data=payload, timeout=30)
"https://oauth.example.com/token", data=payload, timeout=30
)
@patch("app.utils.oauth_helper.requests.post") @patch("app.utils.oauth_helper.requests.post")
@patch("app.utils.oauth_helper.settings") @patch("app.utils.oauth_helper.settings")
@@ -81,9 +80,7 @@ class TestOAuthTokenExchange:
) )
# Verify custom timeout was used # Verify custom timeout was used
mock_post.assert_called_once_with( mock_post.assert_called_once_with("https://oauth.example.com/token", data=payload, timeout=60)
"https://oauth.example.com/token", data=payload, timeout=60
)
@patch("app.utils.oauth_helper.requests.post") @patch("app.utils.oauth_helper.requests.post")
@patch("app.utils.oauth_helper.settings") @patch("app.utils.oauth_helper.settings")
@@ -200,9 +197,7 @@ class TestOAuthTokenExchange:
# Mock error response with invalid JSON # Mock error response with invalid JSON
mock_response = Mock() mock_response = Mock()
mock_response.status_code = 400 mock_response.status_code = 400
mock_response.json.side_effect = requests.exceptions.JSONDecodeError( mock_response.json.side_effect = requests.exceptions.JSONDecodeError("Invalid JSON", "", 0)
"Invalid JSON", "", 0
)
mock_post.return_value = mock_response mock_post.return_value = mock_response
payload = {"grant_type": "authorization_code"} payload = {"grant_type": "authorization_code"}
+91 -127
View File
@@ -6,18 +6,19 @@ These tests verify OCR processing logic with mocked external AI/ML services
""" """
import os import os
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock, Mock
from azure.ai.documentintelligence.models import AnalyzeResult from azure.ai.documentintelligence.models import AnalyzeResult
from app.tasks.process_with_azure_document_intelligence import ( from app.tasks.process_with_azure_document_intelligence import (
process_with_azure_document_intelligence,
get_pdf_page_count,
check_page_rotation,
AZURE_DOC_INTELLIGENCE_LIMITS, AZURE_DOC_INTELLIGENCE_LIMITS,
check_page_rotation,
get_pdf_page_count,
process_with_azure_document_intelligence,
) )
from app.tasks.refine_text_with_gpt import refine_text_with_gpt from app.tasks.refine_text_with_gpt import refine_text_with_gpt
from app.tasks.rotate_pdf_pages import rotate_pdf_pages, determine_rotation_angle from app.tasks.rotate_pdf_pages import determine_rotation_angle, rotate_pdf_pages
@pytest.mark.unit @pytest.mark.unit
@@ -86,22 +87,20 @@ startxref
mock_client.begin_analyze_document.return_value = mock_poller mock_client.begin_analyze_document.return_value = mock_poller
mock_client.get_analyze_result_pdf.return_value = iter([b"searchable pdf content"]) mock_client.get_analyze_result_pdf.return_value = iter([b"searchable pdf content"])
with patch( with (
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client", patch(
mock_client, "app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
), patch( mock_client,
"app.tasks.process_with_azure_document_intelligence.settings" ),
) as mock_settings, patch( patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
"app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages" patch("app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages") as mock_rotate,
) as mock_rotate: ):
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_rotate.delay = MagicMock() mock_rotate.delay = MagicMock()
# Run the task # Run the task
result = process_with_azure_document_intelligence.run( result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
filename=test_pdf.name, file_id=1
)
# Verify results # Verify results
assert result["file"] == test_pdf.name assert result["file"] == test_pdf.name
@@ -124,15 +123,11 @@ startxref
@patch("app.tasks.process_with_azure_document_intelligence.log_task_progress") @patch("app.tasks.process_with_azure_document_intelligence.log_task_progress")
def test_file_not_found_error(self, mock_log, tmp_path): def test_file_not_found_error(self, mock_log, tmp_path):
"""Test that FileNotFoundError is raised when file doesn't exist.""" """Test that FileNotFoundError is raised when file doesn't exist."""
with patch( with patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings:
"app.tasks.process_with_azure_document_intelligence.settings"
) as mock_settings:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
with pytest.raises(FileNotFoundError) as exc_info: with pytest.raises(FileNotFoundError) as exc_info:
process_with_azure_document_intelligence.run( process_with_azure_document_intelligence.run(filename="nonexistent.pdf", file_id=1)
filename="nonexistent.pdf", file_id=1
)
assert "Local file not found" in str(exc_info.value) assert "Local file not found" in str(exc_info.value)
@@ -145,21 +140,16 @@ startxref
test_pdf = tmp_dir / "large.pdf" test_pdf = tmp_dir / "large.pdf"
test_pdf.write_bytes(b"dummy content") test_pdf.write_bytes(b"dummy content")
with patch( with (
"app.tasks.process_with_azure_document_intelligence.settings" patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.process_with_azure_document_intelligence.os.path.getsize") as mock_getsize,
"app.tasks.process_with_azure_document_intelligence.os.path.getsize" ):
) as mock_getsize:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
# Mock file size to be larger than 500 MB # Mock file size to be larger than 500 MB
mock_getsize.return_value = AZURE_DOC_INTELLIGENCE_LIMITS[ mock_getsize.return_value = AZURE_DOC_INTELLIGENCE_LIMITS["max_file_size_bytes"] + 1024
"max_file_size_bytes"
] + 1024
result = process_with_azure_document_intelligence.run( result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
filename=test_pdf.name, file_id=1
)
# Verify error response # Verify error response
assert "error" in result assert "error" in result
@@ -175,19 +165,16 @@ startxref
test_pdf = tmp_dir / "many_pages.pdf" test_pdf = tmp_dir / "many_pages.pdf"
test_pdf.write_bytes(b"%PDF-1.4\n%%EOF") # Minimal valid PDF test_pdf.write_bytes(b"%PDF-1.4\n%%EOF") # Minimal valid PDF
with patch( with (
"app.tasks.process_with_azure_document_intelligence.settings" patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.process_with_azure_document_intelligence.get_pdf_page_count") as mock_page_count,
"app.tasks.process_with_azure_document_intelligence.get_pdf_page_count" ):
) as mock_page_count:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
# Mock page count to exceed limit # Mock page count to exceed limit
mock_page_count.return_value = AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"] + 1 mock_page_count.return_value = AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"] + 1
result = process_with_azure_document_intelligence.run( result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
filename=test_pdf.name, file_id=1
)
# Verify error response # Verify error response
assert "error" in result assert "error" in result
@@ -219,15 +206,14 @@ startxref
mock_client.begin_analyze_document.return_value = mock_poller mock_client.begin_analyze_document.return_value = mock_poller
mock_client.get_analyze_result_pdf.return_value = iter([b"content"]) mock_client.get_analyze_result_pdf.return_value = iter([b"content"])
with patch( with (
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client", patch(
mock_client, "app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
), patch( mock_client,
"app.tasks.process_with_azure_document_intelligence.settings" ),
) as mock_settings, patch( patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
"app.tasks.process_with_azure_document_intelligence.get_pdf_page_count" patch("app.tasks.process_with_azure_document_intelligence.get_pdf_page_count") as mock_page_count,
) as mock_page_count, patch( patch("app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages"),
"app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages"
): ):
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
@@ -235,9 +221,7 @@ startxref
mock_page_count.return_value = None mock_page_count.return_value = None
# Should not raise an error # Should not raise an error
result = process_with_azure_document_intelligence.run( result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
filename=test_pdf.name, file_id=1
)
# Verify processing continued # Verify processing continued
assert "error" not in result assert "error" not in result
@@ -275,21 +259,19 @@ startxref
mock_client.begin_analyze_document.return_value = mock_poller mock_client.begin_analyze_document.return_value = mock_poller
mock_client.get_analyze_result_pdf.return_value = iter([b"content"]) mock_client.get_analyze_result_pdf.return_value = iter([b"content"])
with patch( with (
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client", patch(
mock_client, "app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
), patch( mock_client,
"app.tasks.process_with_azure_document_intelligence.settings" ),
) as mock_settings, patch( patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
"app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages" patch("app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages") as mock_rotate,
) as mock_rotate: ):
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_rotate.delay = MagicMock() mock_rotate.delay = MagicMock()
result = process_with_azure_document_intelligence.run( result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=2)
filename=test_pdf.name, file_id=2
)
# Verify rotation data was passed correctly # Verify rotation data was passed correctly
mock_rotate.delay.assert_called_once() mock_rotate.delay.assert_called_once()
@@ -311,24 +293,21 @@ startxref
test_pdf.write_bytes(b"%PDF-1.4\n%%EOF") test_pdf.write_bytes(b"%PDF-1.4\n%%EOF")
mock_client = Mock() mock_client = Mock()
mock_client.begin_analyze_document.side_effect = Exception( mock_client.begin_analyze_document.side_effect = Exception("Azure API connection failed")
"Azure API connection failed"
)
with patch( with (
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client", patch(
mock_client, "app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
), patch( mock_client,
"app.tasks.process_with_azure_document_intelligence.settings" ),
) as mock_settings: patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
):
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
# Should raise the exception # Should raise the exception
with pytest.raises(Exception) as exc_info: with pytest.raises(Exception) as exc_info:
process_with_azure_document_intelligence.run( process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
filename=test_pdf.name, file_id=1
)
assert "Azure API connection failed" in str(exc_info.value) assert "Azure API connection failed" in str(exc_info.value)
@@ -456,13 +435,11 @@ class TestRefineTextWithGPT:
# Import the module to patch the correct function # Import the module to patch the correct function
from app.tasks import extract_metadata_with_gpt as metadata_module from app.tasks import extract_metadata_with_gpt as metadata_module
with patch( with (
"app.tasks.refine_text_with_gpt.client", mock_client patch("app.tasks.refine_text_with_gpt.client", mock_client),
), patch.object( patch.object(metadata_module, "extract_metadata_with_gpt") as mock_extract,
metadata_module, "extract_metadata_with_gpt" patch("app.tasks.refine_text_with_gpt.settings") as mock_settings,
) as mock_extract, patch( ):
"app.tasks.refine_text_with_gpt.settings"
) as mock_settings:
mock_settings.openai_model = "gpt-4" mock_settings.openai_model = "gpt-4"
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
@@ -481,9 +458,7 @@ class TestRefineTextWithGPT:
assert call_kwargs["messages"][1]["content"] == raw_text assert call_kwargs["messages"][1]["content"] == raw_text
# Verify metadata extraction was queued # Verify metadata extraction was queued
mock_extract.delay.assert_called_once_with( mock_extract.delay.assert_called_once_with(filename, "This is some text with OCR errors")
filename, "This is some text with OCR errors"
)
@patch("app.tasks.refine_text_with_gpt.log_task_progress") @patch("app.tasks.refine_text_with_gpt.log_task_progress")
def test_openai_api_error(self, mock_log): def test_openai_api_error(self, mock_log):
@@ -492,15 +467,12 @@ class TestRefineTextWithGPT:
filename = "test.pdf" filename = "test.pdf"
mock_client = Mock() mock_client = Mock()
mock_client.chat.completions.create.side_effect = Exception( mock_client.chat.completions.create.side_effect = Exception("OpenAI API error")
"OpenAI API error"
)
with patch( with (
"app.tasks.refine_text_with_gpt.client", mock_client patch("app.tasks.refine_text_with_gpt.client", mock_client),
), patch( patch("app.tasks.refine_text_with_gpt.settings") as mock_settings,
"app.tasks.refine_text_with_gpt.settings" ):
) as mock_settings:
mock_settings.openai_model = "gpt-4" mock_settings.openai_model = "gpt-4"
@@ -606,11 +578,10 @@ startxref
rotation_data = {0: 90} # Rotate first page by 90 degrees rotation_data = {0: 90} # Rotate first page by 90 degrees
extracted_text = "Test text" extracted_text = "Test text"
with patch( with (
"app.tasks.rotate_pdf_pages.settings" patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt" ):
) as mock_extract:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
@@ -629,9 +600,7 @@ startxref
assert "0" in result["applied_rotations"] assert "0" in result["applied_rotations"]
# Verify metadata extraction was queued # Verify metadata extraction was queued
mock_extract.delay.assert_called_once_with( mock_extract.delay.assert_called_once_with(test_pdf.name, extracted_text, 1)
test_pdf.name, extracted_text, 1
)
@patch("app.tasks.rotate_pdf_pages.log_task_progress") @patch("app.tasks.rotate_pdf_pages.log_task_progress")
def test_rotate_pdf_pages_no_rotation_needed(self, mock_log, tmp_path): def test_rotate_pdf_pages_no_rotation_needed(self, mock_log, tmp_path):
@@ -644,11 +613,10 @@ startxref
extracted_text = "Test text" extracted_text = "Test text"
with patch( with (
"app.tasks.rotate_pdf_pages.settings" patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt" ):
) as mock_extract:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
@@ -679,11 +647,10 @@ startxref
rotation_data = {0: 0, 1: 0} # All pages have 0 rotation rotation_data = {0: 0, 1: 0} # All pages have 0 rotation
extracted_text = "Test text" extracted_text = "Test text"
with patch( with (
"app.tasks.rotate_pdf_pages.settings" patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt" ):
) as mock_extract:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
@@ -705,11 +672,10 @@ startxref
tmp_dir = tmp_path / "tmp" tmp_dir = tmp_path / "tmp"
tmp_dir.mkdir() tmp_dir.mkdir()
with patch( with (
"app.tasks.rotate_pdf_pages.settings" patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt" ):
) as mock_extract:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
@@ -742,11 +708,10 @@ startxref
rotation_data = {0: 90} rotation_data = {0: 90}
extracted_text = "Test text" extracted_text = "Test text"
with patch( with (
"app.tasks.rotate_pdf_pages.settings" patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt" ):
) as mock_extract:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
@@ -814,11 +779,10 @@ startxref
rotation_data = {"0": "90"} rotation_data = {"0": "90"}
extracted_text = "Test text" extracted_text = "Test text"
with patch( with (
"app.tasks.rotate_pdf_pages.settings" patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
) as mock_settings, patch( patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt" ):
) as mock_extract:
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
mock_extract.delay = MagicMock() mock_extract.delay = MagicMock()
+12 -10
View File
@@ -87,11 +87,12 @@ startxref
original_filename = "Apostille Sverige.pdf" original_filename = "Apostille Sverige.pdf"
# Mock environment and dependencies # Mock environment and dependencies
with patch("app.tasks.process_document.SessionLocal") as mock_session_local, patch( with (
"app.tasks.process_document.settings" patch("app.tasks.process_document.SessionLocal") as mock_session_local,
) as mock_settings, patch("app.tasks.process_document.log_task_progress"), patch( patch("app.tasks.process_document.settings") as mock_settings,
"app.tasks.process_document.extract_metadata_with_gpt" patch("app.tasks.process_document.log_task_progress"),
) as mock_extract: patch("app.tasks.process_document.extract_metadata_with_gpt") as mock_extract,
):
# Setup mocks # Setup mocks
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
@@ -187,11 +188,12 @@ startxref
test_pdf.write_bytes(pdf_content) test_pdf.write_bytes(pdf_content)
# Mock environment and dependencies # Mock environment and dependencies
with patch("app.tasks.process_document.SessionLocal") as mock_session_local, patch( with (
"app.tasks.process_document.settings" patch("app.tasks.process_document.SessionLocal") as mock_session_local,
) as mock_settings, patch("app.tasks.process_document.log_task_progress"), patch( patch("app.tasks.process_document.settings") as mock_settings,
"app.tasks.process_document.extract_metadata_with_gpt" patch("app.tasks.process_document.log_task_progress"),
) as mock_extract: patch("app.tasks.process_document.extract_metadata_with_gpt") as mock_extract,
):
# Setup mocks # Setup mocks
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
+8 -5
View File
@@ -4,11 +4,11 @@ Security tests for path traversal vulnerabilities.
Tests all file path operations to ensure they properly prevent path traversal attacks. Tests all file path operations to ensure they properly prevent path traversal attacks.
""" """
import os
import json import json
import os
import tempfile import tempfile
from pathlib import Path from pathlib import Path
from unittest.mock import Mock, patch, MagicMock from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
@@ -389,9 +389,10 @@ class TestFileUploadSecurity:
def test_sanitize_after_basename(self): def test_sanitize_after_basename(self):
"""Test that sanitization happens after basename extraction.""" """Test that sanitization happens after basename extraction."""
from app.utils.filename_utils import sanitize_filename
import os import os
from app.utils.filename_utils import sanitize_filename
malicious = "../../../passwd.pdf" malicious = "../../../passwd.pdf"
# Step 1: Extract basename (as ui_upload does) # Step 1: Extract basename (as ui_upload does)
@@ -442,10 +443,11 @@ class TestEndToEndPathTraversal:
def test_full_upload_flow_prevents_traversal(self, tmp_path): def test_full_upload_flow_prevents_traversal(self, tmp_path):
"""Test complete upload flow prevents path traversal.""" """Test complete upload flow prevents path traversal."""
from app.utils.filename_utils import sanitize_filename
import os import os
import uuid import uuid
from app.utils.filename_utils import sanitize_filename
# Simulate ui_upload flow # Simulate ui_upload flow
malicious_upload_filename = "../../../etc/passwd" malicious_upload_filename = "../../../etc/passwd"
@@ -472,9 +474,10 @@ class TestEndToEndPathTraversal:
def test_metadata_embedding_flow_prevents_traversal(self, tmp_path): def test_metadata_embedding_flow_prevents_traversal(self, tmp_path):
"""Test metadata embedding flow prevents path traversal.""" """Test metadata embedding flow prevents path traversal."""
from app.utils.filename_utils import sanitize_filename
import os import os
from app.utils.filename_utils import sanitize_filename
# Simulate GPT returning malicious filename # Simulate GPT returning malicious filename
gpt_metadata = { gpt_metadata = {
"filename": "../../../etc/shadow", "filename": "../../../etc/shadow",
+4 -2
View File
@@ -101,9 +101,10 @@ def test_rate_limit_exceeded_returns_429(client):
@pytest.mark.security @pytest.mark.security
def test_rate_limiting_uses_correct_identifier(): def test_rate_limiting_uses_correct_identifier():
"""Test that rate limiting uses IP or user ID as identifier.""" """Test that rate limiting uses IP or user ID as identifier."""
from app.middleware.rate_limit import get_identifier
from fastapi import Request from fastapi import Request
from app.middleware.rate_limit import get_identifier
# Create a mock request # Create a mock request
class MockRequest: class MockRequest:
def __init__(self): def __init__(self):
@@ -157,9 +158,10 @@ def test_limiter_disabled():
@pytest.mark.integration @pytest.mark.integration
def test_rate_limit_exception_handler_registered(): def test_rate_limit_exception_handler_registered():
"""Test that rate limit exception handler is registered.""" """Test that rate limit exception handler is registered."""
from app.main import app
from slowapi.errors import RateLimitExceeded from slowapi.errors import RateLimitExceeded
from app.main import app
# Verify exception handler is registered # Verify exception handler is registered
assert RateLimitExceeded in app.exception_handlers assert RateLimitExceeded in app.exception_handlers
+3 -1
View File
@@ -1,7 +1,9 @@
"""Tests for app/tasks/upload_with_rclone.py module.""" """Tests for app/tasks/upload_with_rclone.py module."""
import os import os
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from app.tasks.upload_with_rclone import upload_with_rclone from app.tasks.upload_with_rclone import upload_with_rclone
+19 -17
View File
@@ -6,19 +6,19 @@ import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.config import Settings
from app.models import ApplicationSettings from app.models import ApplicationSettings
from app.utils.config_loader import convert_setting_value, load_settings_from_db
from app.utils.settings_service import ( from app.utils.settings_service import (
get_setting_from_db, SETTING_METADATA,
save_setting_to_db,
get_all_settings_from_db,
delete_setting_from_db, delete_setting_from_db,
validate_setting_value, get_all_settings_from_db,
get_setting_from_db,
get_setting_metadata, get_setting_metadata,
get_settings_by_category, get_settings_by_category,
SETTING_METADATA, save_setting_to_db,
validate_setting_value,
) )
from app.utils.config_loader import convert_setting_value, load_settings_from_db
from app.config import Settings
@pytest.mark.unit @pytest.mark.unit
@@ -140,9 +140,14 @@ class TestSettingsService:
# Check critical settings are present # Check critical settings are present
critical_settings = [ critical_settings = [
"database_url", "redis_url", "workdir", "debug", "database_url",
"openai_api_key", "azure_ai_key", "redis_url",
"auth_enabled", "session_secret" "workdir",
"debug",
"openai_api_key",
"azure_ai_key",
"auth_enabled",
"session_secret",
] ]
for setting in critical_settings: for setting in critical_settings:
assert setting in SETTING_METADATA, f"Missing metadata for {setting}" assert setting in SETTING_METADATA, f"Missing metadata for {setting}"
@@ -219,9 +224,10 @@ class TestSettingsPrecedence:
def test_db_overrides_default(self, db_session: Session): def test_db_overrides_default(self, db_session: Session):
"""Test that database settings override default values""" """Test that database settings override default values"""
# Create a minimal test settings object # Create a minimal test settings object
from pydantic_settings import BaseSettings
from typing import Optional from typing import Optional
from pydantic_settings import BaseSettings
class TestSettings(BaseSettings): class TestSettings(BaseSettings):
test_value: str = "default" test_value: str = "default"
test_bool: bool = False test_bool: bool = False
@@ -270,10 +276,7 @@ class TestApplicationSettingsModel:
def test_create_setting_record(self, db_session: Session): def test_create_setting_record(self, db_session: Session):
"""Test creating an ApplicationSettings record""" """Test creating an ApplicationSettings record"""
setting = ApplicationSettings( setting = ApplicationSettings(key="test_key", value="test_value")
key="test_key",
value="test_value"
)
db_session.add(setting) db_session.add(setting)
db_session.commit() db_session.commit()
@@ -301,7 +304,7 @@ class TestApplicationSettingsModel:
@pytest.mark.skipif( @pytest.mark.skipif(
True, # Skip for all databases - timestamp update behavior varies True, # Skip for all databases - timestamp update behavior varies
reason="Timestamp update behavior varies by database backend" reason="Timestamp update behavior varies by database backend",
) )
def test_update_timestamp(self, db_session: Session): def test_update_timestamp(self, db_session: Session):
"""Test that updated_at timestamp is updated on modification""" """Test that updated_at timestamp is updated on modification"""
@@ -325,4 +328,3 @@ class TestApplicationSettingsModel:
# Note: SQLite doesn't automatically update onupdate timestamps # Note: SQLite doesn't automatically update onupdate timestamps
# This test is skipped as behavior varies by database backend # This test is skipped as behavior varies by database backend
assert setting.updated_at is not None assert setting.updated_at is not None
+3 -1
View File
@@ -1,7 +1,9 @@
"""Tests for app/utils/config_validator/settings_display.py module.""" """Tests for app/utils/config_validator/settings_display.py module."""
import pytest
from unittest.mock import patch from unittest.mock import patch
import pytest
from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display
+5 -3
View File
@@ -1,12 +1,14 @@
"""Tests for app/utils/setup_wizard.py module.""" """Tests for app/utils/setup_wizard.py module."""
import pytest
from unittest.mock import patch from unittest.mock import patch
import pytest
from app.utils.setup_wizard import ( from app.utils.setup_wizard import (
get_required_settings,
is_setup_required,
get_missing_required_settings, get_missing_required_settings,
get_required_settings,
get_wizard_steps, get_wizard_steps,
is_setup_required,
) )
+9 -2
View File
@@ -172,7 +172,12 @@ class TestStepManager:
# Update an existing step # Update an existing step
now = datetime.now() now = datetime.now()
update_step_status( update_step_status(
db_session, file_record.id, "create_file_record", "success", started_at=now - timedelta(seconds=5), completed_at=now db_session,
file_record.id,
"create_file_record",
"success",
started_at=now - timedelta(seconds=5),
completed_at=now,
) )
# Verify step was updated # Verify step was updated
@@ -201,7 +206,9 @@ class TestStepManager:
now = datetime.now() now = datetime.now()
update_step_status(db_session, file_record.id, "create_file_record", "success", completed_at=now) update_step_status(db_session, file_record.id, "create_file_record", "success", completed_at=now)
update_step_status(db_session, file_record.id, "check_text", "in_progress", started_at=now) update_step_status(db_session, file_record.id, "check_text", "in_progress", started_at=now)
update_step_status(db_session, file_record.id, "extract_text", "failure", error_message="Failed to extract text") update_step_status(
db_session, file_record.id, "extract_text", "failure", error_message="Failed to extract text"
)
# Get all step statuses # Get all step statuses
status_map = get_file_step_status(db_session, file_record.id) status_map = get_file_step_status(db_session, file_record.id)
+5 -12
View File
@@ -5,8 +5,8 @@ Tests the new functionality for storing immutable originals and processed copies
collision handling, and forced Cloud OCR reprocessing. collision handling, and forced Cloud OCR reprocessing.
""" """
import os
import json import json
import os
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
import pytest import pytest
@@ -195,7 +195,7 @@ startxref
local_filename=str(test_pdf), local_filename=str(test_pdf),
original_file_path=str(original_file), original_file_path=str(original_file),
file_size=100, file_size=100,
mime_type="application/pdf" mime_type="application/pdf",
) )
db_session.add(file_record) db_session.add(file_record)
db_session.commit() db_session.commit()
@@ -250,11 +250,7 @@ class TestMetadataAugmentation:
"""Test that persisted metadata includes original and processed paths""" """Test that persisted metadata includes original and processed paths"""
from app.tasks.embed_metadata_into_pdf import persist_metadata from app.tasks.embed_metadata_into_pdf import persist_metadata
metadata = { metadata = {"filename": "2024-01-01_Invoice", "document_type": "Invoice", "tags": ["finance", "2024"]}
"filename": "2024-01-01_Invoice",
"document_type": "Invoice",
"tags": ["finance", "2024"]
}
processed_file = tmp_path / "processed" / "2024-01-01_Invoice.pdf" processed_file = tmp_path / "processed" / "2024-01-01_Invoice.pdf"
processed_file.parent.mkdir(parents=True) processed_file.parent.mkdir(parents=True)
@@ -264,17 +260,14 @@ class TestMetadataAugmentation:
processed_path = str(processed_file) processed_path = str(processed_file)
json_path = persist_metadata( json_path = persist_metadata(
metadata, metadata, str(processed_file), original_file_path=original_path, processed_file_path=processed_path
str(processed_file),
original_file_path=original_path,
processed_file_path=processed_path
) )
# Verify JSON was created # Verify JSON was created
assert os.path.exists(json_path) assert os.path.exists(json_path)
# Verify content # Verify content
with open(json_path, 'r') as f: with open(json_path, "r") as f:
saved_metadata = json.load(f) saved_metadata = json.load(f)
assert "original_file_path" in saved_metadata assert "original_file_path" in saved_metadata
+4 -1
View File
@@ -1,6 +1,8 @@
"""Tests for app/tasks/upload_to_email.py module.""" """Tests for app/tasks/upload_to_email.py module."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -10,4 +12,5 @@ class TestUploadToEmail:
def test_module_imports(self): def test_module_imports(self):
"""Test that the module can be imported.""" """Test that the module can be imported."""
from app.tasks.upload_to_email import upload_to_email from app.tasks.upload_to_email import upload_to_email
assert callable(upload_to_email) assert callable(upload_to_email)
+4 -1
View File
@@ -1,6 +1,8 @@
"""Additional tests for upload_to_ftp task.""" """Additional tests for upload_to_ftp task."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -10,4 +12,5 @@ class TestUploadToFtp:
def test_module_imports(self): def test_module_imports(self):
"""Test that the module can be imported.""" """Test that the module can be imported."""
from app.tasks.upload_to_ftp import upload_to_ftp from app.tasks.upload_to_ftp import upload_to_ftp
assert callable(upload_to_ftp) assert callable(upload_to_ftp)
+6 -4
View File
@@ -3,15 +3,17 @@ Tests for upload tasks including OneDrive, S3, FTP, SFTP, WebDAV, Google Drive,
""" """
import os import os
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from unittest.mock import Mock, patch, MagicMock
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_onedrive import upload_to_onedrive from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_s3 import upload_to_s3 from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_webdav import upload_to_webdav from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_email import upload_to_email
_TEST_CREDENTIAL = "test_pass" # noqa: S105 _TEST_CREDENTIAL = "test_pass" # noqa: S105
+4 -1
View File
@@ -1,7 +1,9 @@
"""Additional tests for upload task modules.""" """Additional tests for upload task modules."""
import os import os
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.unit @pytest.mark.unit
@@ -23,6 +25,7 @@ class TestUploadToNextcloud:
mock_settings.workdir = "/tmp" mock_settings.workdir = "/tmp"
from app.tasks.upload_to_nextcloud import upload_to_nextcloud from app.tasks.upload_to_nextcloud import upload_to_nextcloud
result = upload_to_nextcloud.__wrapped__(mock_self, "/nonexistent/file.pdf") result = upload_to_nextcloud.__wrapped__(mock_self, "/nonexistent/file.pdf")
+7 -5
View File
@@ -1,15 +1,17 @@
"""Tests to increase coverage for upload task modules.""" """Tests to increase coverage for upload task modules."""
import os
import pytest
from unittest.mock import patch, MagicMock
from app.tasks.upload_to_paperless import upload_to_paperless import os
from unittest.mock import MagicMock, patch
import pytest
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_nextcloud import upload_to_nextcloud from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_onedrive import upload_to_onedrive from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_s3 import upload_to_s3 from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_sftp import upload_to_sftp from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_webdav import upload_to_webdav from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_ftp import upload_to_ftp
@pytest.mark.unit @pytest.mark.unit
+3 -2
View File
@@ -1,9 +1,10 @@
"""Comprehensive tests for upload_to_webdav task.""" """Comprehensive tests for upload_to_webdav task."""
import os import os
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from unittest.mock import patch, Mock, MagicMock from requests.exceptions import ConnectionError, RequestException, Timeout
from requests.exceptions import ConnectionError, Timeout, RequestException
from app.tasks.upload_to_webdav import upload_to_webdav from app.tasks.upload_to_webdav import upload_to_webdav
+3 -2
View File
@@ -7,11 +7,12 @@ actual file uploads against it, then verify the files were uploaded successfully
import os import os
import time import time
import pytest
import requests
from pathlib import Path from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
import pytest
import requests
from app.tasks.upload_to_webdav import upload_to_webdav from app.tasks.upload_to_webdav import upload_to_webdav
# Import testcontainers - required for these tests # Import testcontainers - required for these tests
+4 -7
View File
@@ -4,8 +4,9 @@ Tests for app/tasks/uptime_kuma_tasks.py
Tests Uptime Kuma health check ping functionality. Tests Uptime Kuma health check ping functionality.
""" """
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from unittest.mock import Mock, patch, MagicMock
import requests import requests
@@ -43,9 +44,7 @@ class TestUptimeKumaTasks:
# Should return True on success # Should return True on success
assert result is True assert result is True
mock_get.assert_called_once_with( mock_get.assert_called_once_with("https://uptime.example.com/ping/123", timeout=10)
"https://uptime.example.com/ping/123", timeout=10
)
mock_response.raise_for_status.assert_called_once() mock_response.raise_for_status.assert_called_once()
@patch("app.tasks.uptime_kuma_tasks.requests.get") @patch("app.tasks.uptime_kuma_tasks.requests.get")
@@ -93,9 +92,7 @@ class TestUptimeKumaTasks:
# Mock HTTP error # Mock HTTP error
mock_response = Mock() mock_response = Mock()
mock_response.status_code = 500 mock_response.status_code = 500
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("500 Server Error")
"500 Server Error"
)
mock_get.return_value = mock_response mock_get.return_value = mock_response
result = ping_uptime_kuma() result = ping_uptime_kuma()
+9 -16
View File
@@ -29,9 +29,10 @@ class TestURLUploadValidation:
def test_validate_url_scheme_ftp_rejected(self): def test_validate_url_scheme_ftp_rejected(self):
"""Test that FTP URLs are rejected""" """Test that FTP URLs are rejected"""
from app.api.url_upload import URLUploadRequest
from pydantic import ValidationError from pydantic import ValidationError
from app.api.url_upload import URLUploadRequest
with pytest.raises(ValidationError) as exc_info: with pytest.raises(ValidationError) as exc_info:
URLUploadRequest(url="ftp://example.com/file.pdf") URLUploadRequest(url="ftp://example.com/file.pdf")
# Pydantic HttpUrl validates scheme automatically # Pydantic HttpUrl validates scheme automatically
@@ -39,9 +40,10 @@ class TestURLUploadValidation:
def test_validate_url_scheme_file_rejected(self): def test_validate_url_scheme_file_rejected(self):
"""Test that file:// URLs are rejected""" """Test that file:// URLs are rejected"""
from app.api.url_upload import URLUploadRequest
from pydantic import ValidationError from pydantic import ValidationError
from app.api.url_upload import URLUploadRequest
with pytest.raises(ValidationError) as exc_info: with pytest.raises(ValidationError) as exc_info:
URLUploadRequest(url="file:///etc/passwd") URLUploadRequest(url="file:///etc/passwd")
# Pydantic HttpUrl validates scheme automatically # Pydantic HttpUrl validates scheme automatically
@@ -126,17 +128,14 @@ class TestURLUploadValidation:
# Word # Word
assert validate_file_type("application/msword", "file.doc") is True assert validate_file_type("application/msword", "file.doc") is True
assert ( assert (
validate_file_type( validate_file_type("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "file.docx")
"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "file.docx"
)
is True is True
) )
# Excel # Excel
assert validate_file_type("application/vnd.ms-excel", "file.xls") is True assert validate_file_type("application/vnd.ms-excel", "file.xls") is True
assert ( assert (
validate_file_type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "file.xlsx") validate_file_type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "file.xlsx") is True
is True
) )
def test_validate_file_type_images_allowed(self): def test_validate_file_type_images_allowed(self):
@@ -248,9 +247,7 @@ class TestURLUploadEndpoint:
@patch("app.api.url_upload.requests.get") @patch("app.api.url_upload.requests.get")
def test_process_url_blocks_metadata_endpoint(self, mock_requests_get, client): def test_process_url_blocks_metadata_endpoint(self, mock_requests_get, client):
"""Test that cloud metadata endpoints are blocked""" """Test that cloud metadata endpoints are blocked"""
response = client.post( response = client.post("/api/process-url", json={"url": "http://169.254.169.254/latest/meta-data/"})
"/api/process-url", json={"url": "http://169.254.169.254/latest/meta-data/"}
)
assert response.status_code == 400 assert response.status_code == 400
data = response.json() data = response.json()
@@ -341,9 +338,7 @@ class TestURLUploadEndpoint:
@patch("app.api.url_upload.requests.get") @patch("app.api.url_upload.requests.get")
@patch("app.api.url_upload.process_document") @patch("app.api.url_upload.process_document")
def test_process_url_with_custom_filename( def test_process_url_with_custom_filename(self, mock_process_document, mock_requests_get, client, tmp_path):
self, mock_process_document, mock_requests_get, client, tmp_path
):
"""Test URL upload with custom filename""" """Test URL upload with custom filename"""
# Mock successful download # Mock successful download
mock_response = Mock() mock_response = Mock()
@@ -369,9 +364,7 @@ class TestURLUploadEndpoint:
@patch("app.api.url_upload.requests.get") @patch("app.api.url_upload.requests.get")
@patch("app.api.url_upload.process_document") @patch("app.api.url_upload.process_document")
def test_process_url_extracts_filename_from_url( def test_process_url_extracts_filename_from_url(self, mock_process_document, mock_requests_get, client, tmp_path):
self, mock_process_document, mock_requests_get, client, tmp_path
):
"""Test that filename is extracted from URL when not provided""" """Test that filename is extracted from URL when not provided"""
# Mock successful download # Mock successful download
mock_response = Mock() mock_response = Mock()
+2 -1
View File
@@ -1,7 +1,8 @@
"""Additional view tests to increase coverage.""" """Additional view tests to increase coverage."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
_TEST_CREDENTIAL = "test" # noqa: S105 _TEST_CREDENTIAL = "test" # noqa: S105
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/views/dropbox.py module.""" """Tests for app/views/dropbox.py module."""
import pytest import pytest
+4 -2
View File
@@ -1,7 +1,9 @@
"""Tests for app/views/general.py module.""" """Tests for app/views/general.py module."""
import pytest
from unittest.mock import patch, MagicMock
from pathlib import Path from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
@pytest.mark.integration @pytest.mark.integration
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/views/google_drive.py module.""" """Tests for app/views/google_drive.py module."""
import pytest import pytest
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/views/license_routes.py module.""" """Tests for app/views/license_routes.py module."""
import pytest import pytest
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/views/onedrive.py module.""" """Tests for app/views/onedrive.py module."""
import pytest import pytest
+6 -1
View File
@@ -1,6 +1,8 @@
"""Tests for app/views/settings.py module.""" """Tests for app/views/settings.py module."""
from unittest.mock import MagicMock, patch
import pytest import pytest
from unittest.mock import patch, MagicMock
from app.views.settings import require_admin_access from app.views.settings import require_admin_access
@@ -12,6 +14,7 @@ class TestRequireAdminAccess:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_redirects_non_admin_user(self): async def test_redirects_non_admin_user(self):
"""Test that non-admin users are redirected.""" """Test that non-admin users are redirected."""
@require_admin_access @require_admin_access
async def dummy_route(request): async def dummy_route(request):
return {"success": True} return {"success": True}
@@ -25,6 +28,7 @@ class TestRequireAdminAccess:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_redirects_when_no_user(self): async def test_redirects_when_no_user(self):
"""Test that unauthenticated users are redirected.""" """Test that unauthenticated users are redirected."""
@require_admin_access @require_admin_access
async def dummy_route(request): async def dummy_route(request):
return {"success": True} return {"success": True}
@@ -38,6 +42,7 @@ class TestRequireAdminAccess:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_allows_admin_user(self): async def test_allows_admin_user(self):
"""Test that admin users can access the route.""" """Test that admin users can access the route."""
@require_admin_access @require_admin_access
async def dummy_route(request): async def dummy_route(request):
return {"success": True} return {"success": True}
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/views/status.py module.""" """Tests for app/views/status.py module."""
import pytest import pytest
+1
View File
@@ -1,4 +1,5 @@
"""Tests for app/views/wizard.py module.""" """Tests for app/views/wizard.py module."""
import pytest import pytest