Merge pull request #260 from christianlouis/copilot/format-code-with-linters
style: fix black, isort, and flake8 violations across app/ and tests/
This commit is contained in:
+2
-6
@@ -425,16 +425,12 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
|
||||
logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}")
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Neither original nor local file found on disk. Cannot reprocess."
|
||||
status_code=400, detail="Neither original nor local file found on disk. Cannot reprocess."
|
||||
)
|
||||
|
||||
# Queue the file for processing with force_cloud_ocr=True
|
||||
task = process_document.delay(
|
||||
source_file,
|
||||
original_filename=file_record.original_filename,
|
||||
file_id=file_record.id,
|
||||
force_cloud_ocr=True
|
||||
source_file, original_filename=file_record.original_filename, file_id=file_record.id, force_cloud_ocr=True
|
||||
)
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -277,7 +277,7 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
return {
|
||||
"task_id": task.id,
|
||||
"status": "queued",
|
||||
"message": f"File downloaded from URL and queued for processing",
|
||||
"message": "File downloaded from URL and queued for processing",
|
||||
"filename": safe_filename,
|
||||
"size": downloaded_size,
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
|
||||
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
|
||||
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
|
||||
|
||||
# **Ensure all tasks are imported before Celery starts**
|
||||
from app.tasks.process_document import process_document # noqa: F401
|
||||
@@ -33,7 +34,6 @@ from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
|
||||
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
|
||||
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
|
||||
|
||||
celery.conf.task_routes = {
|
||||
"app.tasks.*": {"queue": "default"},
|
||||
|
||||
+22
-6
@@ -33,7 +33,8 @@ class Settings(BaseSettings):
|
||||
paperless_host: Optional[str] = None
|
||||
paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
|
||||
# JSON mapping of metadata field names to Paperless custom field names
|
||||
# Example: {"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
|
||||
# Example: {"absender": "Sender", "empfaenger": "Recipient",
|
||||
# "language": "Language", "correspondent": "Correspondent"}
|
||||
paperless_custom_fields_mapping: Optional[str] = None
|
||||
|
||||
azure_ai_key: str
|
||||
@@ -176,23 +177,35 @@ class Settings(BaseSettings):
|
||||
)
|
||||
max_single_file_size: Optional[int] = Field(
|
||||
default=None,
|
||||
description="Maximum size for a single file chunk in bytes. If set and file exceeds this, it will be split into smaller chunks for processing. Default: None (no splitting).",
|
||||
description=(
|
||||
"Maximum size for a single file chunk in bytes. If set and file exceeds this,"
|
||||
" it will be split into smaller chunks for processing. Default: None (no splitting)."
|
||||
),
|
||||
)
|
||||
|
||||
# Deduplication settings - prevents processing of duplicate files
|
||||
enable_deduplication: bool = Field(
|
||||
default=True,
|
||||
description="Enable deduplication check before processing. If enabled, files with the same SHA-256 hash as previously processed files will not be processed again. Default: True (enabled).",
|
||||
description=(
|
||||
"Enable deduplication check before processing. If enabled, files with the same SHA-256 hash"
|
||||
" as previously processed files will not be processed again. Default: True (enabled)."
|
||||
),
|
||||
)
|
||||
show_deduplication_step: bool = Field(
|
||||
default=True,
|
||||
description="Show the 'Check for Duplicates' step in processing history. If False, the check is still performed but not displayed. Default: True.",
|
||||
description=(
|
||||
"Show the 'Check for Duplicates' step in processing history."
|
||||
" If False, the check is still performed but not displayed. Default: True."
|
||||
),
|
||||
)
|
||||
|
||||
# Processing step timeout - prevents files from getting stuck in "in_progress" state
|
||||
step_timeout: int = Field(
|
||||
default=600,
|
||||
description="Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this, it will be marked as failed. Default: 600 seconds (10 minutes).",
|
||||
description=(
|
||||
"Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this,"
|
||||
" it will be marked as failed. Default: 600 seconds (10 minutes)."
|
||||
),
|
||||
)
|
||||
|
||||
# Security Headers Configuration (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
|
||||
@@ -215,7 +228,10 @@ class Settings(BaseSettings):
|
||||
# Content-Security-Policy (CSP) - Controls resource loading
|
||||
security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.")
|
||||
security_header_csp_value: str = Field(
|
||||
default="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;",
|
||||
default=(
|
||||
"default-src 'self'; script-src 'self' 'unsafe-inline';"
|
||||
" style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
),
|
||||
description="CSP header value. Customize based on your application's resource loading needs.",
|
||||
)
|
||||
|
||||
|
||||
+1
-1
@@ -8,6 +8,7 @@ from fastapi import FastAPI, HTTPException, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from starlette.config import Config
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
||||
@@ -21,7 +22,6 @@ from app.middleware.rate_limit import create_limiter, get_rate_limit_exceeded_ha
|
||||
from app.middleware.security_headers import SecurityHeadersMiddleware
|
||||
from app.utils.config_validator import check_all_configs
|
||||
from app.utils.notification import init_apprise, notify_shutdown, notify_startup
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
|
||||
# Import the routers - now using views directly instead of frontend
|
||||
from app.views import router as frontend_router
|
||||
|
||||
@@ -21,7 +21,6 @@ from typing import Callable
|
||||
|
||||
from fastapi import Request
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -7,10 +7,6 @@ This module provides convenient decorators to apply rate limits to specific endp
|
||||
Import the limiter from main.py state and use these decorators to protect endpoints.
|
||||
"""
|
||||
|
||||
from functools import wraps
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
# Import will happen at runtime to avoid circular dependencies
|
||||
_limiter = None
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# app/models.py
|
||||
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func, Boolean
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
@@ -102,7 +102,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
else:
|
||||
logger.error(f"[{task_id}] Local file {local_file_path} not found, cannot embed metadata.")
|
||||
log_task_progress(
|
||||
task_id, "embed_metadata_into_pdf", "failure", "File not found", file_id=file_id,
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"failure",
|
||||
"File not found",
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Local file not found, cannot embed metadata.\n"
|
||||
f"Tried path: {local_file_path}\n"
|
||||
@@ -199,10 +203,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
logger.info(f"[{task_id}] Persisting metadata to JSON")
|
||||
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
|
||||
json_path = persist_metadata(
|
||||
metadata,
|
||||
final_file_path,
|
||||
original_file_path=original_file_path,
|
||||
processed_file_path=final_file_path
|
||||
metadata, final_file_path, original_file_path=original_file_path, processed_file_path=final_file_path
|
||||
)
|
||||
logger.info(f"[{task_id}] Metadata persisted to {json_path}")
|
||||
log_task_progress(
|
||||
@@ -212,7 +213,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
# Trigger the next step: final storage.
|
||||
logger.info(f"[{task_id}] Queueing final storage task")
|
||||
log_task_progress(
|
||||
task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id,
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"success",
|
||||
"Metadata embedded, queuing finalization",
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Metadata embedded into PDF successfully.\n"
|
||||
f"Original file: {original_file}\n"
|
||||
@@ -245,8 +250,15 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] Failed to embed metadata into {processed_file}: {e}")
|
||||
log_task_progress(
|
||||
task_id, "embed_metadata_into_pdf", "failure", f"Exception: {str(e)}", file_id=file_id,
|
||||
detail=f"Failed to embed metadata into {processed_file}.\nOriginal file: {original_file}\nException: {str(e)}",
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Failed to embed metadata into {processed_file}.\n"
|
||||
f"Original file: {original_file}\nException: {str(e)}"
|
||||
),
|
||||
)
|
||||
# Clean up temporary file in case of error
|
||||
if os.path.exists(processed_file):
|
||||
|
||||
@@ -112,7 +112,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
|
||||
content = completion.choices[0].message.content
|
||||
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
|
||||
log_task_progress(
|
||||
task_id, "call_openai", "success", "Received OpenAI response", file_id=file_id,
|
||||
task_id,
|
||||
"call_openai",
|
||||
"success",
|
||||
"Received OpenAI response",
|
||||
file_id=file_id,
|
||||
detail=f"Raw classification response:\n{content}",
|
||||
)
|
||||
|
||||
@@ -120,7 +124,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
|
||||
if not json_text:
|
||||
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
|
||||
log_task_progress(
|
||||
task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id,
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
"Invalid JSON in response",
|
||||
file_id=file_id,
|
||||
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
|
||||
)
|
||||
return {}
|
||||
@@ -138,16 +146,18 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
|
||||
# 1. Potential locale-specific \w behavior
|
||||
# 2. Files literally named ".." which are valid but problematic
|
||||
# 3. Future code changes that might relax the regex
|
||||
if not re.match(r'^[\w\-\. ]+$', suggested_filename) or ".." in suggested_filename:
|
||||
logger.warning(
|
||||
f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback"
|
||||
)
|
||||
if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
|
||||
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
|
||||
# Reset to empty to trigger fallback to original filename
|
||||
metadata["filename"] = ""
|
||||
|
||||
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
|
||||
log_task_progress(
|
||||
task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id,
|
||||
task_id,
|
||||
"parse_metadata",
|
||||
"success",
|
||||
f"Parsed metadata: {list(metadata.keys())}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
|
||||
)
|
||||
|
||||
@@ -164,7 +174,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] OpenAI classification failed for {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id, "extract_metadata_with_gpt", "failure", f"Exception: {str(e)}", file_id=file_id,
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=f"OpenAI classification failed for {filename}.\nException: {str(e)}",
|
||||
)
|
||||
return {}
|
||||
|
||||
@@ -42,10 +42,7 @@ def monitor_stalled_steps():
|
||||
f"Marked as failed due to timeout."
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
f"[{datetime.utcnow().isoformat()}] "
|
||||
f"No stalled steps found."
|
||||
)
|
||||
logger.debug(f"[{datetime.utcnow().isoformat()}] " f"No stalled steps found.")
|
||||
|
||||
return {"recovered": stalled_count}
|
||||
|
||||
|
||||
@@ -24,7 +24,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def process_document(self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False):
|
||||
def process_document(
|
||||
self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False
|
||||
):
|
||||
"""
|
||||
Process a document file and trigger appropriate text extraction.
|
||||
|
||||
@@ -58,7 +60,10 @@ def process_document(self, original_local_file: str, original_filename: str = No
|
||||
if not os.path.exists(original_local_file):
|
||||
logger.error(f"[{task_id}] File {original_local_file} not found.")
|
||||
log_task_progress(
|
||||
task_id, "process_document", "failure", "File not found",
|
||||
task_id,
|
||||
"process_document",
|
||||
"failure",
|
||||
"File not found",
|
||||
detail=f"File not found on disk: {original_local_file}",
|
||||
)
|
||||
return {"error": "File not found"}
|
||||
@@ -113,13 +118,15 @@ def process_document(self, original_local_file: str, original_filename: str = No
|
||||
else:
|
||||
# Check for duplicate only if this is a new file (not reprocessing)
|
||||
# IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file
|
||||
existing = db.query(FileRecord).filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)).order_by(FileRecord.created_at.asc()).first()
|
||||
existing = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.created_at.asc())
|
||||
.first()
|
||||
)
|
||||
if existing is None:
|
||||
existing = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash)
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
db.query(FileRecord).filter(FileRecord.filehash == filehash).order_by(FileRecord.id.asc()).first()
|
||||
)
|
||||
|
||||
# A file is only a duplicate if it matches a different file's hash
|
||||
|
||||
@@ -101,8 +101,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
"""
|
||||
task_id = self.request.id
|
||||
log_task_progress(
|
||||
task_id, "process_with_azure_document_intelligence", "in_progress",
|
||||
f"Starting OCR for {filename}", file_id=file_id,
|
||||
task_id,
|
||||
"process_with_azure_document_intelligence",
|
||||
"in_progress",
|
||||
f"Starting OCR for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
try:
|
||||
tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
|
||||
@@ -117,8 +120,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
)
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(
|
||||
task_id, "validate_file", "failure",
|
||||
f"File too large: {filename}", file_id=file_id, detail=error_msg,
|
||||
task_id,
|
||||
"validate_file",
|
||||
"failure",
|
||||
f"File too large: {filename}",
|
||||
file_id=file_id,
|
||||
detail=error_msg,
|
||||
)
|
||||
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
|
||||
|
||||
@@ -130,8 +137,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(
|
||||
task_id, "validate_file", "failure",
|
||||
f"Too many pages: {filename}", file_id=file_id, detail=error_msg,
|
||||
task_id,
|
||||
"validate_file",
|
||||
"failure",
|
||||
f"Too many pages: {filename}",
|
||||
file_id=file_id,
|
||||
detail=error_msg,
|
||||
)
|
||||
return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
|
||||
if page_count is None:
|
||||
@@ -140,14 +151,20 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
)
|
||||
|
||||
log_task_progress(
|
||||
task_id, "validate_file", "success",
|
||||
f"File validation passed for {filename}", file_id=file_id,
|
||||
task_id,
|
||||
"validate_file",
|
||||
"success",
|
||||
f"File validation passed for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.")
|
||||
log_task_progress(
|
||||
task_id, "call_azure_ocr", "in_progress",
|
||||
f"Sending {filename} to Azure Document Intelligence", file_id=file_id,
|
||||
task_id,
|
||||
"call_azure_ocr",
|
||||
"in_progress",
|
||||
f"Sending {filename} to Azure Document Intelligence",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Open and send the document for processing
|
||||
@@ -173,8 +190,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters")
|
||||
|
||||
log_task_progress(
|
||||
task_id, "call_azure_ocr", "success",
|
||||
f"Azure OCR completed for {filename}", file_id=file_id,
|
||||
task_id,
|
||||
"call_azure_ocr",
|
||||
"success",
|
||||
f"Azure OCR completed for {filename}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected",
|
||||
)
|
||||
|
||||
@@ -182,8 +202,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id)
|
||||
|
||||
log_task_progress(
|
||||
task_id, "process_with_azure_document_intelligence", "success",
|
||||
f"OCR processing complete for {filename}", file_id=file_id,
|
||||
task_id,
|
||||
"process_with_azure_document_intelligence",
|
||||
"success",
|
||||
f"OCR processing complete for {filename}",
|
||||
file_id=file_id,
|
||||
detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted",
|
||||
)
|
||||
|
||||
@@ -191,7 +214,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}")
|
||||
log_task_progress(
|
||||
task_id, "process_with_azure_document_intelligence", "failure",
|
||||
f"OCR failed for {filename}", file_id=file_id, detail=str(e),
|
||||
task_id,
|
||||
"process_with_azure_document_intelligence",
|
||||
"failure",
|
||||
f"OCR failed for {filename}",
|
||||
file_id=file_id,
|
||||
detail=str(e),
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -63,8 +63,11 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
|
||||
try:
|
||||
task_id = self.request.id
|
||||
log_task_progress(
|
||||
task_id, "rotate_pdf_pages", "in_progress",
|
||||
f"Checking page rotation for {filename}", file_id=file_id,
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"in_progress",
|
||||
f"Checking page rotation for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
pdf_path = os.path.join(settings.workdir, "tmp", filename)
|
||||
if not os.path.exists(pdf_path):
|
||||
@@ -72,12 +75,13 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
|
||||
|
||||
# Skip rotation if no rotation data provided
|
||||
if not rotation_data:
|
||||
logger.info(
|
||||
f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction"
|
||||
)
|
||||
logger.info(f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction")
|
||||
log_task_progress(
|
||||
task_id, "rotate_pdf_pages", "success",
|
||||
"No rotation needed, proceeding to metadata extraction", file_id=file_id,
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"success",
|
||||
"No rotation needed, proceeding to metadata extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
return {"file": filename, "status": "no_rotation_needed"}
|
||||
@@ -95,16 +99,22 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
|
||||
f"[{task_id}] No significant rotations detected in {filename}, proceeding with metadata extraction"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id, "rotate_pdf_pages", "success",
|
||||
"No rotation needed, proceeding to metadata extraction", file_id=file_id,
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"success",
|
||||
"No rotation needed, proceeding to metadata extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
return {"file": filename, "status": "no_rotation_needed"}
|
||||
|
||||
logger.info(f"[{task_id}] Rotating {len(normalized_rotation_data)} pages in {filename}")
|
||||
log_task_progress(
|
||||
task_id, "apply_rotation", "in_progress",
|
||||
f"Rotating {len(normalized_rotation_data)} pages", file_id=file_id,
|
||||
task_id,
|
||||
"apply_rotation",
|
||||
"in_progress",
|
||||
f"Rotating {len(normalized_rotation_data)} pages",
|
||||
file_id=file_id,
|
||||
)
|
||||
applied_rotations = {}
|
||||
|
||||
@@ -144,8 +154,7 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
|
||||
|
||||
if applied_rotations:
|
||||
logger.info(
|
||||
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: "
|
||||
f"{json.dumps(applied_rotations)}"
|
||||
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: " f"{json.dumps(applied_rotations)}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
@@ -157,7 +166,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
|
||||
log_task_progress(
|
||||
task_id, "rotate_pdf_pages", "success",
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"success",
|
||||
f"Rotation complete for {filename}",
|
||||
file_id=file_id,
|
||||
detail={"applied_rotations": applied_rotations},
|
||||
@@ -173,7 +184,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Error rotating PDF {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id, "rotate_pdf_pages", "failure",
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"failure",
|
||||
f"Rotation failed: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail={"error": str(e), "filename": filename},
|
||||
|
||||
@@ -274,8 +274,15 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
|
||||
response_text,
|
||||
)
|
||||
log_task_progress(
|
||||
task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id,
|
||||
detail=f"Failed to upload document to Paperless.\nFile: {file_path}\nError: {exc}\nResponse: {response_text}",
|
||||
task_id,
|
||||
"upload_to_paperless",
|
||||
"failure",
|
||||
error_msg,
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Failed to upload document to Paperless.\n"
|
||||
f"File: {file_path}\nError: {exc}\nResponse: {response_text}"
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -322,9 +329,9 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
|
||||
# Map each metadata field to its corresponding Paperless custom field
|
||||
for metadata_field, paperless_field in field_mapping.items():
|
||||
if metadata_field in metadata and metadata[metadata_field]:
|
||||
# Convert to string to ensure consistent comparison with UNKNOWN_VALUE
|
||||
# Convert to string to ensure consistent comparison
|
||||
value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else ""
|
||||
if value and value != UNKNOWN_VALUE:
|
||||
if value and value != METADATA_UNKNOWN_PLACEHOLDER:
|
||||
custom_fields_to_set[paperless_field] = value
|
||||
logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'")
|
||||
except json.JSONDecodeError as e:
|
||||
|
||||
@@ -103,9 +103,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
|
||||
except (OSError, ValueError) as e:
|
||||
error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(
|
||||
task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg
|
||||
)
|
||||
log_task_progress(task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Optional
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Query, Session
|
||||
|
||||
from app.models import FileRecord, FileProcessingStep
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
|
||||
def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Query:
|
||||
@@ -81,10 +81,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
|
||||
REAL_STEPS.add("check_for_duplicates")
|
||||
|
||||
# Filter to only real steps
|
||||
real_steps_subq = (
|
||||
db.query(FileProcessingStep)
|
||||
.filter(FileProcessingStep.step_name.in_(REAL_STEPS))
|
||||
)
|
||||
real_steps_subq = db.query(FileProcessingStep).filter(FileProcessingStep.step_name.in_(REAL_STEPS))
|
||||
|
||||
if status == "pending":
|
||||
# Files with no real steps (never started processing)
|
||||
@@ -108,8 +105,9 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
|
||||
|
||||
# Get files with failures or in_progress on real steps
|
||||
files_with_issues = (
|
||||
real_steps_subq
|
||||
.filter(or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress"))
|
||||
real_steps_subq.filter(
|
||||
or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress")
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Dict, List
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import FileProcessingStep, FileRecord, ProcessingLog
|
||||
from app.utils.step_manager import get_file_overall_status, get_step_summary
|
||||
from app.utils.step_manager import get_file_overall_status
|
||||
|
||||
|
||||
def get_file_processing_status(db: Session, file_id: int) -> Dict:
|
||||
|
||||
@@ -6,14 +6,11 @@ for files that were processed before the status tracking table was created.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import FileProcessingStep, FileRecord, ProcessingLog
|
||||
from app.utils.step_manager import MAIN_PROCESSING_STEPS
|
||||
from app.models import FileProcessingStep, ProcessingLog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -220,10 +220,9 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
|
||||
|
||||
# Filter to only real steps
|
||||
steps = [
|
||||
s for s in all_steps
|
||||
if s.step_name in REAL_MAIN_STEPS
|
||||
or s.step_name.startswith("queue_")
|
||||
or s.step_name.startswith("upload_to_")
|
||||
s
|
||||
for s in all_steps
|
||||
if s.step_name in REAL_MAIN_STEPS or s.step_name.startswith("queue_") or s.step_name.startswith("upload_to_")
|
||||
]
|
||||
|
||||
if not steps:
|
||||
@@ -341,4 +340,3 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
|
||||
"total_main_steps": main_steps_count,
|
||||
"total_upload_tasks": upload_steps_count,
|
||||
}
|
||||
|
||||
|
||||
@@ -32,9 +32,7 @@ def get_step_timeout() -> int:
|
||||
|
||||
|
||||
def mark_stalled_steps_as_failed(
|
||||
db: Session,
|
||||
timeout_seconds: Optional[int] = None,
|
||||
file_id: Optional[int] = None
|
||||
db: Session, timeout_seconds: Optional[int] = None, file_id: Optional[int] = None
|
||||
) -> int:
|
||||
"""
|
||||
Find and mark any in-progress steps that have exceeded the timeout as failed.
|
||||
@@ -75,8 +73,7 @@ def mark_stalled_steps_as_failed(
|
||||
return 0
|
||||
|
||||
logger.warning(
|
||||
f"Found {len(stalled_steps)} stalled step(s) that exceeded "
|
||||
f"{timeout_seconds}s timeout. Marking as failed."
|
||||
f"Found {len(stalled_steps)} stalled step(s) that exceeded " f"{timeout_seconds}s timeout. Marking as failed."
|
||||
)
|
||||
|
||||
count = 0
|
||||
|
||||
+15
-14
@@ -2,13 +2,11 @@
|
||||
File management views for displaying and managing files.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, Query, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.utils.file_queries import apply_status_filter
|
||||
from app.utils.file_status import get_files_processing_status
|
||||
from app.views.base import APIRouter, get_db, logger, require_login, templates
|
||||
@@ -34,9 +32,9 @@ def files_page(
|
||||
"""
|
||||
try:
|
||||
# Import the model here to avoid circular imports
|
||||
from sqlalchemy import asc, desc, or_
|
||||
from sqlalchemy import asc, desc
|
||||
|
||||
from app.models import FileRecord, ProcessingLog
|
||||
from app.models import FileRecord
|
||||
|
||||
# Start with base query
|
||||
query = db.query(FileRecord)
|
||||
@@ -236,6 +234,7 @@ def _compute_processing_flow(logs):
|
||||
|
||||
# Filter out deduplication step if not enabled or if not showing it
|
||||
from app.config import settings
|
||||
|
||||
if not settings.enable_deduplication or not settings.show_deduplication_step:
|
||||
stages.pop("check_for_duplicates", None)
|
||||
# Update the next pointer for create_file_record
|
||||
@@ -359,16 +358,18 @@ def _compute_step_summary(logs):
|
||||
main_steps = []
|
||||
if settings.enable_deduplication and settings.show_deduplication_step:
|
||||
main_steps.append("check_for_duplicates")
|
||||
main_steps.extend([
|
||||
"create_file_record",
|
||||
"check_text",
|
||||
"extract_text",
|
||||
"process_with_azure_document_intelligence",
|
||||
"extract_metadata_with_gpt",
|
||||
"embed_metadata_into_pdf",
|
||||
"finalize_document_storage",
|
||||
"send_to_all_destinations",
|
||||
])
|
||||
main_steps.extend(
|
||||
[
|
||||
"create_file_record",
|
||||
"check_text",
|
||||
"extract_text",
|
||||
"process_with_azure_document_intelligence",
|
||||
"extract_metadata_with_gpt",
|
||||
"embed_metadata_into_pdf",
|
||||
"finalize_document_storage",
|
||||
"send_to_all_destinations",
|
||||
]
|
||||
)
|
||||
|
||||
upload_prefixes = ["upload_to_", "queue_"]
|
||||
|
||||
|
||||
@@ -14,16 +14,17 @@ These tests exercise the full application stack end-to-end.
|
||||
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
from typing import Generator
|
||||
from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
# Import testcontainers
|
||||
pytest.importorskip("testcontainers", reason="testcontainers not installed")
|
||||
from testcontainers.core.container import DockerContainer
|
||||
from testcontainers.minio import MinioContainer
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
from testcontainers.redis import RedisContainer
|
||||
from testcontainers.minio import MinioContainer
|
||||
|
||||
_TEST_CREDENTIAL = "testpass" # noqa: S105
|
||||
|
||||
@@ -300,6 +301,7 @@ def db_session_real(postgres_container):
|
||||
"""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.database import Base
|
||||
|
||||
# Create engine using PostgreSQL container
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Tests for app/api/common.py module."""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api.common import resolve_file_path
|
||||
|
||||
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
Tests for API error handling - ensuring JSON responses for API routes.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.models import FileRecord
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/api/logs.py module."""
|
||||
import pytest
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import FileRecord, ProcessingLog
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/api/openai.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/api/process.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -44,6 +45,7 @@ class TestProcessEndpoints:
|
||||
def test_processall_endpoint(self, client, tmp_path):
|
||||
"""Test POST /api/processall with no PDF files in workdir."""
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch("app.api.process.settings") as mock_settings:
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
response = client.post("/api/processall")
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Tests for app/api/settings.py module."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.api.settings import require_admin
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Tests for app/api/user.py module."""
|
||||
import pytest
|
||||
|
||||
from hashlib import md5
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api.user import whoami_handler
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Integration tests for auth.py with AUTH_ENABLED=True scenarios."""
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock, AsyncMock
|
||||
import hashlib
|
||||
|
||||
from app.auth import get_gravatar_url, get_current_user, require_login
|
||||
import hashlib
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.auth import get_current_user, get_gravatar_url, require_login
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -20,8 +22,9 @@ class TestRequireLoginWithAuth:
|
||||
return {"success": True}
|
||||
|
||||
# Manually create the decorator behavior
|
||||
from functools import wraps
|
||||
import inspect
|
||||
from functools import wraps
|
||||
|
||||
from fastapi import status
|
||||
from starlette.responses import RedirectResponse
|
||||
|
||||
@@ -45,10 +48,11 @@ class TestRequireLoginWithAuth:
|
||||
@pytest.mark.asyncio
|
||||
async def test_require_login_allows_authenticated_user(self):
|
||||
"""Test that require_login allows through when user exists in session."""
|
||||
from functools import wraps
|
||||
import inspect
|
||||
from starlette.responses import RedirectResponse
|
||||
from functools import wraps
|
||||
|
||||
from fastapi import status
|
||||
from starlette.responses import RedirectResponse
|
||||
|
||||
async def my_route(request):
|
||||
return {"success": True}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Tests for app/tasks/check_credentials.py module."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.tasks.check_credentials import (
|
||||
MockRequest,
|
||||
@@ -95,6 +97,7 @@ class TestUnwrapDecoratedFunction:
|
||||
|
||||
def test_returns_same_function_if_not_decorated(self):
|
||||
"""Test returns same function if not decorated."""
|
||||
|
||||
def my_func():
|
||||
return "hello"
|
||||
|
||||
@@ -103,6 +106,7 @@ class TestUnwrapDecoratedFunction:
|
||||
|
||||
def test_unwraps_decorated_function(self):
|
||||
"""Test unwraps decorated function."""
|
||||
|
||||
def inner():
|
||||
return "hello"
|
||||
|
||||
@@ -116,6 +120,7 @@ class TestUnwrapDecoratedFunction:
|
||||
|
||||
def test_unwraps_multiple_levels(self):
|
||||
"""Test unwraps multiple levels of decoration."""
|
||||
|
||||
def original():
|
||||
return "hello"
|
||||
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
"""Extended tests for app/tasks/check_credentials.py module."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.tasks.check_credentials import (
|
||||
sync_test_openai_connection,
|
||||
sync_test_azure_connection,
|
||||
sync_test_dropbox_token,
|
||||
sync_test_google_drive_token,
|
||||
sync_test_onedrive_token,
|
||||
sync_test_openai_connection,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+23
-20
@@ -1,9 +1,12 @@
|
||||
"""
|
||||
Unit tests for configuration and security validation.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.config import Settings
|
||||
|
||||
|
||||
@@ -24,7 +27,7 @@ class TestConfigurationValidation:
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=True,
|
||||
session_secret=None
|
||||
session_secret=None,
|
||||
)
|
||||
assert "SESSION_SECRET must be set" in str(exc_info.value)
|
||||
|
||||
@@ -41,7 +44,7 @@ class TestConfigurationValidation:
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=True,
|
||||
session_secret="short"
|
||||
session_secret="short",
|
||||
)
|
||||
assert "at least 32 characters" in str(exc_info.value)
|
||||
|
||||
@@ -57,7 +60,7 @@ class TestConfigurationValidation:
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=True,
|
||||
session_secret="a" * 32 # 32 character secret
|
||||
session_secret="a" * 32, # 32 character secret
|
||||
)
|
||||
assert config.auth_enabled is True
|
||||
assert len(config.session_secret) == 32
|
||||
@@ -74,14 +77,12 @@ class TestConfigurationValidation:
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False,
|
||||
session_secret=None
|
||||
session_secret=None,
|
||||
)
|
||||
assert config.auth_enabled is False
|
||||
assert config.session_secret is None
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestBuildMetadataConfiguration:
|
||||
"""Tests for build metadata configuration."""
|
||||
@@ -98,7 +99,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
assert config.version == "1.2.3-test"
|
||||
|
||||
@@ -114,7 +115,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
assert config.build_date == "2026-01-15"
|
||||
|
||||
@@ -130,7 +131,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
assert config.build_date == "2026-01-15T10:30:00Z"
|
||||
|
||||
@@ -146,7 +147,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
assert config.git_sha == "abc1234"
|
||||
|
||||
@@ -154,7 +155,8 @@ class TestBuildMetadataConfiguration:
|
||||
"""Test that git_sha defaults to 'unknown' when not set."""
|
||||
# Mock the file system to ensure no GIT_SHA file exists
|
||||
import app.config
|
||||
monkeypatch.setattr(app.config.os.path, 'dirname', lambda x: str(tmp_path))
|
||||
|
||||
monkeypatch.setattr(app.config.os.path, "dirname", lambda x: str(tmp_path))
|
||||
|
||||
config = Settings(
|
||||
database_url="sqlite:///test.db",
|
||||
@@ -165,7 +167,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
# When no file or env var exists, should return "unknown"
|
||||
assert config.git_sha == "unknown"
|
||||
@@ -173,7 +175,8 @@ class TestBuildMetadataConfiguration:
|
||||
def test_version_default_when_no_file_or_env(self, monkeypatch, tmp_path):
|
||||
"""Test that version defaults to 'unknown' when no VERSION file or env var exists."""
|
||||
import app.config
|
||||
monkeypatch.setattr(app.config.os.path, 'dirname', lambda x: str(tmp_path))
|
||||
|
||||
monkeypatch.setattr(app.config.os.path, "dirname", lambda x: str(tmp_path))
|
||||
|
||||
config = Settings(
|
||||
database_url="sqlite:///test.db",
|
||||
@@ -184,7 +187,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
# When no file or env var exists, should return "unknown"
|
||||
assert config.version == "unknown"
|
||||
@@ -200,7 +203,7 @@ class TestBuildMetadataConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
runtime_info = config.runtime_info
|
||||
# Should contain version, build_date, and git_sha in some form
|
||||
@@ -223,7 +226,7 @@ class TestNotificationConfiguration:
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False,
|
||||
notification_urls="discord://webhook1,telegram://webhook2"
|
||||
notification_urls="discord://webhook1,telegram://webhook2",
|
||||
)
|
||||
assert len(config.notification_urls) == 2
|
||||
assert "discord://webhook1" in config.notification_urls
|
||||
@@ -241,7 +244,7 @@ class TestNotificationConfiguration:
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False,
|
||||
notification_urls=["discord://webhook1", "telegram://webhook2"]
|
||||
notification_urls=["discord://webhook1", "telegram://webhook2"],
|
||||
)
|
||||
assert len(config.notification_urls) == 2
|
||||
|
||||
@@ -263,7 +266,7 @@ class TestSecurityConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
|
||||
# Ensure optional credentials are actually optional (None)
|
||||
@@ -283,7 +286,7 @@ class TestSecurityConfiguration:
|
||||
azure_endpoint="https://test.example.com",
|
||||
gotenberg_url="http://localhost:3000",
|
||||
workdir="/tmp",
|
||||
auth_enabled=False
|
||||
auth_enabled=False,
|
||||
)
|
||||
|
||||
# Should not raise an error
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/utils/config_loader.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.config_loader import convert_setting_value
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
"""Tests for app/utils/config_validator/validators.py module."""
|
||||
import pytest
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.config_validator.validators import (
|
||||
validate_storage_configs,
|
||||
check_all_configs,
|
||||
validate_email_config,
|
||||
validate_notification_config,
|
||||
check_all_configs,
|
||||
validate_storage_configs,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/tasks/convert_to_pdf.py module."""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -11,6 +13,7 @@ class TestConvertToPdfMimeTypes:
|
||||
def test_office_extensions_set(self):
|
||||
"""Test that the task module is importable."""
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
|
||||
assert callable(convert_to_pdf)
|
||||
|
||||
@patch("app.tasks.convert_to_pdf.requests")
|
||||
@@ -25,6 +28,7 @@ class TestConvertToPdfMimeTypes:
|
||||
mock_self.request.id = "test-task-id"
|
||||
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
|
||||
result = convert_to_pdf.__wrapped__(mock_self, "/tmp/test.docx")
|
||||
assert result is None
|
||||
|
||||
@@ -46,5 +50,6 @@ class TestConvertToPdfMimeTypes:
|
||||
mock_self.request.id = "test-task-id"
|
||||
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
|
||||
result = convert_to_pdf.__wrapped__(mock_self, str(test_file))
|
||||
assert result is None
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Tests to boost coverage for various small modules."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -10,11 +12,13 @@ class TestUtilsCompat:
|
||||
def test_imports_hash_file(self):
|
||||
"""Test that hash_file can be imported from utils."""
|
||||
from app.utils import hash_file
|
||||
|
||||
assert callable(hash_file)
|
||||
|
||||
def test_imports_log_task_progress(self):
|
||||
"""Test that log_task_progress can be imported from utils."""
|
||||
from app.utils import log_task_progress
|
||||
|
||||
assert callable(log_task_progress)
|
||||
|
||||
|
||||
@@ -25,31 +29,37 @@ class TestConfigValidatorCompat:
|
||||
def test_imports_validate_email_config(self):
|
||||
"""Test backward compatible import."""
|
||||
from app.utils.config_validator import validate_email_config
|
||||
|
||||
assert callable(validate_email_config)
|
||||
|
||||
def test_imports_validate_storage_configs(self):
|
||||
"""Test backward compatible import."""
|
||||
from app.utils.config_validator import validate_storage_configs
|
||||
|
||||
assert callable(validate_storage_configs)
|
||||
|
||||
def test_imports_mask_sensitive_value(self):
|
||||
"""Test backward compatible import."""
|
||||
from app.utils.config_validator import mask_sensitive_value
|
||||
|
||||
assert callable(mask_sensitive_value)
|
||||
|
||||
def test_imports_get_provider_status(self):
|
||||
"""Test backward compatible import."""
|
||||
from app.utils.config_validator import get_provider_status
|
||||
|
||||
assert callable(get_provider_status)
|
||||
|
||||
def test_imports_dump_all_settings(self):
|
||||
"""Test backward compatible import."""
|
||||
from app.utils.config_validator import dump_all_settings
|
||||
|
||||
assert callable(dump_all_settings)
|
||||
|
||||
def test_imports_check_all_configs(self):
|
||||
"""Test backward compatible import."""
|
||||
from app.utils.config_validator import check_all_configs
|
||||
|
||||
assert callable(check_all_configs)
|
||||
|
||||
|
||||
@@ -60,6 +70,7 @@ class TestCeleryWorkerImport:
|
||||
def test_celery_worker_module_exists(self):
|
||||
"""Test that celery_worker module can be found."""
|
||||
import importlib
|
||||
|
||||
spec = importlib.util.find_spec("app.celery_worker")
|
||||
assert spec is not None
|
||||
|
||||
@@ -71,22 +82,26 @@ class TestSettingsDisplayMasking:
|
||||
def test_dump_all_settings_masks_passwords(self):
|
||||
"""Test that passwords are masked in settings dump."""
|
||||
from app.utils.config_validator.settings_display import dump_all_settings
|
||||
|
||||
# Should not raise
|
||||
dump_all_settings()
|
||||
|
||||
def test_dump_all_settings_masks_tokens(self):
|
||||
"""Test that tokens are masked in settings dump."""
|
||||
from app.utils.config_validator.settings_display import dump_all_settings
|
||||
|
||||
dump_all_settings()
|
||||
|
||||
def test_dump_all_settings_masks_keys(self):
|
||||
"""Test that API keys are masked in settings dump."""
|
||||
from app.utils.config_validator.settings_display import dump_all_settings
|
||||
|
||||
dump_all_settings()
|
||||
|
||||
def test_get_settings_for_display_categories(self):
|
||||
"""Test that all expected categories are returned."""
|
||||
from app.utils.config_validator.settings_display import get_settings_for_display
|
||||
|
||||
result = get_settings_for_display(show_values=True)
|
||||
# Should have multiple categories
|
||||
assert len(result) > 3
|
||||
@@ -102,6 +117,7 @@ class TestNotificationInit:
|
||||
"""Test init_apprise when no URLs configured."""
|
||||
mock_settings.notification_urls = []
|
||||
from app.utils.notification import init_apprise
|
||||
|
||||
result = init_apprise()
|
||||
assert result is not None
|
||||
|
||||
@@ -111,6 +127,7 @@ class TestNotificationInit:
|
||||
"""Test init_apprise with URLs configured."""
|
||||
mock_settings.notification_urls = ["json://localhost"]
|
||||
from app.utils.notification import init_apprise
|
||||
|
||||
result = init_apprise()
|
||||
assert result is not None
|
||||
|
||||
@@ -127,6 +144,7 @@ class TestNotificationFileProcessed:
|
||||
mock_send.return_value = True
|
||||
|
||||
from app.utils.notification import notify_file_processed
|
||||
|
||||
result = notify_file_processed(
|
||||
filename="test.pdf",
|
||||
file_size=1048576,
|
||||
@@ -144,6 +162,7 @@ class TestNotificationFileProcessed:
|
||||
mock_send.return_value = True
|
||||
|
||||
from app.utils.notification import notify_file_processed
|
||||
|
||||
result = notify_file_processed(
|
||||
filename="small.pdf",
|
||||
file_size=512, # Less than 1KB
|
||||
@@ -165,6 +184,7 @@ class TestNotificationCeleryFailure:
|
||||
mock_send.return_value = True
|
||||
|
||||
from app.utils.notification import notify_celery_failure
|
||||
|
||||
result = notify_celery_failure(
|
||||
task_name="process_document",
|
||||
task_id="task-123",
|
||||
@@ -188,6 +208,7 @@ class TestNotificationCredentialFailure:
|
||||
mock_send.return_value = True
|
||||
|
||||
from app.utils.notification import notify_credential_failure
|
||||
|
||||
result = notify_credential_failure(
|
||||
service_name="OpenAI",
|
||||
error="Invalid API key",
|
||||
@@ -208,6 +229,7 @@ class TestNotificationStartupShutdown:
|
||||
mock_send.return_value = True
|
||||
|
||||
from app.utils.notification import notify_startup
|
||||
|
||||
result = notify_startup()
|
||||
assert result is True
|
||||
|
||||
@@ -220,5 +242,6 @@ class TestNotificationStartupShutdown:
|
||||
mock_send.return_value = True
|
||||
|
||||
from app.utils.notification import notify_shutdown
|
||||
|
||||
result = notify_shutdown()
|
||||
assert result is True
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Final tests to push coverage over 60%."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -10,6 +12,7 @@ class TestCheckCredentialsFunctions:
|
||||
def test_sync_test_openai_connection(self):
|
||||
"""Test sync_test_openai_connection."""
|
||||
from app.tasks.check_credentials import sync_test_openai_connection
|
||||
|
||||
result = sync_test_openai_connection()
|
||||
assert isinstance(result, dict)
|
||||
assert "status" in result
|
||||
@@ -17,6 +20,7 @@ class TestCheckCredentialsFunctions:
|
||||
def test_sync_test_azure_connection(self):
|
||||
"""Test sync_test_azure_connection."""
|
||||
from app.tasks.check_credentials import sync_test_azure_connection
|
||||
|
||||
result = sync_test_azure_connection()
|
||||
assert isinstance(result, dict)
|
||||
assert "status" in result
|
||||
@@ -24,6 +28,7 @@ class TestCheckCredentialsFunctions:
|
||||
def test_sync_test_dropbox_token(self):
|
||||
"""Test sync_test_dropbox_token."""
|
||||
from app.tasks.check_credentials import sync_test_dropbox_token
|
||||
|
||||
result = sync_test_dropbox_token()
|
||||
assert isinstance(result, dict)
|
||||
assert "status" in result
|
||||
@@ -31,6 +36,7 @@ class TestCheckCredentialsFunctions:
|
||||
def test_sync_test_google_drive_token(self):
|
||||
"""Test sync_test_google_drive_token."""
|
||||
from app.tasks.check_credentials import sync_test_google_drive_token
|
||||
|
||||
result = sync_test_google_drive_token()
|
||||
assert isinstance(result, dict)
|
||||
assert "status" in result
|
||||
@@ -38,6 +44,7 @@ class TestCheckCredentialsFunctions:
|
||||
def test_sync_test_onedrive_token(self):
|
||||
"""Test sync_test_onedrive_token."""
|
||||
from app.tasks.check_credentials import sync_test_onedrive_token
|
||||
|
||||
result = sync_test_onedrive_token()
|
||||
assert isinstance(result, dict)
|
||||
assert "status" in result
|
||||
@@ -45,29 +52,34 @@ class TestCheckCredentialsFunctions:
|
||||
def test_sync_test_nextcloud_credentials(self):
|
||||
"""Test that check_credentials module has check_credentials task."""
|
||||
from app.tasks.check_credentials import check_credentials
|
||||
|
||||
assert callable(check_credentials)
|
||||
|
||||
def test_sync_test_sftp_credentials(self):
|
||||
"""Test MockRequest scope attribute."""
|
||||
from app.tasks.check_credentials import MockRequest
|
||||
|
||||
req = MockRequest()
|
||||
assert hasattr(req, "session")
|
||||
|
||||
def test_sync_test_email_credentials(self):
|
||||
"""Test MockRequest path_params attribute."""
|
||||
from app.tasks.check_credentials import MockRequest
|
||||
|
||||
req = MockRequest()
|
||||
assert isinstance(req.query_params, dict)
|
||||
|
||||
def test_sync_test_ftp_credentials(self):
|
||||
"""Test MockRequest headers attribute."""
|
||||
from app.tasks.check_credentials import MockRequest
|
||||
|
||||
req = MockRequest()
|
||||
assert isinstance(req.headers, dict)
|
||||
|
||||
def test_sync_test_paperless_credentials(self):
|
||||
"""Test get_failure_state returns dict."""
|
||||
from app.tasks.check_credentials import get_failure_state
|
||||
|
||||
result = get_failure_state()
|
||||
assert isinstance(result, dict)
|
||||
|
||||
@@ -75,7 +87,9 @@ class TestCheckCredentialsFunctions:
|
||||
"""Test save_failure_state accepts dict."""
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.tasks.check_credentials import save_failure_state
|
||||
|
||||
with patch("app.tasks.check_credentials.FAILURE_STATE_FILE", "/tmp/test_failure_state_final.json"):
|
||||
save_failure_state({"test": "value"})
|
||||
if os.path.exists("/tmp/test_failure_state_final.json"):
|
||||
@@ -89,6 +103,7 @@ class TestImapPullInboxes:
|
||||
def test_pull_all_inboxes_is_callable(self):
|
||||
"""Test that pull_all_inboxes is callable."""
|
||||
from app.tasks.imap_tasks import pull_all_inboxes
|
||||
|
||||
assert callable(pull_all_inboxes)
|
||||
|
||||
|
||||
@@ -99,6 +114,7 @@ class TestViewsProviderStatus:
|
||||
def test_get_provider_status_returns_dict(self):
|
||||
"""Test get_provider_status."""
|
||||
from app.utils.config_validator.providers import get_provider_status
|
||||
|
||||
result = get_provider_status()
|
||||
assert isinstance(result, dict)
|
||||
assert len(result) > 0
|
||||
@@ -111,6 +127,7 @@ class TestSettingsService:
|
||||
def test_get_settings_by_category(self):
|
||||
"""Test get_settings_by_category."""
|
||||
from app.utils.settings_service import get_settings_by_category
|
||||
|
||||
result = get_settings_by_category()
|
||||
assert isinstance(result, dict)
|
||||
assert len(result) > 0
|
||||
@@ -118,12 +135,14 @@ class TestSettingsService:
|
||||
def test_get_setting_metadata(self):
|
||||
"""Test get_setting_metadata for a known key."""
|
||||
from app.utils.settings_service import get_setting_metadata
|
||||
|
||||
result = get_setting_metadata("openai_api_key")
|
||||
assert isinstance(result, dict)
|
||||
|
||||
def test_validate_setting_value(self):
|
||||
"""Test validate_setting_value."""
|
||||
from app.utils.settings_service import validate_setting_value
|
||||
|
||||
# Should return tuple of (is_valid, error_message or None)
|
||||
result = validate_setting_value("openai_api_key", "test-key")
|
||||
assert isinstance(result, tuple)
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Tests for app/database.py module."""
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.database import init_db, get_db
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.database import get_db, init_db
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/api/diagnostic.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -7,9 +7,10 @@ and test the complete application workflow from API request to file upload.
|
||||
|
||||
import os
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from unittest.mock import patch
|
||||
|
||||
# Import testcontainers requirement
|
||||
pytest.importorskip("testcontainers", reason="testcontainers not installed")
|
||||
@@ -22,16 +23,16 @@ except ModuleNotFoundError:
|
||||
_has_psycopg2 = False
|
||||
|
||||
from tests.fixtures_integration import (
|
||||
postgres_container,
|
||||
redis_container,
|
||||
gotenberg_container,
|
||||
webdav_container,
|
||||
sftp_container,
|
||||
minio_container,
|
||||
full_infrastructure,
|
||||
celery_app,
|
||||
celery_worker,
|
||||
db_session_real,
|
||||
full_infrastructure,
|
||||
gotenberg_container,
|
||||
minio_container,
|
||||
postgres_container,
|
||||
redis_container,
|
||||
sftp_container,
|
||||
webdav_container,
|
||||
)
|
||||
|
||||
_TEST_CREDENTIAL = "pass" # noqa: S105
|
||||
@@ -117,9 +118,10 @@ class TestEndToEndWithRedis:
|
||||
|
||||
This verifies the Redis broker is working correctly.
|
||||
"""
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
import redis
|
||||
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
|
||||
# Connect to Redis directly
|
||||
r = redis.from_url(redis_container["url"])
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/tasks/embed_metadata_into_pdf.py module."""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.tasks.embed_metadata_into_pdf import persist_metadata
|
||||
from app.utils.filename_utils import get_unique_filepath_with_counter
|
||||
|
||||
@@ -5,9 +5,10 @@ This test module serves as a regression prevention mechanism to ensure
|
||||
that endpoints remain accessible after code refactoring or reorganization.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Test constants
|
||||
TEST_URL = "https://example.com/test.pdf"
|
||||
|
||||
@@ -34,10 +35,7 @@ class TestEndpointRegistration:
|
||||
mock_process_document.delay.return_value = mock_task
|
||||
|
||||
# Make a request to the endpoint - it should not return 404
|
||||
response = client.post(
|
||||
"/api/process-url",
|
||||
json={"url": TEST_URL}
|
||||
)
|
||||
response = client.post("/api/process-url", json={"url": TEST_URL})
|
||||
|
||||
# The endpoint exists if we don't get a 404
|
||||
# We may get other errors (401, 400, 500, etc.) due to validation or missing mocks,
|
||||
@@ -66,10 +64,7 @@ class TestEndpointRegistration:
|
||||
mock_process_document.delay.return_value = mock_task
|
||||
|
||||
# Try POST request
|
||||
response = client.post(
|
||||
"/api/process-url",
|
||||
json={"url": TEST_URL}
|
||||
)
|
||||
response = client.post("/api/process-url", json={"url": TEST_URL})
|
||||
|
||||
# Should not return 405 (Method Not Allowed)
|
||||
assert response.status_code != 405, (
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/tasks/extract_metadata_with_gpt.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.extract_metadata_with_gpt import extract_json_from_text
|
||||
|
||||
+19
-12
@@ -1,10 +1,13 @@
|
||||
"""
|
||||
Tests for file listing, pagination, filtering, and detail endpoints.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.models import FileProcessingStep, FileRecord, ProcessingLog
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@@ -32,7 +35,7 @@ class TestFileListingPagination:
|
||||
original_filename=f"test{i}.pdf",
|
||||
local_filename=f"/tmp/test{i}.pdf",
|
||||
file_size=1024 * (i + 1),
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -53,7 +56,7 @@ class TestFileListingPagination:
|
||||
original_filename=f"test{i}.pdf",
|
||||
local_filename=f"/tmp/test{i}.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -82,7 +85,7 @@ class TestFileListingPagination:
|
||||
original_filename=name,
|
||||
local_filename=f"/tmp/{name}",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -110,7 +113,7 @@ class TestFileListingPagination:
|
||||
original_filename=f"file{i}.pdf",
|
||||
local_filename=f"/tmp/file{i}.pdf",
|
||||
file_size=size,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -131,7 +134,7 @@ class TestFileListingPagination:
|
||||
original_filename=name,
|
||||
local_filename=f"/tmp/{name}",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -160,7 +163,7 @@ class TestFileListingPagination:
|
||||
original_filename=filename,
|
||||
local_filename=f"/tmp/{filename}",
|
||||
file_size=1024,
|
||||
mime_type=mime_type
|
||||
mime_type=mime_type,
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -181,7 +184,7 @@ class TestFileListingPagination:
|
||||
original_filename="test.pdf",
|
||||
local_filename="/tmp/test.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -216,13 +219,17 @@ class TestFileDetailEndpoint:
|
||||
original_filename="test.pdf",
|
||||
local_filename="/tmp/test.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
# Add processing steps (used for status determination)
|
||||
for step_name, status in [("extract_text", "success"), ("extract_metadata_with_gpt", "in_progress"), ("embed_metadata_into_pdf", "success")]:
|
||||
for step_name, status in [
|
||||
("extract_text", "success"),
|
||||
("extract_metadata_with_gpt", "in_progress"),
|
||||
("embed_metadata_into_pdf", "success"),
|
||||
]:
|
||||
step = FileProcessingStep(
|
||||
file_id=file_record.id,
|
||||
step_name=step_name,
|
||||
@@ -237,7 +244,7 @@ class TestFileDetailEndpoint:
|
||||
task_id=f"task_{i}",
|
||||
step_name=f"step_{i}",
|
||||
status=status,
|
||||
message=f"Message {i}"
|
||||
message=f"Message {i}",
|
||||
)
|
||||
db_session.add(log)
|
||||
db_session.commit()
|
||||
@@ -272,7 +279,7 @@ class TestFileDetailEndpoint:
|
||||
original_filename="test.pdf",
|
||||
local_filename="/tmp/test.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
@@ -9,7 +9,7 @@ from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.database import Base
|
||||
from app.models import FileRecord, FileProcessingStep
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
from app.utils.file_queries import apply_status_filter
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.database import Base
|
||||
from app.models import FileRecord, FileProcessingStep
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
from app.utils.step_manager import get_file_overall_status, get_step_summary, initialize_file_steps, update_step_status
|
||||
|
||||
|
||||
@@ -40,10 +40,7 @@ class TestFileStatusCalculation:
|
||||
"""
|
||||
# Create file and initialize steps
|
||||
file_record = FileRecord(
|
||||
filehash="test1",
|
||||
original_filename="test.pdf",
|
||||
local_filename="/tmp/test.pdf",
|
||||
file_size=1024
|
||||
filehash="test1", original_filename="test.pdf", local_filename="/tmp/test.pdf", file_size=1024
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -52,6 +49,7 @@ class TestFileStatusCalculation:
|
||||
|
||||
# Mark all steps as success
|
||||
from app.utils.step_manager import MAIN_PROCESSING_STEPS
|
||||
|
||||
for step_name in MAIN_PROCESSING_STEPS:
|
||||
update_step_status(db_session, file_record.id, step_name, "success")
|
||||
|
||||
@@ -65,10 +63,7 @@ class TestFileStatusCalculation:
|
||||
Test that status shows "processing" when there are in_progress steps.
|
||||
"""
|
||||
file_record = FileRecord(
|
||||
filehash="test2",
|
||||
original_filename="test2.pdf",
|
||||
local_filename="/tmp/test2.pdf",
|
||||
file_size=2048
|
||||
filehash="test2", original_filename="test2.pdf", local_filename="/tmp/test2.pdf", file_size=2048
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -89,10 +84,7 @@ class TestFileStatusCalculation:
|
||||
Test that status shows "failed" when any step has failure status.
|
||||
"""
|
||||
file_record = FileRecord(
|
||||
filehash="test3",
|
||||
original_filename="test3.pdf",
|
||||
local_filename="/tmp/test3.pdf",
|
||||
file_size=3072
|
||||
filehash="test3", original_filename="test3.pdf", local_filename="/tmp/test3.pdf", file_size=3072
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -119,10 +111,7 @@ class TestMetricsCounting:
|
||||
Test that main processing steps are counted correctly.
|
||||
"""
|
||||
file_record = FileRecord(
|
||||
filehash="test4",
|
||||
original_filename="test4.pdf",
|
||||
local_filename="/tmp/test4.pdf",
|
||||
file_size=4096
|
||||
filehash="test4", original_filename="test4.pdf", local_filename="/tmp/test4.pdf", file_size=4096
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -138,6 +127,7 @@ class TestMetricsCounting:
|
||||
|
||||
# Should count each step once
|
||||
from app.utils.step_manager import MAIN_PROCESSING_STEPS
|
||||
|
||||
assert summary["total_main_steps"] == len(MAIN_PROCESSING_STEPS)
|
||||
assert summary["main"]["success"] == 2
|
||||
assert summary["main"]["in_progress"] == 1
|
||||
@@ -147,16 +137,14 @@ class TestMetricsCounting:
|
||||
Test that upload tasks are counted correctly.
|
||||
"""
|
||||
file_record = FileRecord(
|
||||
filehash="test5",
|
||||
original_filename="test5.pdf",
|
||||
local_filename="/tmp/test5.pdf",
|
||||
file_size=5120
|
||||
filehash="test5", original_filename="test5.pdf", local_filename="/tmp/test5.pdf", file_size=5120
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
initialize_file_steps(db_session, file_record.id)
|
||||
from app.utils.step_manager import add_upload_steps
|
||||
|
||||
add_upload_steps(db_session, file_record.id, ["dropbox", "s3", "nextcloud"])
|
||||
|
||||
# Mark upload steps with different statuses
|
||||
@@ -177,16 +165,14 @@ class TestMetricsCounting:
|
||||
Test metrics for a file with multiple successful uploads.
|
||||
"""
|
||||
file_record = FileRecord(
|
||||
filehash="test6",
|
||||
original_filename="test6.pdf",
|
||||
local_filename="/tmp/test6.pdf",
|
||||
file_size=6144
|
||||
filehash="test6", original_filename="test6.pdf", local_filename="/tmp/test6.pdf", file_size=6144
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
|
||||
initialize_file_steps(db_session, file_record.id)
|
||||
from app.utils.step_manager import add_upload_steps
|
||||
|
||||
services = ["dropbox", "s3", "nextcloud", "google_drive", "onedrive", "webdav"]
|
||||
add_upload_steps(db_session, file_record.id, services)
|
||||
|
||||
|
||||
@@ -4,11 +4,12 @@ Tests for app/utils/filename_utils.py
|
||||
Tests filename sanitization and manipulation functions.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -259,7 +260,6 @@ class TestFilenameUtilsEdgeCases:
|
||||
assert "?" not in result
|
||||
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestUniqueFilepathWithCounter:
|
||||
"""Test unique filepath generation with numeric counter suffix"""
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""
|
||||
Test the /files view UI endpoint to ensure template rendering works correctly.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.models import FileRecord
|
||||
|
||||
|
||||
@@ -26,7 +28,7 @@ class TestFilesView:
|
||||
original_filename=f"test{i}.pdf",
|
||||
local_filename=f"/tmp/test{i}.pdf",
|
||||
file_size=1024 * (i + 1),
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -54,7 +56,7 @@ class TestFilesView:
|
||||
original_filename=f"test{i}.pdf",
|
||||
local_filename=f"/tmp/test{i}.pdf",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -89,4 +91,3 @@ class TestFilesView:
|
||||
# Check for drag-and-drop event handlers
|
||||
assert "dragenter" in content or "drag" in content, "Drag event handlers should be present"
|
||||
assert "Drop files anywhere to upload" in content, "Drop message should be present"
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/tasks/finalize_document_storage.py module."""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -18,4 +20,5 @@ class TestFinalizeDocumentStorageHelpers:
|
||||
def test_module_imports(self):
|
||||
"""Test that the module can be imported without errors."""
|
||||
from app.tasks.finalize_document_storage import finalize_document_storage
|
||||
|
||||
assert callable(finalize_document_storage)
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
"""Extended tests for app/tasks/imap_tasks.py module."""
|
||||
import os
|
||||
|
||||
import json
|
||||
import pytest
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch, MagicMock
|
||||
from email.message import EmailMessage
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.imap_tasks import (
|
||||
save_processed_emails,
|
||||
load_processed_emails,
|
||||
fetch_attachments_and_enqueue,
|
||||
find_all_mail_xlist,
|
||||
load_processed_emails,
|
||||
save_processed_emails,
|
||||
)
|
||||
|
||||
|
||||
@@ -41,9 +43,7 @@ class TestFetchAttachmentsExtended:
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Test"
|
||||
# Create attachment with wrong MIME type but .pdf extension
|
||||
msg.add_attachment(
|
||||
b"%PDF-1.4", maintype="application", subtype="octet-stream", filename="invoice.pdf"
|
||||
)
|
||||
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="octet-stream", filename="invoice.pdf")
|
||||
|
||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
|
||||
@@ -1,23 +1,24 @@
|
||||
"""Tests for app/tasks/imap_tasks.py module."""
|
||||
|
||||
import os
|
||||
import json
|
||||
import pytest
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch, MagicMock
|
||||
from email.message import EmailMessage
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.imap_tasks import (
|
||||
load_processed_emails,
|
||||
save_processed_emails,
|
||||
cleanup_old_entries,
|
||||
check_and_pull_mailbox,
|
||||
fetch_attachments_and_enqueue,
|
||||
cleanup_old_entries,
|
||||
email_already_has_label,
|
||||
mark_as_processed_with_star,
|
||||
mark_as_processed_with_label,
|
||||
fetch_attachments_and_enqueue,
|
||||
find_all_mail_folder,
|
||||
get_capabilities,
|
||||
load_processed_emails,
|
||||
mark_as_processed_with_label,
|
||||
mark_as_processed_with_star,
|
||||
save_processed_emails,
|
||||
)
|
||||
|
||||
_TEST_CREDENTIAL = "pass" # noqa: S105
|
||||
|
||||
@@ -5,9 +5,9 @@ Tests task progress logging functionality.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -64,9 +64,7 @@ class TestTaskLogging:
|
||||
mock_processing_log.return_value = mock_log_entry
|
||||
|
||||
# Call without message
|
||||
log_task_progress(
|
||||
task_id="task-456", step_name="upload", status="completed", message=None, file_id=None
|
||||
)
|
||||
log_task_progress(task_id="task-456", step_name="upload", status="completed", message=None, file_id=None)
|
||||
|
||||
# Verify called with None for optional parameters
|
||||
mock_processing_log.assert_called_once_with(
|
||||
@@ -89,9 +87,7 @@ class TestTaskLogging:
|
||||
mock_processing_log.return_value = mock_log_entry
|
||||
|
||||
# Call without file_id
|
||||
log_task_progress(
|
||||
task_id="task-789", step_name="metadata", status="running", message="Extracting metadata"
|
||||
)
|
||||
log_task_progress(task_id="task-789", step_name="metadata", status="running", message="Extracting metadata")
|
||||
|
||||
# file_id should default to None
|
||||
mock_processing_log.assert_called_once()
|
||||
|
||||
@@ -4,8 +4,9 @@ Tests for app/utils/notification.py
|
||||
Tests notification utilities and URL masking.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
_TEST_CREDENTIAL_URL = "https://user:password@example.com/notify" # noqa: S105
|
||||
_TEST_QUERY_URL = "https://example.com/api?key=secret1&password=secret2&public=visible" # noqa: S105
|
||||
@@ -133,8 +134,8 @@ class TestAppriseInitialization:
|
||||
@patch("app.utils.notification.settings")
|
||||
def test_init_apprise_with_configured_urls(self, mock_settings, mock_apprise_class):
|
||||
"""Test Apprise initialization with configured URLs"""
|
||||
from app.utils.notification import init_apprise
|
||||
import app.utils.notification
|
||||
from app.utils.notification import init_apprise
|
||||
|
||||
# Reset global
|
||||
app.utils.notification._apprise = None
|
||||
@@ -162,8 +163,8 @@ class TestAppriseInitialization:
|
||||
@patch("app.utils.notification.settings")
|
||||
def test_init_apprise_no_urls_configured(self, mock_settings, mock_apprise_class):
|
||||
"""Test Apprise initialization without configured URLs"""
|
||||
from app.utils.notification import init_apprise
|
||||
import app.utils.notification
|
||||
from app.utils.notification import init_apprise
|
||||
|
||||
# Reset global
|
||||
app.utils.notification._apprise = None
|
||||
@@ -188,8 +189,8 @@ class TestAppriseInitialization:
|
||||
@patch("app.utils.notification.settings")
|
||||
def test_init_apprise_caches_instance(self, mock_settings, mock_apprise_class):
|
||||
"""Test that Apprise instance is cached"""
|
||||
from app.utils.notification import init_apprise
|
||||
import app.utils.notification
|
||||
from app.utils.notification import init_apprise
|
||||
|
||||
# Reset global
|
||||
app.utils.notification._apprise = None
|
||||
@@ -214,8 +215,8 @@ class TestAppriseInitialization:
|
||||
@patch("app.utils.notification.settings")
|
||||
def test_init_apprise_handles_add_failure(self, mock_settings, mock_apprise_class):
|
||||
"""Test handling when adding notification URL fails"""
|
||||
from app.utils.notification import init_apprise
|
||||
import app.utils.notification
|
||||
from app.utils.notification import init_apprise
|
||||
|
||||
# Reset global
|
||||
app.utils.notification._apprise = None
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
"""Tests for app/utils/notification.py module."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.utils.notification import (
|
||||
_mask_sensitive_url,
|
||||
send_notification,
|
||||
init_apprise,
|
||||
notify_celery_failure,
|
||||
notify_credential_failure,
|
||||
notify_startup,
|
||||
notify_shutdown,
|
||||
notify_file_processed,
|
||||
init_apprise,
|
||||
notify_shutdown,
|
||||
notify_startup,
|
||||
send_notification,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,8 +4,9 @@ Tests for app/utils/oauth_helper.py
|
||||
Tests OAuth token exchange helper functions.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import HTTPException
|
||||
|
||||
@@ -51,9 +52,7 @@ class TestOAuthTokenExchange:
|
||||
assert result["expires_in"] == 3600
|
||||
|
||||
# Verify request was made correctly
|
||||
mock_post.assert_called_once_with(
|
||||
"https://oauth.example.com/token", data=payload, timeout=30
|
||||
)
|
||||
mock_post.assert_called_once_with("https://oauth.example.com/token", data=payload, timeout=30)
|
||||
|
||||
@patch("app.utils.oauth_helper.requests.post")
|
||||
@patch("app.utils.oauth_helper.settings")
|
||||
@@ -81,9 +80,7 @@ class TestOAuthTokenExchange:
|
||||
)
|
||||
|
||||
# Verify custom timeout was used
|
||||
mock_post.assert_called_once_with(
|
||||
"https://oauth.example.com/token", data=payload, timeout=60
|
||||
)
|
||||
mock_post.assert_called_once_with("https://oauth.example.com/token", data=payload, timeout=60)
|
||||
|
||||
@patch("app.utils.oauth_helper.requests.post")
|
||||
@patch("app.utils.oauth_helper.settings")
|
||||
@@ -200,9 +197,7 @@ class TestOAuthTokenExchange:
|
||||
# Mock error response with invalid JSON
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 400
|
||||
mock_response.json.side_effect = requests.exceptions.JSONDecodeError(
|
||||
"Invalid JSON", "", 0
|
||||
)
|
||||
mock_response.json.side_effect = requests.exceptions.JSONDecodeError("Invalid JSON", "", 0)
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
payload = {"grant_type": "authorization_code"}
|
||||
|
||||
+91
-127
@@ -6,18 +6,19 @@ These tests verify OCR processing logic with mocked external AI/ML services
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock, Mock
|
||||
from azure.ai.documentintelligence.models import AnalyzeResult
|
||||
|
||||
from app.tasks.process_with_azure_document_intelligence import (
|
||||
process_with_azure_document_intelligence,
|
||||
get_pdf_page_count,
|
||||
check_page_rotation,
|
||||
AZURE_DOC_INTELLIGENCE_LIMITS,
|
||||
check_page_rotation,
|
||||
get_pdf_page_count,
|
||||
process_with_azure_document_intelligence,
|
||||
)
|
||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages, determine_rotation_angle
|
||||
from app.tasks.rotate_pdf_pages import determine_rotation_angle, rotate_pdf_pages
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -86,22 +87,20 @@ startxref
|
||||
mock_client.begin_analyze_document.return_value = mock_poller
|
||||
mock_client.get_analyze_result_pdf.return_value = iter([b"searchable pdf content"])
|
||||
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
), patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages"
|
||||
) as mock_rotate:
|
||||
with (
|
||||
patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
),
|
||||
patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
|
||||
patch("app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages") as mock_rotate,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_rotate.delay = MagicMock()
|
||||
|
||||
# Run the task
|
||||
result = process_with_azure_document_intelligence.run(
|
||||
filename=test_pdf.name, file_id=1
|
||||
)
|
||||
result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
|
||||
|
||||
# Verify results
|
||||
assert result["file"] == test_pdf.name
|
||||
@@ -124,15 +123,11 @@ startxref
|
||||
@patch("app.tasks.process_with_azure_document_intelligence.log_task_progress")
|
||||
def test_file_not_found_error(self, mock_log, tmp_path):
|
||||
"""Test that FileNotFoundError is raised when file doesn't exist."""
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings:
|
||||
with patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings:
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
|
||||
with pytest.raises(FileNotFoundError) as exc_info:
|
||||
process_with_azure_document_intelligence.run(
|
||||
filename="nonexistent.pdf", file_id=1
|
||||
)
|
||||
process_with_azure_document_intelligence.run(filename="nonexistent.pdf", file_id=1)
|
||||
|
||||
assert "Local file not found" in str(exc_info.value)
|
||||
|
||||
@@ -145,21 +140,16 @@ startxref
|
||||
test_pdf = tmp_dir / "large.pdf"
|
||||
test_pdf.write_bytes(b"dummy content")
|
||||
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.os.path.getsize"
|
||||
) as mock_getsize:
|
||||
with (
|
||||
patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
|
||||
patch("app.tasks.process_with_azure_document_intelligence.os.path.getsize") as mock_getsize,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
# Mock file size to be larger than 500 MB
|
||||
mock_getsize.return_value = AZURE_DOC_INTELLIGENCE_LIMITS[
|
||||
"max_file_size_bytes"
|
||||
] + 1024
|
||||
mock_getsize.return_value = AZURE_DOC_INTELLIGENCE_LIMITS["max_file_size_bytes"] + 1024
|
||||
|
||||
result = process_with_azure_document_intelligence.run(
|
||||
filename=test_pdf.name, file_id=1
|
||||
)
|
||||
result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
|
||||
|
||||
# Verify error response
|
||||
assert "error" in result
|
||||
@@ -175,19 +165,16 @@ startxref
|
||||
test_pdf = tmp_dir / "many_pages.pdf"
|
||||
test_pdf.write_bytes(b"%PDF-1.4\n%%EOF") # Minimal valid PDF
|
||||
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.get_pdf_page_count"
|
||||
) as mock_page_count:
|
||||
with (
|
||||
patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
|
||||
patch("app.tasks.process_with_azure_document_intelligence.get_pdf_page_count") as mock_page_count,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
# Mock page count to exceed limit
|
||||
mock_page_count.return_value = AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"] + 1
|
||||
|
||||
result = process_with_azure_document_intelligence.run(
|
||||
filename=test_pdf.name, file_id=1
|
||||
)
|
||||
result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
|
||||
|
||||
# Verify error response
|
||||
assert "error" in result
|
||||
@@ -219,15 +206,14 @@ startxref
|
||||
mock_client.begin_analyze_document.return_value = mock_poller
|
||||
mock_client.get_analyze_result_pdf.return_value = iter([b"content"])
|
||||
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
), patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.get_pdf_page_count"
|
||||
) as mock_page_count, patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages"
|
||||
with (
|
||||
patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
),
|
||||
patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
|
||||
patch("app.tasks.process_with_azure_document_intelligence.get_pdf_page_count") as mock_page_count,
|
||||
patch("app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages"),
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
@@ -235,9 +221,7 @@ startxref
|
||||
mock_page_count.return_value = None
|
||||
|
||||
# Should not raise an error
|
||||
result = process_with_azure_document_intelligence.run(
|
||||
filename=test_pdf.name, file_id=1
|
||||
)
|
||||
result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
|
||||
|
||||
# Verify processing continued
|
||||
assert "error" not in result
|
||||
@@ -275,21 +259,19 @@ startxref
|
||||
mock_client.begin_analyze_document.return_value = mock_poller
|
||||
mock_client.get_analyze_result_pdf.return_value = iter([b"content"])
|
||||
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
), patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages"
|
||||
) as mock_rotate:
|
||||
with (
|
||||
patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
),
|
||||
patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
|
||||
patch("app.tasks.process_with_azure_document_intelligence.rotate_pdf_pages") as mock_rotate,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_rotate.delay = MagicMock()
|
||||
|
||||
result = process_with_azure_document_intelligence.run(
|
||||
filename=test_pdf.name, file_id=2
|
||||
)
|
||||
result = process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=2)
|
||||
|
||||
# Verify rotation data was passed correctly
|
||||
mock_rotate.delay.assert_called_once()
|
||||
@@ -311,24 +293,21 @@ startxref
|
||||
test_pdf.write_bytes(b"%PDF-1.4\n%%EOF")
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.begin_analyze_document.side_effect = Exception(
|
||||
"Azure API connection failed"
|
||||
)
|
||||
mock_client.begin_analyze_document.side_effect = Exception("Azure API connection failed")
|
||||
|
||||
with patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
), patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.settings"
|
||||
) as mock_settings:
|
||||
with (
|
||||
patch(
|
||||
"app.tasks.process_with_azure_document_intelligence.document_intelligence_client",
|
||||
mock_client,
|
||||
),
|
||||
patch("app.tasks.process_with_azure_document_intelligence.settings") as mock_settings,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
|
||||
# Should raise the exception
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
process_with_azure_document_intelligence.run(
|
||||
filename=test_pdf.name, file_id=1
|
||||
)
|
||||
process_with_azure_document_intelligence.run(filename=test_pdf.name, file_id=1)
|
||||
|
||||
assert "Azure API connection failed" in str(exc_info.value)
|
||||
|
||||
@@ -456,13 +435,11 @@ class TestRefineTextWithGPT:
|
||||
# Import the module to patch the correct function
|
||||
from app.tasks import extract_metadata_with_gpt as metadata_module
|
||||
|
||||
with patch(
|
||||
"app.tasks.refine_text_with_gpt.client", mock_client
|
||||
), patch.object(
|
||||
metadata_module, "extract_metadata_with_gpt"
|
||||
) as mock_extract, patch(
|
||||
"app.tasks.refine_text_with_gpt.settings"
|
||||
) as mock_settings:
|
||||
with (
|
||||
patch("app.tasks.refine_text_with_gpt.client", mock_client),
|
||||
patch.object(metadata_module, "extract_metadata_with_gpt") as mock_extract,
|
||||
patch("app.tasks.refine_text_with_gpt.settings") as mock_settings,
|
||||
):
|
||||
|
||||
mock_settings.openai_model = "gpt-4"
|
||||
mock_extract.delay = MagicMock()
|
||||
@@ -481,9 +458,7 @@ class TestRefineTextWithGPT:
|
||||
assert call_kwargs["messages"][1]["content"] == raw_text
|
||||
|
||||
# Verify metadata extraction was queued
|
||||
mock_extract.delay.assert_called_once_with(
|
||||
filename, "This is some text with OCR errors"
|
||||
)
|
||||
mock_extract.delay.assert_called_once_with(filename, "This is some text with OCR errors")
|
||||
|
||||
@patch("app.tasks.refine_text_with_gpt.log_task_progress")
|
||||
def test_openai_api_error(self, mock_log):
|
||||
@@ -492,15 +467,12 @@ class TestRefineTextWithGPT:
|
||||
filename = "test.pdf"
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.chat.completions.create.side_effect = Exception(
|
||||
"OpenAI API error"
|
||||
)
|
||||
mock_client.chat.completions.create.side_effect = Exception("OpenAI API error")
|
||||
|
||||
with patch(
|
||||
"app.tasks.refine_text_with_gpt.client", mock_client
|
||||
), patch(
|
||||
"app.tasks.refine_text_with_gpt.settings"
|
||||
) as mock_settings:
|
||||
with (
|
||||
patch("app.tasks.refine_text_with_gpt.client", mock_client),
|
||||
patch("app.tasks.refine_text_with_gpt.settings") as mock_settings,
|
||||
):
|
||||
|
||||
mock_settings.openai_model = "gpt-4"
|
||||
|
||||
@@ -606,11 +578,10 @@ startxref
|
||||
rotation_data = {0: 90} # Rotate first page by 90 degrees
|
||||
extracted_text = "Test text"
|
||||
|
||||
with patch(
|
||||
"app.tasks.rotate_pdf_pages.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
|
||||
patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_extract.delay = MagicMock()
|
||||
@@ -629,9 +600,7 @@ startxref
|
||||
assert "0" in result["applied_rotations"]
|
||||
|
||||
# Verify metadata extraction was queued
|
||||
mock_extract.delay.assert_called_once_with(
|
||||
test_pdf.name, extracted_text, 1
|
||||
)
|
||||
mock_extract.delay.assert_called_once_with(test_pdf.name, extracted_text, 1)
|
||||
|
||||
@patch("app.tasks.rotate_pdf_pages.log_task_progress")
|
||||
def test_rotate_pdf_pages_no_rotation_needed(self, mock_log, tmp_path):
|
||||
@@ -644,11 +613,10 @@ startxref
|
||||
|
||||
extracted_text = "Test text"
|
||||
|
||||
with patch(
|
||||
"app.tasks.rotate_pdf_pages.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
|
||||
patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_extract.delay = MagicMock()
|
||||
@@ -679,11 +647,10 @@ startxref
|
||||
rotation_data = {0: 0, 1: 0} # All pages have 0 rotation
|
||||
extracted_text = "Test text"
|
||||
|
||||
with patch(
|
||||
"app.tasks.rotate_pdf_pages.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
|
||||
patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_extract.delay = MagicMock()
|
||||
@@ -705,11 +672,10 @@ startxref
|
||||
tmp_dir = tmp_path / "tmp"
|
||||
tmp_dir.mkdir()
|
||||
|
||||
with patch(
|
||||
"app.tasks.rotate_pdf_pages.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
|
||||
patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_extract.delay = MagicMock()
|
||||
@@ -742,11 +708,10 @@ startxref
|
||||
rotation_data = {0: 90}
|
||||
extracted_text = "Test text"
|
||||
|
||||
with patch(
|
||||
"app.tasks.rotate_pdf_pages.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
|
||||
patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_extract.delay = MagicMock()
|
||||
@@ -814,11 +779,10 @@ startxref
|
||||
rotation_data = {"0": "90"}
|
||||
extracted_text = "Test text"
|
||||
|
||||
with patch(
|
||||
"app.tasks.rotate_pdf_pages.settings"
|
||||
) as mock_settings, patch(
|
||||
"app.tasks.rotate_pdf_pages.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.rotate_pdf_pages.settings") as mock_settings,
|
||||
patch("app.tasks.rotate_pdf_pages.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
mock_extract.delay = MagicMock()
|
||||
|
||||
@@ -87,11 +87,12 @@ startxref
|
||||
original_filename = "Apostille Sverige.pdf"
|
||||
|
||||
# Mock environment and dependencies
|
||||
with patch("app.tasks.process_document.SessionLocal") as mock_session_local, patch(
|
||||
"app.tasks.process_document.settings"
|
||||
) as mock_settings, patch("app.tasks.process_document.log_task_progress"), patch(
|
||||
"app.tasks.process_document.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.process_document.SessionLocal") as mock_session_local,
|
||||
patch("app.tasks.process_document.settings") as mock_settings,
|
||||
patch("app.tasks.process_document.log_task_progress"),
|
||||
patch("app.tasks.process_document.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
# Setup mocks
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
@@ -187,11 +188,12 @@ startxref
|
||||
test_pdf.write_bytes(pdf_content)
|
||||
|
||||
# Mock environment and dependencies
|
||||
with patch("app.tasks.process_document.SessionLocal") as mock_session_local, patch(
|
||||
"app.tasks.process_document.settings"
|
||||
) as mock_settings, patch("app.tasks.process_document.log_task_progress"), patch(
|
||||
"app.tasks.process_document.extract_metadata_with_gpt"
|
||||
) as mock_extract:
|
||||
with (
|
||||
patch("app.tasks.process_document.SessionLocal") as mock_session_local,
|
||||
patch("app.tasks.process_document.settings") as mock_settings,
|
||||
patch("app.tasks.process_document.log_task_progress"),
|
||||
patch("app.tasks.process_document.extract_metadata_with_gpt") as mock_extract,
|
||||
):
|
||||
|
||||
# Setup mocks
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
|
||||
@@ -4,11 +4,11 @@ Security tests for path traversal vulnerabilities.
|
||||
Tests all file path operations to ensure they properly prevent path traversal attacks.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -389,9 +389,10 @@ class TestFileUploadSecurity:
|
||||
|
||||
def test_sanitize_after_basename(self):
|
||||
"""Test that sanitization happens after basename extraction."""
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
import os
|
||||
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
malicious = "../../../passwd.pdf"
|
||||
|
||||
# Step 1: Extract basename (as ui_upload does)
|
||||
@@ -442,10 +443,11 @@ class TestEndToEndPathTraversal:
|
||||
|
||||
def test_full_upload_flow_prevents_traversal(self, tmp_path):
|
||||
"""Test complete upload flow prevents path traversal."""
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# Simulate ui_upload flow
|
||||
malicious_upload_filename = "../../../etc/passwd"
|
||||
|
||||
@@ -472,9 +474,10 @@ class TestEndToEndPathTraversal:
|
||||
|
||||
def test_metadata_embedding_flow_prevents_traversal(self, tmp_path):
|
||||
"""Test metadata embedding flow prevents path traversal."""
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
import os
|
||||
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# Simulate GPT returning malicious filename
|
||||
gpt_metadata = {
|
||||
"filename": "../../../etc/shadow",
|
||||
|
||||
@@ -101,9 +101,10 @@ def test_rate_limit_exceeded_returns_429(client):
|
||||
@pytest.mark.security
|
||||
def test_rate_limiting_uses_correct_identifier():
|
||||
"""Test that rate limiting uses IP or user ID as identifier."""
|
||||
from app.middleware.rate_limit import get_identifier
|
||||
from fastapi import Request
|
||||
|
||||
from app.middleware.rate_limit import get_identifier
|
||||
|
||||
# Create a mock request
|
||||
class MockRequest:
|
||||
def __init__(self):
|
||||
@@ -157,9 +158,10 @@ def test_limiter_disabled():
|
||||
@pytest.mark.integration
|
||||
def test_rate_limit_exception_handler_registered():
|
||||
"""Test that rate limit exception handler is registered."""
|
||||
from app.main import app
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
|
||||
from app.main import app
|
||||
|
||||
# Verify exception handler is registered
|
||||
assert RateLimitExceeded in app.exception_handlers
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/tasks/upload_with_rclone.py module."""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.tasks.upload_with_rclone import upload_with_rclone
|
||||
|
||||
|
||||
+19
-17
@@ -6,19 +6,19 @@ import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import ApplicationSettings
|
||||
from app.utils.config_loader import convert_setting_value, load_settings_from_db
|
||||
from app.utils.settings_service import (
|
||||
get_setting_from_db,
|
||||
save_setting_to_db,
|
||||
get_all_settings_from_db,
|
||||
SETTING_METADATA,
|
||||
delete_setting_from_db,
|
||||
validate_setting_value,
|
||||
get_all_settings_from_db,
|
||||
get_setting_from_db,
|
||||
get_setting_metadata,
|
||||
get_settings_by_category,
|
||||
SETTING_METADATA,
|
||||
save_setting_to_db,
|
||||
validate_setting_value,
|
||||
)
|
||||
from app.utils.config_loader import convert_setting_value, load_settings_from_db
|
||||
from app.config import Settings
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -140,9 +140,14 @@ class TestSettingsService:
|
||||
|
||||
# Check critical settings are present
|
||||
critical_settings = [
|
||||
"database_url", "redis_url", "workdir", "debug",
|
||||
"openai_api_key", "azure_ai_key",
|
||||
"auth_enabled", "session_secret"
|
||||
"database_url",
|
||||
"redis_url",
|
||||
"workdir",
|
||||
"debug",
|
||||
"openai_api_key",
|
||||
"azure_ai_key",
|
||||
"auth_enabled",
|
||||
"session_secret",
|
||||
]
|
||||
for setting in critical_settings:
|
||||
assert setting in SETTING_METADATA, f"Missing metadata for {setting}"
|
||||
@@ -219,9 +224,10 @@ class TestSettingsPrecedence:
|
||||
def test_db_overrides_default(self, db_session: Session):
|
||||
"""Test that database settings override default values"""
|
||||
# Create a minimal test settings object
|
||||
from pydantic_settings import BaseSettings
|
||||
from typing import Optional
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
class TestSettings(BaseSettings):
|
||||
test_value: str = "default"
|
||||
test_bool: bool = False
|
||||
@@ -270,10 +276,7 @@ class TestApplicationSettingsModel:
|
||||
|
||||
def test_create_setting_record(self, db_session: Session):
|
||||
"""Test creating an ApplicationSettings record"""
|
||||
setting = ApplicationSettings(
|
||||
key="test_key",
|
||||
value="test_value"
|
||||
)
|
||||
setting = ApplicationSettings(key="test_key", value="test_value")
|
||||
db_session.add(setting)
|
||||
db_session.commit()
|
||||
|
||||
@@ -301,7 +304,7 @@ class TestApplicationSettingsModel:
|
||||
|
||||
@pytest.mark.skipif(
|
||||
True, # Skip for all databases - timestamp update behavior varies
|
||||
reason="Timestamp update behavior varies by database backend"
|
||||
reason="Timestamp update behavior varies by database backend",
|
||||
)
|
||||
def test_update_timestamp(self, db_session: Session):
|
||||
"""Test that updated_at timestamp is updated on modification"""
|
||||
@@ -325,4 +328,3 @@ class TestApplicationSettingsModel:
|
||||
# Note: SQLite doesn't automatically update onupdate timestamps
|
||||
# This test is skipped as behavior varies by database backend
|
||||
assert setting.updated_at is not None
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/utils/config_validator/settings_display.py module."""
|
||||
import pytest
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
"""Tests for app/utils/setup_wizard.py module."""
|
||||
import pytest
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.setup_wizard import (
|
||||
get_required_settings,
|
||||
is_setup_required,
|
||||
get_missing_required_settings,
|
||||
get_required_settings,
|
||||
get_wizard_steps,
|
||||
is_setup_required,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -172,7 +172,12 @@ class TestStepManager:
|
||||
# Update an existing step
|
||||
now = datetime.now()
|
||||
update_step_status(
|
||||
db_session, file_record.id, "create_file_record", "success", started_at=now - timedelta(seconds=5), completed_at=now
|
||||
db_session,
|
||||
file_record.id,
|
||||
"create_file_record",
|
||||
"success",
|
||||
started_at=now - timedelta(seconds=5),
|
||||
completed_at=now,
|
||||
)
|
||||
|
||||
# Verify step was updated
|
||||
@@ -201,7 +206,9 @@ class TestStepManager:
|
||||
now = datetime.now()
|
||||
update_step_status(db_session, file_record.id, "create_file_record", "success", completed_at=now)
|
||||
update_step_status(db_session, file_record.id, "check_text", "in_progress", started_at=now)
|
||||
update_step_status(db_session, file_record.id, "extract_text", "failure", error_message="Failed to extract text")
|
||||
update_step_status(
|
||||
db_session, file_record.id, "extract_text", "failure", error_message="Failed to extract text"
|
||||
)
|
||||
|
||||
# Get all step statuses
|
||||
status_map = get_file_step_status(db_session, file_record.id)
|
||||
|
||||
@@ -5,8 +5,8 @@ Tests the new functionality for storing immutable originals and processed copies
|
||||
collision handling, and forced Cloud OCR reprocessing.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -195,7 +195,7 @@ startxref
|
||||
local_filename=str(test_pdf),
|
||||
original_file_path=str(original_file),
|
||||
file_size=100,
|
||||
mime_type="application/pdf"
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
db_session.add(file_record)
|
||||
db_session.commit()
|
||||
@@ -250,11 +250,7 @@ class TestMetadataAugmentation:
|
||||
"""Test that persisted metadata includes original and processed paths"""
|
||||
from app.tasks.embed_metadata_into_pdf import persist_metadata
|
||||
|
||||
metadata = {
|
||||
"filename": "2024-01-01_Invoice",
|
||||
"document_type": "Invoice",
|
||||
"tags": ["finance", "2024"]
|
||||
}
|
||||
metadata = {"filename": "2024-01-01_Invoice", "document_type": "Invoice", "tags": ["finance", "2024"]}
|
||||
|
||||
processed_file = tmp_path / "processed" / "2024-01-01_Invoice.pdf"
|
||||
processed_file.parent.mkdir(parents=True)
|
||||
@@ -264,17 +260,14 @@ class TestMetadataAugmentation:
|
||||
processed_path = str(processed_file)
|
||||
|
||||
json_path = persist_metadata(
|
||||
metadata,
|
||||
str(processed_file),
|
||||
original_file_path=original_path,
|
||||
processed_file_path=processed_path
|
||||
metadata, str(processed_file), original_file_path=original_path, processed_file_path=processed_path
|
||||
)
|
||||
|
||||
# Verify JSON was created
|
||||
assert os.path.exists(json_path)
|
||||
|
||||
# Verify content
|
||||
with open(json_path, 'r') as f:
|
||||
with open(json_path, "r") as f:
|
||||
saved_metadata = json.load(f)
|
||||
|
||||
assert "original_file_path" in saved_metadata
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Tests for app/tasks/upload_to_email.py module."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -10,4 +12,5 @@ class TestUploadToEmail:
|
||||
def test_module_imports(self):
|
||||
"""Test that the module can be imported."""
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
|
||||
assert callable(upload_to_email)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Additional tests for upload_to_ftp task."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -10,4 +12,5 @@ class TestUploadToFtp:
|
||||
def test_module_imports(self):
|
||||
"""Test that the module can be imported."""
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
|
||||
assert callable(upload_to_ftp)
|
||||
|
||||
@@ -3,15 +3,17 @@ Tests for upload tasks including OneDrive, S3, FTP, SFTP, WebDAV, Google Drive,
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_s3 import upload_to_s3
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
|
||||
_TEST_CREDENTIAL = "test_pass" # noqa: S105
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Additional tests for upload task modules."""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -23,6 +25,7 @@ class TestUploadToNextcloud:
|
||||
mock_settings.workdir = "/tmp"
|
||||
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
|
||||
result = upload_to_nextcloud.__wrapped__(mock_self, "/nonexistent/file.pdf")
|
||||
|
||||
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""Tests to increase coverage for upload task modules."""
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_s3 import upload_to_s3
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Comprehensive tests for upload_to_webdav task."""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, Mock, MagicMock
|
||||
from requests.exceptions import ConnectionError, Timeout, RequestException
|
||||
from requests.exceptions import ConnectionError, RequestException, Timeout
|
||||
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ actual file uploads against it, then verify the files were uploaded successfully
|
||||
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
|
||||
# Import testcontainers - required for these tests
|
||||
|
||||
@@ -4,8 +4,9 @@ Tests for app/tasks/uptime_kuma_tasks.py
|
||||
Tests Uptime Kuma health check ping functionality.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
import requests
|
||||
|
||||
|
||||
@@ -43,9 +44,7 @@ class TestUptimeKumaTasks:
|
||||
|
||||
# Should return True on success
|
||||
assert result is True
|
||||
mock_get.assert_called_once_with(
|
||||
"https://uptime.example.com/ping/123", timeout=10
|
||||
)
|
||||
mock_get.assert_called_once_with("https://uptime.example.com/ping/123", timeout=10)
|
||||
mock_response.raise_for_status.assert_called_once()
|
||||
|
||||
@patch("app.tasks.uptime_kuma_tasks.requests.get")
|
||||
@@ -93,9 +92,7 @@ class TestUptimeKumaTasks:
|
||||
# Mock HTTP error
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 500
|
||||
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
|
||||
"500 Server Error"
|
||||
)
|
||||
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("500 Server Error")
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
result = ping_uptime_kuma()
|
||||
|
||||
@@ -29,9 +29,10 @@ class TestURLUploadValidation:
|
||||
|
||||
def test_validate_url_scheme_ftp_rejected(self):
|
||||
"""Test that FTP URLs are rejected"""
|
||||
from app.api.url_upload import URLUploadRequest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.api.url_upload import URLUploadRequest
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
URLUploadRequest(url="ftp://example.com/file.pdf")
|
||||
# Pydantic HttpUrl validates scheme automatically
|
||||
@@ -39,9 +40,10 @@ class TestURLUploadValidation:
|
||||
|
||||
def test_validate_url_scheme_file_rejected(self):
|
||||
"""Test that file:// URLs are rejected"""
|
||||
from app.api.url_upload import URLUploadRequest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.api.url_upload import URLUploadRequest
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
URLUploadRequest(url="file:///etc/passwd")
|
||||
# Pydantic HttpUrl validates scheme automatically
|
||||
@@ -126,17 +128,14 @@ class TestURLUploadValidation:
|
||||
# Word
|
||||
assert validate_file_type("application/msword", "file.doc") is True
|
||||
assert (
|
||||
validate_file_type(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "file.docx"
|
||||
)
|
||||
validate_file_type("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "file.docx")
|
||||
is True
|
||||
)
|
||||
|
||||
# Excel
|
||||
assert validate_file_type("application/vnd.ms-excel", "file.xls") is True
|
||||
assert (
|
||||
validate_file_type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "file.xlsx")
|
||||
is True
|
||||
validate_file_type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "file.xlsx") is True
|
||||
)
|
||||
|
||||
def test_validate_file_type_images_allowed(self):
|
||||
@@ -248,9 +247,7 @@ class TestURLUploadEndpoint:
|
||||
@patch("app.api.url_upload.requests.get")
|
||||
def test_process_url_blocks_metadata_endpoint(self, mock_requests_get, client):
|
||||
"""Test that cloud metadata endpoints are blocked"""
|
||||
response = client.post(
|
||||
"/api/process-url", json={"url": "http://169.254.169.254/latest/meta-data/"}
|
||||
)
|
||||
response = client.post("/api/process-url", json={"url": "http://169.254.169.254/latest/meta-data/"})
|
||||
|
||||
assert response.status_code == 400
|
||||
data = response.json()
|
||||
@@ -341,9 +338,7 @@ class TestURLUploadEndpoint:
|
||||
|
||||
@patch("app.api.url_upload.requests.get")
|
||||
@patch("app.api.url_upload.process_document")
|
||||
def test_process_url_with_custom_filename(
|
||||
self, mock_process_document, mock_requests_get, client, tmp_path
|
||||
):
|
||||
def test_process_url_with_custom_filename(self, mock_process_document, mock_requests_get, client, tmp_path):
|
||||
"""Test URL upload with custom filename"""
|
||||
# Mock successful download
|
||||
mock_response = Mock()
|
||||
@@ -369,9 +364,7 @@ class TestURLUploadEndpoint:
|
||||
|
||||
@patch("app.api.url_upload.requests.get")
|
||||
@patch("app.api.url_upload.process_document")
|
||||
def test_process_url_extracts_filename_from_url(
|
||||
self, mock_process_document, mock_requests_get, client, tmp_path
|
||||
):
|
||||
def test_process_url_extracts_filename_from_url(self, mock_process_document, mock_requests_get, client, tmp_path):
|
||||
"""Test that filename is extracted from URL when not provided"""
|
||||
# Mock successful download
|
||||
mock_response = Mock()
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Additional view tests to increase coverage."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
_TEST_CREDENTIAL = "test" # noqa: S105
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/views/dropbox.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Tests for app/views/general.py module."""
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/views/google_drive.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/views/license_routes.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/views/onedrive.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Tests for app/views/settings.py module."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.views.settings import require_admin_access
|
||||
|
||||
@@ -12,6 +14,7 @@ class TestRequireAdminAccess:
|
||||
@pytest.mark.asyncio
|
||||
async def test_redirects_non_admin_user(self):
|
||||
"""Test that non-admin users are redirected."""
|
||||
|
||||
@require_admin_access
|
||||
async def dummy_route(request):
|
||||
return {"success": True}
|
||||
@@ -25,6 +28,7 @@ class TestRequireAdminAccess:
|
||||
@pytest.mark.asyncio
|
||||
async def test_redirects_when_no_user(self):
|
||||
"""Test that unauthenticated users are redirected."""
|
||||
|
||||
@require_admin_access
|
||||
async def dummy_route(request):
|
||||
return {"success": True}
|
||||
@@ -38,6 +42,7 @@ class TestRequireAdminAccess:
|
||||
@pytest.mark.asyncio
|
||||
async def test_allows_admin_user(self):
|
||||
"""Test that admin users can access the route."""
|
||||
|
||||
@require_admin_access
|
||||
async def dummy_route(request):
|
||||
return {"success": True}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/views/status.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Tests for app/views/wizard.py module."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user