gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: fix code formatting with black, isort, and flake8

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-12 03:21:41 +00:00
parent f79bd2cb0c
commit ff9a3ff49f
87 changed files with 874 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+4 -8
View File
@@ -397,9 +397,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
"""
Reprocess a single file with forced Cloud OCR processing.
This endpoint forces Azure Document Intelligence OCR processing regardless
of whether the PDF contains embedded text. Useful for documents with
of whether the PDF contains embedded text. Useful for documents with
low-quality embedded text or when higher quality OCR is needed.
Args:
@@ -425,16 +425,12 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}")
else:
raise HTTPException(
status_code=400,
detail="Neither original nor local file found on disk. Cannot reprocess."
status_code=400, detail="Neither original nor local file found on disk. Cannot reprocess."
)
# Queue the file for processing with force_cloud_ocr=True
task = process_document.delay(
source_file,
original_filename=file_record.original_filename,
file_id=file_record.id,
force_cloud_ocr=True
source_file, original_filename=file_record.original_filename, file_id=file_record.id, force_cloud_ocr=True
)
logger.info(
app/api/url_upload.py
+1 -1
View File
@@ -277,7 +277,7 @@ async def process_url(request: Request, url_request: URLUploadRequest):
return {
"task_id": task.id,
"status": "queued",
"message": f"File downloaded from URL and queued for processing",
"message": "File downloaded from URL and queued for processing",
"filename": safe_filename,
"size": downloaded_size,
}
app/celery_worker.py
+1 -1
View File
@@ -13,6 +13,7 @@ from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
# **Ensure all tasks are imported before Celery starts**
from app.tasks.process_document import process_document # noqa: F401
@@ -33,7 +34,6 @@ from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
celery.conf.task_routes = {
"app.tasks.*": {"queue": "default"},
app/config.py
+22 -6
View File
@@ -33,7 +33,8 @@ class Settings(BaseSettings):
paperless_host: Optional[str] = None
paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
# JSON mapping of metadata field names to Paperless custom field names
# Example: {"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
# Example: {"absender": "Sender", "empfaenger": "Recipient",
# "language": "Language", "correspondent": "Correspondent"}
paperless_custom_fields_mapping: Optional[str] = None
azure_ai_key: str
@@ -176,23 +177,35 @@ class Settings(BaseSettings):
)
max_single_file_size: Optional[int] = Field(
default=None,
description="Maximum size for a single file chunk in bytes. If set and file exceeds this, it will be split into smaller chunks for processing. Default: None (no splitting).",
description=(
"Maximum size for a single file chunk in bytes. If set and file exceeds this,"
" it will be split into smaller chunks for processing. Default: None (no splitting)."
),
)
# Deduplication settings - prevents processing of duplicate files
enable_deduplication: bool = Field(
default=True,
description="Enable deduplication check before processing. If enabled, files with the same SHA-256 hash as previously processed files will not be processed again. Default: True (enabled).",
description=(
"Enable deduplication check before processing. If enabled, files with the same SHA-256 hash"
" as previously processed files will not be processed again. Default: True (enabled)."
),
)
show_deduplication_step: bool = Field(
default=True,
description="Show the 'Check for Duplicates' step in processing history. If False, the check is still performed but not displayed. Default: True.",
description=(
"Show the 'Check for Duplicates' step in processing history."
" If False, the check is still performed but not displayed. Default: True."
),
)
# Processing step timeout - prevents files from getting stuck in "in_progress" state
step_timeout: int = Field(
default=600,
description="Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this, it will be marked as failed. Default: 600 seconds (10 minutes).",
description=(
"Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this,"
" it will be marked as failed. Default: 600 seconds (10 minutes)."
),
)
# Security Headers Configuration (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
@@ -215,7 +228,10 @@ class Settings(BaseSettings):
# Content-Security-Policy (CSP) - Controls resource loading
security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.")
security_header_csp_value: str = Field(
default="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;",
default=(
"default-src 'self'; script-src 'self' 'unsafe-inline';"
" style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
),
description="CSP header value. Customize based on your application's resource loading needs.",
)
app/main.py
+1 -1
View File
@@ -8,6 +8,7 @@ from fastapi import FastAPI, HTTPException, Request, status
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from slowapi.errors import RateLimitExceeded
from starlette.config import Config
from starlette.middleware.sessions import SessionMiddleware
from starlette.middleware.trustedhost import TrustedHostMiddleware
@@ -21,7 +22,6 @@ from app.middleware.rate_limit import create_limiter, get_rate_limit_exceeded_ha
from app.middleware.security_headers import SecurityHeadersMiddleware
from app.utils.config_validator import check_all_configs
from app.utils.notification import init_apprise, notify_shutdown, notify_startup
from slowapi.errors import RateLimitExceeded
# Import the routers - now using views directly instead of frontend
from app.views import router as frontend_router
app/middleware/rate_limit.py
+12 -13
View File
@@ -21,7 +21,6 @@ from typing import Callable
from fastapi import Request
from slowapi import Limiter, _rate_limit_exceeded_handler
from slowapi.errors import RateLimitExceeded
from slowapi.util import get_remote_address
logger = logging.getLogger(__name__)
@@ -30,14 +29,14 @@ logger = logging.getLogger(__name__)
def get_identifier(request: Request) -> str:
"""
Get unique identifier for rate limiting.
Uses authenticated user ID if available, otherwise falls back to IP address.
This provides better rate limiting for authenticated users and prevents
IP-based bypassing for authenticated endpoints.
Args:
request: FastAPI request object
Returns:
Unique identifier string for rate limiting
"""
@@ -50,7 +49,7 @@ def get_identifier(request: Request) -> str:
if identifier:
logger.debug(f"Rate limiting by user: {identifier}")
return f"user:{identifier}"
# Fall back to IP address for unauthenticated requests
ip = get_remote_address(request)
logger.debug(f"Rate limiting by IP: {ip}")
@@ -60,11 +59,11 @@ def get_identifier(request: Request) -> str:
def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
"""
Create and configure the rate limiter.
Args:
redis_url: Redis connection URL for distributed rate limiting
enabled: Whether rate limiting is enabled (default: True)
Returns:
Configured Limiter instance
"""
@@ -76,10 +75,10 @@ def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
default_limits=["10000/minute"], # Effectively unlimited
enabled=False,
)
# Use Redis if available, otherwise fall back to in-memory
storage_uri = redis_url if redis_url else "memory://"
if redis_url:
logger.info(f"Rate limiting enabled with Redis backend: {redis_url}")
else:
@@ -87,7 +86,7 @@ def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
"Rate limiting using in-memory storage (not suitable for production with multiple workers). "
"Configure REDIS_URL for distributed rate limiting."
)
# Create limiter with default limits
# Default: 100 requests per minute per IP/user
limiter = Limiter(
@@ -97,7 +96,7 @@ def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
strategy="fixed-window", # Can be: fixed-window, moving-window, or fixed-window-elastic-expiry
enabled=True,
)
logger.info("Rate limiter initialized successfully")
return limiter
@@ -105,10 +104,10 @@ def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
def get_rate_limit_exceeded_handler() -> Callable:
"""
Get the rate limit exceeded exception handler.
Returns a handler that provides user-friendly 429 responses with
Retry-After header when rate limit is exceeded.
Returns:
Exception handler function
"""
app/middleware/rate_limit_decorators.py
+5 -9
View File
@@ -7,10 +7,6 @@ This module provides convenient decorators to apply rate limits to specific endp
Import the limiter from main.py state and use these decorators to protect endpoints.
"""
from functools import wraps
from fastapi import Request
# Import will happen at runtime to avoid circular dependencies
_limiter = None
@@ -28,13 +24,13 @@ def get_limiter():
def limit(rate_limit: str):
"""
Apply a rate limit to an endpoint.
Args:
rate_limit: Rate limit string (e.g., "10/minute", "100/hour")
Returns:
Decorator function
Example:
@router.post("/login")
@limit("10/minute")
@@ -53,10 +49,10 @@ def limit(rate_limit: str):
def exempt():
"""
Exempt an endpoint from rate limiting.
Returns:
Decorator function
Example:
@router.get("/health")
@exempt()
app/models.py
+3 -3
View File
@@ -1,6 +1,6 @@
# app/models.py
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func, Boolean
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
from app.database import Base
@@ -44,11 +44,11 @@ class FileRecord(Base):
# MIME type or extension (optional)
mime_type = Column(String)
# Deduplication tracking: True if this file is a duplicate of another file
# When a duplicate is detected, this file record is created but marked as duplicate
is_duplicate = Column(Boolean, default=False, nullable=False, index=True)
# If this is a duplicate, record the ID of the original file for reference
duplicate_of_id = Column(Integer, ForeignKey("files.id"), nullable=True)
app/tasks/embed_metadata_into_pdf.py
+26 -14
View File
@@ -35,28 +35,28 @@ def persist_metadata(metadata, final_pdf_path, original_file_path=None, processe
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
the metadata will be saved as "<workdir>/processed/MyFile.json".
Optionally augments the metadata with file path references for traceability.
Args:
metadata: Dictionary of metadata to save
final_pdf_path: Path to the final PDF file
original_file_path: Optional path to the immutable original file
processed_file_path: Optional path to the processed file
Returns:
str: Path to the created JSON file
"""
base, _ = os.path.splitext(final_pdf_path)
json_path = base + ".json"
# Augment metadata with file path references if provided
metadata_with_paths = metadata.copy()
if original_file_path:
metadata_with_paths["original_file_path"] = original_file_path
if processed_file_path:
metadata_with_paths["processed_file_path"] = processed_file_path
with open(json_path, "w", encoding="utf-8") as f:
json.dump(metadata_with_paths, f, ensure_ascii=False, indent=2)
return json_path
@@ -102,7 +102,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
else:
logger.error(f"[{task_id}] Local file {local_file_path} not found, cannot embed metadata.")
log_task_progress(
task_id, "embed_metadata_into_pdf", "failure", "File not found", file_id=file_id,
task_id,
"embed_metadata_into_pdf",
"failure",
"File not found",
file_id=file_id,
detail=(
f"Local file not found, cannot embed metadata.\n"
f"Tried path: {local_file_path}\n"
@@ -199,10 +203,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
logger.info(f"[{task_id}] Persisting metadata to JSON")
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
json_path = persist_metadata(
metadata,
final_file_path,
original_file_path=original_file_path,
processed_file_path=final_file_path
metadata, final_file_path, original_file_path=original_file_path, processed_file_path=final_file_path
)
logger.info(f"[{task_id}] Metadata persisted to {json_path}")
log_task_progress(
@@ -212,7 +213,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
# Trigger the next step: final storage.
logger.info(f"[{task_id}] Queueing final storage task")
log_task_progress(
task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id,
task_id,
"embed_metadata_into_pdf",
"success",
"Metadata embedded, queuing finalization",
file_id=file_id,
detail=(
f"Metadata embedded into PDF successfully.\n"
f"Original file: {original_file}\n"
@@ -229,7 +234,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
try:
original_file_path = Path(original_file).resolve()
workdir_tmp_resolved = workdir_tmp_path.resolve()
# Check if file is within workdir/tmp and exists
if original_file_path.is_relative_to(workdir_tmp_resolved) and original_file_path.exists():
try:
@@ -245,8 +250,15 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as e:
logger.exception(f"[{task_id}] Failed to embed metadata into {processed_file}: {e}")
log_task_progress(
task_id, "embed_metadata_into_pdf", "failure", f"Exception: {str(e)}", file_id=file_id,
detail=f"Failed to embed metadata into {processed_file}.\nOriginal file: {original_file}\nException: {str(e)}",
task_id,
"embed_metadata_into_pdf",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=(
f"Failed to embed metadata into {processed_file}.\n"
f"Original file: {original_file}\nException: {str(e)}"
),
)
# Clean up temporary file in case of error
if os.path.exists(processed_file):
app/tasks/extract_metadata_with_gpt.py
+24 -10
View File
@@ -112,7 +112,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
content = completion.choices[0].message.content
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
log_task_progress(
task_id, "call_openai", "success", "Received OpenAI response", file_id=file_id,
task_id,
"call_openai",
"success",
"Received OpenAI response",
file_id=file_id,
detail=f"Raw classification response:\n{content}",
)
@@ -120,13 +124,17 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id,
task_id,
"extract_metadata_with_gpt",
"failure",
"Invalid JSON in response",
file_id=file_id,
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
)
return {}
metadata = json.loads(json_text)
# SECURITY: Validate filename format from GPT to prevent path traversal
# The prompt requests filenames with only letters, numbers, periods, and underscores
# Enforce this constraint to prevent malicious filenames
@@ -138,16 +146,18 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
# 1. Potential locale-specific \w behavior
# 2. Files literally named ".." which are valid but problematic
# 3. Future code changes that might relax the regex
if not re.match(r'^[\w\-\. ]+$', suggested_filename) or ".." in suggested_filename:
logger.warning(
f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback"
)
if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
# Reset to empty to trigger fallback to original filename
metadata["filename"] = ""
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress(
task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id,
task_id,
"parse_metadata",
"success",
f"Parsed metadata: {list(metadata.keys())}",
file_id=file_id,
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
)
@@ -164,7 +174,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
except Exception as e:
logger.exception(f"[{task_id}] OpenAI classification failed for {filename}: {e}")
log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", f"Exception: {str(e)}", file_id=file_id,
task_id,
"extract_metadata_with_gpt",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=f"OpenAI classification failed for {filename}.\nException: {str(e)}",
)
return {}
app/tasks/monitor_stalled_steps.py
+4 -7
View File
@@ -34,7 +34,7 @@ def monitor_stalled_steps():
try:
with SessionLocal() as db:
stalled_count = mark_stalled_steps_as_failed(db)
if stalled_count > 0:
logger.warning(
f"[{datetime.utcnow().isoformat()}] "
@@ -42,13 +42,10 @@ def monitor_stalled_steps():
f"Marked as failed due to timeout."
)
else:
logger.debug(
f"[{datetime.utcnow().isoformat()}] "
f"No stalled steps found."
)
logger.debug(f"[{datetime.utcnow().isoformat()}] " f"No stalled steps found.")
return {"recovered": stalled_count}
except Exception as e:
logger.error(f"Error in monitor_stalled_steps task: {e}", exc_info=True)
return {"error": str(e), "recovered": 0}
app/tasks/process_document.py
+23 -16
View File
@@ -24,7 +24,9 @@ logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def process_document(self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False):
def process_document(
self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False
):
"""
Process a document file and trigger appropriate text extraction.
@@ -58,7 +60,10 @@ def process_document(self, original_local_file: str, original_filename: str = No
if not os.path.exists(original_local_file):
logger.error(f"[{task_id}] File {original_local_file} not found.")
log_task_progress(
task_id, "process_document", "failure", "File not found",
task_id,
"process_document",
"failure",
"File not found",
detail=f"File not found on disk: {original_local_file}",
)
return {"error": "File not found"}
@@ -71,7 +76,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
else:
logger.info(f"[{task_id}] Computing file hash (deduplication disabled)...")
filehash = hash_file(original_local_file)
# Use provided original_filename or fall back to basename of path
if original_filename is None:
original_filename = os.path.basename(original_local_file)
@@ -81,7 +86,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
mime_type = "application/octet-stream"
logger.info(f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}")
# Log deduplication step result (only if enabled)
if settings.enable_deduplication:
log_task_progress(
@@ -113,13 +118,15 @@ def process_document(self, original_local_file: str, original_filename: str = No
else:
# Check for duplicate only if this is a new file (not reprocessing)
# IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file
existing = db.query(FileRecord).filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)).order_by(FileRecord.created_at.asc()).first()
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.created_at.asc())
.first()
)
if existing is None:
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash)
.order_by(FileRecord.id.asc())
.first()
db.query(FileRecord).filter(FileRecord.filehash == filehash).order_by(FileRecord.id.asc()).first()
)
# A file is only a duplicate if it matches a different file's hash
@@ -215,11 +222,11 @@ def process_document(self, original_local_file: str, original_filename: str = No
# This copy serves as the permanent, untouched reference of the ingested file
original_dir = os.path.join(settings.workdir, "original")
os.makedirs(original_dir, exist_ok=True)
# Use collision-resistant naming with -0001, -0002 suffixes
base_name = os.path.splitext(new_filename)[0]
original_file_path = get_unique_filepath_with_counter(original_dir, base_name, file_ext)
logger.info(f"[{task_id}] Saving immutable original to: {original_file_path}")
log_task_progress(
task_id,
@@ -236,7 +243,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
f"Original saved: {os.path.basename(original_file_path)}",
file_id=new_record.id,
)
# Update the DB with original_file_path
new_record.original_file_path = original_file_path
else:
@@ -308,7 +315,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
)
process_with_azure_document_intelligence.delay(new_filename, file_id)
return {"file": new_local_path, "status": "Queued for forced OCR", "file_id": file_id}
# If the file is not a PDF, skip embedded text check and convert to PDF first
is_pdf = mime_type == "application/pdf" or os.path.splitext(new_local_path)[1].lower() == ".pdf"
if not is_pdf:
@@ -403,7 +410,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
f"Extracted {len(extracted_text)} characters",
file_id=file_id,
)
# Mark Azure OCR as skipped since we extracted text locally
log_task_progress(
task_id,
@@ -438,7 +445,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
"No embedded text, queuing OCR",
file_id=file_id,
)
# Mark local text extraction as skipped since we're using Azure OCR
log_task_progress(
task_id,
@@ -447,7 +454,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
"No embedded text, using Azure OCR instead",
file_id=file_id,
)
log_task_progress(
task_id,
"process_document",
app/tasks/process_with_azure_document_intelligence.py
+43 -16
View File
@@ -101,8 +101,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
"""
task_id = self.request.id
log_task_progress(
task_id, "process_with_azure_document_intelligence", "in_progress",
f"Starting OCR for {filename}", file_id=file_id,
task_id,
"process_with_azure_document_intelligence",
"in_progress",
f"Starting OCR for {filename}",
file_id=file_id,
)
try:
tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
@@ -117,8 +120,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
)
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(
task_id, "validate_file", "failure",
f"File too large: {filename}", file_id=file_id, detail=error_msg,
task_id,
"validate_file",
"failure",
f"File too large: {filename}",
file_id=file_id,
detail=error_msg,
)
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
@@ -130,8 +137,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(
task_id, "validate_file", "failure",
f"Too many pages: {filename}", file_id=file_id, detail=error_msg,
task_id,
"validate_file",
"failure",
f"Too many pages: {filename}",
file_id=file_id,
detail=error_msg,
)
return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
if page_count is None:
@@ -140,14 +151,20 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
)
log_task_progress(
task_id, "validate_file", "success",
f"File validation passed for {filename}", file_id=file_id,
task_id,
"validate_file",
"success",
f"File validation passed for {filename}",
file_id=file_id,
)
logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.")
log_task_progress(
task_id, "call_azure_ocr", "in_progress",
f"Sending {filename} to Azure Document Intelligence", file_id=file_id,
task_id,
"call_azure_ocr",
"in_progress",
f"Sending {filename} to Azure Document Intelligence",
file_id=file_id,
)
# Open and send the document for processing
@@ -173,8 +190,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters")
log_task_progress(
task_id, "call_azure_ocr", "success",
f"Azure OCR completed for {filename}", file_id=file_id,
task_id,
"call_azure_ocr",
"success",
f"Azure OCR completed for {filename}",
file_id=file_id,
detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected",
)
@@ -182,8 +202,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id)
log_task_progress(
task_id, "process_with_azure_document_intelligence", "success",
f"OCR processing complete for {filename}", file_id=file_id,
task_id,
"process_with_azure_document_intelligence",
"success",
f"OCR processing complete for {filename}",
file_id=file_id,
detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted",
)
@@ -191,7 +214,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
except Exception as e:
logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}")
log_task_progress(
task_id, "process_with_azure_document_intelligence", "failure",
f"OCR failed for {filename}", file_id=file_id, detail=str(e),
task_id,
"process_with_azure_document_intelligence",
"failure",
f"OCR failed for {filename}",
file_id=file_id,
detail=str(e),
)
raise
app/tasks/rotate_pdf_pages.py
+28 -15
View File
@@ -63,8 +63,11 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
try:
task_id = self.request.id
log_task_progress(
task_id, "rotate_pdf_pages", "in_progress",
f"Checking page rotation for {filename}", file_id=file_id,
task_id,
"rotate_pdf_pages",
"in_progress",
f"Checking page rotation for {filename}",
file_id=file_id,
)
pdf_path = os.path.join(settings.workdir, "tmp", filename)
if not os.path.exists(pdf_path):
@@ -72,12 +75,13 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
# Skip rotation if no rotation data provided
if not rotation_data:
logger.info(
f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction"
)
logger.info(f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction")
log_task_progress(
task_id, "rotate_pdf_pages", "success",
"No rotation needed, proceeding to metadata extraction", file_id=file_id,
task_id,
"rotate_pdf_pages",
"success",
"No rotation needed, proceeding to metadata extraction",
file_id=file_id,
)
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"}
@@ -95,16 +99,22 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
f"[{task_id}] No significant rotations detected in {filename}, proceeding with metadata extraction"
)
log_task_progress(
task_id, "rotate_pdf_pages", "success",
"No rotation needed, proceeding to metadata extraction", file_id=file_id,
task_id,
"rotate_pdf_pages",
"success",
"No rotation needed, proceeding to metadata extraction",
file_id=file_id,
)
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"}
logger.info(f"[{task_id}] Rotating {len(normalized_rotation_data)} pages in {filename}")
log_task_progress(
task_id, "apply_rotation", "in_progress",
f"Rotating {len(normalized_rotation_data)} pages", file_id=file_id,
task_id,
"apply_rotation",
"in_progress",
f"Rotating {len(normalized_rotation_data)} pages",
file_id=file_id,
)
applied_rotations = {}
@@ -144,8 +154,7 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
if applied_rotations:
logger.info(
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: "
f"{json.dumps(applied_rotations)}"
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: " f"{json.dumps(applied_rotations)}"
)
else:
logger.info(
@@ -157,7 +166,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
log_task_progress(
task_id, "rotate_pdf_pages", "success",
task_id,
"rotate_pdf_pages",
"success",
f"Rotation complete for {filename}",
file_id=file_id,
detail={"applied_rotations": applied_rotations},
@@ -173,7 +184,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
except Exception as e:
logger.error(f"[{task_id}] Error rotating PDF {filename}: {e}")
log_task_progress(
task_id, "rotate_pdf_pages", "failure",
task_id,
"rotate_pdf_pages",
"failure",
f"Rotation failed: {str(e)}",
file_id=file_id,
detail={"error": str(e), "filename": filename},
app/tasks/upload_to_paperless.py
+11 -4
View File
@@ -274,8 +274,15 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
response_text,
)
log_task_progress(
task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id,
detail=f"Failed to upload document to Paperless.\nFile: {file_path}\nError: {exc}\nResponse: {response_text}",
task_id,
"upload_to_paperless",
"failure",
error_msg,
file_id=file_id,
detail=(
f"Failed to upload document to Paperless.\n"
f"File: {file_path}\nError: {exc}\nResponse: {response_text}"
),
)
raise
@@ -322,9 +329,9 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
# Map each metadata field to its corresponding Paperless custom field
for metadata_field, paperless_field in field_mapping.items():
if metadata_field in metadata and metadata[metadata_field]:
# Convert to string to ensure consistent comparison with UNKNOWN_VALUE
# Convert to string to ensure consistent comparison
value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else ""
if value and value != UNKNOWN_VALUE:
if value and value != METADATA_UNKNOWN_PLACEHOLDER:
custom_fields_to_set[paperless_field] = value
logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'")
except json.JSONDecodeError as e:
app/tasks/upload_with_rclone.py
+1 -3
View File
@@ -103,9 +103,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
except (OSError, ValueError) as e:
error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(
task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg
)
log_task_progress(task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg)
raise RuntimeError(error_msg) from e
app/utils/file_queries.py
+10 -12
View File
@@ -10,7 +10,7 @@ from typing import Optional
from sqlalchemy import or_
from sqlalchemy.orm import Query, Session
from app.models import FileRecord, FileProcessingStep
from app.models import FileProcessingStep, FileRecord
def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Query:
@@ -19,13 +19,13 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
This function modifies a SQLAlchemy query to filter files based on their
processing status by examining associated FileProcessingStep entries.
Only tracks "real" processing steps that represent user-facing status:
- Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence,
extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
send_to_all_destinations
- Upload steps: queue_*, upload_to_*
Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored
as they may not complete properly and don't affect the actual status.
@@ -53,7 +53,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
# Define which steps are "real" status-determining steps
# Only high-level logical steps and actual upload destinations (not queue_* steps)
from app.config import settings
REAL_STEPS = {
"create_file_record",
"check_text",
@@ -75,16 +75,13 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
"upload_to_email",
"upload_to_s3",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_STEPS.add("check_for_duplicates")
# Filter to only real steps
real_steps_subq = (
db.query(FileProcessingStep)
.filter(FileProcessingStep.step_name.in_(REAL_STEPS))
)
real_steps_subq = db.query(FileProcessingStep).filter(FileProcessingStep.step_name.in_(REAL_STEPS))
if status == "pending":
# Files with no real steps (never started processing)
@@ -102,14 +99,15 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
# Files where all real steps are either success or skipped (no failures or in_progress)
# Exclude duplicates from completed
query = query.filter(FileRecord.is_duplicate.is_(False))
# Get files that have real steps
files_with_real_steps = real_steps_subq.distinct().subquery()
# Get files with failures or in_progress on real steps
files_with_issues = (
real_steps_subq
.filter(or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress"))
real_steps_subq.filter(
or_(FileProcessingStep.status == "failure", FileProcessingStep.status == "in_progress")
)
.distinct()
.subquery()
)
app/utils/file_status.py
+4 -4
View File
@@ -7,7 +7,7 @@ from typing import Dict, List
from sqlalchemy.orm import Session
from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.utils.step_manager import get_file_overall_status, get_step_summary
from app.utils.step_manager import get_file_overall_status
def get_file_processing_status(db: Session, file_id: int) -> Dict:
@@ -60,7 +60,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
send_to_all_destinations
- Upload steps: upload_to_*
Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored.
Args:
@@ -78,7 +78,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
# Define which steps are "real" status-determining steps
from app.config import settings
REAL_STEPS = {
"create_file_record",
"check_text",
@@ -100,7 +100,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
"upload_to_email",
"upload_to_s3",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_STEPS.add("check_for_duplicates")
app/utils/filename_utils.py
+7 -7
View File
@@ -73,22 +73,22 @@ def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf")
"""
Returns a unique filepath in the specified directory using a numeric counter suffix.
If 'base_filename.pdf' exists, it will append '-0001', '-0002', etc.
This function implements robust collision handling with zero-padded numeric suffixes
as required for document storage organization.
Args:
directory (str): Directory path where the file will be stored
base_filename (str): Base name for the file (without extension)
extension (str): File extension including the dot (default: ".pdf")
Returns:
str: Full path to a unique filename
Examples:
>>> get_unique_filepath_with_counter("/workdir/original", "2024-01-01_Invoice")
"/workdir/original/2024-01-01_Invoice.pdf" # If doesn't exist
>>> get_unique_filepath_with_counter("/workdir/original", "2024-01-01_Invoice")
"/workdir/original/2024-01-01_Invoice-0001.pdf" # If original exists
"""
@@ -96,7 +96,7 @@ def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf")
candidate = os.path.join(directory, base_filename + extension)
if not os.path.exists(candidate):
return candidate
# If base exists, try with counter suffix
counter = 1
while True:
@@ -106,7 +106,7 @@ def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf")
if not os.path.exists(candidate):
return candidate
counter += 1
# Sanity check to prevent infinite loops (very unlikely to reach)
if counter > 9999:
# Fall back to timestamp + UUID if somehow we have 10000 collisions
app/utils/logging.py
+4 -4
View File
@@ -96,7 +96,7 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
detail=detail,
)
db.add(log_entry)
# Update FileProcessingStep table (for status tracking) if file_id is provided
if file_id and step_name:
# Find or create the step record
@@ -105,9 +105,9 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
.filter(FileProcessingStep.file_id == file_id, FileProcessingStep.step_name == step_name)
.first()
)
now = datetime.utcnow()
if not step_record:
# Create new step record
step_record = FileProcessingStep(
@@ -128,5 +128,5 @@ def log_task_progress(task_id, step_name, status, message=None, file_id=None, de
step_record.completed_at = now
if status == "failure":
step_record.error_message = message or detail
db.commit()
app/utils/migrate_logs_to_steps.py
+1 -4
View File
@@ -6,14 +6,11 @@ for files that were processed before the status tracking table was created.
"""
import logging
from datetime import datetime
from typing import Dict, List
from sqlalchemy import func
from sqlalchemy.orm import Session
from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.utils.step_manager import MAIN_PROCESSING_STEPS
from app.models import FileProcessingStep, ProcessingLog
logger = logging.getLogger(__name__)
app/utils/step_manager.py
+10 -12
View File
@@ -167,7 +167,7 @@ def get_file_step_status(db: Session, file_id: int) -> Dict[str, Dict]:
def get_file_overall_status(db: Session, file_id: int) -> Dict:
"""
Get the overall processing status for a file based on its steps.
Only considers "real" processing steps that represent user-facing status.
Ignores diagnostic/internal steps like poll_task, upload_file, etc.
@@ -211,19 +211,18 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
"finalize_document_storage",
"send_to_all_destinations",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_MAIN_STEPS.add("check_for_duplicates")
all_steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
# Filter to only real steps
steps = [
s for s in all_steps
if s.step_name in REAL_MAIN_STEPS
or s.step_name.startswith("queue_")
or s.step_name.startswith("upload_to_")
s
for s in all_steps
if s.step_name in REAL_MAIN_STEPS or s.step_name.startswith("queue_") or s.step_name.startswith("upload_to_")
]
if not steps:
@@ -297,11 +296,11 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"finalize_document_storage",
"send_to_all_destinations",
}
# Add check_for_duplicates if deduplication is enabled
if settings.enable_deduplication:
REAL_MAIN_STEPS.add("check_for_duplicates")
steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0, "skipped": 0}
@@ -318,7 +317,7 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
# Check if it's an upload task (only count actual upload_to_* steps, not queue_* steps)
is_upload = step.step_name.startswith("upload_to_")
# Only count "real" steps
is_real_step = step.step_name in REAL_MAIN_STEPS or is_upload or step.step_name.startswith("queue_")
@@ -341,4 +340,3 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
"total_main_steps": main_steps_count,
"total_upload_tasks": upload_steps_count,
}
app/utils/step_timeout.py
+2 -5
View File
@@ -32,9 +32,7 @@ def get_step_timeout() -> int:
def mark_stalled_steps_as_failed(
db: Session,
timeout_seconds: Optional[int] = None,
file_id: Optional[int] = None
db: Session, timeout_seconds: Optional[int] = None, file_id: Optional[int] = None
) -> int:
"""
Find and mark any in-progress steps that have exceeded the timeout as failed.
@@ -75,8 +73,7 @@ def mark_stalled_steps_as_failed(
return 0
logger.warning(
f"Found {len(stalled_steps)} stalled step(s) that exceeded "
f"{timeout_seconds}s timeout. Marking as failed."
f"Found {len(stalled_steps)} stalled step(s) that exceeded " f"{timeout_seconds}s timeout. Marking as failed."
)
count = 0
app/views/files.py
+17 -16
View File
@@ -2,13 +2,11 @@
File management views for displaying and managing files.
"""
import os
from typing import Optional
from fastapi import Depends, Query, Request
from sqlalchemy.orm import Session
from app.config import settings
from app.utils.file_queries import apply_status_filter
from app.utils.file_status import get_files_processing_status
from app.views.base import APIRouter, get_db, logger, require_login, templates
@@ -34,9 +32,9 @@ def files_page(
"""
try:
# Import the model here to avoid circular imports
from sqlalchemy import asc, desc, or_
from sqlalchemy import asc, desc
from app.models import FileRecord, ProcessingLog
from app.models import FileRecord
# Start with base query
query = db.query(FileRecord)
@@ -233,9 +231,10 @@ def _compute_processing_flow(logs):
"finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
"send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
}
# Filter out deduplication step if not enabled or if not showing it
from app.config import settings
if not settings.enable_deduplication or not settings.show_deduplication_step:
stages.pop("check_for_duplicates", None)
# Update the next pointer for create_file_record
@@ -354,21 +353,23 @@ def _compute_step_summary(logs):
based on timestamp, regardless of input log ordering.
"""
from app.config import settings
# Count statuses for main processing steps (not uploads)
main_steps = []
if settings.enable_deduplication and settings.show_deduplication_step:
main_steps.append("check_for_duplicates")
main_steps.extend([
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
])
main_steps.extend(
[
"create_file_record",
"check_text",
"extract_text",
"process_with_azure_document_intelligence",
"extract_metadata_with_gpt",
"embed_metadata_into_pdf",
"finalize_document_storage",
"send_to_all_destinations",
]
)
upload_prefixes = ["upload_to_", "queue_"]