diff --git a/app/api/files.py b/app/api/files.py index df66c030..510fba09 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -174,16 +174,11 @@ def get_file_details(request: Request, file_id: int, db: DbSession): file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File record with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found") # Get processing logs logs = ( - db.query(ProcessingLog) - .filter(ProcessingLog.file_id == file_id) - .order_by(ProcessingLog.timestamp.desc()) - .all() + db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all() ) # Build log list @@ -204,13 +199,7 @@ def get_file_details(request: Request, file_id: int, db: DbSession): processing_status = _get_file_processing_status(db, file_id) # Check if files exist on disk - files_on_disk = { - "original": ( - os.path.exists(file_record.local_filename) - if file_record.local_filename - else False - ) - } + files_on_disk = {"original": (os.path.exists(file_record.local_filename) if file_record.local_filename else False)} return { "file": { @@ -220,9 +209,7 @@ def get_file_details(request: Request, file_id: int, db: DbSession): "local_filename": file_record.local_filename, "file_size": file_record.file_size, "mime_type": file_record.mime_type, - "created_at": ( - file_record.created_at.isoformat() if file_record.created_at else None - ), + "created_at": (file_record.created_at.isoformat() if file_record.created_at else None), }, "processing_status": processing_status, "logs": log_list, @@ -239,23 +226,17 @@ def delete_file_record(request: Request, file_id: int, db: DbSession): """ # Check if file deletion is allowed if not settings.allow_file_delete: - raise HTTPException( - status_code=403, detail="File deletion is disabled in the configuration" - ) + raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration") try: # Find the file record file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File record with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found") # Log the deletion - logger.info( - f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}" - ) + logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}") # Delete the record db.delete(file_record) @@ -271,9 +252,7 @@ def delete_file_record(request: Request, file_id: int, db: DbSession): except Exception as e: db.rollback() logger.exception(f"Error deleting file record {file_id}: {str(e)}") - raise HTTPException( - status_code=500, detail=f"Error deleting file record: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error deleting file record: {str(e)}") @router.post("/files/bulk-delete") @@ -285,18 +264,14 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession): """ # Check if file deletion is allowed if not settings.allow_file_delete: - raise HTTPException( - status_code=403, detail="File deletion is disabled in the configuration" - ) + raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration") try: # Find all file records file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() if not file_records: - raise HTTPException( - status_code=404, detail="No files found with the provided IDs" - ) + raise HTTPException(status_code=404, detail="No files found with the provided IDs") deleted_count = len(file_records) deleted_ids = [f.id for f in file_records] @@ -321,9 +296,7 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession): except Exception as e: db.rollback() logger.exception(f"Error bulk deleting file records: {str(e)}") - raise HTTPException( - status_code=500, detail=f"Error bulk deleting file records: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error bulk deleting file records: {str(e)}") @router.post("/files/bulk-reprocess") @@ -337,9 +310,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() if not file_records: - raise HTTPException( - status_code=404, detail="No files found with the provided IDs" - ) + raise HTTPException(status_code=404, detail="No files found with the provided IDs") task_ids = [] processed_files = [] @@ -348,9 +319,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): for file_record in file_records: try: # Check if local file exists - if not file_record.local_filename or not os.path.exists( - file_record.local_filename - ): + if not file_record.local_filename or not os.path.exists(file_record.local_filename): errors.append( { "file_id": file_record.id, @@ -361,9 +330,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): continue # Queue the file for processing, passing file_id to skip duplicate check - task = process_document.delay( - file_record.local_filename, file_id=file_record.id - ) + task = process_document.delay(file_record.local_filename, file_id=file_record.id) task_ids.append(task.id) processed_files.append( { @@ -400,9 +367,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession): raise except Exception as e: logger.exception(f"Error bulk reprocessing files: {str(e)}") - raise HTTPException( - status_code=500, detail=f"Error bulk reprocessing files: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}") @router.post("/files/{file_id}/reprocess") @@ -422,14 +387,10 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession): file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") # Check if local file exists - if not file_record.local_filename or not os.path.exists( - file_record.local_filename - ): + if not file_record.local_filename or not os.path.exists(file_record.local_filename): raise HTTPException( status_code=400, detail="Local file not found on disk. Cannot reprocess.", @@ -458,9 +419,7 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession): raise except Exception as e: logger.exception(f"Error reprocessing file {file_id}: {str(e)}") - raise HTTPException( - status_code=500, detail=f"Error reprocessing file: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error reprocessing file: {str(e)}") @router.post("/files/{file_id}/reprocess-with-cloud-ocr") @@ -484,19 +443,13 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession): file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") # Prefer using the original_file_path if available, otherwise fall back to local_filename source_file = None - if file_record.original_file_path and os.path.exists( - file_record.original_file_path - ): + if file_record.original_file_path and os.path.exists(file_record.original_file_path): source_file = file_record.original_file_path - logger.info( - f"Using original file for Cloud OCR reprocessing: {source_file}" - ) + logger.info(f"Using original file for Cloud OCR reprocessing: {source_file}") elif file_record.local_filename and os.path.exists(file_record.local_filename): source_file = file_record.local_filename logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}") @@ -532,9 +485,7 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession): raise except Exception as e: logger.exception(f"Error reprocessing file {file_id} with Cloud OCR: {str(e)}") - raise HTTPException( - status_code=500, detail=f"Error reprocessing file with Cloud OCR: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error reprocessing file with Cloud OCR: {str(e)}") def _extract_text_from_pdf(file_path: str) -> str: @@ -579,31 +530,19 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - if step_name == "process_document": # Full reprocessing with duplicate check bypass - logger.info( - f"Retrying process_document for file {file_id}: local_filename={file_record.local_filename!r}" - ) + logger.info(f"Retrying process_document for file {file_id}: local_filename={file_record.local_filename!r}") if not file_record.local_filename: - logger.error( - f"process_document retry failed for file {file_id}: local_filename is None" - ) - raise HTTPException( - status_code=400, detail="Local file path is None. Cannot retry." - ) + logger.error(f"process_document retry failed for file {file_id}: local_filename is None") + raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry.") exists = os.path.exists(file_record.local_filename) - logger.info( - f"Checking local_filename: {file_record.local_filename!r}, exists={exists}" - ) + logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}") if not exists: error_message = f"Local file not found on disk. Cannot retry. Path checked: local_filename={file_record.local_filename!r} (exists=False)" - logger.error( - f"process_document retry failed for file {file_id}: {error_message}" - ) + logger.error(f"process_document retry failed for file {file_id}: {error_message}") raise HTTPException(status_code=400, detail=error_message) - logger.info( - f"Found file for process_document retry at: {file_record.local_filename!r}" - ) + logger.info(f"Found file for process_document retry at: {file_record.local_filename!r}") task = process_document.delay( file_record.local_filename, original_filename=file_record.original_filename, @@ -621,14 +560,10 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - ) if not file_record.local_filename: logger.error(f"OCR retry failed for file {file_id}: local_filename is None") - raise HTTPException( - status_code=400, detail="Local file path is None. Cannot retry OCR." - ) + raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry OCR.") exists = os.path.exists(file_record.local_filename) - logger.info( - f"Checking local_filename: {file_record.local_filename!r}, exists={exists}" - ) + logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}") if not exists: error_message = f"Local file not found on disk. Cannot retry OCR. Path checked: local_filename={file_record.local_filename!r} (exists=False)" logger.error(f"OCR retry failed for file {file_id}: {error_message}") @@ -644,28 +579,20 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - f"Retrying extract_metadata_with_gpt for file {file_id}: local_filename={file_record.local_filename!r}" ) if not file_record.local_filename: - logger.error( - f"Metadata extraction retry failed for file {file_id}: local_filename is None" - ) + logger.error(f"Metadata extraction retry failed for file {file_id}: local_filename is None") raise HTTPException( status_code=400, detail="Local file path is None. Cannot retry metadata extraction.", ) exists = os.path.exists(file_record.local_filename) - logger.info( - f"Checking local_filename: {file_record.local_filename!r}, exists={exists}" - ) + logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}") if not exists: error_message = f"Local file not found on disk. Cannot retry metadata extraction. Path checked: local_filename={file_record.local_filename!r} (exists=False)" - logger.error( - f"Metadata extraction retry failed for file {file_id}: {error_message}" - ) + logger.error(f"Metadata extraction retry failed for file {file_id}: {error_message}") raise HTTPException(status_code=400, detail=error_message) - logger.info( - f"Found file for metadata extraction retry at: {file_record.local_filename!r}" - ) + logger.info(f"Found file for metadata extraction retry at: {file_record.local_filename!r}") extracted_text = _extract_text_from_pdf(file_record.local_filename) filename = os.path.basename(file_record.local_filename) task = extract_metadata_with_gpt.delay(filename, extracted_text, file_id) @@ -697,36 +624,24 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - # Check 1: local_filename (original tmp location) if file_record.local_filename: exists = os.path.exists(file_record.local_filename) - checked_paths.append( - f"local_filename={file_record.local_filename!r} (exists={exists})" - ) - logger.info( - f"Checking local_filename: {file_record.local_filename!r}, exists={exists}" - ) + checked_paths.append(f"local_filename={file_record.local_filename!r} (exists={exists})") + logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}") if exists: file_path = file_record.local_filename # Check 2: processed_file_path if not file_path and file_record.processed_file_path: exists = os.path.exists(file_record.processed_file_path) - checked_paths.append( - f"processed_file_path={file_record.processed_file_path!r} (exists={exists})" - ) - logger.info( - f"Checking processed_file_path: {file_record.processed_file_path!r}, exists={exists}" - ) + checked_paths.append(f"processed_file_path={file_record.processed_file_path!r} (exists={exists})") + logger.info(f"Checking processed_file_path: {file_record.processed_file_path!r}, exists={exists}") if exists: file_path = file_record.processed_file_path # Check 3: original_file_path (immutable copy in workdir/original/) if not file_path and file_record.original_file_path: exists = os.path.exists(file_record.original_file_path) - checked_paths.append( - f"original_file_path={file_record.original_file_path!r} (exists={exists})" - ) - logger.info( - f"Checking original_file_path: {file_record.original_file_path!r}, exists={exists}" - ) + checked_paths.append(f"original_file_path={file_record.original_file_path!r} (exists={exists})") + logger.info(f"Checking original_file_path: {file_record.original_file_path!r}, exists={exists}") if exists: file_path = file_record.original_file_path @@ -734,25 +649,17 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - if not file_path and file_record.local_filename: workdir = settings.workdir tmp_dir = os.path.join(workdir, "tmp") - fallback_path = os.path.join( - tmp_dir, os.path.basename(file_record.local_filename) - ) + fallback_path = os.path.join(tmp_dir, os.path.basename(file_record.local_filename)) exists = os.path.exists(fallback_path) - checked_paths.append( - f"workdir_tmp_fallback={fallback_path!r} (exists={exists})" - ) - logger.info( - f"Checking workdir/tmp fallback: {fallback_path!r}, exists={exists}" - ) + checked_paths.append(f"workdir_tmp_fallback={fallback_path!r} (exists={exists})") + logger.info(f"Checking workdir/tmp fallback: {fallback_path!r}, exists={exists}") if exists: file_path = fallback_path if not file_path: paths_detail = "; ".join(checked_paths) error_message = f"File not found on disk. Cannot retry metadata embedding. Paths checked: {paths_detail}" - logger.error( - f"embed_metadata_into_pdf retry failed for file {file_id}: {error_message}" - ) + logger.error(f"embed_metadata_into_pdf retry failed for file {file_id}: {error_message}") raise HTTPException(status_code=400, detail=error_message) logger.info(f"Found file for embed_metadata_into_pdf retry at: {file_path!r}") @@ -760,13 +667,9 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - # Pass the full path to the task so it can locate the file task = extract_metadata_task.delay(file_path, extracted_text, file_id) else: - raise HTTPException( - status_code=400, detail=f"Unsupported pipeline step: {step_name}" - ) + raise HTTPException(status_code=400, detail=f"Unsupported pipeline step: {step_name}") - logger.info( - f"Retrying pipeline step: FileID={file_record.id}, Step={step_name}, TaskID={task.id}" - ) + logger.info(f"Retrying pipeline step: FileID={file_record.id}, Step={step_name}, TaskID={task.id}") return { "status": "success", @@ -807,9 +710,7 @@ def retry_subtask( file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") # Pipeline processing steps that can be retried from the failed step pipeline_step_names = { @@ -892,9 +793,7 @@ def retry_subtask( upload_task = task_map[subtask_name] task = upload_task.delay(file_path, file_id) - logger.info( - f"Retrying upload subtask: FileID={file_record.id}, Subtask={subtask_name}, TaskID={task.id}" - ) + logger.info(f"Retrying upload subtask: FileID={file_record.id}, Subtask={subtask_name}, TaskID={task.id}") return { "status": "success", @@ -907,9 +806,7 @@ def retry_subtask( except HTTPException: raise except Exception as e: - logger.exception( - f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}" - ) + logger.exception(f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}") raise HTTPException(status_code=500, detail=f"Error retrying subtask: {str(e)}") @@ -938,18 +835,12 @@ def get_file_preview( file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") if version == "original": # Return the original file from tmp - if not file_record.local_filename or not os.path.exists( - file_record.local_filename - ): - raise HTTPException( - status_code=404, detail="Original file not found on disk" - ) + if not file_record.local_filename or not os.path.exists(file_record.local_filename): + raise HTTPException(status_code=404, detail="Original file not found on disk") file_path = file_record.local_filename @@ -984,18 +875,14 @@ def get_file_preview( return FileResponse( path=file_path, media_type=file_record.mime_type or "application/pdf", - headers={ - "Content-Disposition": f'inline; filename="{file_record.original_filename}"' - }, + headers={"Content-Disposition": f'inline; filename="{file_record.original_filename}"'}, ) except HTTPException: raise except Exception as e: logger.exception(f"Error retrieving file preview: {str(e)}") - raise HTTPException( - status_code=500, detail=f"Error retrieving file preview: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error retrieving file preview: {str(e)}") @router.get("/files/{file_id}/download") @@ -1023,18 +910,12 @@ def download_file( file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: - raise HTTPException( - status_code=404, detail=f"File with ID {file_id} not found" - ) + raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found") if version == "original": # Return the original file from tmp - if not file_record.local_filename or not os.path.exists( - file_record.local_filename - ): - raise HTTPException( - status_code=404, detail="Original file not found on disk" - ) + if not file_record.local_filename or not os.path.exists(file_record.local_filename): + raise HTTPException(status_code=404, detail="Original file not found on disk") file_path = file_record.local_filename @@ -1069,9 +950,7 @@ def download_file( return FileResponse( path=file_path, media_type=file_record.mime_type or "application/pdf", - headers={ - "Content-Disposition": f'attachment; filename="{file_record.original_filename}"' - }, + headers={"Content-Disposition": f'attachment; filename="{file_record.original_filename}"'}, ) except HTTPException: @@ -1189,9 +1068,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): # Check if file splitting is needed (only for PDFs) from app.utils.file_splitting import should_split_file - should_split = is_pdf and should_split_file( - target_path, settings.max_single_file_size - ) + should_split = is_pdf and should_split_file(target_path, settings.max_single_file_size) if should_split: # File needs to be split before processing @@ -1209,9 +1086,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): task_ids = [] for split_file in split_files: split_filename = os.path.basename(split_file) - task = process_document.delay( - split_file, original_filename=split_filename - ) + task = process_document.delay(split_file, original_filename=split_filename) task_ids.append(task.id) logger.info(f"Enqueued split PDF part for processing: {split_file}") @@ -1229,9 +1104,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): except Exception as e: logger.exception(f"Failed to split file {target_path}: {str(e)}") # Fall back to processing the whole file - logger.warning( - f"Falling back to processing whole file due to split error: {str(e)}" - ) + logger.warning(f"Falling back to processing whole file due to split error: {str(e)}") should_split = False if is_pdf and not should_split: @@ -1239,8 +1112,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): task = process_document.delay(target_path, original_filename=safe_filename) logger.info(f"Enqueued PDF for processing: {target_path}") elif mime_type in IMAGE_MIME_TYPES or any( - file_ext.endswith(ext) - for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"] + file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"] ): # If it's an image, convert to PDF first task = convert_to_pdf.delay(target_path, original_filename=safe_filename) @@ -1267,9 +1139,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): logger.info(f"Enqueued office document for PDF conversion: {target_path}") else: # For any other file type, attempt conversion but log a warning - logger.warning( - f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion" - ) + logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") task = convert_to_pdf.delay(target_path, original_filename=safe_filename) return { diff --git a/app/config.py b/app/config.py index 60824f2e..cdeeefad 100644 --- a/app/config.py +++ b/app/config.py @@ -33,9 +33,7 @@ class Settings(BaseSettings): # Making Paperless optional paperless_ngx_api_token: Optional[str] = None paperless_host: Optional[str] = None - paperless_custom_field_absender: Optional[str] = ( - None # Name of the "absender" custom field in Paperless - ) + paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless # JSON mapping of metadata field names to Paperless custom field names # Example: {"absender": "Sender", "empfaenger": "Recipient", # "language": "Language", "correspondent": "Correspondent"} @@ -115,9 +113,7 @@ class Settings(BaseSettings): sftp_private_key_passphrase: Optional[str] = None # Security: Host key verification is enabled by default for security # In development/testing, set to True to disable verification (not recommended) - sftp_disable_host_key_verification: bool = ( - False # Default enforces host key verification - ) + sftp_disable_host_key_verification: bool = False # Default enforces host key verification # Email settings email_host: Optional[str] = None @@ -125,17 +121,13 @@ class Settings(BaseSettings): email_username: Optional[str] = None email_password: Optional[str] = None email_use_tls: bool = True - email_sender: Optional[str] = ( - None # From address, defaults to email_username if not set - ) + email_sender: Optional[str] = None # From address, defaults to email_username if not set email_default_recipient: Optional[str] = None # OneDrive settings onedrive_client_id: Optional[str] = None onedrive_client_secret: Optional[str] = None - onedrive_tenant_id: Optional[str] = ( - "common" # Default to "common" for personal accounts - ) + onedrive_tenant_id: Optional[str] = "common" # Default to "common" for personal accounts onedrive_refresh_token: Optional[str] = None # Required for personal accounts onedrive_folder_path: Optional[str] = None @@ -153,9 +145,7 @@ class Settings(BaseSettings): uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes # HTTP request settings - http_request_timeout: int = ( - 120 # Default timeout for HTTP requests in seconds (handles large file operations) - ) + http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations) # Feature flags allow_file_delete: bool = True # Default to allowing file deletion from database @@ -175,18 +165,12 @@ class Settings(BaseSettings): default_factory=list, description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)", ) - notify_on_task_failure: bool = Field( - default=True, description="Send notifications when Celery tasks fail" - ) + notify_on_task_failure: bool = Field(default=True, description="Send notifications when Celery tasks fail") notify_on_credential_failure: bool = Field( default=True, description="Send notifications when credential checks fail" ) - notify_on_startup: bool = Field( - default=True, description="Send notifications when application starts" - ) - notify_on_shutdown: bool = Field( - default=False, description="Send notifications when application shuts down" - ) + notify_on_startup: bool = Field(default=True, description="Send notifications when application starts") + notify_on_shutdown: bool = Field(default=False, description="Send notifications when application shuts down") notify_on_file_processed: bool = Field( default=True, description="Send notifications when files are successfully processed", @@ -256,9 +240,7 @@ class Settings(BaseSettings): ) # Content-Security-Policy (CSP) - Controls resource loading - security_header_csp_enabled: bool = Field( - default=True, description="Enable CSP header." - ) + security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.") security_header_csp_value: str = Field( default=( "default-src 'self'; script-src 'self' 'unsafe-inline';" @@ -268,9 +250,7 @@ class Settings(BaseSettings): ) # X-Frame-Options - Prevents clickjacking - security_header_x_frame_options_enabled: bool = Field( - default=True, description="Enable X-Frame-Options header." - ) + security_header_x_frame_options_enabled: bool = Field(default=True, description="Enable X-Frame-Options header.") security_header_x_frame_options_value: str = Field( default="DENY", description="X-Frame-Options header value. Options: DENY, SAMEORIGIN, or ALLOW-FROM uri", @@ -335,9 +315,7 @@ class Settings(BaseSettings): if isinstance(data, dict): for key, value in data.items(): if isinstance(value, str) and len(value) >= 2: - if (value[0] == '"' and value[-1] == '"') or ( - value[0] == "'" and value[-1] == "'" - ): + if (value[0] == '"' and value[-1] == '"') or (value[0] == "'" and value[-1] == "'"): data[key] = value[1:-1] return data @@ -372,9 +350,7 @@ class Settings(BaseSettings): return env_build_date # Then try to get build date from BUILD_DATE file - build_date_file = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE" - ) + build_date_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE") if os.path.exists(build_date_file): with open(build_date_file, "r") as f: return f.read().strip() @@ -391,9 +367,7 @@ class Settings(BaseSettings): return env_version # Then try to get version from VERSION file - version_file = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "VERSION" - ) + version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION") if os.path.exists(version_file): with open(version_file, "r") as f: return f.read().strip() @@ -410,9 +384,7 @@ class Settings(BaseSettings): return env_sha # Then try to get from GIT_SHA file - git_sha_file = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "GIT_SHA" - ) + git_sha_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "GIT_SHA") if os.path.exists(git_sha_file): with open(git_sha_file, "r") as f: return f.read().strip() @@ -423,9 +395,7 @@ class Settings(BaseSettings): @property def runtime_info(self) -> str: """Get runtime information from file.""" - runtime_info_file = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "RUNTIME_INFO" - ) + runtime_info_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "RUNTIME_INFO") if os.path.exists(runtime_info_file): with open(runtime_info_file, "r") as f: return f.read().strip() diff --git a/app/main.py b/app/main.py index 899ba559..6a0151a2 100644 --- a/app/main.py +++ b/app/main.py @@ -41,8 +41,7 @@ if settings.auth_enabled and not settings.session_secret: "Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'" ) SESSION_SECRET = ( - settings.session_secret - or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS" + settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS" ) @@ -81,15 +80,11 @@ async def lifespan(app: FastAPI): len(issues) > 0 for provider, issues in config_issues["storage"].items() ) if has_issues: - logging.warning( - "Application started with configuration issues - some features may be unavailable" - ) + logging.warning("Application started with configuration issues - some features may be unavailable") else: logging.info("Application started with valid configuration") - logging.info( - "Router organization: Using refactored API routers from app/api/ directory" - ) + logging.info("Router organization: Using refactored API routers from app/api/ directory") # Initialize notification system init_apprise() @@ -110,9 +105,7 @@ async def lifespan(app: FastAPI): app = FastAPI(title="DocuElevate", lifespan=lifespan) # Initialize rate limiter and attach to app state -limiter = create_limiter( - redis_url=settings.redis_url, enabled=settings.rate_limiting_enabled -) +limiter = create_limiter(redis_url=settings.redis_url, enabled=settings.rate_limiting_enabled) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, get_rate_limit_exceeded_handler()) @@ -152,9 +145,7 @@ static_dir = pathlib.Path(__file__).parents[1] / "frontend" / "static" if os.path.exists(static_dir): app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") else: - print( - f"WARNING: Static directory not found at {static_dir}. Static files will not be served." - ) + print(f"WARNING: Static directory not found at {static_dir}. Static files will not be served.") # Custom exception handlers that return JSON for API routes and HTML for frontend routes @@ -173,9 +164,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): # Handle 404 errors with a custom template if exc.status_code == 404: - return templates.TemplateResponse( - "404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND - ) + return templates.TemplateResponse("404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND) # For other HTTP errors, we could create specific templates or use a generic one # For now, return a simple error page diff --git a/tests/test_file_upload.py b/tests/test_file_upload.py index 4430feba..70826466 100644 --- a/tests/test_file_upload.py +++ b/tests/test_file_upload.py @@ -42,14 +42,10 @@ def mock_celery_tasks(): class TestValidFileUploads: """Tests for successful file uploads with various valid file types.""" - def test_upload_valid_pdf( - self, client: TestClient, sample_pdf_path: str, mock_celery_tasks - ): + def test_upload_valid_pdf(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks): """Test uploading a valid PDF file.""" with open(sample_pdf_path, "rb") as f: - response = client.post( - "/api/ui-upload", files={"file": ("document.pdf", f, "application/pdf")} - ) + response = client.post("/api/ui-upload", files={"file": ("document.pdf", f, "application/pdf")}) assert response.status_code == 200 data = response.json() @@ -182,9 +178,7 @@ class TestInvalidFileUploads: with patch.object(settings, "max_upload_size", small_limit): response = client.post( "/api/ui-upload", - files={ - "file": ("huge.pdf", io.BytesIO(small_content), "application/pdf") - }, + files={"file": ("huge.pdf", io.BytesIO(small_content), "application/pdf")}, ) assert response.status_code == 413 # Request Entity Too Large @@ -236,9 +230,7 @@ class TestInvalidFileUploads: class TestUploadSecurity: """Tests for security aspects of file uploads.""" - def test_path_traversal_prevention_dotdot( - self, client: TestClient, mock_celery_tasks - ): + def test_path_traversal_prevention_dotdot(self, client: TestClient, mock_celery_tasks): """Test that path traversal attempts are prevented.""" # Try to upload a file with path traversal in filename malicious_filename = "../../etc/passwd.pdf" @@ -246,9 +238,7 @@ class TestUploadSecurity: response = client.post( "/api/ui-upload", - files={ - "file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")}, ) assert response.status_code == 200 @@ -259,18 +249,14 @@ class TestUploadSecurity: assert ".." not in data["stored_filename"] assert "/" not in data["stored_filename"] - def test_path_traversal_prevention_absolute( - self, client: TestClient, mock_celery_tasks - ): + def test_path_traversal_prevention_absolute(self, client: TestClient, mock_celery_tasks): """Test that absolute path attempts are prevented.""" malicious_filename = "/etc/shadow.pdf" pdf_content = b"%PDF-1.4\n%EOF" response = client.post( "/api/ui-upload", - files={ - "file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")}, ) assert response.status_code == 200 @@ -280,18 +266,14 @@ class TestUploadSecurity: assert data["original_filename"] == "shadow.pdf" assert not data["stored_filename"].startswith("/") - def test_filename_with_special_characters( - self, client: TestClient, mock_celery_tasks - ): + def test_filename_with_special_characters(self, client: TestClient, mock_celery_tasks): """Test handling of filenames with special characters.""" special_filename = "file name with spaces & special!@#chars.pdf" pdf_content = b"%PDF-1.4\n%EOF" response = client.post( "/api/ui-upload", - files={ - "file": (special_filename, io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": (special_filename, io.BytesIO(pdf_content), "application/pdf")}, ) assert response.status_code == 200 @@ -307,9 +289,7 @@ class TestUploadSecurity: # Stored filename should have UUID and extension assert data["stored_filename"].endswith(".pdf") - def test_path_traversal_prevention_windows_style( - self, client: TestClient, mock_celery_tasks - ): + def test_path_traversal_prevention_windows_style(self, client: TestClient, mock_celery_tasks): """Test that Windows-style path traversal attempts are prevented.""" # Try Windows-style path with backslashes malicious_filename = "..\\..\\..\\windows\\system32\\config.pdf" @@ -317,9 +297,7 @@ class TestUploadSecurity: response = client.post( "/api/ui-upload", - files={ - "file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")}, ) assert response.status_code == 200 @@ -338,18 +316,14 @@ class TestUploadSecurity: assert ".." not in data["original_filename"] assert "/" not in data["original_filename"] - def test_path_traversal_prevention_mixed_separators( - self, client: TestClient, mock_celery_tasks - ): + def test_path_traversal_prevention_mixed_separators(self, client: TestClient, mock_celery_tasks): """Test handling of filenames with mixed path separators.""" malicious_filename = "../path\\to/file.pdf" pdf_content = b"%PDF-1.4\n%EOF" response = client.post( "/api/ui-upload", - files={ - "file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")}, ) assert response.status_code == 200 @@ -376,9 +350,7 @@ class TestUploadErrorHandling: response = client.post( "/api/ui-upload", - files={ - "file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf")}, ) assert response.status_code == 500 @@ -386,9 +358,7 @@ class TestUploadErrorHandling: def test_upload_celery_task_failure(self, client: TestClient, mock_celery_tasks): """Test handling when Celery task queueing fails.""" - mock_celery_tasks["process_document"].side_effect = Exception( - "Celery connection failed" - ) + mock_celery_tasks["process_document"].side_effect = Exception("Celery connection failed") pdf_content = b"%PDF-1.4\n%EOF" @@ -397,9 +367,7 @@ class TestUploadErrorHandling: with pytest.raises(Exception): client.post( "/api/ui-upload", - files={ - "file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf") - }, + files={"file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf")}, ) @@ -440,19 +408,14 @@ class TestUploadFilenameHandling: response = client.post( "/api/ui-upload", - files={ - "file": ("NOEXTENSION", io.BytesIO(content), "application/octet-stream") - }, + files={"file": ("NOEXTENSION", io.BytesIO(content), "application/octet-stream")}, ) assert response.status_code == 200 data = response.json() assert data["original_filename"] == "NOEXTENSION" # Stored filename should be just the UUID without extension - assert ( - "." not in data["stored_filename"] - or data["stored_filename"].count(".") == 0 - ) + assert "." not in data["stored_filename"] or data["stored_filename"].count(".") == 0 @pytest.mark.integration @@ -465,9 +428,7 @@ class TestUploadMimeTypeDetection: response = client.post( "/api/ui-upload", - files={ - "file": ("doc.pdf", io.BytesIO(pdf_content), "application/octet-stream") - }, + files={"file": ("doc.pdf", io.BytesIO(pdf_content), "application/octet-stream")}, ) assert response.status_code == 200 @@ -499,9 +460,7 @@ class TestUploadMimeTypeDetection: class TestFileSplitting: """Tests for file splitting functionality when MAX_SINGLE_FILE_SIZE is configured.""" - def test_pdf_splitting_when_configured( - self, client: TestClient, sample_pdf_path: str, mock_celery_tasks - ): + def test_pdf_splitting_when_configured(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks): """Test that PDFs are split when they exceed MAX_SINGLE_FILE_SIZE.""" from app.config import settings @@ -516,9 +475,7 @@ class TestFileSplitting: ] # Mock should_split_file to return True - with patch( - "app.utils.file_splitting.should_split_file", return_value=True - ): + with patch("app.utils.file_splitting.should_split_file", return_value=True): with open(sample_pdf_path, "rb") as f: response = client.post( "/api/ui-upload", @@ -539,9 +496,7 @@ class TestFileSplitting: # Verify each split file was queued for processing assert mock_celery_tasks["process_document"].call_count == 3 - def test_no_splitting_when_not_configured( - self, client: TestClient, sample_pdf_path: str, mock_celery_tasks - ): + def test_no_splitting_when_not_configured(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks): """Test that PDFs are not split when MAX_SINGLE_FILE_SIZE is None.""" from app.config import settings @@ -564,9 +519,7 @@ class TestFileSplitting: # Verify file was processed directly without splitting mock_celery_tasks["process_document"].assert_called_once() - def test_no_splitting_for_small_files( - self, client: TestClient, sample_pdf_path: str, mock_celery_tasks - ): + def test_no_splitting_for_small_files(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks): """Test that small PDFs are not split even when MAX_SINGLE_FILE_SIZE is configured.""" from app.config import settings @@ -588,9 +541,7 @@ class TestFileSplitting: # Verify file was processed directly mock_celery_tasks["process_document"].assert_called_once() - def test_splitting_fallback_on_error( - self, client: TestClient, sample_pdf_path: str, mock_celery_tasks - ): + def test_splitting_fallback_on_error(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks): """Test that if splitting fails, the file is processed as a whole.""" from app.config import settings diff --git a/tests/test_request_size_limit.py b/tests/test_request_size_limit.py index 72e60af9..99827f57 100644 --- a/tests/test_request_size_limit.py +++ b/tests/test_request_size_limit.py @@ -93,9 +93,7 @@ class TestRequestSizeLimitMiddleware: # May get 200/401/403 but not 413 assert response.status_code != 413 - def test_middleware_error_message_contains_limit_and_config_hint( - self, client: TestClient - ): + def test_middleware_error_message_contains_limit_and_config_hint(self, client: TestClient): """413 response body contains limit details and config variable name.""" from app.config import settings @@ -132,9 +130,7 @@ class TestFileUploadSizeLimitStreaming: mock_conv.delay.return_value = task yield - def test_upload_rejected_when_content_length_declared_too_large( - self, client: TestClient - ): + def test_upload_rejected_when_content_length_declared_too_large(self, client: TestClient): """Upload is rejected early via Content-Length check before reading data.""" from app.config import settings @@ -150,9 +146,7 @@ class TestFileUploadSizeLimitStreaming: ) assert response.status_code == 413 - def test_upload_rejected_mid_stream_when_data_exceeds_limit( - self, client: TestClient - ): + def test_upload_rejected_mid_stream_when_data_exceeds_limit(self, client: TestClient): """Upload is rejected mid-stream when actual data exceeds max_upload_size.""" from app.config import settings @@ -162,9 +156,7 @@ class TestFileUploadSizeLimitStreaming: large_content = b"x" * (small_limit + 1) response = client.post( "/api/ui-upload", - files={ - "file": ("big.pdf", io.BytesIO(large_content), "application/pdf") - }, + files={"file": ("big.pdf", io.BytesIO(large_content), "application/pdf")}, ) assert response.status_code == 413 assert "too large" in response.json()["detail"].lower()