""" File management views for displaying and managing files. """ from datetime import datetime, timezone from typing import Optional from fastapi import Depends, HTTPException, Query, Request from sqlalchemy.orm import Session from app.utils.cache import cache_get, cache_set from app.utils.file_queries import apply_status_filter from app.utils.file_status import get_files_processing_status from app.views.base import APIRouter, get_db, logger, require_login, templates router = APIRouter() # Error message constants _FILE_NOT_FOUND = "File not found" @router.get("/files") @require_login def files_page( request: Request, db: Session = Depends(get_db), page: int = Query(1, ge=1), per_page: int = Query(25, ge=1, le=200), sort_by: str = Query("created_at"), sort_order: str = Query("desc"), search: Optional[str] = Query(None), mime_type: Optional[str] = Query(None), status: Optional[str] = Query(None), date_from: Optional[str] = Query(None), date_to: Optional[str] = Query(None), storage_provider: Optional[str] = Query(None), tags: Optional[str] = Query(None), ocr_quality: Optional[str] = Query(None), ): """ Return the 'files.html' template with server-side pagination, sorting, and filtering """ from app.config import settings try: # Import the model here to avoid circular imports from sqlalchemy import asc, desc from app.models import FileProcessingStep, FileRecord # Start with base query query = db.query(FileRecord) # Apply search filter if search: query = query.filter(FileRecord.original_filename.ilike(f"%{search}%")) # Apply MIME type filter if mime_type: query = query.filter(FileRecord.mime_type == mime_type) # Apply date range filters if date_from: try: dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc) query = query.filter(FileRecord.created_at >= dt_from) except ValueError: pass # Silently ignore invalid dates in view if date_to: try: dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc) query = query.filter(FileRecord.created_at <= dt_to) except ValueError: pass # Apply storage provider filter if storage_provider: step_name = f"upload_to_{storage_provider}" uploaded_file_ids = ( db.query(FileProcessingStep.file_id) .filter( FileProcessingStep.step_name == step_name, FileProcessingStep.status == "success", ) .distinct() .subquery() ) query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id))) # Apply tags filter (AND logic) if tags: tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()] for tag in tag_list: # Escape SQL LIKE wildcards to prevent unintended pattern matching escaped_tag = tag.replace("%", r"\%").replace("_", r"\_") query = query.filter(FileRecord.ai_metadata.ilike(f"%{escaped_tag}%")) # Apply OCR quality filter if ocr_quality == "poor": # Files scored below the configured threshold threshold = settings.text_quality_threshold query = query.filter( FileRecord.ocr_quality_score.isnot(None), FileRecord.ocr_quality_score < threshold, ) elif ocr_quality == "good": threshold = settings.text_quality_threshold query = query.filter( FileRecord.ocr_quality_score.isnot(None), FileRecord.ocr_quality_score >= threshold, ) elif ocr_quality == "unchecked": query = query.filter(FileRecord.ocr_quality_score.is_(None)) # Apply status filter (before pagination for correct counts) query = apply_status_filter(query, db, status) # Get total count before pagination total_items = query.count() # Apply sorting sort_column = { "id": FileRecord.id, "original_filename": FileRecord.original_filename, "file_size": FileRecord.file_size, "mime_type": FileRecord.mime_type, "created_at": FileRecord.created_at, }.get(sort_by, FileRecord.created_at) if sort_order == "asc": query = query.order_by(asc(sort_column)) else: query = query.order_by(desc(sort_column)) # Apply pagination offset = (page - 1) * per_page files = query.offset(offset).limit(per_page).all() # Get processing status for all files efficiently (avoids N+1) file_ids = [f.id for f in files] statuses = get_files_processing_status(db, file_ids) # Add status to each file files_with_status = [] for file in files: file.processing_status = statuses.get(file.id, {}).get("status", "pending") files_with_status.append(file) # Calculate pagination info total_pages = (total_items + per_page - 1) // per_page # Get unique MIME types for filter dropdown (cached) mime_types = cache_get("mime_types") if mime_types is None: raw = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all() mime_types = [mt[0] for mt in raw if mt[0]] cache_set("mime_types", mime_types, ttl=120) # Debug output logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})") return templates.TemplateResponse( "files.html", { "request": request, "files": files_with_status, "pagination": { "page": page, "per_page": per_page, "total": total_items, "pages": total_pages, }, "sort_by": sort_by, "sort_order": sort_order, "search": search or "", "mime_type": mime_type or "", "status": status or "", "date_from": date_from or "", "date_to": date_to or "", "storage_provider": storage_provider or "", "tags": tags or "", "ocr_quality": ocr_quality or "", "ocr_quality_threshold": settings.text_quality_threshold, "mime_types": mime_types, "upload_concurrency": settings.upload_concurrency, "upload_queue_delay_ms": settings.upload_queue_delay_ms, }, ) except Exception as e: # Log any errors logger.error(f"Error retrieving files: {str(e)}") # Return error message to template return templates.TemplateResponse( "files.html", { "request": request, "files": [], "pagination": {"page": 1, "per_page": per_page, "total": 0, "pages": 0}, "error": str(e), "upload_concurrency": settings.upload_concurrency, "upload_queue_delay_ms": settings.upload_queue_delay_ms, }, ) @router.get("/files/{file_id}") @require_login def file_view_page(request: Request, file_id: int, db: Session = Depends(get_db)): """ Return the document view page — document-centric view with metadata, preview, and extracted text. Process-oriented details are available via /files/{file_id}/detail. """ try: import json import os from app.models import FileRecord file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: return templates.TemplateResponse( "file_view.html", {"request": request, "file": None, "error": f"File with ID {file_id} not found"}, ) from app.config import settings # Resolve the expected base directory to guard against path traversal in DB values. workdir = os.path.realpath(settings.workdir) def _safe_exists(path: str | None) -> bool: """Return True only when *path* exists and resides within workdir.""" if not path: return False resolved = os.path.realpath(path) try: common = os.path.commonpath([resolved, workdir]) except ValueError: return False return common == workdir and os.path.exists(resolved) # Check whether the backing files exist on disk original_file_exists = _safe_exists(file_record.original_file_path) processed_file_exists = _safe_exists(file_record.processed_file_path) # Load AI metadata — JSON sidecar file first, then DB column gpt_metadata = None if file_record.processed_file_path: metadata_path = os.path.splitext(os.path.realpath(file_record.processed_file_path))[0] + ".json" if _safe_exists(metadata_path): try: with open(metadata_path, "r", encoding="utf-8") as f: gpt_metadata = json.load(f) except Exception as e: logger.warning(f"Failed to load metadata sidecar for file {file_id}: {e}") if gpt_metadata is None and file_record.ai_metadata: try: gpt_metadata = json.loads(file_record.ai_metadata) except Exception as e: logger.warning(f"Failed to parse ai_metadata for file {file_id}: {e}") # Quick processing status (no logs needed) try: from app.utils.step_manager import get_step_summary as _get_step_summary step_summary = _get_step_summary(db, file_id) except Exception: step_summary = None # Resolve the pipeline assigned to this file (explicit or system default) pipeline_info = _resolve_pipeline(db, file_record) return templates.TemplateResponse( "file_view.html", { "request": request, "file": file_record, "gpt_metadata": gpt_metadata, "original_file_exists": original_file_exists, "processed_file_exists": processed_file_exists, "step_summary": step_summary, "pipeline_info": pipeline_info, }, ) except Exception as e: logger.error(f"Error retrieving file view {file_id}: {str(e)}") return templates.TemplateResponse("file_view.html", {"request": request, "file": None, "error": str(e)}) @router.get("/files/{file_id}/detail") @require_login def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_db)): """ Return the file detail page showing processing history and file information """ try: import json import os from app.models import FileRecord, ProcessingLog # Find the file record file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: return templates.TemplateResponse( "file_detail.html", {"request": request, "file": None, "error": f"File with ID {file_id} not found"} ) # Get processing logs logs = ( db.query(ProcessingLog) .filter(ProcessingLog.file_id == file_id) .order_by(ProcessingLog.timestamp.asc()) .all() ) # Check if original file exists (use persisted path from database) original_file_exists = False if file_record.original_file_path and os.path.exists(file_record.original_file_path): original_file_exists = True # Check if processed file exists (use persisted path from database) processed_file_exists = False if file_record.processed_file_path and os.path.exists(file_record.processed_file_path): processed_file_exists = True # Load metadata from JSON file if it exists gpt_metadata = None if file_record.processed_file_path: # Metadata JSON file is stored alongside the processed PDF metadata_path = os.path.splitext(file_record.processed_file_path)[0] + ".json" if os.path.exists(metadata_path): try: with open(metadata_path, "r", encoding="utf-8") as f: gpt_metadata = json.load(f) logger.debug(f"Loaded GPT metadata from {metadata_path}") except Exception as e: logger.warning(f"Failed to load metadata from {metadata_path}: {e}") # Resolve the pipeline assigned to this file (explicit or system default) pipeline_info = _resolve_pipeline(db, file_record) # Compute processing flow for visualization — filter to pipeline steps when available flow_data = _compute_processing_flow(logs, pipeline_steps=pipeline_info["steps"] if pipeline_info else None) # Compute step-aligned summary from status table (preferred) or fallback to logs try: from app.utils.step_manager import get_step_summary as get_step_summary_from_table step_summary = get_step_summary_from_table(db, file_id) except Exception: # Fallback to log-based computation if status table not available step_summary = _compute_step_summary(logs) return templates.TemplateResponse( "file_detail.html", { "request": request, "file": file_record, "logs": logs, "original_file_exists": original_file_exists, "processed_file_exists": processed_file_exists, "gpt_metadata": gpt_metadata, "flow_data": flow_data, "step_summary": step_summary, "pipeline_info": pipeline_info, }, ) except Exception as e: logger.error(f"Error retrieving file details: {str(e)}") return templates.TemplateResponse("file_detail.html", {"request": request, "file": None, "error": str(e)}) # --------------------------------------------------------------------------- # Pipeline ↔ Celery-log stage mapping # --------------------------------------------------------------------------- # Maps each pipeline step_type to the set of Celery task log stage keys that # implement it. Used to filter the flow visualization when a pipeline is # assigned to a file. # # ⚠️ MAINTENANCE NOTE: When a new step type is added to PIPELINE_STEP_TYPES # in app/api/pipelines.py it MUST also be added here, otherwise the flow # visualization will silently skip its Celery-task stages for files using that # step type. The test ``TestPipelineInfoInViews::test_step_type_mapping_is_complete`` # enforces this invariant automatically. _STEP_TYPE_TO_STAGES: dict[str, list[str]] = { "convert_to_pdf": ["convert_to_pdf"], "check_duplicates": ["check_for_duplicates"], "ocr": ["check_text", "extract_text", "process_with_ocr"], "extract_metadata": ["extract_metadata_with_gpt"], "embed_metadata": ["embed_metadata_into_pdf"], "compute_embedding": ["compute_embedding"], "send_to_destinations": ["finalize_document_storage", "send_to_all_destinations"], # "classify" is defined in PIPELINE_STEP_TYPES but has no Celery log stages yet. # When a classify task is implemented, add its stage key(s) here. "classify": [], } # These internal bookkeeping stages are always shown in the flow regardless of # which pipeline steps are defined. _ALWAYS_SHOW_STAGES: frozenset[str] = frozenset({"create_file_record"}) def _resolve_pipeline(db: Session, file_record) -> dict | None: """Resolve the pipeline information for a file. If the file has an explicit ``pipeline_id``, load that pipeline. Otherwise fall back to the active system-default pipeline (``owner_id IS NULL``, ``is_default=True``). Returns a dict with keys: id, name, description, is_default, is_system, is_explicit, steps or ``None`` when no pipeline exists in the database. """ from app.models import Pipeline, PipelineStep pipeline = None if file_record.pipeline_id: pipeline = db.query(Pipeline).filter(Pipeline.id == file_record.pipeline_id).first() if pipeline is None: pipeline = ( db.query(Pipeline) .filter( Pipeline.owner_id.is_(None), Pipeline.is_default.is_(True), Pipeline.is_active.is_(True), ) .first() ) if pipeline is None: return None steps = db.query(PipelineStep).filter(PipelineStep.pipeline_id == pipeline.id).order_by(PipelineStep.position).all() return { "id": pipeline.id, "name": pipeline.name, "description": pipeline.description, "is_default": pipeline.is_default, "is_system": pipeline.owner_id is None, # True when the file has a pipeline explicitly assigned (not inferred default) "is_explicit": bool(file_record.pipeline_id), "steps": steps, } def _compute_processing_flow(logs, pipeline_steps=None): """ Compute the processing flow structure from logs for visualization. Returns a structured representation of the processing pipeline with branches. Detects upload sub-tasks and organizes them as branches under the parent upload stage. Args: logs: list of ProcessingLog objects (ordered by timestamp asc) pipeline_steps: optional list of PipelineStep objects for the assigned pipeline. When provided, the set of stages shown is filtered to only those that correspond to the pipeline's enabled steps (plus bookkeeping stages like ``create_file_record`` and any stage that actually ran in the logs). """ # Define the full catalogue of main processing stages stages = { "convert_to_pdf": {"label": "Convert to PDF", "next": ["check_for_duplicates", "create_file_record"]}, "check_for_duplicates": {"label": "Check for Duplicates", "next": ["create_file_record"]}, "create_file_record": {"label": "Create File Record", "next": ["check_text"]}, "check_text": { "label": "Check Embedded Text", "next": ["extract_text", "process_with_ocr"], }, "extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]}, "process_with_ocr": { "label": "OCR Processing", "next": ["extract_metadata_with_gpt"], }, "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]}, "embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]}, "finalize_document_storage": { "label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations", "compute_embedding"], }, "send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True}, "compute_embedding": {"label": "Compute Embedding", "next": []}, } # Filter out deduplication step if not enabled or if not showing it from app.config import settings if not settings.enable_deduplication or not settings.show_deduplication_step: stages.pop("check_for_duplicates", None) # Update the next pointer for create_file_record if "create_file_record" in stages: stages["create_file_record"]["next"] = ["check_text"] # When a pipeline is assigned, filter stages to only those relevant to the # pipeline's enabled steps plus always-show bookkeeping stages and any stage # that actually produced log entries (so nothing already-run is hidden). if pipeline_steps is not None: # Collect Celery stage keys that the pipeline's enabled steps map to allowed: set[str] = set(_ALWAYS_SHOW_STAGES) for ps in pipeline_steps: if ps.enabled: allowed.update(_STEP_TYPE_TO_STAGES.get(ps.step_type, [])) # Pre-scan logs so we can also keep any stage that already ran ran_stages: set[str] = set() for log in logs: ran_stages.add(log.step_name) allowed.update(ran_stages) stages = {k: v for k, v in stages.items() if k in allowed} # Define upload sub-tasks (branches) upload_tasks = { "upload_to_dropbox": "Dropbox", "upload_to_nextcloud": "Nextcloud", "upload_to_paperless": "Paperless-ngx", "upload_to_google_drive": "Google Drive", "upload_to_onedrive": "OneDrive", "upload_to_s3": "S3 Storage", "upload_to_webdav": "WebDAV", "upload_to_ftp": "FTP Storage", "upload_to_sftp": "SFTP Storage", "upload_to_email": "Email", "queue_dropbox": "Dropbox", "queue_nextcloud": "Nextcloud", "queue_paperless": "Paperless-ngx", "queue_google_drive": "Google Drive", "queue_onedrive": "OneDrive", "queue_s3": "S3 Storage", "queue_webdav": "WebDAV", "queue_ftp": "FTP Storage", "queue_sftp": "SFTP Storage", "queue_email": "Email", } # Create a map of step names to their log entries step_map = {} upload_branches = {} for log in logs: step_name = log.step_name # Check if this is an upload sub-task if step_name in upload_tasks: # Extract the actual upload task name (remove queue_ prefix if present) upload_key = step_name.replace("queue_", "upload_to_") if upload_key not in upload_branches: upload_branches[upload_key] = [] upload_branches[upload_key].append( {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id} ) else: # Normalize legacy OCR step name for backward compatibility with old log entries if step_name == "process_with_azure_document_intelligence": step_name = "process_with_ocr" # Regular processing step if step_name not in step_map: step_map[step_name] = [] step_map[step_name].append( {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id} ) # Build the flow structure flow = [] for stage_key, stage_info in stages.items(): stage_logs = step_map.get(stage_key, []) # Determine overall status for this stage if stage_logs: latest_log = stage_logs[-1] status = latest_log["status"] message = latest_log["message"] timestamp = latest_log["timestamp"] task_id = latest_log["task_id"] else: status = "not_run" message = None timestamp = None task_id = None stage_data = { "key": stage_key, "label": stage_info["label"], "status": status, "message": message, "timestamp": timestamp, "task_id": task_id, "can_retry": status == "failure", "is_branch_parent": stage_info.get("has_branches", False), } # If this is the upload stage, add branches if stage_info.get("has_branches") and upload_branches: branches = [] for upload_key, upload_logs in upload_branches.items(): latest_upload = upload_logs[-1] upload_name = upload_tasks.get(upload_key, upload_key.replace("upload_to_", "").title()) branches.append( { "key": upload_key, "label": upload_name, "status": latest_upload["status"], "message": latest_upload["message"], "timestamp": latest_upload["timestamp"], "task_id": latest_upload["task_id"], "can_retry": latest_upload["status"] == "failure", } ) stage_data["branches"] = branches flow.append(stage_data) return flow def _compute_step_summary(logs): """ Compute a step-aligned summary from logs showing queued, success, and failure counts. Returns a dictionary with main step counts and upload branch counts. Note: This function is order-independent - it selects the latest status per step based on timestamp, regardless of input log ordering. """ from app.config import settings # Count statuses for main processing steps (not uploads) main_steps = [] if settings.enable_deduplication and settings.show_deduplication_step: main_steps.append("check_for_duplicates") main_steps.extend( [ "create_file_record", "check_text", "extract_text", "process_with_ocr", "extract_metadata_with_gpt", "embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations", ] ) upload_prefixes = ["upload_to_", "queue_"] main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0} upload_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0} # Track latest status for each step by comparing timestamps (order-independent) main_steps_seen = {} # {step_name: (timestamp, status)} upload_tasks_seen = {} # {step_name: (timestamp, status)} for log in logs: step_name = log.step_name status = log.status.lower() # Normalize status if status == "pending": status = "queued" # Normalize legacy OCR step name for backward compatibility if step_name == "process_with_azure_document_intelligence": step_name = "process_with_ocr" # Check if it's an upload task is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes) if is_upload: # Track latest status for each unique upload task by timestamp if step_name not in upload_tasks_seen or log.timestamp > upload_tasks_seen[step_name][0]: upload_tasks_seen[step_name] = (log.timestamp, status) elif step_name in main_steps: # Track latest status for main steps by timestamp if step_name not in main_steps_seen or log.timestamp > main_steps_seen[step_name][0]: main_steps_seen[step_name] = (log.timestamp, status) # Count main step statuses from latest status per step for _, task_status in main_steps_seen.values(): if task_status in main_counts: main_counts[task_status] += 1 # Count upload task statuses from latest status per task for _, task_status in upload_tasks_seen.values(): if task_status in upload_counts: upload_counts[task_status] += 1 return { "main": main_counts, "uploads": upload_counts, "total_main_steps": len(main_steps_seen), "total_upload_tasks": len(upload_tasks_seen), } @router.get("/files/{file_id}/preview/original") @require_login def preview_original_file(request: Request, file_id: int, db: Session = Depends(get_db)): """ Serve the original (pre-processing) PDF file for preview """ import os from fastapi import status from fastapi.responses import FileResponse from app.models import FileRecord file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) if not file_record.original_file_path or not os.path.exists(file_record.original_file_path): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Original file not found on disk") return FileResponse( path=file_record.original_file_path, media_type="application/pdf", headers={"Content-Disposition": "inline"}, ) @router.get("/files/{file_id}/preview/processed") @require_login def preview_processed_file(request: Request, file_id: int, db: Session = Depends(get_db)): """ Serve the processed (with embedded metadata) PDF file for preview """ import os from fastapi import status from fastapi.responses import FileResponse from app.models import FileRecord file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) if not file_record.processed_file_path or not os.path.exists(file_record.processed_file_path): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Processed file not found on disk") return FileResponse( path=file_record.processed_file_path, media_type="application/pdf", headers={"Content-Disposition": "inline"}, ) @router.get("/files/{file_id}/text/original") @require_login def get_original_text(request: Request, file_id: int, db: Session = Depends(get_db)): """ Extract and return text from the original PDF file on-demand """ import os from fastapi import status from fastapi.responses import JSONResponse from app.models import FileRecord file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) if not file_record.original_file_path or not os.path.exists(file_record.original_file_path): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Original file not found on disk") try: # Extract text from PDF using pypdf from pypdf import PdfReader # Upgraded from PyPDF2 to fix CVE-2023-36464 reader = PdfReader(file_record.original_file_path) text = "" for page in reader.pages: text += page.extract_text() + "\n\n" if not text.strip(): text = "(No text could be extracted from this PDF - it may be a scanned image without OCR)" return JSONResponse(content={"text": text.strip(), "page_count": len(reader.pages)}) except Exception as e: logger.error(f"Error extracting text from original file {file_id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to extract text: {str(e)}" ) @router.get("/files/{file_id}/text/processed") @require_login def get_processed_text(request: Request, file_id: int, db: Session = Depends(get_db)): """ Extract and return text from the processed PDF file on-demand """ import os from fastapi import status from fastapi.responses import JSONResponse from app.models import FileRecord file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) if not file_record.processed_file_path or not os.path.exists(file_record.processed_file_path): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Processed file not found on disk") try: # Extract text from PDF using pypdf from pypdf import PdfReader # Upgraded from PyPDF2 to fix CVE-2023-36464 reader = PdfReader(file_record.processed_file_path) text = "" for page in reader.pages: text += page.extract_text() + "\n\n" if not text.strip(): text = "(No text could be extracted from this PDF - it may be a scanned image without OCR)" return JSONResponse(content={"text": text.strip(), "page_count": len(reader.pages)}) except Exception as e: logger.error(f"Error extracting text from processed file {file_id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to extract text: {str(e)}" ) @router.get("/files/{file_id}/text/default-language") @require_login def get_default_language_text(request: Request, file_id: int, db: Session = Depends(get_db)): """Return the persisted default-language translation for the file view.""" from fastapi import status from fastapi.responses import JSONResponse from app.models import FileRecord file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_FILE_NOT_FOUND) if not file_record.default_language_text: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No default-language translation available", ) return JSONResponse( content={ "text": file_record.default_language_text, "language_code": file_record.default_language_code, "detected_language": file_record.detected_language, } ) @router.get("/duplicates") @require_login def duplicates_page( request: Request, db: Session = Depends(get_db), page: int = Query(1, ge=1), per_page: int = Query(25, ge=1, le=200), ): """Render the duplicate-document management page. Passes exact-duplicate group data (server-side) plus the configured near-duplicate threshold so the JS finder can pre-populate the form. """ from app.config import settings from app.models import FileRecord try: # Find hashes that have at least one is_duplicate=True record dup_hashes_query = db.query(FileRecord.filehash).filter(FileRecord.is_duplicate.is_(True)).distinct() total_groups = dup_hashes_query.count() offset = (page - 1) * per_page dup_hashes = [row.filehash for row in dup_hashes_query.offset(offset).limit(per_page).all()] groups = [] total_duplicate_files = 0 for filehash in dup_hashes: original = ( db.query(FileRecord) .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)) .order_by(FileRecord.id.asc()) .first() ) duplicates = ( db.query(FileRecord) .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True)) .order_by(FileRecord.id.asc()) .all() ) total_duplicate_files += len(duplicates) def _to_dict(f: FileRecord) -> dict: return { "id": f.id, "original_filename": f.original_filename, "filehash": f.filehash, "file_size": f.file_size, "mime_type": f.mime_type, "is_duplicate": f.is_duplicate, "duplicate_of_id": f.duplicate_of_id, "created_at": f.created_at.isoformat() if f.created_at else None, } groups.append( { "filehash": filehash, "original": _to_dict(original) if original else None, "duplicates": [_to_dict(d) for d in duplicates], "duplicate_count": len(duplicates), } ) total_pages = max(1, (total_groups + per_page - 1) // per_page) return templates.TemplateResponse( "duplicates.html", { "request": request, "groups": groups, "total_groups": total_groups, "total_duplicate_files": total_duplicate_files, "pagination": { "page": page, "per_page": per_page, "total": total_groups, "pages": total_pages, }, "near_duplicate_threshold": settings.near_duplicate_threshold, }, ) except Exception as e: logger.error(f"Error rendering duplicates page: {e}") return templates.TemplateResponse( "duplicates.html", { "request": request, "groups": [], "total_groups": 0, "total_duplicate_files": 0, "pagination": {"page": 1, "per_page": per_page, "total": 0, "pages": 1}, "near_duplicate_threshold": 0.85, "error": str(e), }, ) @router.get("/similarity") @require_login def similarity_dashboard_page( request: Request, db: Session = Depends(get_db), ): """Render the corpus-wide similarity dashboard. Passes the configured threshold and embedding coverage stats so the template can display them immediately while the JS fetches the actual pairs from the API asynchronously. """ from app.config import settings from app.models import FileRecord try: total_files = db.query(FileRecord).count() files_with_embedding = ( db.query(FileRecord).filter(FileRecord.embedding.isnot(None), FileRecord.embedding != "").count() ) files_with_ocr = db.query(FileRecord).filter(FileRecord.ocr_text.isnot(None), FileRecord.ocr_text != "").count() return templates.TemplateResponse( "similarity_dashboard.html", { "request": request, "default_threshold": settings.near_duplicate_threshold, "embedding_model": settings.embedding_model, "total_files": total_files, "files_with_embedding": files_with_embedding, "files_with_ocr": files_with_ocr, "files_missing_embedding": files_with_ocr - files_with_embedding, }, ) except Exception as e: logger.error(f"Error rendering similarity dashboard: {e}") return templates.TemplateResponse( "similarity_dashboard.html", { "request": request, "default_threshold": 0.85, "embedding_model": "text-embedding-3-small", "total_files": 0, "files_with_embedding": 0, "files_with_ocr": 0, "files_missing_embedding": 0, "error": str(e), }, )