diff --git a/app/views/files.py b/app/views/files.py index f259c566..2e5b165c 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -2,6 +2,7 @@ File management views for displaying and managing files. """ +import os from typing import Optional from fastapi import Depends, Query, Request @@ -133,6 +134,7 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d Return the file detail page showing processing history and file information """ try: + import json import os from app.models import FileRecord, ProcessingLog @@ -153,24 +155,28 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d .all() ) - # Check if file exists on disk - file_exists = os.path.exists(file_record.local_filename) if file_record.local_filename else False + # Check if original file exists (use persisted path from database) + original_file_exists = False + if file_record.original_file_path and os.path.exists(file_record.original_file_path): + original_file_exists = True - # Check if processed file exists - processed_exists = False - workdir = settings.workdir - processed_dir = os.path.join(workdir, "processed") - if os.path.exists(processed_dir): - base_filename = os.path.splitext(file_record.original_filename)[0] - potential_paths = [ - os.path.join(processed_dir, f"{file_record.filehash}.pdf"), - os.path.join(processed_dir, f"{base_filename}_processed.pdf"), - os.path.join(processed_dir, file_record.original_filename), - ] - for path in potential_paths: - if os.path.exists(path): - processed_exists = True - break + # Check if processed file exists (use persisted path from database) + processed_file_exists = False + if file_record.processed_file_path and os.path.exists(file_record.processed_file_path): + processed_file_exists = True + + # Load metadata from JSON file if it exists + gpt_metadata = None + if file_record.processed_file_path: + # Metadata JSON file is stored alongside the processed PDF + metadata_path = os.path.splitext(file_record.processed_file_path)[0] + ".json" + if os.path.exists(metadata_path): + try: + with open(metadata_path, "r", encoding="utf-8") as f: + gpt_metadata = json.load(f) + logger.debug(f"Loaded GPT metadata from {metadata_path}") + except Exception as e: + logger.warning(f"Failed to load metadata from {metadata_path}: {e}") # Compute processing flow for visualization flow_data = _compute_processing_flow(logs) @@ -190,8 +196,9 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d "request": request, "file": file_record, "logs": logs, - "file_exists": file_exists, - "processed_exists": processed_exists, + "original_file_exists": original_file_exists, + "processed_file_exists": processed_file_exists, + "gpt_metadata": gpt_metadata, "flow_data": flow_data, "step_summary": step_summary, }, @@ -396,3 +403,137 @@ def _compute_step_summary(logs): "total_main_steps": len(main_steps_seen), "total_upload_tasks": len(upload_tasks_seen), } + + +@router.get("/files/{file_id}/preview/original") +@require_login +def preview_original_file(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Serve the original (pre-processing) PDF file for preview + """ + import os + + from fastapi import HTTPException, status + from fastapi.responses import FileResponse + + from app.models import FileRecord + + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + if not file_record.original_file_path or not os.path.exists(file_record.original_file_path): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Original file not found on disk") + + return FileResponse( + path=file_record.original_file_path, + media_type="application/pdf", + headers={"Content-Disposition": "inline"}, + ) + + +@router.get("/files/{file_id}/preview/processed") +@require_login +def preview_processed_file(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Serve the processed (with embedded metadata) PDF file for preview + """ + import os + + from fastapi import HTTPException, status + from fastapi.responses import FileResponse + + from app.models import FileRecord + + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + if not file_record.processed_file_path or not os.path.exists(file_record.processed_file_path): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Processed file not found on disk") + + return FileResponse( + path=file_record.processed_file_path, + media_type="application/pdf", + headers={"Content-Disposition": "inline"}, + ) + + +@router.get("/files/{file_id}/text/original") +@require_login +def get_original_text(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Extract and return text from the original PDF file on-demand + """ + import os + + from fastapi import HTTPException, status + from fastapi.responses import JSONResponse + + from app.models import FileRecord + + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + if not file_record.original_file_path or not os.path.exists(file_record.original_file_path): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Original file not found on disk") + + try: + # Extract text from PDF using PyPDF2 + from PyPDF2 import PdfReader + + reader = PdfReader(file_record.original_file_path) + text = "" + for page in reader.pages: + text += page.extract_text() + "\n\n" + + if not text.strip(): + text = "(No text could be extracted from this PDF - it may be a scanned image without OCR)" + + return JSONResponse(content={"text": text.strip(), "page_count": len(reader.pages)}) + except Exception as e: + logger.error(f"Error extracting text from original file {file_id}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to extract text: {str(e)}" + ) + + +@router.get("/files/{file_id}/text/processed") +@require_login +def get_processed_text(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Extract and return text from the processed PDF file on-demand + """ + import os + + from fastapi import HTTPException, status + from fastapi.responses import JSONResponse + + from app.models import FileRecord + + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + if not file_record.processed_file_path or not os.path.exists(file_record.processed_file_path): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Processed file not found on disk") + + try: + # Extract text from PDF using PyPDF2 + from PyPDF2 import PdfReader + + reader = PdfReader(file_record.processed_file_path) + text = "" + for page in reader.pages: + text += page.extract_text() + "\n\n" + + if not text.strip(): + text = "(No text could be extracted from this PDF - it may be a scanned image without OCR)" + + return JSONResponse(content={"text": text.strip(), "page_count": len(reader.pages)}) + except Exception as e: + logger.error(f"Error extracting text from processed file {file_id}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to extract text: {str(e)}" + ) diff --git a/docs/FileDetailPageLayout.md b/docs/FileDetailPageLayout.md new file mode 100644 index 00000000..c7b932db --- /dev/null +++ b/docs/FileDetailPageLayout.md @@ -0,0 +1,111 @@ +# File Detail Page - Technical Reference + +## API Endpoints + +``` +GET /files/{id}/detail → Main detail page (template view) +GET /files/{id}/preview/original → Serve original PDF file +GET /files/{id}/preview/processed → Serve processed PDF file +GET /files/{id}/text/original → Extract text from original PDF (on-demand) +GET /files/{id}/text/processed → Extract text from processed PDF (on-demand) +POST /api/files/{id}/retry → Retry full processing +POST /api/files/{id}/retry-subtask → Retry specific task +``` + +## Text Extraction + +Text extraction is performed **on-demand** when the user clicks "View Extracted Text": +- Uses PyPDF2 to extract text from PDF files in real-time +- Returns JSON: `{"text": "...", "page_count": 3}` +- Client-side caching prevents re-extraction on subsequent views +- Loading indicator shown during extraction +- Graceful error handling if extraction fails + +## Template Context Variables + +| Variable | Type | Description | +|----------|------|-------------| +| `file` | FileRecord | Database record with file metadata | +| `gpt_metadata` | dict | Extracted metadata from JSON file (or None) | +| `extracted_text` | str | Text content from OCR/extraction (or None) | +| `original_file_exists` | bool | True if original file exists on disk | +| `processed_file_exists` | bool | True if processed file exists on disk | +| `logs` | List[ProcessingLog] | Processing history records | +| `step_summary` | dict | Aggregated processing status counts | +| `flow_data` | dict | Processing flow visualization data | + +## File System Structure + +``` +workdir/ +├── tmp/ +│ └── {uuid}.pdf ← Temporary ingestion file +├── original/ +│ └── {uuid}.pdf ← Immutable original copy (original_file_path) +└── processed/ + ├── 2024-01-15_Invoice.pdf ← Processed with metadata (processed_file_path) + └── 2024-01-15_Invoice.json ← GPT metadata JSON +``` + +## Metadata JSON Schema + +Stored as `{processed_file_path_without_extension}.json`: + +```json +{ + "document_type": "Invoice", + "filename": "2024-01-15_Company_Invoice", + "date": "2024-01-15", + "absender": "Test Company GmbH", + "empfaenger": "Customer Inc", + "betrag": "€ 1,234.56", + "kontonummer": "DE89370400440532013000", + "tags": ["invoice", "payment", "2024"], + "original_file_path": "/workdir/original/{uuid}.pdf", + "processed_file_path": "/workdir/processed/2024-01-15_Invoice.pdf" +} +``` + +## JavaScript Functions + +```javascript +toggleMetadata() // Show/hide JSON metadata view +toggleTextModal(modalId) // Open/close text extraction modals +window.onclick // Close modal when clicking outside +``` + +## Database Schema + +```sql +CREATE TABLE files ( + id INTEGER PRIMARY KEY, + filehash TEXT NOT NULL, + original_filename TEXT, + local_filename TEXT NOT NULL, + original_file_path TEXT, -- Points to workdir/original/ + processed_file_path TEXT, -- Points to workdir/processed/ + file_size INTEGER, + mime_type TEXT, + created_at DATETIME +); + +CREATE TABLE processing_logs ( + id INTEGER PRIMARY KEY, + file_id INTEGER, + task_id TEXT, + step_name TEXT, -- e.g., "extract_text", "process_with_azure_document_intelligence" + status TEXT, -- "pending", "in_progress", "success", "failure" + message TEXT, + detail TEXT, -- May contain extracted text + timestamp DATETIME +); +``` + +## Processing Stages + +Text extraction logs to check for `extracted_text`: +- `extract_text` - Local PDF text extraction +- `process_with_azure_document_intelligence` - Azure OCR processing + +Both may contain extracted text in the `detail` field when `status = 'success'`. + diff --git a/docs/UserGuide.md b/docs/UserGuide.md index 0f18a82a..a0de2635 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -95,13 +95,52 @@ The **Files** page provides access to all processed documents: When you click on a file, you'll see a comprehensive detail view with the following sections: +#### File Information +This section displays metadata about the file: #### File Information This section displays metadata about the file: - File ID and original filename - File hash (SHA-256) - File size and MIME type - Creation timestamp -- Local path and disk status +- Original file path and status (shows if the immutable original is available) +- Processed file path and status (shows if the final processed file is available) + +#### Extracted Metadata (GPT) +If metadata has been extracted by GPT, this section displays structured information including: +- **Document Type**: Classification (Invoice, Receipt, Contract, etc.) +- **Suggested Filename**: AI-recommended filename based on content +- **Document Date**: Extracted date from document +- **Sender (Absender)**: Sender or issuing party information +- **Recipient (Empfänger)**: Recipient information +- **Amount (Betrag)**: Financial amounts (for invoices, receipts) +- **Account Number**: Bank account information +- **Tags**: Document categories and labels + +**Show JSON**: Click this button to toggle the full metadata in JSON format. This is useful for: +- Viewing all extracted fields at once +- Debugging metadata extraction issues +- Copying metadata for external use +- Understanding the complete data structure + +#### Document Previews +View your documents side-by-side in embedded PDF viewers: +- **Original Document**: The immutable original file as first ingested, before any processing +- **Processed Document**: The final file with embedded metadata + +**Features**: +- In-browser PDF rendering for immediate viewing +- Side-by-side comparison of original vs processed versions +- Full 600px height previews for detailed review +- **View Extracted Text** buttons below each preview to see the full text content + +**Text Extraction Modals**: Click "View Extracted Text" to open a fullscreen modal showing: +- Complete extracted text from OCR or PDF text layer +- Dark-themed, scrollable display for easy reading +- Copy-friendly pre-formatted text +- Close by clicking the close button or clicking outside the modal + +**Note**: The original file is stored in an immutable archive (`workdir/original/`) and is never modified. The processed file is stored in `workdir/processed/` with the suggested filename and embedded metadata. #### Processing History View the complete processing history with a timeline showing: @@ -138,11 +177,23 @@ If your file is still available on disk, you can preview it directly in the brow - **Processed File**: The file after metadata has been embedded (if processing completed) Both previews support: -- In-browser PDF viewing -- Opening in a new tab for full-screen viewing +- In-browser PDF viewing with embedded viewer - Side-by-side comparison of original and processed versions +- Full text extraction viewing via modal overlays -**Note**: The original file is stored in an immutable archive and is never modified. This ensures you always have access to the file exactly as it was uploaded, which is valuable for: +**View Extracted Text**: Each preview includes a button to view the complete extracted text in a fullscreen modal. When you click this button: +- The system extracts text from the PDF file on-demand using PyPDF2 +- A loading indicator shows while extraction is in progress +- The extracted text is displayed in a scrollable, copy-friendly format +- The text is cached so subsequent views load instantly + +This is useful for: +- Verifying OCR accuracy +- Searching within document content +- Copying text for external use +- Reading documents without downloading them + +**Note**: The original file is stored in an immutable archive (`workdir/original/`) and is never modified. This ensures you always have access to the file exactly as it was uploaded, which is valuable for: - Auditing and compliance - Debugging processing issues - Reprocessing with improved algorithms diff --git a/frontend/templates/file_detail.html b/frontend/templates/file_detail.html index 57a11962..58225f9d 100644 --- a/frontend/templates/file_detail.html +++ b/frontend/templates/file_detail.html @@ -2,6 +2,12 @@ {% block title %}File Details{% endblock %} {% block head_extra %} + + + {% if file %} {% endif %} {% endblock %} @@ -683,13 +948,15 @@ {{ file.created_at.strftime('%Y-%m-%d %H:%M:%S') if file.created_at else 'N/A' }}
- Local Path - {{ file.local_filename }} + Original File Path + + {{ file.original_file_path if file.original_file_path else 'N/A' }} +
- File on Disk + Original File Status - {% if file_exists %} + {% if original_file_exists %} File exists @@ -700,6 +967,217 @@ {% endif %}
+
+ Processed File Path + + {{ file.processed_file_path if file.processed_file_path else 'N/A' }} + +
+
+ Processed File Status + + {% if processed_file_exists %} + + File exists + + {% else %} + + File not found + + {% endif %} + +
+ + + + + {% if gpt_metadata %} +
+
+

Extracted Metadata (GPT)

+ +
+ +
+ {% if gpt_metadata.document_type %} +
+ Document Type + {{ gpt_metadata.document_type }} +
+ {% endif %} + + {% if gpt_metadata.filename %} +
+ Suggested Filename + {{ gpt_metadata.filename }} +
+ {% endif %} + + {% if gpt_metadata.date %} +
+ Document Date + {{ gpt_metadata.date }} +
+ {% endif %} + + {% if gpt_metadata.absender %} +
+ Sender (Absender) + {{ gpt_metadata.absender }} +
+ {% endif %} + + {% if gpt_metadata.empfaenger %} +
+ Recipient (Empfänger) + {{ gpt_metadata.empfaenger }} +
+ {% endif %} + + {% if gpt_metadata.betrag %} +
+ Amount (Betrag) + {{ gpt_metadata.betrag }} +
+ {% endif %} + + {% if gpt_metadata.kontonummer %} +
+ Account Number + {{ gpt_metadata.kontonummer }} +
+ {% endif %} + + {% if gpt_metadata.tags %} +
+ Tags + + {% if gpt_metadata.tags is string %} + {{ gpt_metadata.tags }} + {% else %} + {{ ', '.join(gpt_metadata.tags) }} + {% endif %} + +
+ {% endif %} +
+ + + +
+ {% endif %} + + +
+

Document Previews

+ +
+ +
+

Original Document

+ {% if original_file_exists %} +
+
+ + Loading... + +
+
+
+ +

Loading PDF...

+
+
+
+ + {% else %} +
+ +

Original file not available

+
+ {% endif %} +
+ + +
+

Processed Document

+ {% if processed_file_exists %} +
+
+ + Loading... + +
+
+
+ +

Loading PDF...

+
+
+
+ + {% else %} +
+ +

Processed file not available yet

+
+ {% endif %} +
+
+
+ + + + + + + diff --git a/tests/test_file_detail_enhancements.py b/tests/test_file_detail_enhancements.py new file mode 100644 index 00000000..944f7c27 --- /dev/null +++ b/tests/test_file_detail_enhancements.py @@ -0,0 +1,229 @@ +""" +Tests for file detail page enhancements. + +Tests the new features: +- GPT metadata display +- PDF preview endpoints +- Text modal functionality +- Using persisted original_file_path and processed_file_path +""" + +import json +import os +import tempfile +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.models import FileRecord + + +@pytest.fixture +def sample_metadata(): + """Sample GPT metadata""" + return { + "document_type": "Invoice", + "filename": "2024-01-15_Company_Invoice", + "date": "2024-01-15", + "absender": "Test Company GmbH", + "empfaenger": "Customer Inc", + "betrag": "€ 1,234.56", + "kontonummer": "DE89370400440532013000", + "tags": ["invoice", "payment", "2024"], + } + + +@pytest.mark.integration +def test_file_detail_page_with_metadata(client: TestClient, db_session, sample_pdf_file): + """Test file detail page displays GPT metadata correctly""" + from app.models import FileRecord + + # Create a file record with paths + file_record = FileRecord( + filehash="test123abc", + original_filename="test_invoice.pdf", + local_filename=str(sample_pdf_file), + file_size=1024, + mime_type="application/pdf", + original_file_path=str(sample_pdf_file), + processed_file_path=None, # Not processed yet + ) + db_session.add(file_record) + db_session.commit() + db_session.refresh(file_record) + + # Get detail page + response = client.get(f"/files/{file_record.id}/detail") + assert response.status_code == 200 + + html = response.text + # Check for file info display + assert "File Information" in html + assert "test_invoice.pdf" in html + assert "Original File Path" in html + + +@pytest.mark.integration +def test_file_detail_with_gpt_metadata(client: TestClient, db_session, sample_pdf_file, sample_metadata, tmp_path): + """Test file detail page with GPT metadata JSON""" + from app.models import FileRecord + + # Create processed file path and metadata JSON + processed_file = tmp_path / "2024-01-15_Company_Invoice.pdf" + processed_file.write_bytes(sample_pdf_file.read_bytes()) + + metadata_file = tmp_path / "2024-01-15_Company_Invoice.json" + metadata_file.write_text(json.dumps(sample_metadata, indent=2)) + + file_record = FileRecord( + filehash="test456def", + original_filename="invoice_001.pdf", + local_filename=str(sample_pdf_file), + file_size=1024, + mime_type="application/pdf", + original_file_path=str(sample_pdf_file), + processed_file_path=str(processed_file), + ) + db_session.add(file_record) + db_session.commit() + db_session.refresh(file_record) + + # Get detail page + response = client.get(f"/files/{file_record.id}/detail") + assert response.status_code == 200 + + html = response.text + # Check for metadata display + assert "Extracted Metadata (GPT)" in html + assert "Invoice" in html # document_type + assert "2024-01-15_Company_Invoice" in html # filename + assert "Test Company GmbH" in html # absender + assert "€ 1,234.56" in html # betrag + + +@pytest.mark.integration +def test_preview_original_file_endpoint(client: TestClient, db_session, sample_pdf_file): + """Test original file preview endpoint""" + from app.models import FileRecord + + file_record = FileRecord( + filehash="test789ghi", + original_filename="original.pdf", + local_filename=str(sample_pdf_file), + file_size=1024, + mime_type="application/pdf", + original_file_path=str(sample_pdf_file), + processed_file_path=None, + ) + db_session.add(file_record) + db_session.commit() + db_session.refresh(file_record) + + # Request preview + response = client.get(f"/files/{file_record.id}/preview/original") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/pdf" + + +@pytest.mark.integration +def test_preview_processed_file_endpoint(client: TestClient, db_session, sample_pdf_file, tmp_path): + """Test processed file preview endpoint""" + from app.models import FileRecord + + # Create processed file + processed_file = tmp_path / "processed.pdf" + processed_file.write_bytes(sample_pdf_file.read_bytes()) + + file_record = FileRecord( + filehash="test101jkl", + original_filename="doc.pdf", + local_filename=str(sample_pdf_file), + file_size=1024, + mime_type="application/pdf", + original_file_path=str(sample_pdf_file), + processed_file_path=str(processed_file), + ) + db_session.add(file_record) + db_session.commit() + db_session.refresh(file_record) + + # Request preview + response = client.get(f"/files/{file_record.id}/preview/processed") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/pdf" + + +@pytest.mark.integration +def test_preview_missing_file_returns_404(client: TestClient, db_session, sample_pdf_file): + """Test preview endpoint returns 404 when file doesn't exist""" + from app.models import FileRecord + + file_record = FileRecord( + filehash="test202mno", + original_filename="missing.pdf", + local_filename=str(sample_pdf_file), + file_size=1024, + mime_type="application/pdf", + original_file_path="/nonexistent/path/original.pdf", + processed_file_path="/nonexistent/path/processed.pdf", + ) + db_session.add(file_record) + db_session.commit() + db_session.refresh(file_record) + + # Request original (missing) + response = client.get(f"/files/{file_record.id}/preview/original") + assert response.status_code == 404 + + # Request processed (missing) + response = client.get(f"/files/{file_record.id}/preview/processed") + assert response.status_code == 404 + + +@pytest.mark.integration +def test_file_detail_shows_file_status_indicators(client: TestClient, db_session, sample_pdf_file): + """Test file detail page shows correct status indicators for original and processed files""" + from app.models import FileRecord + + file_record = FileRecord( + filehash="test303pqr", + original_filename="status_test.pdf", + local_filename=str(sample_pdf_file), + file_size=1024, + mime_type="application/pdf", + original_file_path=str(sample_pdf_file), # Exists + processed_file_path="/nonexistent/processed.pdf", # Doesn't exist + ) + db_session.add(file_record) + db_session.commit() + db_session.refresh(file_record) + + response = client.get(f"/files/{file_record.id}/detail") + assert response.status_code == 200 + + html = response.text + # Check for status indicators + assert "Original File Status" in html + assert "Processed File Status" in html + # Original should show as exists + assert html.count("File exists") >= 1 + # Processed should show as not found + assert "File not found" in html + + +@pytest.mark.unit +def test_metadata_json_structure(sample_metadata): + """Test metadata JSON structure is valid""" + # Verify all expected fields are present + assert "document_type" in sample_metadata + assert "filename" in sample_metadata + assert "date" in sample_metadata + assert "absender" in sample_metadata + assert "tags" in sample_metadata + + # Verify it's JSON serializable + json_str = json.dumps(sample_metadata) + parsed = json.loads(json_str) + assert parsed == sample_metadata