gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add comprehensive processing logging system

Browse Source 
- Added database logging to all major processing tasks
- Created API endpoints for retrieving processing logs
- Updated frontend to display processing logs per file
- Logging includes: process_document, convert_to_pdf, extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-06 22:30:18 +00:00
parent 0d36872539
commit 1903dc5bcd
9 changed files with 509 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/process_document.py
+41 -8
View File
@@ -4,6 +4,7 @@ import os
import uuid
import shutil
import mimetypes
import logging
import PyPDF2 # Replace fitz with PyPDF2
from app.config import settings
@@ -13,11 +14,13 @@ from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.celery_app import celery
from app.database import SessionLocal
from app.models import FileRecord
from app.utils import hash_file
from app.utils import hash_file, log_task_progress
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def process_document(original_local_file: str):
@celery.task(base=BaseTaskWithRetry, bind=True)
def process_document(self, original_local_file: str):
"""
Process a document file and trigger appropriate text extraction.
@@ -28,24 +31,34 @@ def process_document(original_local_file: str):
- Check for embedded text. If present, run local GPT extraction
- Otherwise, queue Azure Document Intelligence processing
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting document processing: {original_local_file}")
log_task_progress(task_id, "process_document", "in_progress", f"Processing file: {original_local_file}")
if not os.path.exists(original_local_file):
print(f"[ERROR] File {original_local_file} not found.")
logger.error(f"[{task_id}] File {original_local_file} not found.")
log_task_progress(task_id, "process_document", "failure", "File not found")
return {"error": "File not found"}
# 0. Compute the file hash and check for duplicates
logger.info(f"[{task_id}] Computing file hash...")
log_task_progress(task_id, "hash_file", "in_progress", "Computing file hash")
filehash = hash_file(original_local_file)
original_filename = os.path.basename(original_local_file)
file_size = os.path.getsize(original_local_file)
mime_type, _ = mimetypes.guess_type(original_local_file)
if not mime_type:
mime_type = "application/octet-stream"
logger.info(f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}")
log_task_progress(task_id, "hash_file", "success", f"Hash: {filehash[:10]}..., Size: {file_size} bytes")
# Acquire DB session in the task
with SessionLocal() as db:
existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none()
if existing:
print(f"[INFO] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
logger.info(f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
log_task_progress(task_id, "process_document", "success", "Duplicate file detected, skipping", file_id=existing.id)
return {
"status": "duplicate_file",
"file_id": existing.id,
@@ -53,6 +66,8 @@ def process_document(original_local_file: str):
}
# Not a duplicate -> insert a new record
logger.info(f"[{task_id}] Creating new file record in database")
log_task_progress(task_id, "create_file_record", "in_progress", "Creating file record")
new_record = FileRecord(
filehash=filehash,
original_filename=original_filename,
@@ -63,6 +78,8 @@ def process_document(original_local_file: str):
db.add(new_record)
db.commit()
db.refresh(new_record)
logger.info(f"[{task_id}] File record created with ID: {new_record.id}")
log_task_progress(task_id, "create_file_record", "success", f"File record ID: {new_record.id}", file_id=new_record.id)
# 1. Generate a UUID-based filename and place it in /workdir/tmp
file_ext = os.path.splitext(original_local_file)[1]
@@ -73,14 +90,19 @@ def process_document(original_local_file: str):
os.makedirs(tmp_dir, exist_ok=True)
new_local_path = os.path.join(tmp_dir, new_filename)
logger.info(f"[{task_id}] Copying file to: {new_local_path}")
log_task_progress(task_id, "copy_file", "in_progress", f"Copying file to {new_filename}", file_id=new_record.id)
# Copy the file instead of moving it
shutil.copy(original_local_file, new_local_path)
log_task_progress(task_id, "copy_file", "success", f"File copied to {new_filename}", file_id=new_record.id)
# Update the DB with final local filename
new_record.local_filename = new_local_path
db.commit()
# 2. Check for embedded text (outside the DB session to avoid long open transactions)
logger.info(f"[{task_id}] Checking for embedded text in PDF")
log_task_progress(task_id, "check_text", "in_progress", "Checking for embedded text", file_id=new_record.id)
with open(new_local_path, 'rb') as file:
pdf_reader = PyPDF2.PdfReader(file)
has_text = False
@@ -90,19 +112,30 @@ def process_document(original_local_file: str):
break
if has_text:
print(f"[INFO] PDF {original_local_file} contains embedded text. Processing locally.")
logger.info(f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally.")
log_task_progress(task_id, "check_text", "success", "Embedded text found, extracting locally", file_id=new_record.id)
# Extract text locally
logger.info(f"[{task_id}] Extracting text from PDF")
log_task_progress(task_id, "extract_text", "in_progress", "Extracting text locally", file_id=new_record.id)
extracted_text = ""
with open(new_local_path, 'rb') as file:
pdf_reader = PyPDF2.PdfReader(file)
for page in pdf_reader.pages:
extracted_text += page.extract_text() + "\n"
logger.info(f"[{task_id}] Extracted {len(extracted_text)} characters")
log_task_progress(task_id, "extract_text", "success", f"Extracted {len(extracted_text)} characters", file_id=new_record.id)
# Call metadata extraction directly
logger.info(f"[{task_id}] Queueing metadata extraction")
log_task_progress(task_id, "process_document", "success", "Queued for metadata extraction", file_id=new_record.id)
extract_metadata_with_gpt.delay(new_filename, extracted_text)
return {"file": new_local_path, "status": "Text extracted locally"}
return {"file": new_local_path, "status": "Text extracted locally", "file_id": new_record.id}
# 3. If no embedded text, queue Azure Document Intelligence processing
logger.info(f"[{task_id}] No embedded text found. Queueing Azure Document Intelligence processing")
log_task_progress(task_id, "check_text", "success", "No embedded text, queuing OCR", file_id=new_record.id)
log_task_progress(task_id, "process_document", "success", "Queued for OCR processing", file_id=new_record.id)
process_with_azure_document_intelligence.delay(new_filename)
return {"file": new_local_path, "status": "Queued for OCR"}
return {"file": new_local_path, "status": "Queued for OCR", "file_id": new_record.id}