gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: streamline file record creation and duplicate checking in document processing

Browse Source
This commit is contained in:
Christian Krakau-Louis
2025-03-28 16:53:25 +01:00
parent aae9a89d6b
commit 7481581e4e
1 changed files with 31 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/process_document.py
+8 -7
View File
@@ -46,20 +46,19 @@ def process_document(original_local_file: str):
mime_type = "application/octet-stream" mime_type = "application/octet-stream"
# Acquire DB session in the task # Acquire DB session in the task
new_record = None
with SessionLocal() as db: with SessionLocal() as db:
# Keep all FileRecord operations within this session scope
task_logger(f"Checking for duplicate files", step_name="check_duplicates", task_id=task_id) task_logger(f"Checking for duplicate files", step_name="check_duplicates", task_id=task_id)
existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none() existing_record = db.query(FileRecord).filter(FileRecord.filehash == filehash).one_or_none()
if existing: if existing_record:
task_logger(f"Duplicate file detected (hash={filehash[:10]}...). Skipping processing.", task_logger(f"Duplicate file detected (hash={filehash[:10]}...). Skipping processing.",
step_name="process_document", task_id=task_id, file_id=existing.id, status="success") step_name="process_document", task_id=task_id, file_id=existing_record.id, status="success")
return { return {
"status": "duplicate_file", "status": "duplicate_file",
"file_id": existing.id, "file_id": existing_record.id,
"detail": "File already processed." "detail": "File already processed."
} }
else:
# Not a duplicate -> insert a new record
task_logger(f"Creating file record for {original_local_file}", step_name="create_file_record", task_id=task_id) task_logger(f"Creating file record for {original_local_file}", step_name="create_file_record", task_id=task_id)
new_record = FileRecord( new_record = FileRecord(
filehash=filehash, filehash=filehash,
@@ -89,6 +88,8 @@ def process_document(original_local_file: str):
new_record.local_filename = new_local_path new_record.local_filename = new_local_path
db.commit() db.commit()
# Perform all further interactions with existing_record/new_record here
# 2. Check for embedded text (outside the DB session to avoid long open transactions) # 2. Check for embedded text (outside the DB session to avoid long open transactions)
task_logger(f"Checking for embedded text", step_name="check_embedded_text", task_id=task_id, file_id=new_record.id) task_logger(f"Checking for embedded text", step_name="check_embedded_text", task_id=task_id, file_id=new_record.id)
pdf_doc = fitz.open(new_local_path) pdf_doc = fitz.open(new_local_path)