refactor: streamline file record creation and duplicate checking in document processing
This commit is contained in:
@@ -46,20 +46,19 @@ def process_document(original_local_file: str):
|
||||
mime_type = "application/octet-stream"
|
||||
|
||||
# Acquire DB session in the task
|
||||
new_record = None
|
||||
with SessionLocal() as db:
|
||||
# Keep all FileRecord operations within this session scope
|
||||
task_logger(f"Checking for duplicate files", step_name="check_duplicates", task_id=task_id)
|
||||
existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none()
|
||||
if existing:
|
||||
existing_record = db.query(FileRecord).filter(FileRecord.filehash == filehash).one_or_none()
|
||||
if existing_record:
|
||||
task_logger(f"Duplicate file detected (hash={filehash[:10]}...). Skipping processing.",
|
||||
step_name="process_document", task_id=task_id, file_id=existing.id, status="success")
|
||||
step_name="process_document", task_id=task_id, file_id=existing_record.id, status="success")
|
||||
return {
|
||||
"status": "duplicate_file",
|
||||
"file_id": existing.id,
|
||||
"file_id": existing_record.id,
|
||||
"detail": "File already processed."
|
||||
}
|
||||
|
||||
# Not a duplicate -> insert a new record
|
||||
else:
|
||||
task_logger(f"Creating file record for {original_local_file}", step_name="create_file_record", task_id=task_id)
|
||||
new_record = FileRecord(
|
||||
filehash=filehash,
|
||||
@@ -89,6 +88,8 @@ def process_document(original_local_file: str):
|
||||
new_record.local_filename = new_local_path
|
||||
db.commit()
|
||||
|
||||
# Perform all further interactions with existing_record/new_record here
|
||||
|
||||
# 2. Check for embedded text (outside the DB session to avoid long open transactions)
|
||||
task_logger(f"Checking for embedded text", step_name="check_embedded_text", task_id=task_id, file_id=new_record.id)
|
||||
pdf_doc = fitz.open(new_local_path)
|
||||
|
||||
Reference in New Issue
Block a user