Fix DetachedInstanceError in process_document task by storing file_id before session closes
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
+123
-28
@@ -9,7 +9,9 @@ import PyPDF2 # Replace fitz with PyPDF2
|
||||
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
|
||||
from app.tasks.process_with_azure_document_intelligence import (
|
||||
process_with_azure_document_intelligence,
|
||||
)
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.celery_app import celery
|
||||
from app.database import SessionLocal
|
||||
@@ -33,7 +35,12 @@ def process_document(self, original_local_file: str):
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting document processing: {original_local_file}")
|
||||
log_task_progress(task_id, "process_document", "in_progress", f"Processing file: {original_local_file}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"in_progress",
|
||||
f"Processing file: {original_local_file}",
|
||||
)
|
||||
|
||||
if not os.path.exists(original_local_file):
|
||||
logger.error(f"[{task_id}] File {original_local_file} not found.")
|
||||
@@ -49,25 +56,42 @@ def process_document(self, original_local_file: str):
|
||||
mime_type, _ = mimetypes.guess_type(original_local_file)
|
||||
if not mime_type:
|
||||
mime_type = "application/octet-stream"
|
||||
|
||||
logger.info(f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}")
|
||||
log_task_progress(task_id, "hash_file", "success", f"Hash: {filehash[:10]}..., Size: {file_size} bytes")
|
||||
|
||||
logger.info(
|
||||
f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"hash_file",
|
||||
"success",
|
||||
f"Hash: {filehash[:10]}..., Size: {file_size} bytes",
|
||||
)
|
||||
|
||||
# Acquire DB session in the task
|
||||
with SessionLocal() as db:
|
||||
existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none()
|
||||
if existing:
|
||||
logger.info(f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
|
||||
log_task_progress(task_id, "process_document", "success", "Duplicate file detected, skipping", file_id=existing.id)
|
||||
logger.info(
|
||||
f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing."
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Duplicate file detected, skipping",
|
||||
file_id=existing.id,
|
||||
)
|
||||
return {
|
||||
"status": "duplicate_file",
|
||||
"file_id": existing.id,
|
||||
"detail": "File already processed."
|
||||
"detail": "File already processed.",
|
||||
}
|
||||
|
||||
# Not a duplicate -> insert a new record
|
||||
logger.info(f"[{task_id}] Creating new file record in database")
|
||||
log_task_progress(task_id, "create_file_record", "in_progress", "Creating file record")
|
||||
log_task_progress(
|
||||
task_id, "create_file_record", "in_progress", "Creating file record"
|
||||
)
|
||||
new_record = FileRecord(
|
||||
filehash=filehash,
|
||||
original_filename=original_filename,
|
||||
@@ -79,7 +103,13 @@ def process_document(self, original_local_file: str):
|
||||
db.commit()
|
||||
db.refresh(new_record)
|
||||
logger.info(f"[{task_id}] File record created with ID: {new_record.id}")
|
||||
log_task_progress(task_id, "create_file_record", "success", f"File record ID: {new_record.id}", file_id=new_record.id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"create_file_record",
|
||||
"success",
|
||||
f"File record ID: {new_record.id}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
|
||||
# 1. Generate a UUID-based filename and place it in /workdir/tmp
|
||||
file_ext = os.path.splitext(original_local_file)[1]
|
||||
@@ -91,19 +121,40 @@ def process_document(self, original_local_file: str):
|
||||
new_local_path = os.path.join(tmp_dir, new_filename)
|
||||
|
||||
logger.info(f"[{task_id}] Copying file to: {new_local_path}")
|
||||
log_task_progress(task_id, "copy_file", "in_progress", f"Copying file to {new_filename}", file_id=new_record.id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"copy_file",
|
||||
"in_progress",
|
||||
f"Copying file to {new_filename}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
# Copy the file instead of moving it
|
||||
shutil.copy(original_local_file, new_local_path)
|
||||
log_task_progress(task_id, "copy_file", "success", f"File copied to {new_filename}", file_id=new_record.id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"copy_file",
|
||||
"success",
|
||||
f"File copied to {new_filename}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
|
||||
# Update the DB with final local filename
|
||||
new_record.local_filename = new_local_path
|
||||
db.commit()
|
||||
|
||||
# Store file_id before session closes to avoid DetachedInstanceError
|
||||
file_id = new_record.id
|
||||
|
||||
# 2. Check for embedded text (outside the DB session to avoid long open transactions)
|
||||
logger.info(f"[{task_id}] Checking for embedded text in PDF")
|
||||
log_task_progress(task_id, "check_text", "in_progress", "Checking for embedded text", file_id=new_record.id)
|
||||
with open(new_local_path, 'rb') as file:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"in_progress",
|
||||
"Checking for embedded text",
|
||||
file_id=file_id,
|
||||
)
|
||||
with open(new_local_path, "rb") as file:
|
||||
pdf_reader = PyPDF2.PdfReader(file)
|
||||
has_text = False
|
||||
for page in pdf_reader.pages:
|
||||
@@ -112,30 +163,74 @@ def process_document(self, original_local_file: str):
|
||||
break
|
||||
|
||||
if has_text:
|
||||
logger.info(f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally.")
|
||||
log_task_progress(task_id, "check_text", "success", "Embedded text found, extracting locally", file_id=new_record.id)
|
||||
logger.info(
|
||||
f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally."
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"success",
|
||||
"Embedded text found, extracting locally",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Extract text locally
|
||||
logger.info(f"[{task_id}] Extracting text from PDF")
|
||||
log_task_progress(task_id, "extract_text", "in_progress", "Extracting text locally", file_id=new_record.id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"in_progress",
|
||||
"Extracting text locally",
|
||||
file_id=file_id,
|
||||
)
|
||||
extracted_text = ""
|
||||
with open(new_local_path, 'rb') as file:
|
||||
with open(new_local_path, "rb") as file:
|
||||
pdf_reader = PyPDF2.PdfReader(file)
|
||||
for page in pdf_reader.pages:
|
||||
extracted_text += page.extract_text() + "\n"
|
||||
|
||||
|
||||
logger.info(f"[{task_id}] Extracted {len(extracted_text)} characters")
|
||||
log_task_progress(task_id, "extract_text", "success", f"Extracted {len(extracted_text)} characters", file_id=new_record.id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"success",
|
||||
f"Extracted {len(extracted_text)} characters",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Call metadata extraction directly
|
||||
logger.info(f"[{task_id}] Queueing metadata extraction")
|
||||
log_task_progress(task_id, "process_document", "success", "Queued for metadata extraction", file_id=new_record.id)
|
||||
extract_metadata_with_gpt.delay(new_filename, extracted_text, new_record.id)
|
||||
return {"file": new_local_path, "status": "Text extracted locally", "file_id": new_record.id}
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for metadata extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
extract_metadata_with_gpt.delay(new_filename, extracted_text, file_id)
|
||||
return {
|
||||
"file": new_local_path,
|
||||
"status": "Text extracted locally",
|
||||
"file_id": file_id,
|
||||
}
|
||||
|
||||
# 3. If no embedded text, queue Azure Document Intelligence processing
|
||||
logger.info(f"[{task_id}] No embedded text found. Queueing Azure Document Intelligence processing")
|
||||
log_task_progress(task_id, "check_text", "success", "No embedded text, queuing OCR", file_id=new_record.id)
|
||||
log_task_progress(task_id, "process_document", "success", "Queued for OCR processing", file_id=new_record.id)
|
||||
process_with_azure_document_intelligence.delay(new_filename, new_record.id)
|
||||
return {"file": new_local_path, "status": "Queued for OCR", "file_id": new_record.id}
|
||||
logger.info(
|
||||
f"[{task_id}] No embedded text found. Queueing Azure Document Intelligence processing"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"success",
|
||||
"No embedded text, queuing OCR",
|
||||
file_id=file_id,
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for OCR processing",
|
||||
file_id=file_id,
|
||||
)
|
||||
process_with_azure_document_intelligence.delay(new_filename, file_id)
|
||||
return {"file": new_local_path, "status": "Queued for OCR", "file_id": file_id}
|
||||
|
||||
Reference in New Issue
Block a user