gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: handle file in processed directory when retrying embed_metadata_into_pdf

Browse Source 
- Update _retry_pipeline_step to check for file in tmp, processed, and fallback locations
- Pass full path to extract_metadata_with_gpt instead of just basename
- Update extract_metadata_with_gpt to handle both basename and full path parameters
- Add test case for retrying when file is in processed directory

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-13 21:06:20 +00:00
parent 80bb9cec10
commit 42d35c7c6f
3 changed files with 86 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/extract_metadata_with_gpt.py
+16 -5
View File
@@ -47,17 +47,28 @@ def extract_json_from_text(text):
@celery.task(base=BaseTaskWithRetry, bind=True)
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
"""Uses OpenAI to classify document metadata."""
"""
Uses OpenAI to classify document metadata.
Args:
filename: Can be either a basename (e.g., "file.pdf") or a full path (e.g., "/workdir/processed/file.pdf")
cleaned_text: The extracted text from the document
file_id: Optional file ID for tracking
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
log_task_progress(
task_id, "extract_metadata_with_gpt", "in_progress", f"Extracting metadata for {filename}", file_id=file_id
task_id, "extract_metadata_with_gpt", "in_progress", f"Extracting metadata for {os.path.basename(filename)}", file_id=file_id
)
# Get file_id from database if not provided
if file_id is None:
tmp_dir = os.path.join(settings.workdir, "tmp")
file_path = os.path.join(tmp_dir, filename)
# Handle both basename and full path
if os.path.isabs(filename):
file_path = filename
else:
file_path = os.path.join(tmp_dir, filename)
if os.path.exists(file_path):
with SessionLocal() as db:
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
@@ -162,14 +173,14 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
)
# Trigger the next step: embedding metadata into the PDF
# Pass the original filename (UUID-based) so embed_metadata_into_pdf can find the file on disk
# Pass the filename (can be basename or full path) so embed_metadata_into_pdf can find the file on disk
logger.info(f"[{task_id}] Queueing metadata embedding task")
log_task_progress(
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
)
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
return {"s3_file": filename, "metadata": metadata}
return {"s3_file": os.path.basename(filename), "metadata": metadata}
except Exception as e:
logger.exception(f"[{task_id}] OpenAI classification failed for {filename}: {e}")