gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: fix code formatting with black, isort, and flake8

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-12 03:21:41 +00:00
parent f79bd2cb0c
commit ff9a3ff49f
87 changed files with 874 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/embed_metadata_into_pdf.py
+26 -14
View File
@@ -35,28 +35,28 @@ def persist_metadata(metadata, final_pdf_path, original_file_path=None, processe
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
the metadata will be saved as "<workdir>/processed/MyFile.json".
Optionally augments the metadata with file path references for traceability.
Args:
metadata: Dictionary of metadata to save
final_pdf_path: Path to the final PDF file
original_file_path: Optional path to the immutable original file
processed_file_path: Optional path to the processed file
Returns:
str: Path to the created JSON file
"""
base, _ = os.path.splitext(final_pdf_path)
json_path = base + ".json"
# Augment metadata with file path references if provided
metadata_with_paths = metadata.copy()
if original_file_path:
metadata_with_paths["original_file_path"] = original_file_path
if processed_file_path:
metadata_with_paths["processed_file_path"] = processed_file_path
with open(json_path, "w", encoding="utf-8") as f:
json.dump(metadata_with_paths, f, ensure_ascii=False, indent=2)
return json_path
@@ -102,7 +102,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
else:
logger.error(f"[{task_id}] Local file {local_file_path} not found, cannot embed metadata.")
log_task_progress(
task_id, "embed_metadata_into_pdf", "failure", "File not found", file_id=file_id,
task_id,
"embed_metadata_into_pdf",
"failure",
"File not found",
file_id=file_id,
detail=(
f"Local file not found, cannot embed metadata.\n"
f"Tried path: {local_file_path}\n"
@@ -199,10 +203,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
logger.info(f"[{task_id}] Persisting metadata to JSON")
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
json_path = persist_metadata(
metadata,
final_file_path,
original_file_path=original_file_path,
processed_file_path=final_file_path
metadata, final_file_path, original_file_path=original_file_path, processed_file_path=final_file_path
)
logger.info(f"[{task_id}] Metadata persisted to {json_path}")
log_task_progress(
@@ -212,7 +213,11 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
# Trigger the next step: final storage.
logger.info(f"[{task_id}] Queueing final storage task")
log_task_progress(
task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id,
task_id,
"embed_metadata_into_pdf",
"success",
"Metadata embedded, queuing finalization",
file_id=file_id,
detail=(
f"Metadata embedded into PDF successfully.\n"
f"Original file: {original_file}\n"
@@ -229,7 +234,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
try:
original_file_path = Path(original_file).resolve()
workdir_tmp_resolved = workdir_tmp_path.resolve()
# Check if file is within workdir/tmp and exists
if original_file_path.is_relative_to(workdir_tmp_resolved) and original_file_path.exists():
try:
@@ -245,8 +250,15 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
except Exception as e:
logger.exception(f"[{task_id}] Failed to embed metadata into {processed_file}: {e}")
log_task_progress(
task_id, "embed_metadata_into_pdf", "failure", f"Exception: {str(e)}", file_id=file_id,
detail=f"Failed to embed metadata into {processed_file}.\nOriginal file: {original_file}\nException: {str(e)}",
task_id,
"embed_metadata_into_pdf",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=(
f"Failed to embed metadata into {processed_file}.\n"
f"Original file: {original_file}\nException: {str(e)}"
),
)
# Clean up temporary file in case of error
if os.path.exists(processed_file):
app/tasks/extract_metadata_with_gpt.py
+24 -10
View File
@@ -112,7 +112,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
content = completion.choices[0].message.content
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
log_task_progress(
task_id, "call_openai", "success", "Received OpenAI response", file_id=file_id,
task_id,
"call_openai",
"success",
"Received OpenAI response",
file_id=file_id,
detail=f"Raw classification response:\n{content}",
)
@@ -120,13 +124,17 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id,
task_id,
"extract_metadata_with_gpt",
"failure",
"Invalid JSON in response",
file_id=file_id,
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
)
return {}
metadata = json.loads(json_text)
# SECURITY: Validate filename format from GPT to prevent path traversal
# The prompt requests filenames with only letters, numbers, periods, and underscores
# Enforce this constraint to prevent malicious filenames
@@ -138,16 +146,18 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
# 1. Potential locale-specific \w behavior
# 2. Files literally named ".." which are valid but problematic
# 3. Future code changes that might relax the regex
if not re.match(r'^[\w\-\. ]+$', suggested_filename) or ".." in suggested_filename:
logger.warning(
f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback"
)
if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
# Reset to empty to trigger fallback to original filename
metadata["filename"] = ""
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress(
task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id,
task_id,
"parse_metadata",
"success",
f"Parsed metadata: {list(metadata.keys())}",
file_id=file_id,
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
)
@@ -164,7 +174,11 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
except Exception as e:
logger.exception(f"[{task_id}] OpenAI classification failed for {filename}: {e}")
log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", f"Exception: {str(e)}", file_id=file_id,
task_id,
"extract_metadata_with_gpt",
"failure",
f"Exception: {str(e)}",
file_id=file_id,
detail=f"OpenAI classification failed for {filename}.\nException: {str(e)}",
)
return {}
app/tasks/monitor_stalled_steps.py
+4 -7
View File
@@ -34,7 +34,7 @@ def monitor_stalled_steps():
try:
with SessionLocal() as db:
stalled_count = mark_stalled_steps_as_failed(db)
if stalled_count > 0:
logger.warning(
f"[{datetime.utcnow().isoformat()}] "
@@ -42,13 +42,10 @@ def monitor_stalled_steps():
f"Marked as failed due to timeout."
)
else:
logger.debug(
f"[{datetime.utcnow().isoformat()}] "
f"No stalled steps found."
)
logger.debug(f"[{datetime.utcnow().isoformat()}] " f"No stalled steps found.")
return {"recovered": stalled_count}
except Exception as e:
logger.error(f"Error in monitor_stalled_steps task: {e}", exc_info=True)
return {"error": str(e), "recovered": 0}
app/tasks/process_document.py
+23 -16
View File
@@ -24,7 +24,9 @@ logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def process_document(self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False):
def process_document(
self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False
):
"""
Process a document file and trigger appropriate text extraction.
@@ -58,7 +60,10 @@ def process_document(self, original_local_file: str, original_filename: str = No
if not os.path.exists(original_local_file):
logger.error(f"[{task_id}] File {original_local_file} not found.")
log_task_progress(
task_id, "process_document", "failure", "File not found",
task_id,
"process_document",
"failure",
"File not found",
detail=f"File not found on disk: {original_local_file}",
)
return {"error": "File not found"}
@@ -71,7 +76,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
else:
logger.info(f"[{task_id}] Computing file hash (deduplication disabled)...")
filehash = hash_file(original_local_file)
# Use provided original_filename or fall back to basename of path
if original_filename is None:
original_filename = os.path.basename(original_local_file)
@@ -81,7 +86,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
mime_type = "application/octet-stream"
logger.info(f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}")
# Log deduplication step result (only if enabled)
if settings.enable_deduplication:
log_task_progress(
@@ -113,13 +118,15 @@ def process_document(self, original_local_file: str, original_filename: str = No
else:
# Check for duplicate only if this is a new file (not reprocessing)
# IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file
existing = db.query(FileRecord).filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)).order_by(FileRecord.created_at.asc()).first()
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
.order_by(FileRecord.created_at.asc())
.first()
)
if existing is None:
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash)
.order_by(FileRecord.id.asc())
.first()
db.query(FileRecord).filter(FileRecord.filehash == filehash).order_by(FileRecord.id.asc()).first()
)
# A file is only a duplicate if it matches a different file's hash
@@ -215,11 +222,11 @@ def process_document(self, original_local_file: str, original_filename: str = No
# This copy serves as the permanent, untouched reference of the ingested file
original_dir = os.path.join(settings.workdir, "original")
os.makedirs(original_dir, exist_ok=True)
# Use collision-resistant naming with -0001, -0002 suffixes
base_name = os.path.splitext(new_filename)[0]
original_file_path = get_unique_filepath_with_counter(original_dir, base_name, file_ext)
logger.info(f"[{task_id}] Saving immutable original to: {original_file_path}")
log_task_progress(
task_id,
@@ -236,7 +243,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
f"Original saved: {os.path.basename(original_file_path)}",
file_id=new_record.id,
)
# Update the DB with original_file_path
new_record.original_file_path = original_file_path
else:
@@ -308,7 +315,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
)
process_with_azure_document_intelligence.delay(new_filename, file_id)
return {"file": new_local_path, "status": "Queued for forced OCR", "file_id": file_id}
# If the file is not a PDF, skip embedded text check and convert to PDF first
is_pdf = mime_type == "application/pdf" or os.path.splitext(new_local_path)[1].lower() == ".pdf"
if not is_pdf:
@@ -403,7 +410,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
f"Extracted {len(extracted_text)} characters",
file_id=file_id,
)
# Mark Azure OCR as skipped since we extracted text locally
log_task_progress(
task_id,
@@ -438,7 +445,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
"No embedded text, queuing OCR",
file_id=file_id,
)
# Mark local text extraction as skipped since we're using Azure OCR
log_task_progress(
task_id,
@@ -447,7 +454,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
"No embedded text, using Azure OCR instead",
file_id=file_id,
)
log_task_progress(
task_id,
"process_document",
app/tasks/process_with_azure_document_intelligence.py
+43 -16
View File
@@ -101,8 +101,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
"""
task_id = self.request.id
log_task_progress(
task_id, "process_with_azure_document_intelligence", "in_progress",
f"Starting OCR for {filename}", file_id=file_id,
task_id,
"process_with_azure_document_intelligence",
"in_progress",
f"Starting OCR for {filename}",
file_id=file_id,
)
try:
tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
@@ -117,8 +120,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
)
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(
task_id, "validate_file", "failure",
f"File too large: {filename}", file_id=file_id, detail=error_msg,
task_id,
"validate_file",
"failure",
f"File too large: {filename}",
file_id=file_id,
detail=error_msg,
)
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
@@ -130,8 +137,12 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(
task_id, "validate_file", "failure",
f"Too many pages: {filename}", file_id=file_id, detail=error_msg,
task_id,
"validate_file",
"failure",
f"Too many pages: {filename}",
file_id=file_id,
detail=error_msg,
)
return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
if page_count is None:
@@ -140,14 +151,20 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
)
log_task_progress(
task_id, "validate_file", "success",
f"File validation passed for {filename}", file_id=file_id,
task_id,
"validate_file",
"success",
f"File validation passed for {filename}",
file_id=file_id,
)
logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.")
log_task_progress(
task_id, "call_azure_ocr", "in_progress",
f"Sending {filename} to Azure Document Intelligence", file_id=file_id,
task_id,
"call_azure_ocr",
"in_progress",
f"Sending {filename} to Azure Document Intelligence",
file_id=file_id,
)
# Open and send the document for processing
@@ -173,8 +190,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters")
log_task_progress(
task_id, "call_azure_ocr", "success",
f"Azure OCR completed for {filename}", file_id=file_id,
task_id,
"call_azure_ocr",
"success",
f"Azure OCR completed for {filename}",
file_id=file_id,
detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected",
)
@@ -182,8 +202,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id)
log_task_progress(
task_id, "process_with_azure_document_intelligence", "success",
f"OCR processing complete for {filename}", file_id=file_id,
task_id,
"process_with_azure_document_intelligence",
"success",
f"OCR processing complete for {filename}",
file_id=file_id,
detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted",
)
@@ -191,7 +214,11 @@ def process_with_azure_document_intelligence(self, filename: str, file_id: int =
except Exception as e:
logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}")
log_task_progress(
task_id, "process_with_azure_document_intelligence", "failure",
f"OCR failed for {filename}", file_id=file_id, detail=str(e),
task_id,
"process_with_azure_document_intelligence",
"failure",
f"OCR failed for {filename}",
file_id=file_id,
detail=str(e),
)
raise
app/tasks/rotate_pdf_pages.py
+28 -15
View File
@@ -63,8 +63,11 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
try:
task_id = self.request.id
log_task_progress(
task_id, "rotate_pdf_pages", "in_progress",
f"Checking page rotation for {filename}", file_id=file_id,
task_id,
"rotate_pdf_pages",
"in_progress",
f"Checking page rotation for {filename}",
file_id=file_id,
)
pdf_path = os.path.join(settings.workdir, "tmp", filename)
if not os.path.exists(pdf_path):
@@ -72,12 +75,13 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
# Skip rotation if no rotation data provided
if not rotation_data:
logger.info(
f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction"
)
logger.info(f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction")
log_task_progress(
task_id, "rotate_pdf_pages", "success",
"No rotation needed, proceeding to metadata extraction", file_id=file_id,
task_id,
"rotate_pdf_pages",
"success",
"No rotation needed, proceeding to metadata extraction",
file_id=file_id,
)
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"}
@@ -95,16 +99,22 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
f"[{task_id}] No significant rotations detected in {filename}, proceeding with metadata extraction"
)
log_task_progress(
task_id, "rotate_pdf_pages", "success",
"No rotation needed, proceeding to metadata extraction", file_id=file_id,
task_id,
"rotate_pdf_pages",
"success",
"No rotation needed, proceeding to metadata extraction",
file_id=file_id,
)
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"}
logger.info(f"[{task_id}] Rotating {len(normalized_rotation_data)} pages in {filename}")
log_task_progress(
task_id, "apply_rotation", "in_progress",
f"Rotating {len(normalized_rotation_data)} pages", file_id=file_id,
task_id,
"apply_rotation",
"in_progress",
f"Rotating {len(normalized_rotation_data)} pages",
file_id=file_id,
)
applied_rotations = {}
@@ -144,8 +154,7 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
if applied_rotations:
logger.info(
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: "
f"{json.dumps(applied_rotations)}"
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: " f"{json.dumps(applied_rotations)}"
)
else:
logger.info(
@@ -157,7 +166,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
log_task_progress(
task_id, "rotate_pdf_pages", "success",
task_id,
"rotate_pdf_pages",
"success",
f"Rotation complete for {filename}",
file_id=file_id,
detail={"applied_rotations": applied_rotations},
@@ -173,7 +184,9 @@ def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=Non
except Exception as e:
logger.error(f"[{task_id}] Error rotating PDF {filename}: {e}")
log_task_progress(
task_id, "rotate_pdf_pages", "failure",
task_id,
"rotate_pdf_pages",
"failure",
f"Rotation failed: {str(e)}",
file_id=file_id,
detail={"error": str(e), "filename": filename},
app/tasks/upload_to_paperless.py
+11 -4
View File
@@ -274,8 +274,15 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
response_text,
)
log_task_progress(
task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id,
detail=f"Failed to upload document to Paperless.\nFile: {file_path}\nError: {exc}\nResponse: {response_text}",
task_id,
"upload_to_paperless",
"failure",
error_msg,
file_id=file_id,
detail=(
f"Failed to upload document to Paperless.\n"
f"File: {file_path}\nError: {exc}\nResponse: {response_text}"
),
)
raise
@@ -322,9 +329,9 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
# Map each metadata field to its corresponding Paperless custom field
for metadata_field, paperless_field in field_mapping.items():
if metadata_field in metadata and metadata[metadata_field]:
# Convert to string to ensure consistent comparison with UNKNOWN_VALUE
# Convert to string to ensure consistent comparison
value = str(metadata[metadata_field]) if metadata[metadata_field] is not None else ""
if value and value != UNKNOWN_VALUE:
if value and value != METADATA_UNKNOWN_PLACEHOLDER:
custom_fields_to_set[paperless_field] = value
logger.debug(f"[{task_id}] Mapping {metadata_field}='{value}' to field '{paperless_field}'")
except json.JSONDecodeError as e:
app/tasks/upload_with_rclone.py
+1 -3
View File
@@ -103,9 +103,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
except (OSError, ValueError) as e:
error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(
task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg
)
log_task_progress(task_id, "upload_with_rclone", "failure", f"Upload failed for {filename}", detail=error_msg)
raise RuntimeError(error_msg) from e