gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(duplication): implement duplicate file handling and detection across processing steps

Browse Source
This commit is contained in:
Christian Krakau-Louis
2026-02-12 02:37:51 +01:00
parent 70757e5644
commit 9a2c2d20b1
12 changed files with 325 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/convert_to_pdf.py
+129 -5
View File
@@ -2,7 +2,10 @@
import logging
import mimetypes
import os
from typing import Optional, Tuple
import filetype
import puremagic
import requests
from celery import shared_task
@@ -13,8 +16,115 @@ from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def _detect_mime_type_from_magic(file_path: str) -> Optional[str]:
"""
Detect MIME type from file headers using platform-agnostic libraries.
Args:
file_path: Path to the file on disk.
Returns:
Detected MIME type or None if unknown.
"""
try:
matches = puremagic.from_file(file_path)
if matches:
return matches[0].mime_type
except puremagic.PureError:
pass
guess = filetype.guess(file_path)
if guess:
return guess.mime
return None
def _detect_mime_type(file_path: str, original_filename: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
"""
Detect MIME type using extension, original filename, or magic bytes.
Args:
file_path: Path to the file on disk.
original_filename: Optional original filename provided at upload time.
Returns:
Tuple of detected MIME type and encoding.
"""
mime_type, encoding = mimetypes.guess_type(file_path)
if mime_type:
return mime_type, encoding
if original_filename:
mime_type, encoding = mimetypes.guess_type(original_filename)
if mime_type:
return mime_type, encoding
return _detect_mime_type_from_magic(file_path), encoding
def _detect_extension(file_path: str, original_filename: Optional[str], mime_type: Optional[str]) -> str:
"""
Detect file extension from file path, original filename, or MIME type.
Args:
file_path: Path to the file on disk.
original_filename: Optional original filename provided at upload time.
mime_type: Detected MIME type if available.
Returns:
File extension with leading dot (e.g., ".pdf") or empty string if unknown.
"""
file_ext = os.path.splitext(file_path)[1].lower()
if file_ext:
return file_ext
if original_filename:
file_ext = os.path.splitext(original_filename)[1].lower()
if file_ext:
return file_ext
if mime_type:
return mimetypes.guess_extension(mime_type) or ""
try:
matches = puremagic.from_file(file_path)
if matches and matches[0].extension:
return f".{matches[0].extension.lstrip('.')}"
except puremagic.PureError:
pass
guess = filetype.guess(file_path)
if guess and guess.extension:
return f".{guess.extension.lstrip('.')}"
return ""
def _build_filename(file_path: str, original_filename: Optional[str], file_ext: str) -> str:
"""
Build a filename for upload that includes a valid extension when possible.
Args:
file_path: Path to the file on disk.
original_filename: Optional original filename provided at upload time.
file_ext: Detected file extension.
Returns:
Filename to send to Gotenberg.
"""
if original_filename and os.path.splitext(original_filename)[1]:
return original_filename
base_name = os.path.basename(file_path)
if file_ext and not base_name.lower().endswith(file_ext):
return f"{base_name}{file_ext}"
return base_name
@shared_task(bind=True)
def convert_to_pdf(self, file_path, original_filename=None):
def convert_to_pdf(self, file_path: str, original_filename: Optional[str] = None) -> Optional[str]:
"""
Converts a file to PDF using Gotenberg's API.
Determines the appropriate Gotenberg endpoint based on the file's MIME type.
@@ -35,9 +145,23 @@ def convert_to_pdf(self, file_path, original_filename=None):
return
# Try to guess the MIME type based on file content and extension
mime_type, encoding = mimetypes.guess_type(file_path)
file_ext = os.path.splitext(file_path)[1].lower()
mime_type, encoding = _detect_mime_type(file_path, original_filename)
file_ext = _detect_extension(file_path, original_filename, mime_type)
logger.info(f"[{task_id}] Guessed MIME type for '{file_path}' is: {mime_type}, extension: {file_ext}")
if not mime_type and not file_ext:
log_task_progress(
task_id,
"detect_file_type",
"failure",
"Unable to determine file type for conversion",
detail=(
f"File: {file_path}\n"
f"Original filename: {original_filename or 'N/A'}\n"
"No extension and no detectable magic header."
),
)
logger.error(f"[{task_id}] Unable to determine file type for conversion: {file_path}")
return None
log_task_progress(task_id, "detect_file_type", "success", f"File type: {mime_type or file_ext}")
# Determine which Gotenberg endpoint to use
@@ -91,7 +215,7 @@ def convert_to_pdf(self, file_path, original_filename=None):
or file_ext in IMAGE_EXTENSIONS
):
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
files = {"files": (os.path.basename(file_path), open(file_path, "rb"))}
files = {"files": (_build_filename(file_path, original_filename, file_ext), open(file_path, "rb"))}
# Add some quality settings for better PDF output
form_data = {
@@ -176,7 +300,7 @@ def convert_to_pdf(self, file_path, original_filename=None):
# Fallback to LibreOffice for everything else
else:
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
files = {"files": (os.path.basename(file_path), open(file_path, "rb"))}
files = {"files": (_build_filename(file_path, original_filename, file_ext), open(file_path, "rb"))}
logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
if not endpoint:
app/tasks/process_document.py
+79 -14
View File
@@ -7,6 +7,7 @@ import shutil
import uuid
import PyPDF2 # Replace fitz with PyPDF2
from PyPDF2.errors import PdfReadError
from app.celery_app import celery
from app.config import settings
@@ -112,25 +113,44 @@ def process_document(self, original_local_file: str, original_filename: str = No
else:
# Check for duplicate only if this is a new file (not reprocessing)
# IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file
existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none()
existing = db.query(FileRecord).filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)).order_by(FileRecord.created_at.asc()).first()
if existing is None:
existing = (
db.query(FileRecord)
.filter(FileRecord.filehash == filehash)
.order_by(FileRecord.id.asc())
.first()
)
# A file is only a duplicate if it matches a different file's hash
# (not its own hash when reprocessing)
if existing and existing.id != file_id and settings.enable_deduplication:
logger.info(f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
# Log the deduplication result without creating a new database record
# This avoids UNIQUE constraint violations on filehash
duplicate_record = FileRecord(
filehash=filehash,
original_filename=original_filename,
local_filename="",
file_size=file_size,
mime_type=mime_type,
is_duplicate=True,
duplicate_of_id=existing.id,
)
db.add(duplicate_record)
db.commit()
db.refresh(duplicate_record)
if settings.enable_deduplication and settings.show_deduplication_step:
log_task_progress(
task_id,
"check_for_duplicates",
"success",
f"Duplicate detected - matching file ID {existing.id}",
file_id=existing.id,
file_id=duplicate_record.id,
detail=(
f"Duplicate file detected.\n"
f"File hash: {filehash}\n"
f"Original file record ID: {existing.id}\n"
f"This file record ID: {duplicate_record.id}\n"
f"Original filename: {original_filename}"
),
)
@@ -139,7 +159,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
"process_document",
"success",
"Duplicate file detected, skipping",
file_id=existing.id,
file_id=duplicate_record.id,
detail=(
f"Duplicate file detected.\n"
f"File hash: {filehash}\n"
@@ -149,7 +169,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
)
return {
"status": "duplicate_file",
"file_id": existing.id,
"file_id": duplicate_record.id,
"original_file_id": existing.id,
"detail": "File already processed.",
}
@@ -289,6 +309,30 @@ def process_document(self, original_local_file: str, original_filename: str = No
process_with_azure_document_intelligence.delay(new_filename, file_id)
return {"file": new_local_path, "status": "Queued for forced OCR", "file_id": file_id}
# If the file is not a PDF, skip embedded text check and convert to PDF first
is_pdf = mime_type == "application/pdf" or os.path.splitext(new_local_path)[1].lower() == ".pdf"
if not is_pdf:
logger.info(f"[{task_id}] Non-PDF file detected, queuing PDF conversion before OCR")
log_task_progress(
task_id,
"check_text",
"skipped",
"Non-PDF file detected, converting to PDF",
file_id=file_id,
)
log_task_progress(
task_id,
"process_document",
"success",
"Queued for PDF conversion",
file_id=file_id,
)
celery.send_task(
"app.tasks.convert_to_pdf.convert_to_pdf",
args=[new_local_path, original_filename],
)
return {"file": new_local_path, "status": "Queued for PDF conversion", "file_id": file_id}
logger.info(f"[{task_id}] Checking for embedded text in PDF")
log_task_progress(
task_id,
@@ -297,13 +341,34 @@ def process_document(self, original_local_file: str, original_filename: str = No
"Checking for embedded text",
file_id=file_id,
)
with open(new_local_path, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
has_text = False
for page in pdf_reader.pages:
if page.extract_text().strip():
has_text = True
break
try:
with open(new_local_path, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
has_text = False
for page in pdf_reader.pages:
if page.extract_text().strip():
has_text = True
break
except PdfReadError as exc:
logger.warning(f"[{task_id}] PDF read error during embedded text check: {exc}")
log_task_progress(
task_id,
"check_text",
"in_progress",
"PDF read error, retrying embedded text check",
file_id=file_id,
detail=str(exc),
)
raise self.retry(
exc=exc,
countdown=10,
kwargs={
"original_local_file": original_local_file,
"original_filename": original_filename,
"file_id": file_id,
"force_cloud_ocr": force_cloud_ocr,
},
)
if has_text:
logger.info(f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally.")
app/tasks/upload_to_paperless.py
+40 -2
View File
@@ -4,6 +4,7 @@ import json
import logging
import os
import time
from typing import Optional
import requests
@@ -55,7 +56,23 @@ def normalize_metadata_value(value) -> str:
return str_value
def poll_task_for_document_id(task_id: str) -> int:
def _is_duplicate_error(result_message: str) -> bool:
"""
Determine whether a Paperless task failure indicates a duplicate document.
Args:
result_message: Failure result string from Paperless.
Returns:
True if the message indicates a duplicate, otherwise False.
"""
if not result_message:
return False
lowered = result_message.lower()
return "duplicate" in lowered and "not consuming" in lowered
def poll_task_for_document_id(task_id: str) -> Optional[int]:
"""
Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
or until we run out of attempts.
@@ -92,7 +109,11 @@ def poll_task_for_document_id(task_id: str) -> int:
return int(doc_str)
raise RuntimeError(f"Task {task_id} completed but no doc ID found. Task info: {task_info}")
elif status == "FAILURE":
raise RuntimeError(f"Task {task_id} failed: {task_info.get('result')}")
result_message = task_info.get("result")
if _is_duplicate_error(result_message):
logger.info("Task %s reported duplicate document: %s", task_id, result_message)
return None
raise RuntimeError(f"Task {task_id} failed: {result_message}")
attempts += 1
time.sleep(POLL_INTERVAL_SEC)
@@ -266,6 +287,23 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
logger.info(f"[{task_id}] Polling for document ID")
log_task_progress(task_id, "poll_task", "in_progress", "Waiting for Paperless processing", file_id=file_id)
doc_id = poll_task_for_document_id(raw_task_id)
if doc_id is None:
logger.info(f"[{task_id}] Paperless reported duplicate for {file_path}; skipping upload")
log_task_progress(
task_id,
"upload_to_paperless",
"skipped",
"Duplicate document detected by Paperless - skipping",
file_id=file_id,
detail="Paperless reported duplicate; document was not consumed.",
)
return {
"status": "Duplicate",
"paperless_task_id": raw_task_id,
"paperless_document_id": None,
"file_path": file_path,
}
logger.info(f"[{task_id}] Document {file_path} successfully ingested => ID={doc_id}")
log_task_progress(
task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id