gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed to workdir variable

Browse Source
This commit is contained in:
Christian Krakau-Louis
2025-02-11 22:01:42 +01:00
parent 3afc6746b8
commit fb217a8d8f
5 changed files with 47 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config.py
+11 -2
View File
@@ -13,8 +13,17 @@ class Settings(BaseSettings):
redis_url: str
s3_bucket_name: str
openai_api_key: str
workdir: str
dropbox_app_key: str
dropbox_app_secret: str
dropbox_token_file_path: str
dropbox_folder: str
nextcloud_upload_url: str
nextcloud_username: str
nextcloud_password: str
paperless_ngx_url: str
paperless_ngx_api_token: str
paperless_host: str
class Config:
env_file = ".env"
app/main.py
+7 -3
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env python3
from fastapi import FastAPI, HTTPException
from .tasks.upload_to_s3 import upload_to_s3
import os
from fastapi import FastAPI, HTTPException
from app.config import settings
from app.tasks.upload_to_s3 import upload_to_s3
app = FastAPI(title="Document Processing API")
@@ -17,9 +18,12 @@ def process(file_path: str):
This enqueues the first task (upload_to_s3), which handles the full pipeline.
"""
# If file_path is not absolute, treat it as relative to settings.workdir.
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, file_path)
if not os.path.exists(file_path):
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = upload_to_s3.delay(file_path)
return {"task_id": task.id, "status": "queued"}
app/tasks/embed_metadata_into_pdf.py
+19 -8
View File
@@ -29,8 +29,8 @@ def unique_filepath(directory, base_filename, extension=".pdf"):
def persist_metadata(metadata, final_pdf_path):
"""
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
For example, if final_pdf_path is "/var/docparse/working/processed/MyFile.pdf",
the metadata will be saved as "/var/docparse/working/processed/MyFile.json".
For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
the metadata will be saved as "<workdir>/processed/MyFile.json".
"""
base, _ = os.path.splitext(final_pdf_path)
json_path = base + ".json"
@@ -49,14 +49,14 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
- keywords: a comma‐separated list from the "tags" field
After processing, the file is moved to
/var/docparse/working/processed/<suggested_filename.pdf>
<workdir>/processed/<suggested_filename.pdf>
where <suggested_filename.pdf> is derived from metadata["filename"].
The output PDF is saved incrementally while preserving its original encryption.
Additionally, the metadata is persisted to a JSON file with the same base name.
"""
# Check for file existence; if not found, try the known shared directory.
# Check for file existence; if not found, try the known shared tmp directory.
if not os.path.exists(local_file_path):
alt_path = os.path.join("/var/docparse/working/tmp", os.path.basename(local_file_path))
alt_path = os.path.join(settings.workdir, "tmp", os.path.basename(local_file_path))
if os.path.exists(alt_path):
local_file_path = alt_path
else:
@@ -93,14 +93,17 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
# Remove any extension and then add .pdf
suggested_filename = os.path.splitext(suggested_filename)[0]
# Define the final directory and ensure it exists.
final_dir = "/var/docparse/working/processed"
# Define the final directory based on settings.workdir and ensure it exists.
final_dir = os.path.join(settings.workdir, "processed")
os.makedirs(final_dir, exist_ok=True)
# Get a unique filepath in case of collisions.
final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
# Move the processed file using shutil.move to handle cross-device moves.
shutil.move(processed_file, final_file_path)
# Ensure the temporary file is deleted if it still exists.
if os.path.exists(processed_file):
os.remove(processed_file)
# Persist the metadata into a JSON file with the same base name.
json_path = persist_metadata(metadata, final_file_path)
@@ -109,9 +112,17 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
# Trigger the next step: final storage.
finalize_document_storage.delay(original_file, final_file_path, metadata)
# After triggering final storage, delete the original file if it is in workdir/tmp.
workdir_tmp = os.path.join(settings.workdir, "tmp")
if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
try:
os.remove(original_file)
print(f"[INFO] Deleted original file from {original_file}")
except Exception as e:
print(f"[ERROR] Could not delete original file {original_file}: {e}")
return {"file": final_file_path, "metadata_file": json_path, "status": "Metadata embedded"}
except Exception as e:
print(f"[ERROR] Failed to embed metadata into {processed_file}: {e}")
return {"error": str(e)}
app/tasks/process_with_textract.py
+4 -5
View File
@@ -64,13 +64,13 @@ def create_searchable_pdf(tmp_file_path, extracted_pages):
def process_with_textract(s3_filename: str):
"""
Processes a PDF document using Textract and overlays invisible OCR text onto
the local temporary file (already stored under /var/docparse/working/tmp).
the local temporary file (already stored under <workdir>/tmp).
Steps:
1. Start a Textract text detection job.
2. Poll until the job succeeds and organize the Textract Blocks into pages
(each page is a list of (text, bounding-box) tuples).
3. Use the local tmp file at /var/docparse/working/tmp/<s3_filename> to add the OCR overlay.
3. Use the local tmp file at <workdir>/tmp/<s3_filename> to add the OCR overlay.
4. Delete the S3 object.
5. Trigger downstream metadata extraction by calling extract_metadata_with_gpt.
"""
@@ -106,8 +106,8 @@ def process_with_textract(s3_filename: str):
raise Exception("Textract job failed")
time.sleep(3)
# Use the existing local tmp file (from /var/docparse/working/tmp).
tmp_file_path = os.path.join("/var/docparse/working/tmp", s3_filename)
# Use the local tmp file located under the workdir configuration.
tmp_file_path = os.path.join(settings.workdir, "tmp", s3_filename)
if not os.path.exists(tmp_file_path):
raise Exception(f"Local file not found: {tmp_file_path}")
logger.info(f"Processing local file {tmp_file_path} with OCR overlay.")
@@ -128,4 +128,3 @@ def process_with_textract(s3_filename: str):
except Exception as e:
logger.error(f"Error processing {s3_filename}: {e}")
raise
app/tasks/upload_to_s3.py
+6 -4
View File
@@ -38,10 +38,13 @@ def upload_to_s3(original_local_file: str):
file_ext = os.path.splitext(original_local_file)[1] # Preserve original file extension
file_uuid = str(uuid.uuid4())
new_filename = f"{file_uuid}{file_ext}"
new_local_path = os.path.join("/var/docparse/working/tmp", new_filename)
# Ensure the target directory exists
os.makedirs(os.path.dirname(new_local_path), exist_ok=True)
# Construct the new local path using settings.workdir and a 'tmp' subdirectory
tmp_dir = os.path.join(settings.workdir, "tmp")
new_local_path = os.path.join(tmp_dir, new_filename)
# Ensure the target tmp directory exists
os.makedirs(tmp_dir, exist_ok=True)
# Copy the file instead of moving it
shutil.copy(original_local_file, new_local_path)
@@ -59,4 +62,3 @@ def upload_to_s3(original_local_file: str):
except Exception as e:
print(f"[ERROR] Failed to upload {new_local_path} to S3: {e}")
return {"error": str(e)}