Changed to workdir variable

app/config.py

@@ -13,8 +13,17 @@ class Settings(BaseSettings):
     redis_url: str
     s3_bucket_name: str
     openai_api_key: str
-
+    workdir: str
-
+    dropbox_app_key: str
+    dropbox_app_secret: str
+    dropbox_token_file_path: str
+    dropbox_folder: str
+    nextcloud_upload_url: str
+    nextcloud_username: str
+    nextcloud_password: str
+    paperless_ngx_url: str
+    paperless_ngx_api_token: str
+    paperless_host: str
 
     class Config:
         env_file = ".env"

app/main.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 
-from fastapi import FastAPI, HTTPException
-from .tasks.upload_to_s3 import upload_to_s3
 import os
+from fastapi import FastAPI, HTTPException
+from app.config import settings
+from app.tasks.upload_to_s3 import upload_to_s3
 
 app = FastAPI(title="Document Processing API")
 
@@ -17,9 +18,12 @@ def process(file_path: str):
     This enqueues the first task (upload_to_s3), which handles the full pipeline.
     """
 
+    # If file_path is not absolute, treat it as relative to settings.workdir.
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, file_path)
+
     if not os.path.exists(file_path):
         raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
 
     task = upload_to_s3.delay(file_path)
     return {"task_id": task.id, "status": "queued"}
-

app/tasks/embed_metadata_into_pdf.py

@@ -29,8 +29,8 @@ def unique_filepath(directory, base_filename, extension=".pdf"):
 def persist_metadata(metadata, final_pdf_path):
     """
     Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
-    For example, if final_pdf_path is "/var/docparse/working/processed/MyFile.pdf",
+    For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
-    the metadata will be saved as "/var/docparse/working/processed/MyFile.json".
+    the metadata will be saved as "<workdir>/processed/MyFile.json".
     """
     base, _ = os.path.splitext(final_pdf_path)
     json_path = base + ".json"
@@ -49,14 +49,14 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
       - keywords: a comma‐separated list from the "tags" field
 
     After processing, the file is moved to
-      /var/docparse/working/processed/<suggested_filename.pdf>
+      <workdir>/processed/<suggested_filename.pdf>
     where <suggested_filename.pdf> is derived from metadata["filename"].
     The output PDF is saved incrementally while preserving its original encryption.
     Additionally, the metadata is persisted to a JSON file with the same base name.
     """
-    # Check for file existence; if not found, try the known shared directory.
+    # Check for file existence; if not found, try the known shared tmp directory.
     if not os.path.exists(local_file_path):
-        alt_path = os.path.join("/var/docparse/working/tmp", os.path.basename(local_file_path))
+        alt_path = os.path.join(settings.workdir, "tmp", os.path.basename(local_file_path))
         if os.path.exists(alt_path):
             local_file_path = alt_path
         else:
@@ -93,14 +93,17 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
         suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
         # Remove any extension and then add .pdf
         suggested_filename = os.path.splitext(suggested_filename)[0]
-        # Define the final directory and ensure it exists.
+        # Define the final directory based on settings.workdir and ensure it exists.
-        final_dir = "/var/docparse/working/processed"
+        final_dir = os.path.join(settings.workdir, "processed")
         os.makedirs(final_dir, exist_ok=True)
         # Get a unique filepath in case of collisions.
         final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
 
         # Move the processed file using shutil.move to handle cross-device moves.
         shutil.move(processed_file, final_file_path)
+        # Ensure the temporary file is deleted if it still exists.
+        if os.path.exists(processed_file):
+            os.remove(processed_file)
 
         # Persist the metadata into a JSON file with the same base name.
         json_path = persist_metadata(metadata, final_file_path)
@@ -109,9 +112,17 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
         # Trigger the next step: final storage.
         finalize_document_storage.delay(original_file, final_file_path, metadata)
 
+        # After triggering final storage, delete the original file if it is in workdir/tmp.
+        workdir_tmp = os.path.join(settings.workdir, "tmp")
+        if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
+            try:
+                os.remove(original_file)
+                print(f"[INFO] Deleted original file from {original_file}")
+            except Exception as e:
+                print(f"[ERROR] Could not delete original file {original_file}: {e}")
+
         return {"file": final_file_path, "metadata_file": json_path, "status": "Metadata embedded"}
 
     except Exception as e:
         print(f"[ERROR] Failed to embed metadata into {processed_file}: {e}")
         return {"error": str(e)}
-

app/tasks/process_with_textract.py

@@ -64,13 +64,13 @@ def create_searchable_pdf(tmp_file_path, extracted_pages):
 def process_with_textract(s3_filename: str):
     """
     Processes a PDF document using Textract and overlays invisible OCR text onto
-    the local temporary file (already stored under /var/docparse/working/tmp).
+    the local temporary file (already stored under <workdir>/tmp).
     
     Steps:
       1. Start a Textract text detection job.
       2. Poll until the job succeeds and organize the Textract Blocks into pages
          (each page is a list of (text, bounding-box) tuples).
-      3. Use the local tmp file at /var/docparse/working/tmp/<s3_filename> to add the OCR overlay.
+      3. Use the local tmp file at <workdir>/tmp/<s3_filename> to add the OCR overlay.
       4. Delete the S3 object.
       5. Trigger downstream metadata extraction by calling extract_metadata_with_gpt.
     """
@@ -106,8 +106,8 @@ def process_with_textract(s3_filename: str):
                 raise Exception("Textract job failed")
             time.sleep(3)
 
-        # Use the existing local tmp file (from /var/docparse/working/tmp).
+        # Use the local tmp file located under the workdir configuration.
-        tmp_file_path = os.path.join("/var/docparse/working/tmp", s3_filename)
+        tmp_file_path = os.path.join(settings.workdir, "tmp", s3_filename)
         if not os.path.exists(tmp_file_path):
             raise Exception(f"Local file not found: {tmp_file_path}")
         logger.info(f"Processing local file {tmp_file_path} with OCR overlay.")
@@ -128,4 +128,3 @@ def process_with_textract(s3_filename: str):
     except Exception as e:
         logger.error(f"Error processing {s3_filename}: {e}")
         raise
-

app/tasks/upload_to_s3.py

@@ -38,10 +38,13 @@ def upload_to_s3(original_local_file: str):
     file_ext = os.path.splitext(original_local_file)[1]  # Preserve original file extension
     file_uuid = str(uuid.uuid4())
     new_filename = f"{file_uuid}{file_ext}"
-    new_local_path = os.path.join("/var/docparse/working/tmp", new_filename)
 
-    # Ensure the target directory exists
+    # Construct the new local path using settings.workdir and a 'tmp' subdirectory
-    os.makedirs(os.path.dirname(new_local_path), exist_ok=True)
+    tmp_dir = os.path.join(settings.workdir, "tmp")
+    new_local_path = os.path.join(tmp_dir, new_filename)
+
+    # Ensure the target tmp directory exists
+    os.makedirs(tmp_dir, exist_ok=True)
 
     # Copy the file instead of moving it
     shutil.copy(original_local_file, new_local_path)
@@ -59,4 +62,3 @@ def upload_to_s3(original_local_file: str):
     except Exception as e:
         print(f"[ERROR] Failed to upload {new_local_path} to S3: {e}")
         return {"error": str(e)}
-