Initial plan

.deepsource.toml

@@ -1,10 +0,0 @@
-version = 1
-
-[[analyzers]]
-name = "python"
-
-  [analyzers.meta]
-  runtime_version = "3.x.x"
-
-[[analyzers]]
-name = "javascript"

.env.demo

@@ -1,152 +1,56 @@
-# **Core Settings**
-WORKDIR=/workdir
+# **Config Variables**
 DATABASE_URL=sqlite:///./app/database.db
 REDIS_URL=redis://redis:6379/0
-EXTERNAL_HOSTNAME=docuelevate.example.com
-GOTENBERG_URL=http://gotenberg:3000
-ALLOW_FILE_DELETE=true  # Allow deletion of file records
+WORKDIR=/workdir
+AWS_REGION="eu-central-1"
+AZURE_REGION="eastus"
+AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
+S3_BUCKET_NAME=<your_bucket_name>
+NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
+NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
+PAPERLESS_NGX_URL=https://paperless.example.com/api/documents/post_document/
+PAPERLESS_HOST=https://paperless.example.com
 
-# **Authentication**
-AUTH_ENABLED=true
-# Generate a secure random string, for example:
-# python -c "import secrets; print(secrets.token_hex(32))"
-SESSION_SECRET=b39fd43f68d0491ca942f28a16e484b1e763fe9accf4445ca2669a5f3b179eb4
-ADMIN_USERNAME=admin
-ADMIN_PASSWORD=your_secure_password
-
-# **OpenID Connect/Authentik Settings**
-AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
-AUTHENTIK_CLIENT_SECRET=<yourAuthentikClientSecret>
-AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docuelevate/.well-known/openid-configuration>
-OAUTH_PROVIDER_NAME="Authentik SSO"
-
-# **AI/ML Services**
-# OpenAI
+# **Tokens/API Credentials**
+AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY>"
+AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
 OPENAI_API_KEY="<OPENAI_API_KEY>"
 OPENAI_BASE_URL=https://api.openai.com/v1
 OPENAI_MODEL=gpt-4o-mini
-
-# Azure AI
-AZURE_REGION="eastus"
-AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
+PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
+DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
+DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
+DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
 AZURE_AI_KEY=<AZURE_AI_KEY>
 
-# **Email Settings**
-EMAIL_HOST=smtp.example.com
-EMAIL_PORT=587
-EMAIL_USERNAME=docuelevate@example.com
-EMAIL_PASSWORD=your_secure_email_password
-EMAIL_USE_TLS=True
-EMAIL_SENDER=DocuElevate System <docuelevate@example.com>
-EMAIL_DEFAULT_RECIPIENT=recipient@example.com
+# **User Credentials**
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=your_secure_password
+NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
+NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
+IMAP1_USERNAME=<IMAP1_USERNAME>
+IMAP1_PASSWORD=<IMAP1_PASSWORD>
+IMAP2_USERNAME=<IMAP2_USERNAME>
+IMAP2_PASSWORD=<IMAP2_PASSWORD>
 
 # **IMAP Settings**
 IMAP1_HOST=mail.example.com
 IMAP1_PORT=993
-IMAP1_USERNAME=<IMAP1_USERNAME>
-IMAP1_PASSWORD=<IMAP1_PASSWORD>
 IMAP1_SSL=true
 IMAP1_POLL_INTERVAL_MINUTES=5
 IMAP1_DELETE_AFTER_PROCESS=false
 
 IMAP2_HOST=imap.gmail.com
 IMAP2_PORT=993
-IMAP2_USERNAME=<IMAP2_USERNAME>
-IMAP2_PASSWORD=<IMAP2_PASSWORD>
 IMAP2_SSL=true
 IMAP2_POLL_INTERVAL_MINUTES=10
 IMAP2_DELETE_AFTER_PROCESS=false
 
-# **Storage/Document Services**
-# Amazon S3
-AWS_REGION=us-east-1
-AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
-AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
-S3_BUCKET_NAME=my-document-bucket
-S3_FOLDER_PREFIX=documents/uploads/2023/  # Organizes files in this subfolder
-S3_STORAGE_CLASS=STANDARD
-S3_ACL=private
+GOTENBERG_URL=http://gotenberg:3000
 
-# NextCloud
-NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
-NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
-NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
-NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
-
-# Paperless-ngx
-PAPERLESS_HOST=https://paperless.example.com
-PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
-
-# Dropbox
-DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
-DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
-DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
-DROPBOX_FOLDER="/Documents/Uploads"
-
-# Google Drive
-# Service Account Method:
-GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"}
-GOOGLE_DRIVE_FOLDER_ID=<YOUR_FOLDER_ID>
-GOOGLE_DRIVE_DELEGATE_TO=<OPTIONAL_USER_EMAIL>
-
-# OAuth Method (Alternative):
-GOOGLE_DRIVE_USE_OAUTH=false  # Set to true to use OAuth instead of service account
-GOOGLE_DRIVE_CLIENT_ID=your-oauth-client-id  # Required for OAuth method
-GOOGLE_DRIVE_CLIENT_SECRET=your-oauth-client-secret  # Required for OAuth method
-GOOGLE_DRIVE_REFRESH_TOKEN=your-oauth-refresh-token  # Required for OAuth method
-
-# OneDrive
-ONEDRIVE_CLIENT_ID=your-client-id
-ONEDRIVE_CLIENT_SECRET=your-client-secret
-ONEDRIVE_TENANT_ID=common
-ONEDRIVE_REFRESH_TOKEN=your-refresh-token
-ONEDRIVE_FOLDER_PATH=Documents/Uploads
-
-# WebDAV
-WEBDAV_URL=https://webdav.example.com/path
-WEBDAV_USERNAME=webdav_user
-WEBDAV_PASSWORD=your_secure_webdav_password
-WEBDAV_FOLDER=/Documents/Uploads
-WEBDAV_VERIFY_SSL=True
-
-# FTP
-FTP_HOST=ftp.example.com
-FTP_PORT=21
-FTP_USERNAME=ftp_user
-FTP_PASSWORD=your_secure_ftp_password
-FTP_FOLDER=/Documents/Uploads
-FTP_USE_TLS=True
-FTP_ALLOW_PLAINTEXT=True
-
-# SFTP
-SFTP_HOST=sftp.example.com
-SFTP_PORT=22
-SFTP_USERNAME=sftp_user
-SFTP_PASSWORD=your_secure_sftp_password
-# SFTP_PRIVATE_KEY=/path/to/private_key.pem
-# SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
-SFTP_FOLDER=/Documents/Uploads
-
-# **Notification Settings**
-# Configure notification services using Apprise URL format
-# See https://github.com/caronc/apprise#supported-notifications
-# Examples:
-# - Discord: discord://webhook_id/webhook_token
-# - Telegram: tgram://bot_token/chat_id
-# - Email: mailto://user:pass@example.com
-# - Pushover: pover://user_key/app_token
-# - Slack: slack://tokenA/tokenB/tokenC
-# - Matrix: matrix://username:password@domain/#room
-
-# You can specify multiple notification URLs by separating them with commas
-NOTIFICATION_URLS=discord://webhook_id/webhook_token,mailto://user:pass@gmail.com,tgram://bot_token/chat_id
-
-# Control when notifications are sent
-NOTIFY_ON_TASK_FAILURE=True
-NOTIFY_ON_CREDENTIAL_FAILURE=True
-NOTIFY_ON_STARTUP=True
-NOTIFY_ON_SHUTDOWN=False
-
-# Uptime Kuma
-UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
-UPTIME_KUMA_PING_INTERVAL=5
+# ** needed for Authentik **
+AUTH_ENABLED=true
+SESSION_SECRET=<atLeast32Characters>
+AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
+AUTHENTIK_CLIENT_SECRET=<yourAuthentikAppClientSecret>
+AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/document-parser/.well-known/openid-configuration>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
-
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -5,15 +5,8 @@
 
 version: 2
 updates:
-  - package-ecosystem: "pip"
-    directory: "/"
+  - package-ecosystem: "" # See documentation for possible values
+    directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
-    open-pull-requests-limit: 10
-    
-  - package-ecosystem: "npm"
-    directory: "/frontend/static"
-    schedule:
-      interval: "monthly"
-    open-pull-requests-limit: 5
-
+      

.github/workflows/codeql.yaml

@@ -1,41 +0,0 @@
-name: "CodeQL Security Scanning"
-
-on:
-  push:
-    branches: [ "main", "develop" ]
-  pull_request:
-    branches: [ "main", "develop" ]
-  schedule:
-    - cron: '0 0 * * 1'  # Run every Monday at midnight
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'python', 'javascript' ]
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        queries: security-and-quality
-
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
-      with:
-        category: "/language:${{matrix.language}}"

.github/workflows/docker-build.yaml

@@ -9,67 +9,46 @@ on:
     branches:
       - main
       - develop
-    tags:
-      - 'v*'
-      - '[0-9]+.*'
   pull_request:
     branches:
       - main
 
-env:
-  IMAGE_NAME: christianlouis/document-processor
-
 jobs:
-  docker:
+  build:
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract Git Tag (if applicable)
-        if: startsWith(github.ref, 'refs/tags/')
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
-
-      - name: Extract metadata for tags
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.IMAGE_NAME }}
-            ghcr.io/${{ github.repository_owner }}/document-processor
-
-      - name: Build and Push Docker Image with Provenance and SBOM
-        uses: docker/build-push-action@v6
+      - name: Build and Push
+        uses: docker/build-push-action@v4
         with:
+          # Specify target platforms
+          platforms: linux/amd64
           context: .
           file: Dockerfile
-          platforms: linux/amd64
           push: true
-          sbom: true
-          provenance: mode=max
           tags: |
-            ${{ env.IMAGE_NAME }}:latest
-            ${{ env.IMAGE_NAME }}:${{ github.sha }}
+            christianlouis/document-processor:latest
+            christianlouis/document-processor:${{ github.sha }}
             ghcr.io/${{ github.repository_owner }}/document-processor:latest
             ghcr.io/${{ github.repository_owner }}/document-processor:${{ github.sha }}
-            ${{ startsWith(github.ref, 'refs/tags/') && format('{0}:{1}', env.IMAGE_NAME, env.VERSION) || '' }}
-            ${{ startsWith(github.ref, 'refs/tags/') && format('ghcr.io/{0}/document-processor:{1}', github.repository_owner, env.VERSION) || '' }}
+          # Cache options (optional)
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/docker-ci.yml

@@ -1,35 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches:
-      - "main"
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ vars.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_PAT }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: cloud
-          endpoint: "christianlouis/cklbuilder"
-          install: true
-
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          tags: "${{ vars.DOCKER_USER }}/docuelevate:latest"
-          outputs: ${{ github.event_name == 'pull_request' && 'type=cacheonly' || 'type=registry' }}
-          provenance: mode=max
-          sbom: true

.github/workflows/tests.yaml

@@ -17,40 +17,24 @@ jobs:
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          pip install -r requirements.txt
+          pip install pytest flake8 black mypy pylint
 
-      - name: Run Tests
-        run: pytest tests/ -v --cov=app --cov-report=xml --cov-report=term
-
-      - name: Upload Coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-          fail_ci_if_error: false
+      # - name: Run Tests
+      #   run: pytest tests/
 
       - name: Run Linter (Flake8)
-        run: flake8 app/ --max-line-length=120 --extend-ignore=E203,W503
-        continue-on-error: false
+        run: flake8 app/
+        continue-on-error: true
 
       - name: Run Code Formatter (Black)
-        run: black --check app/ --line-length=120
-        continue-on-error: false
+        run: black --check app/
+        continue-on-error: true
 
       - name: Run Type Checker (Mypy)
-        run: mypy app/ --ignore-missing-imports
+        run: mypy app/
         continue-on-error: true
 
       - name: Run Linter (Pylint)
-        run: pylint app/ --max-line-length=120 --disable=C0111,C0103,R0903
+        run: pylint app/
         continue-on-error: true
-      
-      - name: Run Security Linter (Bandit)
-        run: bandit -r app/ -ll -f json -o bandit-report.json
-        continue-on-error: true
-      
-      - name: Upload Bandit Report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: bandit-report
-          path: bandit-report.json

.gitignore

@@ -25,34 +25,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
-
-# Environment files - NEVER commit these!
 .env
-.env.local
-.env.*.local
-*.env
-
-# Secrets and credentials
-*secret*
-*credentials*.json
-!frontend/static/*  # Allow static files even if they match patterns
-!docs/*  # Allow documentation files
-
-# Private keys
-*.pem
-*.key
-*.p12
-*.pfx
-id_rsa*
-ssh_host_*
-
-# Database files - may contain sensitive data
-*.db
-*.sqlite
-*.sqlite3
-database.db
-db.sqlite3
-db.sqlite3-journal
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -86,6 +59,8 @@ cover/
 # Django stuff:
 *.log
 local_settings.py
+db.sqlite3
+db.sqlite3-journal
 
 # Flask stuff:
 instance/

.pre-commit-config.yaml

@@ -1,71 +0,0 @@
-# Pre-commit hooks for code quality and security
-# Install: pip install pre-commit
-# Setup: pre-commit install
-# Run manually: pre-commit run --all-files
-
-repos:
-  # General file checks
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
-      - id: check-json
-      - id: check-added-large-files
-        args: ['--maxkb=1000']
-      - id: check-merge-conflict
-      - id: detect-private-key
-      - id: detect-aws-credentials
-        args: ['--allow-missing-credentials']
-
-  # Python code formatting
-  - repo: https://github.com/psf/black
-    rev: 24.1.1
-    hooks:
-      - id: black
-        args: ['--line-length=120']
-        language_version: python3.11
-
-  # Import sorting
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        args: ['--profile=black', '--line-length=120']
-
-  # Linting
-  - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
-    hooks:
-      - id: flake8
-        args: ['--max-line-length=120', '--extend-ignore=E203,W503']
-
-  # Security linting
-  - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.6
-    hooks:
-      - id: bandit
-        args: ['-ll', '-r', 'app/']
-        exclude: 'tests/'
-
-  # Type checking
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
-    hooks:
-      - id: mypy
-        args: ['--ignore-missing-imports']
-        additional_dependencies: ['types-requests']
-
-  # Secret detection
-  - repo: https://github.com/Yelp/detect-secrets
-    rev: v1.4.0
-    hooks:
-      - id: detect-secrets
-        args: ['--baseline', '.secrets.baseline']
-        exclude: |
-          (?x)^(
-            .+\.lock|
-            .+\.json|
-            .env.demo
-          )$

.readthedocs.yaml

@@ -1,22 +0,0 @@
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Set the OS, Python version, and other tools you might need
-build:
-  os: ubuntu-24.04
-  tools:
-    python: "3.13"
-
-# Build documentation with Mkdocs
-mkdocs:
-   configuration: mkdocs.yml
-
-# Optionally, but recommended,
-# declare the Python requirements required to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-    install:
-    - requirements: docs/requirements.txt

AGENTIC_CODING.md

@@ -1,661 +0,0 @@
-# Agentic Coding Guide for DocuElevate
-
-**Version:** 1.0  
-**Last Updated:** 2026-02-06
-
-This guide helps AI coding agents work effectively with the DocuElevate codebase. It provides context, conventions, and best practices for autonomous code contributions.
-
----
-
-## 🎯 Project Overview
-
-### What is DocuElevate?
-DocuElevate is an intelligent document processing system that:
-- Ingests documents from multiple sources (email, web upload, API)
-- Processes documents (OCR, PDF conversion, metadata extraction)
-- Stores documents in various cloud storage providers
-- Uses AI (OpenAI, Azure) for intelligent document classification and metadata extraction
-
-### Tech Stack
-```
-Backend:  FastAPI, SQLAlchemy, Celery, Redis
-Frontend: Jinja2 templates, Tailwind CSS
-AI/ML:    OpenAI API, Azure Document Intelligence
-Storage:  Dropbox, Google Drive, OneDrive, S3, Nextcloud, Paperless-NGX
-Auth:     Authentik (OAuth2), Basic Auth
-Infra:    Docker, Docker Compose, Alembic (migrations)
-```
-
-### Key Directories
-```
-DocuElevate/
-├── app/
-│   ├── api/          # REST API endpoints
-│   ├── tasks/        # Celery background tasks
-│   ├── routes/       # Deprecated - being migrated to api/
-│   ├── views/        # UI routes and templates
-│   ├── utils/        # Utility functions
-│   ├── config.py     # Configuration (Pydantic Settings)
-│   ├── database.py   # SQLAlchemy setup
-│   ├── models.py     # Database models
-│   ├── main.py       # FastAPI app initialization
-│   └── auth.py       # Authentication logic
-├── frontend/
-│   ├── static/       # CSS, JS, images
-│   └── templates/    # Jinja2 HTML templates
-├── tests/           # Pytest test suite
-├── docs/            # User documentation
-├── migrations/      # Alembic database migrations
-└── docker/          # Docker configuration
-```
-
----
-
-## 🤖 Agent Guidelines
-
-### Before Making Changes
-
-1. **Understand the Context**
-   - Read relevant documentation in `docs/`
-   - Check `TODO.md` for current priorities
-   - Review `SECURITY_AUDIT.md` for security considerations
-   - Check `ROADMAP.md` for feature direction
-
-2. **Check Existing Patterns**
-   - Look at similar existing code first
-   - Follow the established patterns in the codebase
-   - Don't introduce new patterns without good reason
-
-3. **Identify Dependencies**
-   - Check if your change affects multiple modules
-   - Ensure you understand the Celery task flow
-   - Consider impact on database schema
-
-### Code Conventions
-
-#### Python Style
-```python
-# Use Black formatting (line length: 120)
-# Use type hints
-def process_document(file_path: str, metadata: Dict[str, Any]) -> DocumentMetadata:
-    """
-    Process a document and extract metadata.
-    
-    Args:
-        file_path: Absolute path to the document file
-        metadata: Additional metadata to include
-        
-    Returns:
-        DocumentMetadata object with extracted information
-        
-    Raises:
-        FileNotFoundError: If file doesn't exist
-        ProcessingError: If processing fails
-    """
-    pass
-
-# Use descriptive variable names
-user_document_path = Path("/workdir/documents/invoice.pdf")
-ocr_result = extract_text_from_pdf(user_document_path)
-
-# Prefer explicit over implicit
-if storage_provider == "dropbox":
-    upload_to_dropbox(file_path, metadata)
-elif storage_provider == "google_drive":
-    upload_to_google_drive(file_path, metadata)
-else:
-    raise ValueError(f"Unknown storage provider: {storage_provider}")
-```
-
-#### Configuration
-```python
-# Always use settings from config.py
-from app.config import settings
-
-# Good
-api_key = settings.openai_api_key
-
-# Bad - never hardcode
-api_key = "sk-abc123..."
-
-# Check if optional services are configured
-if settings.dropbox_app_key:
-    # Dropbox is configured
-    upload_to_dropbox()
-```
-
-#### Error Handling
-```python
-# Use appropriate exception types
-from fastapi import HTTPException, status
-
-# API endpoints should return HTTP errors
-@router.get("/files/{file_id}")
-async def get_file(file_id: int):
-    file = get_file_from_db(file_id)
-    if not file:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"File with ID {file_id} not found"
-        )
-    return file
-
-# Tasks should log and handle errors gracefully
-@celery_app.task(bind=True, max_retries=3)
-def process_document_task(self, file_path: str):
-    try:
-        result = process_document(file_path)
-        return result
-    except TemporaryError as e:
-        logger.warning(f"Temporary error processing {file_path}: {e}")
-        raise self.retry(exc=e, countdown=60)
-    except PermanentError as e:
-        logger.error(f"Permanent error processing {file_path}: {e}")
-        # Don't retry permanent errors
-        return {"error": str(e)}
-```
-
-#### Testing
-```python
-# Mark tests appropriately
-@pytest.mark.unit
-def test_hash_file():
-    """Unit test for file hashing utility."""
-    pass
-
-@pytest.mark.integration
-def test_upload_api_endpoint(client):
-    """Integration test for upload API."""
-    pass
-
-@pytest.mark.requires_external
-@pytest.mark.skip(reason="Requires OpenAI API key")
-def test_openai_metadata_extraction():
-    """Test actual OpenAI integration."""
-    pass
-
-# Use fixtures for common setup
-def test_document_processing(sample_pdf_path, db_session):
-    """Test uses fixtures from conftest.py"""
-    pass
-```
-
----
-
-## 📝 Common Tasks
-
-### Adding a New API Endpoint
-
-1. Create endpoint in `app/api/`:
-```python
-# app/api/my_feature.py
-from fastapi import APIRouter, HTTPException
-from app.database import get_db
-from app.models import MyModel
-
-router = APIRouter(prefix="/api/my-feature", tags=["my-feature"])
-
-@router.get("/")
-async def list_items(db=Depends(get_db)):
-    """List all items."""
-    items = db.query(MyModel).all()
-    return items
-```
-
-2. Register router in `app/api/__init__.py`:
-```python
-from app.api import my_feature
-
-router.include_router(my_feature.router)
-```
-
-3. Add tests in `tests/test_api_my_feature.py`
-
-### Adding a New Celery Task
-
-1. Create task in `app/tasks/`:
-```python
-# app/tasks/my_task.py
-from app.celery_app import celery_app
-import logging
-
-logger = logging.getLogger(__name__)
-
-@celery_app.task(bind=True, max_retries=3)
-def my_background_task(self, param: str):
-    """
-    Description of what this task does.
-    
-    Args:
-        param: Description of parameter
-    """
-    try:
-        logger.info(f"Processing task with param: {param}")
-        # Task logic here
-        return {"status": "success"}
-    except Exception as e:
-        logger.error(f"Task failed: {e}")
-        raise self.retry(exc=e, countdown=60)
-```
-
-2. Import in `app/tasks/__init__.py`
-3. Add tests in `tests/test_tasks.py`
-
-### Adding a Database Model
-
-1. Define model in `app/models.py`:
-```python
-class MyModel(Base):
-    __tablename__ = "my_table"
-    
-    id = Column(Integer, primary_key=True, index=True)
-    name = Column(String, nullable=False)
-    created_at = Column(DateTime, default=datetime.utcnow)
-```
-
-2. Create migration:
-```bash
-cd /path/to/DocuElevate
-alembic revision --autogenerate -m "Add MyModel table"
-alembic upgrade head
-```
-
-3. Add model to tests fixtures
-
-### Adding a Storage Provider
-
-1. Create provider module in `app/tasks/storage/`:
-```python
-# app/tasks/storage/my_provider.py
-from app.config import settings
-import logging
-
-logger = logging.getLogger(__name__)
-
-def upload_to_my_provider(file_path: str, metadata: dict) -> str:
-    """
-    Upload file to My Provider.
-    
-    Args:
-        file_path: Local path to file
-        metadata: Document metadata
-        
-    Returns:
-        URL or ID of uploaded file
-        
-    Raises:
-        ProviderError: If upload fails
-    """
-    if not settings.my_provider_api_key:
-        raise ValueError("MY_PROVIDER_API_KEY not configured")
-    
-    # Implementation
-    pass
-```
-
-2. Add configuration to `app/config.py`:
-```python
-class Settings(BaseSettings):
-    # ... existing settings ...
-    my_provider_api_key: Optional[str] = None
-    my_provider_endpoint: Optional[str] = None
-```
-
-3. Add to `.env.demo`:
-```bash
-# My Provider
-MY_PROVIDER_API_KEY=your_api_key_here
-MY_PROVIDER_ENDPOINT=https://api.myprovider.com
-```
-
-4. Add validator in `app/utils/config_validator/`
-5. Add tests with mocked API calls
-
----
-
-## 🔒 Security Best Practices
-
-### What to NEVER Do
-- ❌ Hardcode API keys, passwords, or secrets
-- ❌ Log sensitive data (passwords, tokens, API keys)
-- ❌ Accept unsanitized user input for file paths
-- ❌ Disable security features without documentation
-- ❌ Commit `.env` files or credentials
-
-### What to ALWAYS Do
-- ✅ Use `settings` from `app/config.py` for all configuration
-- ✅ Validate and sanitize all user inputs
-- ✅ Use parameterized database queries (SQLAlchemy handles this)
-- ✅ Check file paths for directory traversal (`Path.resolve()`)
-- ✅ Use appropriate HTTP status codes (401, 403, 404, etc.)
-- ✅ Log security-relevant events
-- ✅ Add rate limiting for sensitive endpoints
-- ✅ Use HTTPS in production (documented in deployment guide)
-
-### Input Validation Example
-```python
-from pathlib import Path
-from fastapi import HTTPException, status
-
-def validate_file_path(file_path: str, base_dir: str = "/workdir") -> Path:
-    """Validate file path is within allowed directory."""
-    try:
-        path = Path(file_path).resolve()
-        base = Path(base_dir).resolve()
-        
-        # Ensure path is within base directory
-        if not path.is_relative_to(base):
-            raise ValueError("Path outside allowed directory")
-        
-        return path
-    except Exception as e:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail=f"Invalid file path: {e}"
-        )
-```
-
----
-
-## 🧪 Testing Strategy
-
-### Test Coverage Goals
-- **Target:** 80% overall coverage
-- **Critical modules:** 90%+ (auth, config, database)
-- **Tasks:** 70%+ (complex to test with external services)
-- **API endpoints:** 85%+
-
-### Test Types
-```python
-# Unit tests - fast, isolated, no external dependencies
-@pytest.mark.unit
-def test_hash_file_empty(tmp_path):
-    """Test hashing an empty file."""
-    file = tmp_path / "empty.txt"
-    file.write_text("")
-    assert hash_file(str(file)) == "expected_hash"
-
-# Integration tests - test multiple components together
-@pytest.mark.integration
-def test_upload_and_process(client, sample_pdf):
-    """Test full upload and processing flow."""
-    response = client.post("/api/upload", files={"file": sample_pdf})
-    assert response.status_code == 200
-
-# External service tests - skipped by default
-@pytest.mark.requires_external
-@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="No API key")
-def test_real_openai_extraction():
-    """Test actual OpenAI API (skipped in CI)."""
-    pass
-```
-
-### Running Tests
-```bash
-# All tests
-pytest
-
-# Specific category
-pytest -m unit
-pytest -m integration
-
-# With coverage
-pytest --cov=app --cov-report=html
-
-# Specific file
-pytest tests/test_api.py -v
-
-# Skip external services
-pytest -m "not requires_external"
-```
-
----
-
-## 🚀 Performance Considerations
-
-### Async/Await
-- FastAPI endpoints are async by default
-- Use `async def` for I/O-bound operations
-- Use regular `def` for CPU-bound operations
-
-```python
-# Good - async for I/O
-@router.get("/files")
-async def list_files(db: Session = Depends(get_db)):
-    files = db.query(FileRecord).all()
-    return files
-
-# Also good - sync for CPU-heavy
-@router.post("/hash")
-def hash_large_file(file: UploadFile):
-    return compute_hash(file.file.read())
-```
-
-### Database Queries
-```python
-# Good - single query with join
-files = db.query(FileRecord).options(
-    joinedload(FileRecord.metadata)
-).filter(FileRecord.user_id == user_id).all()
-
-# Bad - N+1 queries
-files = db.query(FileRecord).filter(FileRecord.user_id == user_id).all()
-for file in files:
-    metadata = file.metadata  # Triggers separate query each time
-```
-
-### Celery Tasks
-```python
-# Long-running tasks should update progress
-@celery_app.task(bind=True)
-def process_large_batch(self, file_ids: List[int]):
-    total = len(file_ids)
-    for i, file_id in enumerate(file_ids):
-        process_file(file_id)
-        self.update_state(
-            state='PROGRESS',
-            meta={'current': i + 1, 'total': total}
-        )
-```
-
----
-
-## 📚 Documentation Requirements
-
-### Code Documentation
-```python
-def complex_function(param1: str, param2: int = 10) -> Dict[str, Any]:
-    """
-    One-line summary of what the function does.
-    
-    More detailed explanation if needed. Can span multiple
-    lines and include examples.
-    
-    Args:
-        param1: Description of param1
-        param2: Description of param2, defaults to 10
-        
-    Returns:
-        Dictionary containing:
-            - key1: Description
-            - key2: Description
-            
-    Raises:
-        ValueError: If param1 is empty
-        FileNotFoundError: If file doesn't exist
-        
-    Examples:
-        >>> result = complex_function("test", 5)
-        >>> print(result['key1'])
-        'value'
-    """
-    pass
-```
-
-### API Documentation
-- Use FastAPI's automatic OpenAPI generation
-- Add descriptions to endpoints
-- Document request/response models
-- Include example requests/responses
-
-```python
-@router.post(
-    "/upload",
-    response_model=UploadResponse,
-    status_code=status.HTTP_201_CREATED,
-    summary="Upload a document",
-    description="Upload a document for processing. Supports PDF, images, and Office documents.",
-    responses={
-        201: {"description": "Document uploaded successfully"},
-        400: {"description": "Invalid file format"},
-        413: {"description": "File too large"},
-    }
-)
-async def upload_document(
-    file: UploadFile = File(..., description="Document file to upload"),
-    tags: List[str] = Query([], description="Optional tags for the document"),
-):
-    """Upload endpoint implementation."""
-    pass
-```
-
----
-
-## 🐛 Debugging
-
-### Logging
-```python
-import logging
-
-logger = logging.getLogger(__name__)
-
-# Use appropriate log levels
-logger.debug("Detailed information for debugging")
-logger.info("General information about operation")
-logger.warning("Warning about potential issue")
-logger.error("Error that needs attention")
-logger.critical("Critical error that needs immediate attention")
-
-# Include context in logs
-logger.info(f"Processing document: {file_id}, user: {user_id}")
-
-# Don't log sensitive data
-logger.info(f"User authenticated")  # Good
-logger.info(f"Password: {password}")  # BAD!
-```
-
-### Common Issues
-
-1. **Import Errors**
-   - Check if module is in `__init__.py`
-   - Verify Python path includes project root
-   - Look for circular imports
-
-2. **Database Issues**
-   - Check if migrations are up to date: `alembic upgrade head`
-   - Verify DATABASE_URL is set correctly
-   - Check if tables exist: `sqlite3 app/database.db .schema`
-
-3. **Celery Issues**
-   - Verify Redis is running: `redis-cli ping`
-   - Check Celery worker logs
-   - Ensure tasks are imported in `celery_worker.py`
-
-4. **Test Failures**
-   - Check if test database is clean (use fixtures)
-   - Verify environment variables are set in `conftest.py`
-   - Run single test to isolate issue: `pytest tests/test_file.py::test_name -v`
-
----
-
-## 🔄 Git Workflow
-
-### Branch Names
-- `feature/description` - New features
-- `bugfix/description` - Bug fixes
-- `hotfix/description` - Urgent production fixes
-- `refactor/description` - Code refactoring
-- `docs/description` - Documentation updates
-
-### Commit Messages
-```
-type(scope): Short description (max 72 chars)
-
-Longer description if needed. Explain:
-- What changed
-- Why it changed
-- Any breaking changes
-
-Fixes #123
-```
-
-Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`
-
-### Pull Requests
-1. Create PR with descriptive title
-2. Fill out PR template
-3. Link related issues
-4. Ensure CI passes
-5. Request reviews
-6. Address feedback
-7. Squash merge when approved
-
----
-
-## ✅ Pre-commit Checklist
-
-Before submitting code:
-
-- [ ] Code follows style guide (Black formatted)
-- [ ] All tests pass (`pytest`)
-- [ ] New code has tests
-- [ ] Coverage doesn't decrease
-- [ ] Documentation updated if needed
-- [ ] No secrets or credentials in code
-- [ ] Linting passes (`flake8`, `pylint`)
-- [ ] Type hints added (`mypy` clean)
-- [ ] CHANGELOG.md updated (if user-facing)
-- [ ] Security scan passed (`bandit`)
-
-Run full check:
-```bash
-pytest --cov=app
-black app/ tests/
-flake8 app/ --max-line-length=120
-mypy app/
-bandit -r app/
-```
-
----
-
-## 🤝 Agent Collaboration
-
-### When to Ask for Help
-- Breaking changes needed
-- Unsure about architecture decision
-- Security implications unclear
-- Performance impact unknown
-- Tests consistently failing
-
-### How to Document Changes
-1. Update relevant documentation
-2. Add comments for complex logic
-3. Update TODO.md if introducing tech debt
-4. Note breaking changes in commit message
-5. Update API documentation if endpoints changed
-
----
-
-## 📞 Resources
-
-- **Main README:** [README.md](README.md)
-- **API Docs:** http://localhost:8000/docs (when running)
-- **User Guide:** [docs/UserGuide.md](docs/UserGuide.md)
-- **Deployment:** [docs/DeploymentGuide.md](docs/DeploymentGuide.md)
-- **Troubleshooting:** [docs/Troubleshooting.md](docs/Troubleshooting.md)
-- **GitHub Issues:** Track bugs and features
-- **GitHub Discussions:** Questions and community
-
----
-
-*This guide is a living document. Improvements welcome via PR!*

ANALYSIS_SUMMARY.md

@@ -1,348 +0,0 @@
-# Repository Analysis & Improvement Summary
-
-**Date:** 2026-02-06  
-**Repository:** christianlouis/DocuElevate  
-**Current Version:** v0.3.2
-
-## Executive Summary
-
-This document summarizes the comprehensive analysis and improvements made to prepare the DocuElevate repository for secure, maintainable, and agentic development.
-
----
-
-## 🔍 Analysis Conducted
-
-### Repository Structure
-- ✅ Analyzed all key components (app/, frontend/, tests/, docs/)
-- ✅ Identified 25+ Celery tasks for document processing
-- ✅ Mapped 11 API modules and route organization
-- ✅ Reviewed database models and migration setup
-- ✅ Examined CI/CD workflows and build configuration
-
-### Security Audit
-- ✅ Scanned dependencies for known vulnerabilities
-- ✅ Identified 3 critical security issues
-- ✅ Reviewed authentication and session management
-- ✅ Checked for hardcoded credentials (none found)
-- ✅ Examined file handling for path traversal risks
-
-### Code Quality Assessment
-- ✅ Evaluated testing coverage (initially <5%)
-- ✅ Reviewed linting and formatting setup
-- ✅ Identified code duplication in storage providers
-- ✅ Found Pydantic V1 deprecation warnings
-- ✅ Noted missing type hints in several modules
-
----
-
-## 🛡️ Security Improvements
-
-### Critical Vulnerabilities Fixed
-
-1. **Authlib Vulnerability** ✅
-   - **Issue:** CVE affecting versions < 1.6.5
-   - **Risk:** Denial of Service via oversized JOSE segments, JWS/JWT bypass
-   - **Fix:** Updated requirements.txt to require authlib>=1.6.5
-
-2. **Starlette DoS Vulnerability** ✅
-   - **Issue:** O(n^2) DoS via Range header merging
-   - **Risk:** Performance degradation, potential service disruption
-   - **Fix:** Updated requirements.txt to require starlette>=0.49.1
-
-3. **Weak SESSION_SECRET Default** ✅
-   - **Issue:** Predictable default secret key in main.py
-   - **Risk:** Session hijacking, authentication bypass
-   - **Fix:** Enhanced validation, clear insecure marking, error on missing
-
-### Security Enhancements Added
-
-- ✅ Enhanced .gitignore to prevent credential leaks
-- ✅ Added CodeQL security scanning workflow
-- ✅ Added Bandit security linting
-- ✅ Created SECURITY_AUDIT.md with findings
-- ✅ Added pre-commit secret detection hooks
-- ✅ Documented security best practices
-
----
-
-## 🧪 Testing Infrastructure
-
-### Created Test Framework
-```
-tests/
-├── conftest.py          # Shared fixtures and configuration
-├── test_utils.py        # Existing utility tests (3 tests)
-├── test_config.py       # Configuration validation (8 tests)
-└── test_api.py          # API integration tests (8 tests - 6 need fixes)
-```
-
-### Test Configuration
-- ✅ pytest.ini with coverage and marker configuration
-- ✅ Fixtures for test database, sample files, mock responses
-- ✅ Test categorization (unit, integration, security, requires_external)
-- ✅ Coverage reporting configured (HTML, XML, terminal)
-
-### Test Results
-- **Total Tests:** 19 tests created
-- **Passing:** 13 tests (68%)
-- **Needs Fixes:** 6 API tests (auth configuration issues)
-- **Coverage:** Not measured yet (requires fixes first)
-
----
-
-## 📊 CI/CD Improvements
-
-### GitHub Actions Workflows
-
-**Enhanced tests.yaml:**
-- ✅ Enabled pytest execution (was commented out)
-- ✅ Added coverage reporting with Codecov
-- ✅ Made Flake8 and Black checks blocking
-- ✅ Added Bandit security scanning
-- ✅ Improved linting configuration (line length: 120)
-
-**New codeql.yaml:**
-- ✅ Security scanning for Python and JavaScript
-- ✅ Scheduled weekly scans
-- ✅ Runs on PRs and main branch pushes
-- ✅ Uses security-and-quality queries
-
-**Pre-commit Hooks (.pre-commit-config.yaml):**
-- ✅ File checks (trailing whitespace, large files, etc.)
-- ✅ Black formatting (line length: 120)
-- ✅ isort import sorting
-- ✅ Flake8 linting
-- ✅ Bandit security scanning
-- ✅ mypy type checking
-- ✅ Secret detection with detect-secrets
-
----
-
-## 📚 Documentation Created
-
-### Planning Documents
-
-1. **ROADMAP.md** (6.6 KB)
-   - Vision through v2.0+ (2027)
-   - Short-term goals (Q1-Q2 2026)
-   - Medium-term goals (Q3-Q4 2026)
-   - Long-term strategic initiatives
-   - Technology debt tracking
-
-2. **MILESTONES.md** (9.1 KB)
-   - Detailed release planning
-   - Version history and EOL policy
-   - v0.3.3 through v2.0.0 roadmap
-   - Breaking changes documentation
-   - Support policy
-
-3. **TODO.md** (8.4 KB)
-   - Prioritized task list (Critical → Low)
-   - Known bugs tracking
-   - Technical debt inventory
-   - Completed tasks log
-   - Task status notation system
-
-4. **AGENTIC_CODING.md** (17.3 KB)
-   - Comprehensive coding guide for AI agents
-   - Project overview and tech stack
-   - Code conventions and patterns
-   - Common task examples (API, tasks, models, providers)
-   - Security best practices
-   - Testing strategy
-   - Performance considerations
-   - Debugging guide
-
-5. **SECURITY_AUDIT.md** (4.5 KB)
-   - Security findings and remediation
-   - Fixed vulnerabilities documentation
-   - Ongoing security measures
-   - Recommendations by priority
-
-### Updated Documentation
-
-6. **CONTRIBUTING.md** (Enhanced)
-   - Added references to all new docs
-   - Linked testing guidelines
-   - Referenced agentic coding guide
-   - Added security policy links
-
----
-
-## 📈 Code Quality Improvements
-
-### Dependency Management
-- ✅ Fixed vulnerable packages (authlib, starlette)
-- ✅ Added version constraints for security
-- ✅ Updated requirements-dev.txt with testing tools
-- ✅ Added security scanning tools (bandit, safety)
-
-### Linting Configuration
-- ✅ Standardized line length to 120 characters
-- ✅ Configured Flake8 to ignore E203, W503 (Black compatibility)
-- ✅ Set up mypy with ignore-missing-imports
-- ✅ Configured Pylint with reasonable defaults
-
-### Testing Tools Added
-```
-pytest>=8.0.0
-pytest-cov>=4.1.0
-pytest-asyncio>=0.23.0
-pytest-mock>=3.12.0
-httpx>=0.26.0
-```
-
----
-
-## 🤖 Agentic Coding Readiness
-
-### Documentation Completeness
-- ✅ **Project Overview:** Clear description of purpose and architecture
-- ✅ **Tech Stack:** Fully documented with versions
-- ✅ **Directory Structure:** Explained with purpose of each component
-- ✅ **Code Conventions:** Python style, configuration, error handling
-- ✅ **Common Tasks:** Step-by-step guides for frequent operations
-- ✅ **Security Guidelines:** What to do and what to avoid
-- ✅ **Testing Strategy:** How to write and run tests
-- ✅ **Git Workflow:** Branch naming, commit messages, PR process
-
-### Agent-Friendly Features
-- ✅ Clear code examples for common patterns
-- ✅ Comprehensive error handling guidance
-- ✅ Security checklist and best practices
-- ✅ Pre-commit checklist for quality assurance
-- ✅ Debugging tips for common issues
-- ✅ Performance considerations documented
-- ✅ Resource links for more information
-
----
-
-## 📋 Remaining Work
-
-### High Priority (Next 2 Weeks)
-- [ ] Fix API integration test failures (auth configuration)
-- [ ] Add tests for file upload functionality
-- [ ] Add mocked tests for OCR and metadata extraction
-- [ ] Achieve 60% test coverage
-- [ ] Fix critical Flake8 violations
-- [ ] Run Black formatter on entire codebase
-- [ ] Add type hints to core modules
-
-### Medium Priority (Next Month)
-- [ ] Fix Pydantic V1 → V2 migration warnings
-- [ ] Migrate from PyPDF2 to pypdf (modern fork)
-- [ ] Consolidate storage provider code
-- [ ] Add API pagination
-- [ ] Implement retry logic for Celery tasks
-- [ ] Add performance benchmarks
-
-### Documentation Enhancements
-- [ ] Add architecture diagram
-- [ ] Create video tutorials
-- [ ] Add more code examples
-- [ ] Document all environment variables
-- [ ] Create troubleshooting guide for tests
-
----
-
-## 📊 Metrics
-
-### Before Improvements
-- **Test Coverage:** <5% (only 3 tests)
-- **Security Issues:** 3 critical vulnerabilities
-- **CI/CD:** Tests disabled, linting non-blocking
-- **Documentation:** Good user docs, limited dev docs
-- **Code Quality:** Some linting, no pre-commit hooks
-
-### After Improvements
-- **Test Coverage:** 68% passing (13/19 tests), 6 need fixes
-- **Security Issues:** All 3 critical issues fixed
-- **CI/CD:** Tests enabled, security scanning added
-- **Documentation:** Comprehensive guides for developers and agents
-- **Code Quality:** Pre-commit hooks, strict linting, type checking
-
-### Target (Next Month)
-- **Test Coverage:** 80% overall coverage
-- **Security:** Regular automated scans, 0 known issues
-- **CI/CD:** All checks blocking, green builds
-- **Documentation:** Video tutorials, architecture diagrams
-- **Code Quality:** 100% type hints, zero warnings
-
----
-
-## 🎯 Key Achievements
-
-1. ✅ **Eliminated Critical Security Vulnerabilities**
-   - Fixed 3 high-severity CVEs
-   - Enhanced secret management
-   - Added automated security scanning
-
-2. ✅ **Established Testing Infrastructure**
-   - Created comprehensive test framework
-   - Added 16 new tests
-   - Configured coverage reporting
-
-3. ✅ **Improved CI/CD Pipeline**
-   - Enabled automated testing
-   - Added security scanning (CodeQL, Bandit)
-   - Made quality checks blocking
-
-4. ✅ **Created Comprehensive Documentation**
-   - 42KB of new documentation
-   - Complete agentic coding guide
-   - Clear roadmap and milestones
-
-5. ✅ **Prepared for Agentic Development**
-   - Clear patterns and conventions
-   - Comprehensive examples
-   - Pre-commit quality checks
-
----
-
-## 🔗 Document Links
-
-- [ROADMAP.md](ROADMAP.md) - Long-term vision and features
-- [MILESTONES.md](MILESTONES.md) - Release planning
-- [TODO.md](TODO.md) - Current tasks and priorities
-- [AGENTIC_CODING.md](AGENTIC_CODING.md) - Comprehensive coding guide
-- [SECURITY_AUDIT.md](SECURITY_AUDIT.md) - Security findings
-- [CONTRIBUTING.md](CONTRIBUTING.md) - Contribution guidelines
-
----
-
-## 📞 Next Steps for Maintainers
-
-1. **Review and Merge PR**
-   - Review all changes in this PR
-   - Test locally if needed
-   - Merge when satisfied
-
-2. **Configure Branch Protection**
-   - Require passing tests
-   - Require security scans
-   - Require code review
-
-3. **Set Up Codecov**
-   - Configure Codecov token
-   - Set coverage thresholds
-   - Add status badge to README
-
-4. **Enable Pre-commit Hooks**
-   - Install for all contributors
-   - Document in onboarding
-
-5. **Work Through TODO.md**
-   - Fix API test failures first
-   - Increase test coverage
-   - Address code quality issues
-
-6. **Schedule Regular Reviews**
-   - Weekly TODO.md updates
-   - Monthly security audits
-   - Quarterly roadmap reviews
-
----
-
-**Prepared by:** GitHub Copilot Agent  
-**Review Status:** Ready for maintainer review  
-**Recommended Action:** Merge and continue with TODO.md priorities

BUILD_DATE

@@ -1 +0,0 @@
-2025-04-11

CODE_OF_CONDUCT.md

@@ -1,133 +0,0 @@
-
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official email address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-code-of-conduct@fret.de.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -1,129 +0,0 @@
-# Contributing to DocuElevate
-
-Thank you for your interest in contributing to DocuElevate! This document provides guidelines and instructions for contributing to the project.
-
-## Code of Conduct
-
-By participating in this project, you agree to abide by the [Code of Conduct](CODE_OF_CONDUCT.md).
-
-## How to Contribute
-
-### Reporting Bugs
-
-If you find a bug in the codebase, please submit an issue on GitHub with:
-
-1. A clear title and description
-2. Steps to reproduce the issue
-3. Expected behavior
-4. Actual behavior
-5. Environment information (OS, Docker version, etc.)
-
-### Feature Requests
-
-We welcome feature requests! Please submit an issue with:
-
-1. A clear title and description
-2. The problem the feature would solve
-3. Any ideas you have for implementing the feature
-
-### Pull Requests
-
-1. Fork the repository
-2. Create a new branch for your changes
-3. Make your changes
-4. Run the tests to ensure everything works
-5. Submit a pull request with a clear description of the changes
-
-## Development Environment
-
-### Setting Up Your Environment
-
-```bash
-# Clone the repository
-git clone https://github.com/christianlouis/document-processor.git
-cd document-processor
-
-# Create a virtual environment
-python -m venv venv
-source venv/bin/activate  # On Windows: venv\Scripts\activate
-
-# Install dependencies
-pip install -r requirements.txt
-pip install -r requirements-dev.txt
-```
-
-### Running Tests
-
-```bash
-pytest
-```
-
-### Code Style
-
-We use:
-- Black for Python code formatting
-- Flake8 for linting
-- isort for import sorting
-
-```bash
-# Format code
-black .
-
-# Check linting
-flake8
-
-# Sort imports
-isort .
-```
-
-## Project Structure
-
-```
-DocuElevate/
-├── app/               # Main application code
-│   ├── api/          # REST API endpoints (organized by feature)
-│   ├── tasks/        # Celery background tasks
-│   ├── views/        # UI routes and template rendering
-│   ├── utils/        # Utility functions and helpers
-│   ├── config.py     # Configuration management (Pydantic)
-│   ├── database.py   # Database setup and session management
-│   ├── models.py     # SQLAlchemy models
-│   ├── main.py       # FastAPI app initialization
-│   └── auth.py       # Authentication logic
-├── frontend/         # Frontend assets
-│   ├── static/       # CSS, JavaScript, images
-│   └── templates/    # Jinja2 HTML templates
-├── tests/            # Test suite
-├── docs/             # User and developer documentation
-├── migrations/       # Alembic database migrations
-└── docker/           # Docker configuration files
-```
-
-## 📚 Additional Resources
-
-### Documentation
-- **[AGENTIC_CODING.md](AGENTIC_CODING.md)** - Comprehensive guide for AI agents and developers
-- **[README.md](README.md)** - Project overview and quickstart
-- **[ROADMAP.md](ROADMAP.md)** - Future features and long-term vision
-- **[MILESTONES.md](MILESTONES.md)** - Release planning and versioning
-- **[TODO.md](TODO.md)** - Current tasks and priorities
-- **[SECURITY.md](SECURITY.md)** - Security policy
-- **[SECURITY_AUDIT.md](SECURITY_AUDIT.md)** - Security findings and improvements
-
-### Testing
-- All new features must include tests
-- Aim for 80% code coverage
-- See [AGENTIC_CODING.md#testing-strategy](AGENTIC_CODING.md#testing-strategy) for detailed testing guidelines
-
-### Security
-- Never commit secrets or credentials
-- Follow guidelines in [SECURITY_AUDIT.md](SECURITY_AUDIT.md)
-- Report security issues per [SECURITY.md](SECURITY.md)
-
-## 🤝 Getting Help
-
-- **GitHub Issues:** Bug reports and feature requests
-- **GitHub Discussions:** Questions and community support
-- **Documentation:** Check `docs/` directory for guides
-
-Thank you for contributing to DocuElevate!

Dockerfile

@@ -1,42 +1,30 @@
-# Use multi-stage build for a smaller final image
-FROM python:3.14.1 AS builder
+
+# Stage 1: Build dependencies
+FROM python:3.11 AS builder
 
 WORKDIR /app
 
-# Copy requirements first for better layer caching
 COPY requirements.txt /app/
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Second stage for the actual runtime
-FROM python:3.14.1-slim
+# Stage 2: Final image
+FROM python:3.11-slim
 
 WORKDIR /app
 
-# Copy installed packages from builder stage
-COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
+# Copy installed dependencies
+COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
 COPY --from=builder /usr/local/bin /usr/local/bin
 
-# Copy application code
+# Copy application files correctly
 COPY ./app /app/app
-COPY ./VERSION /app/VERSION
 COPY ./frontend /app/frontend
-COPY ./BUILD_DATE /app/BUILD_DATE
-COPY ./LICENSE /app/LICENSE
 
-# Create runtime_info directory
-RUN mkdir -p /app/runtime_info
-
-# Create necessary directories
-RUN mkdir -p /workdir
-
-# Set environment variables
+# Set Python path explicitly
 ENV PYTHONPATH=/app
-ENV PYTHONUNBUFFERED=1
 
-# Expose the port the app runs on
+# Expose API port
 EXPOSE 8000
+WORKDIR /app
 
-# Default command
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
-
-

LICENSE

@@ -1,4 +1,4 @@
-Apache License
+                                 Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
@@ -175,7 +175,18 @@ Apache License
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2025 Christian Krakau-Louis <christian@docuelevate.org>
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MILESTONES.md

@@ -1,341 +0,0 @@
-# DocuElevate Milestones
-
-**Last Updated:** 2026-02-06
-
-This document outlines the release milestones, versioning strategy, and detailed feature breakdown for DocuElevate.
-
-## Versioning Strategy
-
-DocuElevate follows [Semantic Versioning 2.0.0](https://semver.org/):
-- **MAJOR.MINOR.PATCH** (e.g., 1.2.3)
-- **MAJOR:** Breaking changes or major architectural shifts
-- **MINOR:** New features, backward-compatible
-- **PATCH:** Bug fixes, security patches, backward-compatible
-
-### Release Cadence
-- **Patch releases:** As needed for critical bugs/security
-- **Minor releases:** Every 6-8 weeks
-- **Major releases:** Every 12-18 months
-
----
-
-## Current Release: v0.3.2 (February 2026)
-
-### Status: Stable
-- Production-ready document processing
-- Multi-provider storage support
-- Basic web UI and REST API
-- OAuth2 authentication
-
----
-
-## Upcoming Milestones
-
-### v0.3.3 - Security & Testing Hardening (February 2026)
-**Target Date:** February 15, 2026  
-**Status:** 🚧 In Progress  
-**Theme:** Security, Quality, Testing
-
-#### Goals
-- [x] Fix critical security vulnerabilities (authlib, starlette)
-- [x] Implement comprehensive test suite
-- [x] Add security scanning (CodeQL, Bandit)
-- [x] Improve CI/CD pipeline
-- [ ] Achieve 60% test coverage
-- [ ] Add pre-commit hooks
-- [ ] Update all dependencies to latest secure versions
-
-#### Deliverables
-- [x] SECURITY_AUDIT.md documentation
-- [x] pytest configuration and fixtures
-- [x] API integration tests
-- [x] Configuration validation tests
-- [ ] Task processing tests
-- [ ] Storage provider integration tests
-- [x] Updated CI/CD workflows
-- [ ] Security best practices guide
-
-#### Breaking Changes
-- None
-
----
-
-### v0.4.0 - Enhanced Search & UI Improvements (April 2026)
-**Target Date:** April 1, 2026  
-**Status:** 📋 Planned  
-**Theme:** User Experience, Search, Performance
-
-#### Goals
-- Implement full-text search across documents
-- Responsive mobile interface
-- Dark mode support
-- Document preview in browser
-- Performance optimizations
-- Improved error handling and user feedback
-
-#### Deliverables
-- Full-text search API and UI
-- Advanced filtering capabilities
-- Responsive CSS framework integration
-- Dark mode toggle
-- In-browser document viewer
-- Loading states and progress indicators
-- Performance benchmarks
-- Mobile-optimized interface
-
-#### Breaking Changes
-- API response format changes for search endpoints (documented)
-
-#### Migration Path
-- Search endpoint changes will be versioned (/api/v1/search → /api/v2/search)
-- Old endpoints deprecated but functional for 2 releases
-
----
-
-### v0.4.5 - Workflow Automation (June 2026)
-**Target Date:** June 1, 2026  
-**Status:** 📋 Planned  
-**Theme:** Automation, Integration, Webhooks
-
-#### Goals
-- Custom processing pipelines
-- Conditional routing based on document type
-- Webhook support for external integrations
-- Rule-based classification
-- Scheduled batch processing
-
-#### Deliverables
-- Pipeline configuration UI
-- Webhook management interface
-- Rule engine for document routing
-- Batch processing scheduler
-- Integration examples and templates
-- Webhook payload documentation
-
-#### Breaking Changes
-- None
-
----
-
-### v0.5.0 - Advanced AI & Multi-language (August 2026)
-**Target Date:** August 1, 2026  
-**Status:** 📋 Planned  
-**Theme:** AI Enhancement, Internationalization
-
-#### Goals
-- Custom AI model support
-- Multi-language OCR
-- Document similarity detection
-- Duplicate detection
-- UI internationalization (i18n)
-- API localization
-
-#### Deliverables
-- Custom model integration API
-- Multi-language OCR configuration
-- Similarity algorithm implementation
-- Duplicate detection service
-- Translation framework (10+ languages)
-- Localized documentation
-
-#### Breaking Changes
-- Configuration file format changes (auto-migration script provided)
-
----
-
-### v1.0.0 - Enterprise Edition (November 2026)
-**Target Date:** November 1, 2026  
-**Status:** 📋 Planned  
-**Theme:** Enterprise Features, Scalability, Multi-tenancy
-
-This is our first major release, marking production-ready enterprise capabilities.
-
-#### Goals
-- Multi-tenancy and organization management
-- Role-based access control (RBAC)
-- Horizontal scaling support
-- Comprehensive audit logging
-- SLA monitoring and alerting
-- Professional support offerings
-
-#### Deliverables
-- **Multi-tenancy**
-  - Organization/team management UI
-  - Per-tenant configuration and branding
-  - Resource quotas and billing integration
-  - Tenant isolation at database level
-
-- **Access Control**
-  - RBAC with customizable roles
-  - Permission management UI
-  - API key management per organization
-  - SSO integration (SAML, LDAP)
-
-- **Scalability**
-  - Horizontal scaling documentation
-  - Load balancer configuration
-  - Distributed caching
-  - Database replication support
-  - Message queue clustering
-
-- **Observability**
-  - Comprehensive audit logs
-  - Prometheus metrics export
-  - Grafana dashboards
-  - APM integration (New Relic, DataDog)
-  - SLA monitoring
-
-- **Documentation**
-  - Enterprise deployment guide
-  - High availability setup
-  - Disaster recovery procedures
-  - Security compliance guide
-  - Professional services offerings
-
-#### Breaking Changes
-- Database schema migration (automatic with Alembic)
-- Configuration file restructure (migration tool provided)
-- API v1 deprecated (v2 required for new features)
-
-#### Migration Path
-- Detailed migration guide provided
-- Automated migration scripts
-- Rollback procedures documented
-- Migration support via GitHub Discussions
-
----
-
-### v1.1.0 - Collaboration & Analytics (January 2027)
-**Target Date:** January 15, 2027  
-**Status:** 📋 Planned  
-**Theme:** Collaboration, Reporting, Analytics
-
-#### Goals
-- Document sharing with expiring links
-- Comments and annotations
-- Version history
-- Analytics dashboard
-- Cost analysis
-- Export reports
-
-#### Deliverables
-- Sharing interface with permissions
-- Comment system with threading
-- Version control and diff viewer
-- Analytics dashboard with charts
-- Cost breakdown by provider
-- Report generation (PDF, CSV, Excel)
-- User activity tracking
-
-#### Breaking Changes
-- None
-
----
-
-### v2.0.0 - On-Premise AI & Platform Expansion (Q3 2027)
-**Target Date:** Q3 2027  
-**Status:** 🔮 Future  
-**Theme:** Self-hosting, Privacy, Platform Diversity
-
-#### Goals
-- Self-hosted AI models (no cloud dependencies)
-- Local LLM integration
-- Desktop and mobile applications
-- Offline-first capabilities
-- Enhanced privacy features
-- Plugin marketplace
-
-#### Deliverables
-- Tesseract/EasyOCR integration
-- Ollama/LLaMA support
-- Desktop app (Windows, Mac, Linux)
-- Mobile apps (iOS, Android)
-- Browser extensions (Chrome, Firefox)
-- Plugin SDK and marketplace
-- Offline mode
-
-#### Breaking Changes
-- Major API restructure (v3)
-- New authentication system
-- Configuration format change
-- Minimum Python version: 3.12
-
----
-
-## Release Process
-
-### Pre-release Checklist
-- [ ] All tests passing
-- [ ] Security scan passed
-- [ ] Code review completed
-- [ ] Documentation updated
-- [ ] CHANGELOG.md updated
-- [ ] Migration guide (if breaking changes)
-- [ ] Release notes drafted
-- [ ] Version numbers bumped
-- [ ] Docker images built and tested
-
-### Release Artifacts
-- Source code (GitHub)
-- Docker images (Docker Hub)
-- PyPI package (future)
-- Helm charts (future)
-- Documentation site update
-
-### Post-release
-- [ ] GitHub release created
-- [ ] Blog post published
-- [ ] Social media announcement
-- [ ] Community notification
-- [ ] Support documentation updated
-- [ ] Monitor for critical issues
-
----
-
-## Version History
-
-| Version | Release Date | Theme | Status |
-|---------|-------------|-------|--------|
-| v0.1.0 | 2024-Q1 | Initial Release | Released |
-| v0.2.0 | 2024-Q3 | Multi-provider Support | Released |
-| v0.3.0 | 2025-Q4 | UI & Authentication | Released |
-| v0.3.2 | 2026-02 | Current Stable | Released |
-| v0.3.3 | 2026-02 | Security & Testing | In Progress |
-| v0.4.0 | 2026-04 | Search & UX | Planned |
-| v0.5.0 | 2026-08 | Advanced AI | Planned |
-| v1.0.0 | 2026-11 | Enterprise | Planned |
-| v2.0.0 | 2027-Q3 | Platform Expansion | Future |
-
----
-
-## Support & EOL Policy
-
-### Active Support
-- Current stable release: Full support (bug fixes, security patches, features)
-- Previous minor release: Security patches only
-- Older versions: Community support only
-
-### End of Life (EOL)
-- Minor versions: EOL when 2 newer minor versions released
-- Major versions: EOL 18 months after next major version
-
-### Security Patches
-- Critical vulnerabilities: Patched within 48 hours
-- High severity: Patched within 1 week
-- Medium/Low: Included in next regular release
-
----
-
-## Contributing to Milestones
-
-Want to contribute to a specific milestone?
-
-1. Check the [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) board
-2. Look for issues tagged with milestone labels
-3. Read [CONTRIBUTING.md](CONTRIBUTING.md)
-4. Comment on the issue you'd like to work on
-5. Submit a PR linked to the issue
-
----
-
-*This milestone document is updated regularly. For real-time status, check our [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) board.*

NOTICE

@@ -1,138 +0,0 @@
-DocuElevate
-Copyright 2025 Christian Krakau-Louis
-
-This product includes software developed for the DocuElevate project.
-
-================================================================================
-
-This software includes third-party components with their own licenses:
-
-SPECIAL NOTICE REGARDING LGPL SOFTWARE:
---------------------------------------------------------------------------------
-DocuElevate incorporates Paramiko, which is licensed under the GNU Lesser General 
-Public License (LGPL) version 2.1. In accordance with the LGPL:
-
-1. The complete source code for Paramiko can be obtained from:
-   https://github.com/paramiko/paramiko
-
-2. This software is distributed in the hope that it will be useful, but WITHOUT 
-   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
-   FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License 
-   for more details.
-
-3. A copy of the GNU Lesser General Public License version 2.1 can be found at:
-   frontend/static/licenses/lgpl.txt and at https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
-
-4. Users have the right to obtain the source code of Paramiko and to modify and
-   redistribute it under the terms of the LGPL.
-
-# Python Dependencies
---------------------------------------------------------------------------------
-FastAPI (MIT License)
-Copyright (c) 2018 Sebastián Ramírez
-https://github.com/tiangolo/fastapi
-
-Celery (BSD License)
-Copyright (c) 2015-2016 Ask Solem & contributors
-https://github.com/celery/celery
-
-Uvicorn (BSD License)
-Copyright (c) 2017-present, Encode OSS Ltd.
-https://github.com/encode/uvicorn
-
-SQLAlchemy (MIT License)
-Copyright (c) 2005-2023 SQLAlchemy authors and contributors
-https://github.com/sqlalchemy/sqlalchemy
-
-Pydantic (MIT License)
-Copyright (c) 2017-present Pydantic Services Inc.
-https://github.com/pydantic/pydantic
-
-OpenAI (MIT License)
-Copyright (c) 2023 OpenAI
-https://github.com/openai/openai-python
-
-PyPDF2 (BSD License)
-Copyright (c) 2006-2008, Mathieu Fenniak
-https://github.com/py-pdf/PyPDF2
-
-Requests (Apache 2.0 License)
-Copyright 2019 Kenneth Reitz
-https://github.com/psf/requests
-
-Dropbox (MIT License)
-Copyright (c) 2015-2021 Dropbox, Inc.
-https://github.com/dropbox/dropbox-sdk-python
-
-Azure AI Document Intelligence (MIT License)
-Copyright (c) Microsoft Corporation
-https://github.com/Azure/azure-sdk-for-python
-
-Authlib (BSD License)
-Copyright (c) 2017-present, Hsiaoming Yang
-https://github.com/lepture/authlib
-
-python-dotenv (BSD License)
-Copyright (c) 2014, Saurabh Kumar
-https://github.com/theskumar/python-dotenv
-
-Starlette (BSD License)
-Copyright (c) 2018-present, Encode OSS Ltd.
-https://github.com/encode/starlette
-
-Alembic (MIT License)
-Copyright (c) 2009-2023 Michael Bayer
-https://github.com/sqlalchemy/alembic
-
-Google API Client (Apache 2.0 License)
-Copyright 2014 Google LLC
-https://github.com/googleapis/google-api-python-client
-
-Microsoft Graph Core (MIT License)
-Copyright (c) Microsoft Corporation
-https://github.com/microsoftgraph/msgraph-sdk-python-core
-
-MSAL (MIT License)
-Copyright (c) Microsoft Corporation
-https://github.com/AzureAD/microsoft-authentication-library-for-python
-
-Boto3 (Apache 2.0 License)
-Copyright Amazon.com, Inc. or its affiliates
-https://github.com/boto/boto3
-
-Paramiko (LGPL-2.1 License)
-Copyright (c) 2003-2009 Robey Pointer
-https://github.com/paramiko/paramiko
-
-Apprise (MIT License)
-Copyright (C) 2019-2024 Chris Caron
-https://github.com/caronc/apprise
-
-# Docker Images
---------------------------------------------------------------------------------
-Redis (BSD License)
-Copyright (c) 2006-2020, Salvatore Sanfilippo
-https://redis.io/
-
-Gotenberg (MIT License)
-Copyright (c) 2019 Julien Neuhart
-https://github.com/gotenberg/gotenberg
-
-# Frontend Libraries
---------------------------------------------------------------------------------
-Tailwind CSS (MIT License)
-Copyright (c) Tailwind Labs, Inc.
-https://github.com/tailwindlabs/tailwindcss
-
-Alpine.js (MIT License)
-Copyright (c) 2019-2021 Caleb Porzio and contributors
-https://github.com/alpinejs/alpine
-
-Font Awesome (Font Awesome Free License)
-https://github.com/FortAwesome/Font-Awesome
-
-# For a complete list of all dependencies and their licenses
---------------------------------------------------------------------------------
-See the attribution page in the application or run:
-pip install pip-licenses
-pip-licenses

README.md

@@ -1,20 +1,11 @@
-<div align="center">
-  <img src="frontend/static/logo_writing.svg" alt="DocuElevate Logo" width="280" />
-  <p>Intelligent Document Processing & Management</p>
-</div>
-
-# DocuElevate
-
-<div align="center">
-  <a href="https://www.docuelevate.org"><img src="frontend/static/hero.png" alt="DocuElevate Logo" width="80%" /></a>
-</div>
+# Document Processing System
 
 ## Overview
 
-DocuElevate automates the handling, extraction, and processing of documents using a variety of services, including:
+This project automates the handling, extraction, and processing of documents using a variety of services, including:
 
 - **OpenAI** for metadata extraction and text refinement.  
-- **Dropbox**, **Nextcloud**, and **Google Drive** for file storage and uploads.  
+- **Dropbox** and **Nextcloud** for file storage and uploads.  
 - **Paperless NGX** for document indexing and management.  
 - **Azure Document Intelligence** for OCR on PDFs.  
 - **Gotenberg** for file-to-PDF conversions.  
@@ -24,62 +15,10 @@ It is designed for flexibility and configurability through environment variables
 
 The project includes a **UI** for uploading and managing files, and an API documentation page is available at `/docs` (powered by **FastAPI**).
 
-## Documentation Index
-
-- [User Guide](docs/UserGuide.md) - How to use DocuElevate
-- [API Documentation](docs/API.md) - API reference
-- [Deployment Guide](docs/DeploymentGuide.md) - How to deploy DocuElevate
-- [Configuration Guide](docs/ConfigurationGuide.md) - Available configuration options
-- [Development Guide](CONTRIBUTING.md) - How to contribute to DocuElevate
-- [Troubleshooting](docs/Troubleshooting.md) - Common issues and solutions
-
-## Screenshots
-
-<div align="center">
-  <img src="docs/upload-view.png" alt="DocuElevate Upload Interface" width="80%" />
-  <p><em>Upload interface for adding new documents</em></p>
-  
-  <img src="docs/files-view.png" alt="DocuElevate Files View" width="80%" />
-  <p><em>Files view with processed documents and metadata</em></p>
-</div>
-
-## Workflow Process
-
-DocuElevate follows a streamlined document processing workflow:
-
-<div align="center">
-  <img src="docs/workflow-diagram.png" alt="DocuElevate Workflow" width="90%" />
-</div>
-
-### Document Ingestion
-Documents enter DocuElevate through three possible channels:
-1. **Web Upload**: Users manually upload files via the web interface
-2. **Email Attachments**: Automatic polling of configured IMAP mailboxes (supports multiple accounts)
-3. **API**: Direct programmatic uploads via the REST API
-
-### Processing Pipeline
-Every document goes through the following steps:
-1. **PDF Conversion**: Non-PDF files are converted to PDF format using Gotenberg
-2. **OCR Processing**: Azure Document Intelligence extracts text from images/scans
-3. **Metadata Extraction**: OpenAI analyzes document content to identify:
-   - Document type (invoice, receipt, contract, etc.)
-   - Key entities (dates, names, amounts, account numbers)
-   - Important data points specific to the document type
-4. **Enrichment**: Metadata is attached to the document in a structured format
-
-### Distribution
-Processed documents with their metadata can be automatically sent to:
-- **Dropbox**: For cloud storage and sharing
-- **Nextcloud**: For self-hosted file storage
-- **Google Drive**: For Google Workspace integration
-- **Paperless-NGX**: For advanced document management with search capabilities
-
-Users can choose to send documents to any combination of these destinations through configuration settings or manual selection.
-
 ## Features
 
 - **Document Upload & Storage**:  
-  - Manual uploads (via API or UI) to Dropbox, Nextcloud, Google Drive, or Paperless.  
+  - Manual uploads (via API or UI) to Dropbox, Nextcloud, or Paperless.  
 - **OCR Processing (Azure)**:  
   - Extract text from scanned PDFs using Azure Document Intelligence.  
 - **Metadata Extraction (OpenAI)**:  
@@ -95,79 +34,170 @@ Users can choose to send documents to any combination of these destinations thro
 
 ## Frameworks Used
 
-- **FastAPI**: High-performance web framework for APIs.
-- **Celery**: Task queue for asynchronous processing.
-- **Redis**: Message broker and result backend.
-- **SQLAlchemy**: ORM for database interactions.
-- **Tailwind CSS**: Utility-first CSS framework.
-- **Docker**: Containerization for easy deployment.
+- **FastAPI**: A modern, fast (high-performance) web framework for building APIs with Python.  
+- **Celery**: A distributed task queue for asynchronous processing.  
+- **SQLAlchemy**: A powerful ORM for database interactions.  
+- **Jinja2**: A templating engine for rendering HTML pages.  
+- **Tailwind CSS**: A utility-first CSS framework for styling the UI.  
 
-## Quick Start
+## Environment Variables
 
-For detailed installation and deployment instructions, please refer to the [Deployment Guide](docs/DeploymentGuide.md).
+The `.env` file drives all configuration. This table breaks down key variables—some are optional, depending on which services you actually use.
 
-```bash
-# Clone the repository
-git clone <repository_url>
-cd document-processor
+### Core Settings
 
-# Configure environment variables
-cp .env.example .env
-# Edit .env with your settings
+| **Variable**           | **Description**                                          | **Example**                    |
+|------------------------|----------------------------------------------------------|--------------------------------|
+| `DATABASE_URL`         | Path/URL to the SQLite database (or other SQL backend). | `sqlite:///./app/database.db`  |
+| `REDIS_URL`            | URL for Redis, used by Celery for broker & result store. | `redis://redis:6379/0`         |
+| `WORKDIR`              | Working directory for the application.                  | `/workdir`                     |
+| `GOTENBERG_URL`        | Gotenberg PDF processing URL.                           | `http://gotenberg:3000`        |
 
-# Run with Docker Compose
-docker-compose up -d
+### IMAP Configuration (Multiple Mailboxes)
+
+| **Variable**                  | **Description**                                              | **Example**       |
+|-------------------------------|--------------------------------------------------------------|-------------------|
+| `IMAP1_HOST`                  | Hostname for first IMAP server.                             | `mail.example.com`|
+| `IMAP1_PORT`                  | Port number (usually `993`).                                | `993`             |
+| `IMAP1_USERNAME`              | IMAP login (first mailbox).                                 | `user@example.com`|
+| `IMAP1_PASSWORD`              | IMAP password (first mailbox).                              | `*******`         |
+| `IMAP1_SSL`                   | Use SSL (`true`/`false`).                                   | `true`            |
+| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail.                  | `5`               |
+| `IMAP1_DELETE_AFTER_PROCESS`  | Delete emails after processing (`true`/`false`).            | `false`           |
+| `IMAP2_HOST`                  | Hostname for second IMAP server (optional).                 | `imap.gmail.com`  |
+| `IMAP2_PORT`                  | Port number for second mailbox.                             | `993`             |
+| `IMAP2_USERNAME`              | IMAP login for second mailbox.                              | `you@gmail.com`   |
+| `IMAP2_PASSWORD`              | IMAP password for second mailbox.                           | `*******`         |
+| `IMAP2_SSL`                   | Use SSL for second mailbox (`true`/`false`).                | `true`            |
+| `IMAP2_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll second mailbox.                | `10`              |
+| `IMAP2_DELETE_AFTER_PROCESS`  | Delete emails after processing (`true`/`false`) for mailbox.| `false`           |
+
+### OpenAI & Azure Document Intelligence
+
+| **Variable**          | **Description**                                                    | **How to Obtain**                                                                                              |
+|-----------------------|--------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|
+| `OPENAI_API_KEY`      | API key for OpenAI services (used for metadata extraction/refinement). | [OpenAI platform](https://platform.openai.com/account/api-keys)                                                |
+| `OPENAI_BASE_URL`     | Base URL for OpenAI API (optional, defaults to OpenAI's endpoint). | `https://api.openai.com/v1`                                              |
+| `OPENAI_MODEL`        | OpenAI model to use for tasks (e.g., GPT-4).                       | `gpt-4`                                                                                                        |
+| `AZURE_AI_KEY`        | Azure Document Intelligence key (for OCR).                        | [Azure Portal](https://portal.azure.com/)                                                                        |
+| `AZURE_REGION`        | Azure region of your Document Intelligence instance.              | e.g. `eastus`, `westeurope`                                                                                     |
+| `AZURE_ENDPOINT`      | Endpoint URL for Document Intelligence.                           | e.g. `https://<yourendpoint>.cognitiveservices.azure.com/`                                                      |
+
+### Authentik
+
+| **Variable**            | **Description**                                               |
+|-------------------------|---------------------------------------------------------------|
+| `AUTH_ENABLED`          | Enable or disable authentication (`true`/`false`).           |
+| `AUTHENTIK_CLIENT_ID`   | Client ID for Authentik OAuth2.                              |
+| `AUTHENTIK_CLIENT_SECRET` | Client secret for Authentik OAuth2.                        |
+| `AUTHENTIK_CONFIG_URL`  | Configuration URL for Authentik OpenID Connect.             |
+
+### Paperless NGX
+
+| **Variable**                  | **Description**                                     |
+|-------------------------------|-----------------------------------------------------|
+| `PAPERLESS_NGX_API_TOKEN`     | API token for Paperless NGX.                       |
+| `PAPERLESS_HOST`              | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). |
+
+### Dropbox
+
+| **Variable**            | **Description**                                  | **How to Obtain**                                                                 |
+|-------------------------|--------------------------------------------------|------------------------------------------------------------------------------------|
+| `DROPBOX_APP_KEY`       | Dropbox API app key.                             | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create)        |
+| `DROPBOX_APP_SECRET`    | Dropbox API app secret.                          | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create)        |
+| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox.                | Follow Dropbox OAuth flow to retrieve                                              |
+| `DROPBOX_FOLDER`        | Default folder path for Dropbox uploads.         | e.g. `"/Documents/Uploads"`                                                        |
+
+### Nextcloud
+
+| **Variable**            | **Description**                                               |
+|-------------------------|---------------------------------------------------------------|
+| `NEXTCLOUD_UPLOAD_URL`  | Nextcloud WebDAV URL (e.g. `https://nc.example.com/remote.php/dav/files/<USERNAME>`). |
+| `NEXTCLOUD_USERNAME`    | Nextcloud login username.                                    |
+| `NEXTCLOUD_PASSWORD`    | Nextcloud login password.                                    |
+| `NEXTCLOUD_FOLDER`      | Destination folder in Nextcloud (e.g. `"/Documents/Uploads"`). |
+
+## Running as a Docker Container
+
+This project uses Celery (with Redis) for asynchronous task management and Gotenberg for PDF conversion. The `docker-compose.yml` file defines these services:
+
+- **API Service**: Runs the FastAPI application via `uvicorn`.
+- **Worker Service**: Runs the Celery worker for processing tasks (PDF conversions, OCR, etc.).
+- **Redis**: Provides the message broker & result backend for Celery.
+- **Gotenberg**: Offers PDF conversion capabilities.
+
+### Running the Application with Docker Compose
+
+1. **Install Docker and Docker Compose** on your system.  
+2. **Clone the repository** and navigate into it:
+   ```bash
+   git clone <repository_url>
+   cd <repository_name>
+   ```
+3. **Create and configure the `.env` file**:
+   - Fill in the variables from the tables above.  
+   - (At minimum, you need `DATABASE_URL`, `REDIS_URL`, `WORKDIR`, plus whichever service creds you plan to use.)
+4. **Launch the services**:
+   ```bash
+   docker-compose up -d
+   ```
+5. The API will be available at **`http://localhost:8000`**, and the API documentation is available at **`http://localhost:8000/docs`**.
+
+### Services in `docker-compose.yml`
+
+Below is the default structure (simplified):
+
+```yaml
+services:
+  api:
+    image: christianlouis/document-processor:latest
+    container_name: document_api
+    working_dir: /workdir
+    command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
+    environment:
+      - PYTHONPATH=/app
+    env_file:
+      - .env
+    ports:
+      - "8000:8000"
+    depends_on:
+      - redis
+      - worker
+    volumes:
+      - /var/docparse/workdir:/workdir
+
+  worker:
+    image: christianlouis/document-processor:latest
+    container_name: document_worker
+    working_dir: /workdir
+    command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
+    env_file:
+      - .env
+    environment:
+      - PYTHONPATH=/app
+    depends_on:
+      - redis
+      - gotenberg
+    volumes:
+      - /var/docparse/workdir:/workdir
+
+  gotenberg:
+    image: gotenberg/gotenberg:latest
+    container_name: gotenberg
+
+  redis:
+    image: redis:alpine
+    container_name: document_redis
+    restart: always
 ```
 
-The API will be available at **`http://localhost:8000`**, and the API documentation is available at **`http://localhost:8000/docs`**.
+## To-Do List
 
-## License
+- **Make upload targets configurable** (e.g., easily choose only Dropbox, Nextcloud, or Paperless).  
 
-This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
+---
 
-## Third-Party Software
-
-This project uses various third-party libraries and components. See [NOTICE](NOTICE) for attributions and the [attribution page](frontend/templates/attribution.html) in the application for more details.
-
-### LGPL Compliance
-
-This project uses Paramiko which is licensed under LGPL-2.1. In accordance with the LGPL license:
-
-- The source code for Paramiko can be obtained from https://github.com/paramiko/paramiko
-- A copy of the LGPL license is available in the application at `/licenses/lgpl.txt`
-- Users have the right to modify and redistribute Paramiko under the terms of the LGPL
-
-## Dependency Licenses
-
-The following is a summary of the licenses used by our direct dependencies:
-
-| Dependency | License |
-|------------|---------|
-| FastAPI    | MIT     |
-| Celery     | BSD     |
-| Uvicorn    | BSD     |
-| SQLAlchemy | MIT     |
-| Pydantic   | MIT     |
-| OpenAI     | MIT     |
-| PyPDF2     | BSD     |
-| Requests   | Apache 2.0 |
-| Dropbox    | MIT     |
-| Azure AI Document Intelligence | MIT |
-| Authlib    | BSD     |
-| Starlette  | BSD     |
-| Alembic    | MIT     |
-| Google API Client | Apache 2.0 |
-| Microsoft Graph Core | MIT |
-| MSAL       | MIT     |
-| Boto3      | Apache 2.0 |
-| Paramiko   | LGPL-2.1|
-| Apprise    | MIT     |
-| Redis      | BSD     |
-| Gotenberg  | MIT     |
-
-For a comprehensive list of all dependencies and their licenses, run:
-
-```
-pip install pip-licenses
-pip-licenses
-```
+**Questions or Issues?**  
+- Feel free to open an issue or pull request.  
+- For local testing or development, use `docker-compose up` and watch the logs via `docker-compose logs -f`.  
+- Ensure your `.env` aligns with the environment variables listed above. If you see unexpected errors, check for typos or missing values.

ROADMAP.md

@@ -1,213 +0,0 @@
-# DocuElevate Roadmap
-
-**Last Updated:** 2026-02-06  
-**Version:** 1.0
-
-## Vision
-
-DocuElevate aims to be the premier open-source intelligent document processing platform, providing seamless integration with cloud storage providers, advanced AI-powered metadata extraction, and enterprise-grade security and scalability.
-
-## Current Status (v0.3.2)
-
-### Core Features ✅
-- Multi-provider document storage (Dropbox, Google Drive, OneDrive, Nextcloud, S3, etc.)
-- IMAP email integration for document ingestion
-- OCR processing via Azure Document Intelligence
-- AI-powered metadata extraction via OpenAI
-- PDF conversion via Gotenberg
-- Web UI for document upload and management
-- REST API with OpenAPI documentation
-- Celery-based async task processing
-- OAuth2 authentication via Authentik
-
-## Short-term Goals (Q1-Q2 2026) - v0.4.x to v0.5.x
-
-### Quality & Stability 🎯
-- **Test Coverage** (High Priority)
-  - [ ] Achieve 80% code coverage for core modules
-  - [ ] Add integration tests for all storage providers
-  - [ ] Add end-to-end workflow tests
-  - [ ] Performance benchmarks and load testing
-
-- **Code Quality** (High Priority)
-  - [ ] Enable strict linting in CI/CD
-  - [ ] Refactor large modules for better maintainability
-  - [ ] Add comprehensive type hints
-  - [ ] Improve error handling and user feedback
-
-- **Security** (Critical Priority)
-  - [x] Fix known vulnerabilities in dependencies
-  - [ ] Implement rate limiting on API endpoints
-  - [ ] Add CSRF protection
-  - [ ] Security audit by external party
-  - [ ] Implement API key rotation
-  - [ ] Add audit logging for sensitive operations
-
-### Features - v0.4.0
-- **Enhanced Search & Filtering**
-  - [ ] Full-text search across documents
-  - [ ] Advanced filtering by metadata, tags, date ranges
-  - [ ] Saved search queries
-  - [ ] Bulk operations on search results
-
-- **Improved UI/UX**
-  - [ ] Responsive mobile interface
-  - [ ] Dark mode support
-  - [ ] Document preview in browser
-  - [ ] Drag-and-drop file upload
-  - [ ] Progress indicators for long-running tasks
-  - [ ] Real-time notifications via WebSocket
-
-### Features - v0.5.0
-- **Workflow Automation**
-  - [ ] Custom processing pipelines
-  - [ ] Conditional routing based on document type
-  - [ ] Scheduled batch processing
-  - [ ] Webhook support for external integrations
-  - [ ] Rule-based document classification
-
-- **Advanced AI Features**
-  - [ ] Custom AI models for specialized document types
-  - [ ] Multi-language OCR support
-  - [ ] Document similarity detection
-  - [ ] Automatic duplicate detection
-  - [ ] Intelligent document splitting
-
-## Medium-term Goals (Q3-Q4 2026) - v1.0.x
-
-### Enterprise Features - v1.0.0
-- **Multi-tenancy**
-  - [ ] Organization/team management
-  - [ ] Role-based access control (RBAC)
-  - [ ] Per-tenant configuration
-  - [ ] Resource quotas and limits
-  - [ ] Audit logs per organization
-
-- **Scalability**
-  - [ ] Horizontal scaling support
-  - [ ] Distributed task processing
-  - [ ] Caching layer (Redis/Memcached)
-  - [ ] Database connection pooling
-  - [ ] Message queue optimization
-
-- **Advanced Integrations**
-  - [ ] Microsoft SharePoint integration
-  - [ ] Slack/Teams bot integration
-  - [ ] Zapier/Make.com integration
-  - [ ] Custom webhook receivers
-  - [ ] GraphQL API
-
-### Features - v1.1.0
-- **Collaboration**
-  - [ ] Document sharing with expiring links
-  - [ ] Comments and annotations
-  - [ ] Version history and rollback
-  - [ ] Real-time collaborative editing metadata
-  - [ ] Activity feed
-
-- **Reporting & Analytics**
-  - [ ] Processing statistics dashboard
-  - [ ] Storage usage analytics
-  - [ ] AI confidence scores and accuracy tracking
-  - [ ] Cost analysis per provider
-  - [ ] Export reports (PDF, CSV, Excel)
-
-## Long-term Goals (2027+) - v2.0+
-
-### Strategic Initiatives
-- **On-Premise AI Models**
-  - [ ] Self-hosted OCR (Tesseract, EasyOCR)
-  - [ ] Local LLM integration (Ollama, LLaMA)
-  - [ ] GPU acceleration support
-  - [ ] Model fine-tuning interface
-  - [ ] Hybrid cloud/on-premise processing
-
-- **Advanced Document Management**
-  - [ ] Document lifecycle management
-  - [ ] Retention policies and auto-deletion
-  - [ ] Compliance templates (GDPR, HIPAA, SOC2)
-  - [ ] Digital signature support
-  - [ ] Encryption at rest and in transit
-
-- **Platform Expansion**
-  - [ ] Desktop applications (Electron)
-  - [ ] Mobile apps (iOS/Android)
-  - [ ] Browser extensions
-  - [ ] Command-line interface (CLI)
-  - [ ] VS Code extension for developers
-
-### Research & Innovation
-- [ ] Machine learning for custom document types
-- [ ] Blockchain for document provenance
-- [ ] Federated learning for privacy-preserving AI
-- [ ] Edge computing support
-- [ ] Quantum-resistant encryption
-
-## Community & Ecosystem
-
-### Developer Experience
-- [ ] Plugin system for custom processors
-- [ ] Marketplace for extensions
-- [ ] SDK for multiple languages (Python, JavaScript, Go)
-- [ ] Template library for common workflows
-- [ ] Video tutorials and courses
-
-### Documentation
-- [x] User guide
-- [x] API documentation
-- [x] Deployment guide
-- [ ] Architecture deep-dive
-- [ ] Contributing guide enhancements
-- [ ] Video walkthroughs
-- [ ] Internationalization (i18n) of docs
-
-### Community Building
-- [ ] Regular community calls
-- [ ] Bug bounty program
-- [ ] Ambassador program
-- [ ] Annual conference/meetup
-- [ ] Certification program
-
-## Technology Debt
-
-### Refactoring Needed
-- [ ] Migrate from PyPDF2 to pypdf (modern fork)
-- [ ] Standardize error handling across modules
-- [ ] Consolidate configuration management
-- [ ] Optimize database queries
-- [ ] Reduce code duplication in storage providers
-
-### Performance Optimization
-- [ ] Profile and optimize hot paths
-- [ ] Implement lazy loading for UI
-- [ ] Add CDN for static assets
-- [ ] Optimize Docker image size
-- [ ] Database indexing strategy
-
-## Deprecation Notice
-
-### Planned Deprecations
-- None currently planned
-
-### Migration Guides
-- Will be provided for any breaking changes
-
-## How to Contribute
-
-See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines. Roadmap items are open for discussion and contributions!
-
-### Priority Labels
-- 🔴 Critical - Security, data loss, or major bugs
-- 🟠 High - Important features or significant improvements
-- 🟡 Medium - Nice-to-have features or minor improvements
-- 🟢 Low - Future considerations or research items
-
-## Feedback & Requests
-
-- **GitHub Issues:** Feature requests and bug reports
-- **GitHub Discussions:** General questions and ideas
-- **Email:** [Maintainer contact from repository]
-
----
-
-*This roadmap is a living document and may change based on community feedback, technical constraints, and strategic priorities.*

SECURITY.md

@@ -1,58 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-| Version | Supported          |
-| ------- | ------------------ |
-| 0.4.x   | :white_check_mark: |
-| 0.3.x   | :white_check_mark: |
-| 0.2.x   | :white_check_mark: |
-| < 0.2   | :x:                |
-
-Each version will be supported for six months after release or until a new release has been made, whichever is longer.
-
-## Reporting a Vulnerability
-
-We take the security of our document-processor seriously. If you believe you've found a security vulnerability, please follow these steps:
-
-### How to Report
-
-1. **Do NOT disclose the vulnerability publicly** until it has been addressed by our team.
-2. Email your findings to [security@christianlouis.de](mailto:security@christianlouis.de). Encrypt your message if it contains sensitive details.
-3. Include as much information as possible:
-   - Type of vulnerability
-   - Full paths of source files related to the vulnerability
-   - Step-by-step instructions to reproduce the issue
-   - Proof of concept code, if possible
-   - Impact of the vulnerability
-
-### What to Expect
-
-- A confirmation email within 48 hours acknowledging your report.
-- An assessment and validation of the reported vulnerability within 1 week.
-- Regular updates about the progress of addressing the vulnerability.
-- Credit for discovering and reporting the vulnerability (if desired).
-
-### Disclosure Policy
-
-- Please allow us reasonable time to resolve the issue before making any public disclosures.
-- We aim to address confirmed vulnerabilities within 30-90 days, depending on complexity.
-- Once the vulnerability is fixed, we'll publish a security advisory with details and credit.
-
-## Security Best Practices
-
-When using document-processor:
-- Keep your installation up-to-date with the latest security patches
-- Use strong access controls and authentication mechanisms
-- Validate all inputs from untrusted sources
-- Follow the principle of least privilege when configuring permissions
-
-## Security Updates
-
-Security updates will be released as part of our regular versioning process. Critical security fixes may be released as out-of-band updates.
-
-## Acknowledgments
-
-We'd like to thank the following individuals for responsibly reporting security issues:
-
-*This list will be updated as contributions are received.*

SECURITY_AUDIT.md

@@ -1,123 +0,0 @@
-# Security Audit Report
-
-**Date:** 2026-02-06  
-**Status:** Completed Initial Assessment
-
-## Executive Summary
-
-This document tracks security vulnerabilities found in DocuElevate and their remediation status.
-
-## Critical Vulnerabilities (Fixed) ✅
-
-### 1. Outdated Authlib with Known Vulnerabilities
-**Status:** ✅ FIXED  
-**Severity:** HIGH  
-**Description:** Authlib version 1.3.2 had two critical vulnerabilities:
-- CVE: Denial of Service via Oversized JOSE Segments
-- CVE: JWS/JWT accepts unknown crit headers (RFC violation → possible authz bypass)
-
-**Fix:** Updated `requirements.txt` to require `authlib>=1.6.5`
-
-### 2. Starlette DoS Vulnerability
-**Status:** ✅ FIXED  
-**Severity:** MEDIUM  
-**Description:** Starlette 0.41.3 vulnerable to O(n^2) DoS via Range header merging in `FileResponse`
-
-**Fix:** Updated `requirements.txt` to require `starlette>=0.49.1`
-
-### 3. Weak SESSION_SECRET Default
-**Status:** ✅ FIXED  
-**Severity:** HIGH  
-**Description:** Default SESSION_SECRET value in `app/main.py` was a predictable string that could be exploited if not overridden
-
-**Fix:** 
-- Enhanced validation in `app/main.py` to raise error if auth is enabled without proper secret
-- Updated default to be clearly marked as insecure for development only
-- Added generation instructions in error message
-
-## Medium Risk Issues (Fixed) ✅
-
-### 4. Insufficient .gitignore Protection
-**Status:** ✅ FIXED  
-**Severity:** MEDIUM  
-**Description:** .gitignore didn't adequately protect against accidentally committing sensitive files (credentials, private keys, secrets)
-
-**Fix:** Enhanced `.gitignore` with comprehensive patterns for:
-- Various environment file formats
-- Credential JSON files
-- Private keys (.pem, .key, .pfx, etc.)
-- SSH keys
-- Explicit exclusion of patterns where needed
-
-## Best Practices Implemented
-
-### Dependency Management
-- ✅ Version pinning for security-critical packages (authlib, starlette)
-- ✅ Advisory database checks integrated into development workflow
-- ⏳ TODO: Add automated dependency vulnerability scanning in CI/CD
-
-### Authentication & Secrets
-- ✅ Strong validation for SESSION_SECRET (minimum 32 characters)
-- ✅ Error-on-missing for critical security settings when auth enabled
-- ✅ Clear documentation of secret generation methods
-- ✅ .env.demo file for configuration examples (no real secrets)
-
-### Configuration Security
-- ✅ All secrets loaded from environment variables
-- ✅ No hardcoded credentials in codebase
-- ✅ Proper masking in configuration validators
-
-## Ongoing Security Measures
-
-### CI/CD Security
-- ⏳ **TODO:** Add CodeQL scanning to GitHub Actions
-- ⏳ **TODO:** Add Bandit (Python security linter) to CI pipeline
-- ⏳ **TODO:** Add dependency vulnerability scanning (Safety, pip-audit)
-- ⏳ **TODO:** Make security scans blocking (fail on critical issues)
-
-### Code Security
-- ⏳ **TODO:** Implement rate limiting on API endpoints
-- ⏳ **TODO:** Add CSRF protection for state-changing operations
-- ⏳ **TODO:** Implement request size limits
-- ⏳ **TODO:** Add input sanitization for all user inputs
-- ⏳ **TODO:** Implement proper API key rotation mechanisms
-
-### Infrastructure Security
-- ✅ TrustedHostMiddleware configured
-- ✅ ProxyHeadersMiddleware for reverse proxy setup
-- ⏳ **TODO:** Add security headers (HSTS, CSP, X-Frame-Options)
-- ⏳ **TODO:** Implement proper CORS configuration
-- ⏳ **TODO:** Add request logging with sensitive data masking
-
-## Recommendations
-
-### High Priority
-1. **Enable CodeQL scanning** - Automated security vulnerability detection
-2. **Implement rate limiting** - Prevent abuse and DoS attacks
-3. **Add comprehensive input validation** - Prevent injection attacks
-4. **Implement API authentication** - Secure all API endpoints properly
-
-### Medium Priority
-1. **Add security headers** - Improve browser-side security
-2. **Implement audit logging** - Track security-relevant events
-3. **Add automated security testing** - Integration with CI/CD
-4. **Document security architecture** - Security design decisions
-
-### Low Priority
-1. **Security training documentation** - For contributors
-2. **Penetration testing** - Professional security assessment
-3. **Bug bounty program** - Community security contributions
-
-## Security Contact
-
-For security issues, please follow the guidelines in [SECURITY.md](SECURITY.md).
-
-## Audit History
-
-| Date | Auditor | Scope | Critical Issues | Status |
-|------|---------|-------|-----------------|--------|
-| 2026-02-06 | Automated Agent | Dependencies, Auth, Config | 3 | Fixed |
-
----
-
-**Next Audit Due:** 2026-05-06 (Quarterly)

TODO.md

@@ -1,267 +0,0 @@
-# DocuElevate TODO List
-
-**Last Updated:** 2026-02-06  
-**Current Version:** v0.3.2
-
-This document tracks actionable tasks for the current development cycle. For long-term planning, see [ROADMAP.md](ROADMAP.md) and [MILESTONES.md](MILESTONES.md).
-
----
-
-## 🔴 Critical Priority (This Week)
-
-### Security
-- [x] Fix authlib vulnerability (upgrade to 1.6.5+)
-- [x] Fix starlette DoS vulnerability (upgrade to 0.49.1+)
-- [x] Improve SESSION_SECRET validation
-- [ ] Run security audit with Bandit
-- [ ] Review all direct file path operations for path traversal vulnerabilities
-- [ ] Add rate limiting middleware to API endpoints
-- [ ] Implement CSRF token for state-changing operations
-
-### Testing
-- [x] Set up pytest infrastructure
-- [x] Create test fixtures and conftest.py
-- [x] Add basic API integration tests
-- [x] Add configuration validation tests
-- [ ] Fix API integration tests (auth configuration issues)
-- [ ] Add tests for file upload functionality
-- [ ] Add tests for OCR processing (mocked)
-- [ ] Add tests for metadata extraction (mocked)
-- [ ] Add tests for storage provider integrations (mocked)
-- [ ] Achieve 60% code coverage
-
----
-
-## 🟠 High Priority (This Sprint - 2 Weeks)
-
-### Code Quality
-- [ ] Fix all critical Flake8 violations
-- [ ] Run Black formatter on entire codebase
-- [ ] Add type hints to core modules (config.py, database.py, models.py)
-- [ ] Refactor large functions in tasks/ directory
-- [ ] Add docstrings to all public functions and classes
-- [ ] Remove unused imports and dead code
-
-### CI/CD
-- [x] Enable tests in GitHub Actions
-- [x] Add coverage reporting
-- [x] Add CodeQL scanning
-- [ ] Add dependency scanning (Dependabot or similar)
-- [ ] Make linting checks blocking (once critical issues fixed)
-- [ ] Add build status badges to README.md
-
-### Documentation
-- [x] Create ROADMAP.md
-- [x] Create MILESTONES.md
-- [x] Create TODO.md
-- [x] Create SECURITY_AUDIT.md
-- [ ] Create AGENTIC_CODING.md
-- [ ] Update CONTRIBUTING.md with testing guidelines
-- [ ] Add architecture diagram to docs/
-- [ ] Document all environment variables in docs/ConfigurationGuide.md
-- [ ] Add troubleshooting section for common test failures
-
----
-
-## 🟡 Medium Priority (Next Month)
-
-### Features
-- [ ] Implement retry logic for failed Celery tasks
-- [ ] Add pagination to file list endpoint
-- [ ] Add bulk delete functionality
-- [ ] Implement file download endpoint
-- [ ] Add document preview functionality
-- [ ] Add search/filter functionality to UI
-- [ ] Implement notification system for task completion
-- [ ] Add support for configuring custom metadata fields
-
-### Refactoring
-- [ ] Consolidate storage provider code (reduce duplication)
-- [ ] Create base class for storage providers
-- [ ] Standardize error responses across all API endpoints
-- [ ] Move hardcoded strings to constants
-- [ ] Extract common validation logic into utilities
-- [ ] Optimize database queries (add indexes)
-- [ ] Reduce Docker image size
-
-### Testing
-- [ ] Add end-to-end tests for complete workflows
-- [ ] Add performance tests for large file processing
-- [ ] Add tests for edge cases (empty files, corrupted PDFs, etc.)
-- [ ] Add stress tests for concurrent uploads
-- [ ] Set up test data fixtures
-- [ ] Add mock servers for external APIs
-
----
-
-## 🟢 Low Priority (Backlog)
-
-### Features
-- [ ] Add file versioning support
-- [ ] Implement document tagging system
-- [ ] Add custom metadata templates
-- [ ] Support for additional storage providers (Box, Mega, etc.)
-- [ ] Add support for zip file uploads
-- [ ] Implement folder organization
-- [ ] Add audit log viewer in UI
-- [ ] Support for scheduled document processing
-
-### UI/UX
-- [ ] Improve mobile responsiveness
-- [ ] Add dark mode
-- [ ] Add loading spinners for async operations
-- [ ] Improve error messages for users
-- [ ] Add drag-and-drop file upload
-- [ ] Add file type icons
-- [ ] Implement toast notifications
-- [ ] Add keyboard shortcuts
-
-### Developer Experience
-- [ ] Create development Docker Compose setup
-- [ ] Add hot-reload for development
-- [ ] Create seed data script for testing
-- [ ] Add debug toolbar for FastAPI
-- [ ] Create CLI tool for common operations
-- [ ] Add profiling tools
-- [ ] Create contributor onboarding guide
-
----
-
-## 🐛 Known Bugs
-
-### High Priority
-- [ ] Investigate session timeout issues with Authentik
-- [ ] Fix intermittent Redis connection failures
-- [ ] Handle large file uploads (>100MB) gracefully
-- [ ] Fix timezone handling in task scheduling
-
-### Medium Priority
-- [ ] PDF rotation not persisting in some cases
-- [ ] Metadata extraction fails for non-English documents
-- [ ] UI refresh needed after file upload
-- [ ] Error messages not showing in UI sometimes
-
-### Low Priority
-- [ ] Static files caching issues in production
-- [ ] Minor CSS alignment issues on some browsers
-- [ ] Log files growing too large over time
-
----
-
-## 📚 Documentation Tasks
-
-### User Documentation
-- [ ] Create video tutorial for basic usage
-- [ ] Add screenshots to all documentation pages
-- [ ] Create FAQ document
-- [ ] Write integration guides for each storage provider
-- [ ] Create quickstart guide (5 minutes to first document)
-- [ ] Document all API endpoints with examples
-- [ ] Add Postman collection
-
-### Developer Documentation
-- [ ] Document project architecture
-- [ ] Create database schema diagram
-- [ ] Document Celery task flow
-- [ ] Add code comments for complex logic
-- [ ] Create API versioning strategy document
-- [ ] Document testing strategy
-- [ ] Add examples for extending the system
-
----
-
-## 🔧 Technical Debt
-
-### Refactoring Needed
-- [ ] Replace PyPDF2 with pypdf (modern maintained fork)
-- [ ] Migrate from string-based task names to explicit imports in Celery
-- [ ] Standardize logging format across all modules
-- [ ] Remove duplicated configuration loading code
-- [ ] Consolidate error handling patterns
-- [ ] Extract magic numbers into constants
-- [ ] Improve variable naming in legacy code sections
-
-### Performance Optimization
-- [ ] Profile slow API endpoints
-- [ ] Optimize database queries (N+1 problem in file list)
-- [ ] Implement caching for frequently accessed data
-- [ ] Lazy-load heavy dependencies
-- [ ] Optimize Docker image layers
-- [ ] Reduce memory usage in OCR processing
-- [ ] Add database connection pooling
-
----
-
-## 📦 Dependencies to Update
-
-### Security Updates
-- [x] authlib → 1.6.5+
-- [x] starlette → 0.49.1+
-- [ ] Review all dependencies for known vulnerabilities
-- [ ] Update pinned versions in requirements.txt
-
-### Regular Updates
-- [ ] fastapi → latest stable
-- [ ] celery → latest stable
-- [ ] sqlalchemy → latest stable
-- [ ] pydantic → latest stable (check for breaking changes)
-- [ ] Check all dependencies for major version updates
-
----
-
-## ✅ Completed (Recent)
-
-### 2026-02-06
-- [x] Created comprehensive test infrastructure
-- [x] Fixed critical security vulnerabilities
-- [x] Added security scanning workflows
-- [x] Created ROADMAP.md and MILESTONES.md
-- [x] Enhanced .gitignore for security
-- [x] Improved SESSION_SECRET handling
-- [x] Created SECURITY_AUDIT.md
-- [x] Set up pytest with coverage
-- [x] Added API and configuration tests
-- [x] Updated CI/CD workflows
-- [x] Added pre-commit hooks configuration
-- [x] Created TODO.md (this file)
-
----
-
-## 📋 How to Use This TODO
-
-### For Contributors
-1. Pick a task from the appropriate priority section
-2. Check if there's a related GitHub issue; if not, create one
-3. Assign yourself to the issue
-4. Move task to "In Progress" (add your name)
-5. Submit PR when complete
-6. Move task to "Completed" section with date
-
-### For Maintainers
-- Review and update priorities weekly
-- Add new tasks as they're identified
-- Archive completed tasks monthly
-- Link tasks to GitHub issues/PRs
-- Update status in standups/meetings
-
-### Task Status Notation
-- `[ ]` - Not started
-- `[~]` - In progress (add contributor name: `[~@username]`)
-- `[x]` - Completed
-- `[!]` - Blocked (add reason in note)
-
----
-
-## 🔗 Related Documents
-
-- [ROADMAP.md](ROADMAP.md) - Long-term vision and features
-- [MILESTONES.md](MILESTONES.md) - Release planning and versions
-- [CONTRIBUTING.md](CONTRIBUTING.md) - Contribution guidelines
-- [SECURITY.md](SECURITY.md) - Security policy
-- [SECURITY_AUDIT.md](SECURITY_AUDIT.md) - Security audit results
-- [GitHub Issues](https://github.com/christianlouis/DocuElevate/issues) - Bug reports and feature requests
-- [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) - Sprint boards
-
----
-
-*This TODO list is reviewed and updated regularly. Last review: 2026-02-06*

VERSION

@@ -1 +0,0 @@
-0.4.5-dev

app/__init__.py

@@ -1,3 +0,0 @@
-"""
-Document processor application package.
-"""

app/api.py

@@ -0,0 +1,76 @@
+# app/api.py
+from fastapi import APIRouter, Request, HTTPException, status, Depends
+from hashlib import md5
+from sqlalchemy.orm import Session
+from typing import List
+
+from app.auth import require_login
+from app.database import SessionLocal
+from app.models import FileRecord
+
+router = APIRouter()
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+@router.get("/whoami")
+async def whoami(request: Request):
+    """
+    Returns user info if logged in, else 401.
+    """
+    user = request.session.get("user")
+    if not user:
+        raise HTTPException(status_code=401, detail="Not logged in")
+
+    email = user.get("email")
+    if not email:
+        raise HTTPException(status_code=400, detail="User has no email in session")
+
+    # Generate Gravatar URL from email
+    email_hash = md5(email.strip().lower().encode()).hexdigest()
+    gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
+
+    return {
+        "email": email,
+        "picture": gravatar_url
+    }
+
+@router.get("/files")
+@require_login
+def list_files_api(request: Request, db: Session = Depends(get_db)):
+    """
+    Returns a JSON list of all FileRecord entries.
+    Protected by `@require_login`, so only logged-in sessions can access.
+    
+    Example response:
+    [
+      {
+        "id": 123,
+        "filehash": "abc123...",
+        "original_filename": "example.pdf",
+        "local_filename": "/workdir/tmp/<uuid>.pdf",
+        "file_size": 1048576,
+        "mime_type": "application/pdf",
+        "created_at": "2025-05-01T12:34:56.789000"
+      },
+      ...
+    ]
+    """
+    files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all()
+    # Return a simple list of dicts
+    result = []
+    for f in files:
+        result.append({
+            "id": f.id,
+            "filehash": f.filehash,
+            "original_filename": f.original_filename,
+            "local_filename": f.local_filename,
+            "file_size": f.file_size,
+            "mime_type": f.mime_type,
+            "created_at": f.created_at.isoformat() if f.created_at else None
+        })
+    return result

app/api/__init__.py

@@ -1,33 +0,0 @@
-"""
-API Router module that combines all API endpoints
-"""
-from fastapi import APIRouter
-import logging
-
-# Import all the individual routers
-from app.api.user import router as user_router
-from app.api.files import router as files_router
-from app.api.process import router as process_router
-from app.api.diagnostic import router as diagnostic_router
-from app.api.onedrive import router as onedrive_router
-from app.api.dropbox import router as dropbox_router
-from app.api.openai import router as openai_router
-from app.api.azure import router as azure_router
-from app.api.google_drive import router as google_drive_router
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-# Create the main router that includes all the others
-router = APIRouter()
-
-# Include all the routers
-router.include_router(user_router)
-router.include_router(files_router)
-router.include_router(process_router)
-router.include_router(diagnostic_router)
-router.include_router(onedrive_router)
-router.include_router(dropbox_router)
-router.include_router(openai_router)
-router.include_router(azure_router)
-router.include_router(google_drive_router)

app/api/azure.py

@@ -1,124 +0,0 @@
-"""
-Azure AI API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException, status
-import logging
-import os
-
-from app.auth import require_login
-from app.config import settings
-
-# Import the Azure modules including the administration client
-from azure.core.credentials import AzureKeyCredential
-from azure.ai.documentintelligence import DocumentIntelligenceClient
-from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
-import azure.core.exceptions
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.get("/azure/test")
-@require_login
-async def test_azure_connection(request: Request):
-    """
-    Test if the configured Azure Document Intelligence connection is valid.
-    Uses the DocumentIntelligenceAdministrationClient for testing the connection.
-    """
-    try:
-        logger.info("Testing Azure Document Intelligence connection")
-        
-        # Check if Azure configuration is present
-        if not settings.azure_endpoint or not settings.azure_ai_key:
-            logger.warning("Azure Document Intelligence configuration is incomplete")
-            missing = []
-            if not settings.azure_endpoint:
-                missing.append("endpoint")
-            if not settings.azure_ai_key:
-                missing.append("API key")
-                
-            return {
-                "status": "error",
-                "message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}"
-            }
-        
-        # Try to initialize the admin client and make a request to list operations
-        try:
-            # Initialize the admin client with credentials
-            admin_client = DocumentIntelligenceAdministrationClient(
-                endpoint=settings.azure_endpoint,
-                credential=AzureKeyCredential(settings.azure_ai_key)
-            )
-            
-            # Test the connection by listing operations - this is a documented method in the admin client
-            operations = list(admin_client.list_operations())
-            
-            # Successfully initialized client and made a request
-            logger.info("Azure Document Intelligence Admin connection successfully tested")
-            
-            # Return success with available operations info
-            operations_info = []
-            try:
-                for op in operations:
-                    if hasattr(op, 'operation_id') and op.operation_id:
-                        op_info = {
-                            "id": op.operation_id,
-                            "status": op.status if hasattr(op, 'status') else "Unknown",
-                            "created": str(op.created_on) if hasattr(op, 'created_on') else "Unknown",
-                            "kind": op.kind if hasattr(op, 'kind') else "Unknown"
-                        }
-                        operations_info.append(op_info)
-                
-                operation_count = len(operations_info)
-                return {
-                    "status": "success",
-                    "message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.",
-                    "endpoint": settings.azure_endpoint,
-                    "operations_count": operation_count,
-                    "recent_operations": operations_info[:3] if operations_info else []
-                }
-            except Exception as e:
-                # If error occurs while processing operations info, still return success
-                logger.warning(f"Connected to Azure but couldn't parse operations: {e}")
-                return {
-                    "status": "success",
-                    "message": "Azure Document Intelligence connection is valid, but couldn't retrieve operations details.",
-                    "endpoint": settings.azure_endpoint
-                }
-                
-        except azure.core.exceptions.ClientAuthenticationError as e:
-            logger.error(f"Azure authentication error: {e}")
-            return {
-                "status": "error",
-                "message": f"Authentication error: Invalid API key or credentials",
-                "detail": str(e)
-            }
-        except azure.core.exceptions.ServiceRequestError as e:
-            logger.error(f"Azure service request error: {e}")
-            return {
-                "status": "error",
-                "message": f"Service request error: Could not reach the Azure endpoint",
-                "detail": str(e)
-            }
-        except ValueError as e:
-            logger.error(f"Azure configuration value error: {e}")
-            return {
-                "status": "error",
-                "message": f"Configuration error: {str(e)}",
-                "detail": str(e)
-            }
-        except Exception as e:
-            logger.error(f"Azure connection test failed with unexpected error: {e}")
-            return {
-                "status": "error",
-                "message": f"Connection test failed with unexpected error",
-                "detail": str(e)
-            }
-    
-    except Exception as e:
-        logger.exception("Unexpected error testing Azure Document Intelligence connection")
-        return {
-            "status": "error",
-            "message": f"Unexpected error: {str(e)}"
-        }

app/api/common.py

@@ -1,36 +0,0 @@
-"""
-Common utilities for API routes
-"""
-import logging
-import os
-from sqlalchemy.orm import Session
-from fastapi import Depends
-
-from app.database import SessionLocal
-from app.config import settings
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-def get_db():
-    """Database dependency injection for routes"""
-    db = SessionLocal()
-    try:
-        yield db
-    finally:
-        db.close()
-
-def resolve_file_path(file_path: str, subfolder: str = None) -> str:
-    """
-    Resolves a file path to an absolute path.
-    If the path is not absolute, it will be joined with the workdir path.
-    Optionally, can include a subfolder like 'processed'.
-    
-    Returns the absolute file path.
-    """
-    if not os.path.isabs(file_path):
-        if subfolder:
-            file_path = os.path.join(settings.workdir, subfolder, file_path)
-        else:
-            file_path = os.path.join(settings.workdir, file_path)
-    return file_path

app/api/diagnostic.py

@@ -1,100 +0,0 @@
-"""
-Diagnostic API endpoints
-"""
-from fastapi import APIRouter, Request, Depends
-import logging
-
-from app.auth import require_login, get_current_user
-from app.config import settings
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.get("/diagnostic/settings")
-@require_login
-async def diagnostic_settings(request: Request, current_user: dict = Depends(get_current_user)):
-    """
-    API endpoint to dump settings to the log and view basic config information
-    This endpoint doesn't expose sensitive information like passwords or tokens
-    """
-    from app.utils.config_validator import dump_all_settings, get_settings_for_display
-    # Dump full settings to log for admin to see
-    dump_all_settings()
-    
-    # Return safe subset of settings for API response
-    safe_settings = {
-        "workdir": settings.workdir,
-        "external_hostname": settings.external_hostname,
-        "configured_services": {
-            "email": bool(getattr(settings, 'email_host', None)),
-            "s3": bool(getattr(settings, 's3_bucket_name', None)),
-            "dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
-            "onedrive": bool(getattr(settings, 'onedrive_refresh_token', None)),
-            "nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
-            "sftp": bool(getattr(settings, 'sftp_host', None)),
-            "paperless": bool(getattr(settings, 'paperless_host', None)),
-            "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
-            "uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)),
-            "auth": bool(getattr(settings, 'authentik_config_url', None)),
-            "openai": bool(getattr(settings, 'openai_api_key', None)),
-            "azure": bool(getattr(settings, 'azure_api_key', None) and getattr(settings, 'azure_endpoint', None)),
-        },
-        "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
-    }
-    
-    return {
-        "status": "success",
-        "settings": safe_settings,
-        "message": "Full settings have been dumped to application logs"
-    }
-
-@router.post("/diagnostic/test-notification")
-@require_login
-async def test_notification(request: Request):
-    # Add request_time to request.state
-    import datetime
-    request.state.request_time = datetime.datetime.utcnow().isoformat()
-    """
-    Send a test notification through all configured notification channels
-    """
-    from app.utils.notification import send_notification
-    
-    try:
-        notification_urls = getattr(settings, 'notification_urls', [])
-        if not notification_urls:
-            return {
-                "status": "warning",
-                "message": "No notification services configured. Add notification URLs to your configuration."
-            }
-        
-        # Send a test notification
-        hostname = settings.external_hostname or "Document Processor"
-        result = send_notification(
-            title=f"Test Notification from {hostname}",
-            message=f"This is a test notification sent at {request.state.request_time}. If you're receiving this, notifications are working!",
-            notification_type="success",
-            tags=["test", "notification", "diagnostic"]
-        )
-        
-        if result:
-            logger.info("Test notification sent successfully")
-            return {
-                "status": "success",
-                "message": f"Test notification sent successfully to {len(notification_urls)} service(s)",
-                "services_count": len(notification_urls)
-            }
-        else:
-            logger.warning("Test notification send attempt returned False")
-            return {
-                "status": "error",
-                "message": "Failed to send test notification. Check application logs for details."
-            }
-    
-    except Exception as e:
-        logger.exception(f"Error sending test notification: {e}")
-        return {
-            "status": "error",
-            "message": f"Error sending notification: {str(e)}"
-        }

app/api/dropbox.py

@@ -1,342 +0,0 @@
-"""
-Dropbox API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException, status, Form
-import logging
-import os
-import requests
-import json
-from datetime import datetime, timedelta
-from typing import Optional
-
-from app.auth import require_login
-from app.config import settings
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.post("/dropbox/exchange-token")
-@require_login
-async def exchange_dropbox_token(
-    request: Request,
-    client_id: str = Form(...),
-    client_secret: str = Form(...),
-    redirect_uri: str = Form(...),
-    code: str = Form(...),
-    folder_path: str = Form(None)
-):
-    """
-    Exchange an authorization code for a refresh token from Dropbox.
-    This is done on the server to avoid exposing client secret in the browser.
-    """
-    try:
-        logger.info("Starting Dropbox token exchange process")
-        
-        # Prepare the token request
-        token_url = "https://api.dropboxapi.com/oauth2/token"
-        
-        payload = {
-            'client_id': client_id,
-            'client_secret': client_secret,
-            'code': code,
-            'redirect_uri': redirect_uri,
-            'grant_type': 'authorization_code'
-        }
-        
-        # Log request details (excluding secret)
-        safe_payload = payload.copy()
-        safe_payload['client_secret'] = '[REDACTED]' 
-        safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
-        logger.info(f"Token exchange request payload: {safe_payload}")
-        
-        # Make the token request
-        logger.info("Sending POST request to Dropbox for token exchange")
-        response = requests.post(token_url, data=payload)
-        
-        # Check if the request was successful
-        logger.info(f"Token exchange response status: {response.status_code}")
-        
-        if response.status_code != 200:
-            # Log the error response for debugging
-            try:
-                error_json = response.json()
-                logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
-                error_detail = error_json
-            except Exception as json_err:
-                logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
-                logger.error(f"Raw response content: {response.content[:500]}")  # Limit log size
-                error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
-            
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail=f"Token exchange failed: {error_detail}"
-            )
-            
-        # Return the token response
-        token_data = response.json()
-        
-        # Validate the token response
-        if "refresh_token" not in token_data:
-            logger.error(f"Dropbox returned success but no refresh token found in response: {token_data.keys()}")
-            raise HTTPException(
-                status_code=status.HTTP_502_BAD_GATEWAY,
-                detail="Dropbox OAuth server returned success but no refresh token was included"
-            )
-        
-        # Calculate token length for logging
-        refresh_token_length = len(token_data.get("refresh_token", ""))
-        access_token_length = len(token_data.get("access_token", ""))
-        
-        logger.info(f"Successfully exchanged authorization code for Dropbox tokens. "
-                   f"Refresh token length: {refresh_token_length}, "
-                   f"Access token length: {access_token_length}")
-        
-        # Return just what's needed by the frontend
-        return {
-            "refresh_token": token_data["refresh_token"],
-            "access_token": token_data["access_token"],
-            "expires_in": token_data.get("expires_in", 14400)
-        }
-        
-    except HTTPException:
-        # Re-raise HTTP exceptions as they already have appropriate status codes
-        raise
-    except Exception as e:
-        logger.exception(f"Unexpected error during Dropbox token exchange: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to exchange token: {str(e)}"
-        )
-
-@router.post("/dropbox/update-settings")
-@require_login
-async def update_dropbox_settings(
-    request: Request,
-    app_key: str = Form(None),
-    app_secret: str = Form(None),
-    refresh_token: str = Form(...),
-    folder_path: str = Form(None)
-):
-    """
-    Update Dropbox settings in memory
-    """
-    try:
-        logger.info("Updating Dropbox settings in memory")
-        
-        # Update settings in memory
-        if refresh_token:
-            settings.dropbox_refresh_token = refresh_token
-            logger.info("Updated DROPBOX_REFRESH_TOKEN in memory")
-            
-        if app_key:
-            settings.dropbox_app_key = app_key
-            logger.info("Updated DROPBOX_APP_KEY in memory")
-            
-        if app_secret:
-            settings.dropbox_app_secret = app_secret
-            logger.info("Updated DROPBOX_APP_SECRET in memory")
-            
-        if folder_path:
-            settings.dropbox_folder = folder_path
-            logger.info("Updated DROPBOX_FOLDER in memory")
-        
-        # Test token validity would be here, but we'll skip it for now
-        
-        return {
-            "status": "success",
-            "message": "Dropbox settings have been updated in memory"
-        }
-        
-    except Exception as e:
-        logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to update Dropbox settings: {str(e)}"
-        )
-
-@router.get("/dropbox/test-token")
-@require_login
-async def test_dropbox_token(request: Request):
-    """
-    Test if the configured Dropbox token is valid and return expiration information.
-    """
-    try:
-        logger.info("Testing Dropbox token validity")
-        
-        if not settings.dropbox_refresh_token or not settings.dropbox_app_key or not settings.dropbox_app_secret:
-            logger.warning("Dropbox credentials not fully configured")
-            return {
-                "status": "error", 
-                "message": "Dropbox credentials are not fully configured"
-            }
-        
-        # Check token validity by getting current account info
-        headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
-        response = requests.post(
-            "https://api.dropboxapi.com/2/users/get_current_account",
-            headers=headers
-        )
-        
-        # If token is invalid, try refreshing it
-        if response.status_code == 401:
-            logger.info("Dropbox access token invalid or expired, trying to refresh")
-            
-            # Get a new access token using the refresh token
-            refresh_url = "https://api.dropbox.com/oauth2/token"
-            refresh_data = {
-                "grant_type": "refresh_token",
-                "refresh_token": settings.dropbox_refresh_token,
-                "client_id": settings.dropbox_app_key,
-                "client_secret": settings.dropbox_app_secret
-            }
-            
-            refresh_response = requests.post(refresh_url, data=refresh_data)
-            
-            if refresh_response.status_code != 200:
-                logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
-                return {
-                    "status": "error",
-                    "message": "Refresh token has expired or is invalid",
-                    "needs_reauth": True
-                }
-            
-            token_info = refresh_response.json()
-            access_token = token_info.get("access_token")
-            
-            # Try again with the new access token
-            headers = {"Authorization": f"Bearer {access_token}"}
-            response = requests.post(
-                "https://api.dropboxapi.com/2/users/get_current_account",
-                headers=headers
-            )
-        
-        if response.status_code != 200:
-            logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
-            return {
-                "status": "error",
-                "message": f"Token validation failed with status {response.status_code}: {response.text}"
-            }
-        
-        # Get account info
-        account_info = response.json()
-        account_email = account_info.get("email", "Unknown account")
-        account_name = account_info.get("name", {}).get("display_name", "Unknown user")
-        
-        # Dropbox refresh tokens don't expire, but we should note that in our response
-        token_info = {
-            "expires_in_human": "Never expires (perpetual token)",
-            "is_perpetual": True
-        }
-        
-        logger.info(f"Successfully connected to Dropbox as {account_email}")
-        
-        return {
-            "status": "success",
-            "message": f"Dropbox connection successful",
-            "account": account_email,
-            "account_name": account_name,
-            "token_info": token_info
-        }
-        
-    except Exception as e:
-        logger.exception(f"Unexpected error testing Dropbox token: {str(e)}")
-        return {
-            "status": "error",
-            "message": f"Connection error: {str(e)}"
-        }
-
-@router.post("/dropbox/save-settings")
-@require_login
-async def save_dropbox_settings(
-    request: Request,
-    app_key: str = Form(None),
-    app_secret: str = Form(None),
-    refresh_token: str = Form(...),
-    folder_path: str = Form(None)
-):
-    """
-    Save Dropbox settings to the .env file
-    """
-    try:
-        # Get the path to the .env file
-        env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
-        
-        if not os.path.exists(env_path):
-            logger.error(f".env file not found at {env_path}")
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail="Could not find .env file to update"
-            )
-            
-        logger.info(f"Updating Dropbox settings in {env_path}")
-        
-        # Read the current .env file
-        with open(env_path, "r") as f:
-            env_lines = f.readlines()
-            
-        # Define settings to update
-        dropbox_settings = {
-            "DROPBOX_REFRESH_TOKEN": refresh_token,
-        }
-        
-        # Only update these if provided
-        if app_key:
-            dropbox_settings["DROPBOX_APP_KEY"] = app_key
-        if app_secret:
-            dropbox_settings["DROPBOX_APP_SECRET"] = app_secret
-        if folder_path:
-            dropbox_settings["DROPBOX_FOLDER"] = folder_path
-            
-        # Process each line and update or add settings
-        updated = set()
-        new_env_lines = []
-        for line in env_lines:
-            line = line.rstrip()
-            is_updated = False
-            for key, value in dropbox_settings.items():
-                if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
-                    if line.startswith("# "):  # Uncomment if commented out
-                        line = line[2:]
-                    new_env_lines.append(f"{key}={value}")
-                    updated.add(key)
-                    is_updated = True
-                    break
-            if not is_updated:
-                new_env_lines.append(line)
-                
-        # Add any settings that weren't updated (they weren't in the file)
-        for key, value in dropbox_settings.items():
-            if key not in updated:
-                new_env_lines.append(f"{key}={value}")
-                
-        # Write the updated .env file
-        with open(env_path, "w") as f:
-            f.write("\n".join(new_env_lines) + "\n")
-            
-        # Update the settings in memory
-        if refresh_token:
-            settings.dropbox_refresh_token = refresh_token
-        if app_key:
-            settings.dropbox_app_key = app_key
-        if app_secret:
-            settings.dropbox_app_secret = app_secret
-        if folder_path:
-            settings.dropbox_folder = folder_path
-            
-        logger.info("Successfully updated Dropbox settings")
-        
-        return {
-            "status": "success",
-            "message": "Dropbox settings have been saved"
-        }
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.exception(f"Unexpected error saving Dropbox settings: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to save Dropbox settings: {str(e)}"
-        )

app/api/files.py

@@ -1,201 +0,0 @@
-"""
-File-related API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File
-from sqlalchemy.orm import Session
-import logging
-import os
-import uuid
-import mimetypes
-
-from app.auth import require_login
-from app.models import FileRecord
-from app.config import settings
-from app.api.common import get_db
-from app.tasks.process_document import process_document
-from app.tasks.convert_to_pdf import convert_to_pdf
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.get("/files")
-@require_login
-def list_files_api(request: Request, db: Session = Depends(get_db)):
-    """
-    Returns a JSON list of all FileRecord entries.
-    Protected by `@require_login`, so only logged-in sessions can access.
-    
-    Example response:
-    [
-      {
-        "id": 123,
-        "filehash": "abc123...",
-        "original_filename": "example.pdf",
-        "local_filename": "/workdir/tmp/<uuid>.pdf",
-        "file_size": 1048576,
-        "mime_type": "application/pdf",
-        "created_at": "2025-05-01T12:34:56.789000"
-      },
-      ...
-    ]
-    """
-    files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all()
-    # Return a simple list of dicts
-    result = []
-    for f in files:
-        result.append({
-            "id": f.id,
-            "filehash": f.filehash,
-            "original_filename": f.original_filename,
-            "local_filename": f.local_filename,
-            "file_size": f.file_size,
-            "mime_type": f.mime_type,
-            "created_at": f.created_at.isoformat() if f.created_at else None
-        })
-    return result
-
-@router.delete("/files/{file_id}")
-@require_login
-def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)):
-    """
-    Delete a file record from the database.
-    This only removes the database entry, not the actual file.
-    """
-    # Check if file deletion is allowed
-    if not settings.allow_file_delete:
-        raise HTTPException(
-            status_code=403,
-            detail="File deletion is disabled in the configuration"
-        )
-
-    try:
-        # Find the file record
-        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
-        
-        if not file_record:
-            raise HTTPException(
-                status_code=404,
-                detail=f"File record with ID {file_id} not found"
-            )
-        
-        # Log the deletion
-        logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")
-        
-        # Delete the record
-        db.delete(file_record)
-        db.commit()
-        
-        return {
-            "status": "success",
-            "message": f"File record {file_id} deleted successfully"
-        }
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        db.rollback()
-        logger.exception(f"Error deleting file record {file_id}: {str(e)}")
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error deleting file record: {str(e)}"
-        )
-
-@router.post("/ui-upload")
-@require_login
-async def ui_upload(request: Request, file: UploadFile = File(...)):
-    """Endpoint to accept a user-uploaded file and enqueue it for processing."""
-    workdir = settings.workdir
-    
-    # Extract just the filename without any path components to prevent path traversal
-    safe_filename = os.path.basename(file.filename)
-    
-    # Generate a unique filename with UUID to prevent overwriting and filename conflicts
-    unique_id = str(uuid.uuid4())
-    # Keep the original extension if present
-    if "." in safe_filename:
-        file_extension = safe_filename.rsplit(".", 1)[1]
-        target_filename = f"{unique_id}.{file_extension}"
-    else:
-        target_filename = unique_id
-    
-    # Store both the safe original name and the unique name
-    target_path = os.path.join(workdir, target_filename)
-    
-    try:
-        with open(target_path, "wb") as f:
-            content = await file.read()
-            f.write(content)
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Failed to save file: {e}"
-        )
-
-    # Log the mapping between original and safe filename
-    logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
-    
-    # Check file size
-    file_size = os.path.getsize(target_path)
-    max_size = 500 * 1024 * 1024  # 500MB
-    if file_size > max_size:
-        # Remove the file if it's too large
-        os.remove(target_path)
-        raise HTTPException(
-            status_code=413,
-            detail=f"File too large: {file_size} bytes (max {max_size} bytes)"
-        )
-    
-    # Same set of allowed file types as in the IMAP task
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-    
-    # Image MIME types that need conversion
-    IMAGE_MIME_TYPES = {
-        'image/jpeg', 'image/jpg', 'image/png', 
-        'image/gif', 'image/bmp', 'image/tiff',
-        'image/webp', 'image/svg+xml'
-    }
-    
-    # Determine if the file is a PDF or needs conversion
-    mime_type, _ = mimetypes.guess_type(target_path)
-    file_ext = os.path.splitext(target_path)[1].lower()
-    
-    # Check if it's a PDF by extension or MIME type
-    is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"
-    
-    if is_pdf:
-        # If it's a PDF, process directly
-        task = process_document.delay(target_path)
-        logger.info(f"Enqueued PDF for processing: {target_path}")
-    elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']):
-        # If it's an image, convert to PDF first
-        task = convert_to_pdf.delay(target_path)
-        logger.info(f"Enqueued image for PDF conversion: {target_path}")
-    elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']):
-        # If it's an office document, convert to PDF first
-        task = convert_to_pdf.delay(target_path)
-        logger.info(f"Enqueued office document for PDF conversion: {target_path}")
-    else:
-        # For any other file type, attempt conversion but log a warning
-        logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
-        task = convert_to_pdf.delay(target_path)
-    
-    return {
-        "task_id": task.id, 
-        "status": "queued", 
-        "original_filename": safe_filename,
-        "stored_filename": target_filename
-    }

app/api/google_drive.py

@@ -1,506 +0,0 @@
-"""
-Google Drive API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException, status, Form
-import logging
-import os
-import requests
-import json
-from typing import Optional
-from datetime import datetime, timedelta
-
-from app.auth import require_login
-from app.config import settings
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.post("/google-drive/exchange-token")
-@require_login
-async def exchange_google_drive_token(
-    request: Request,
-    client_id: str = Form(...),
-    client_secret: str = Form(...),
-    redirect_uri: str = Form(...),
-    code: str = Form(...),
-    folder_id: Optional[str] = Form(None)
-):
-    """
-    Exchange an authorization code for refresh and access tokens from Google.
-    This is done on the server to avoid exposing client secret in the browser.
-    """
-    try:
-        logger.info("Starting Google Drive token exchange process")
-        
-        # Prepare the token request
-        token_url = "https://oauth2.googleapis.com/token"
-        
-        payload = {
-            'client_id': client_id,
-            'client_secret': client_secret,
-            'code': code,
-            'redirect_uri': redirect_uri,
-            'grant_type': 'authorization_code'
-        }
-        
-        # Log request details (excluding secret)
-        safe_payload = payload.copy()
-        safe_payload['client_secret'] = '[REDACTED]' 
-        safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
-        logger.info(f"Token exchange request payload: {safe_payload}")
-        
-        # Make the token request
-        logger.info("Sending POST request to Google for token exchange")
-        response = requests.post(token_url, data=payload)
-        
-        # Check if the request was successful
-        logger.info(f"Token exchange response status: {response.status_code}")
-        
-        if response.status_code != 200:
-            # Log the error response for debugging
-            try:
-                error_json = response.json()
-                logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
-                error_detail = error_json
-            except Exception as json_err:
-                logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
-                logger.error(f"Raw response content: {response.content[:500]}")  # Limit log size
-                error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
-            
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail=f"Token exchange failed: {error_detail}"
-            )
-            
-        # Return the token response
-        token_data = response.json()
-        
-        # Validate the token response
-        if "refresh_token" not in token_data:
-            logger.error(f"Google returned success but no refresh token found in response: {token_data.keys()}")
-            raise HTTPException(
-                status_code=status.HTTP_502_BAD_GATEWAY,
-                detail="Google OAuth server returned success but no refresh token was included"
-            )
-        
-        # Calculate token length for logging
-        refresh_token_length = len(token_data.get("refresh_token", ""))
-        access_token_length = len(token_data.get("access_token", ""))
-        
-        logger.info(f"Successfully exchanged authorization code for Google Drive tokens. "
-                   f"Refresh token length: {refresh_token_length}, "
-                   f"Access token length: {access_token_length}")
-        
-        # Return just what's needed by the frontend
-        return {
-            "refresh_token": token_data["refresh_token"],
-            "access_token": token_data["access_token"],
-            "expires_in": token_data.get("expires_in", 3600)
-        }
-        
-    except HTTPException:
-        # Re-raise HTTP exceptions as they already have appropriate status codes
-        raise
-    except Exception as e:
-        logger.exception(f"Unexpected error during Google Drive token exchange: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to exchange token: {str(e)}"
-        )
-
-@router.post("/google-drive/update-settings")
-@require_login
-async def update_google_drive_settings(
-    request: Request,
-    client_id: str = Form(None),
-    client_secret: str = Form(None),
-    refresh_token: str = Form(...),
-    folder_id: str = Form(None),
-    use_oauth: str = Form("true")
-):
-    """
-    Update Google Drive settings in memory
-    """
-    try:
-        logger.info("Updating Google Drive settings in memory")
-        
-        # Convert use_oauth string to boolean
-        use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t")
-        
-        # Update settings in memory
-        if refresh_token:
-            settings.google_drive_refresh_token = refresh_token
-            logger.info("Updated GOOGLE_DRIVE_REFRESH_TOKEN in memory")
-            
-        if client_id:
-            settings.google_drive_client_id = client_id
-            logger.info("Updated GOOGLE_DRIVE_CLIENT_ID in memory")
-            
-        if client_secret:
-            settings.google_drive_client_secret = client_secret
-            logger.info("Updated GOOGLE_DRIVE_CLIENT_SECRET in memory")
-            
-        if folder_id:
-            settings.google_drive_folder_id = folder_id
-            logger.info("Updated GOOGLE_DRIVE_FOLDER_ID in memory")
-        
-        # Set the OAuth flag
-        settings.google_drive_use_oauth = use_oauth_bool
-        logger.info(f"Updated GOOGLE_DRIVE_USE_OAUTH in memory to {use_oauth_bool}")
-        
-        return {
-            "status": "success",
-            "message": "Google Drive settings have been updated in memory"
-        }
-        
-    except Exception as e:
-        logger.exception(f"Unexpected error updating Google Drive settings: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to update Google Drive settings: {str(e)}"
-        )
-
-@router.get("/google-drive/test-token")
-@require_login
-async def test_google_drive_token(request: Request):
-    """
-    Test if the configured Google Drive token is valid.
-    Tests both OAuth and service account approaches based on configuration.
-    """
-    try:
-        from app.tasks.upload_to_google_drive import get_drive_service_oauth, get_google_drive_service
-        
-        logger.info("Testing Google Drive token validity")
-        
-        # Check if OAuth is enabled and configured
-        if getattr(settings, 'google_drive_use_oauth', False):
-            if not (settings.google_drive_client_id and 
-                  settings.google_drive_client_secret and
-                  settings.google_drive_refresh_token):
-                logger.warning("Google Drive OAuth credentials not fully configured")
-                return {
-                    "status": "error", 
-                    "message": "Google Drive OAuth credentials are not fully configured"
-                }
-            
-            try:
-                # Test OAuth connection
-                service = get_drive_service_oauth()
-                
-                # Get credentials for checking token validity
-                import google.oauth2.credentials
-                from google.auth.transport.requests import Request
-                
-                credentials = google.oauth2.credentials.Credentials(
-                    token=None,
-                    refresh_token=settings.google_drive_refresh_token,
-                    token_uri="https://oauth2.googleapis.com/token",
-                    client_id=settings.google_drive_client_id,
-                    client_secret=settings.google_drive_client_secret
-                )
-                
-                # Force a refresh to update the token expiration
-                if not credentials.valid:
-                    credentials.refresh(Request())
-                
-                # Get token expiration info
-                expiration_info = {}
-                if hasattr(credentials, 'expiry') and credentials.expiry:
-                    now = datetime.now()
-                    expiry = credentials.expiry
-                    time_left = expiry - now
-                    expiration_info = {
-                        "expires_at": expiry.isoformat(),
-                        "expires_in_seconds": max(0, int(time_left.total_seconds())),
-                        "expires_in_human": format_time_remaining(time_left)
-                    }
-                
-                # Test basic API operation
-                about = service.about().get(fields="user").execute()
-                user_email = about.get("user", {}).get("emailAddress", "Unknown")
-                
-                logger.info(f"Successfully connected to Google Drive as {user_email}")
-                
-                return {
-                    "status": "success",
-                    "message": f"OAuth token is valid! Connected as {user_email}",
-                    "account": user_email,
-                    "auth_type": "oauth",
-                    "token_info": expiration_info
-                }
-            except Exception as e:
-                error_msg = str(e)
-                logger.error(f"Google Drive OAuth token test failed: {error_msg}")
-                
-                # Check if this is a token-related error
-                if "invalid_grant" in error_msg.lower() or "token" in error_msg.lower():
-                    return {
-                        "status": "error",
-                        "message": f"OAuth token validation failed: {error_msg}",
-                        "needs_reauth": True
-                    }
-                return {
-                    "status": "error",
-                    "message": f"Connection error: {error_msg}"
-                }
-        else:
-            # Test service account connection
-            if not settings.google_drive_credentials_json:
-                logger.warning("Google Drive service account credentials not configured")
-                return {
-                    "status": "error", 
-                    "message": "Google Drive service account credentials are not configured"
-                }
-                
-            try:
-                service = get_google_drive_service()
-                about = service.about().get(fields="user").execute()
-                
-                # For service accounts, try to show the delegated user if available
-                user_email = about.get("user", {}).get("emailAddress", "Unknown")
-                delegated_user = getattr(settings, 'google_drive_delegate_to', None)
-                
-                if delegated_user:
-                    user_display = f"{user_email} (delegating as {delegated_user})"
-                else:
-                    user_display = user_email
-                    
-                logger.info(f"Successfully connected to Google Drive using service account as {user_display}")
-                
-                return {
-                    "status": "success",
-                    "message": f"Service account is valid! Connected as {user_display}",
-                    "account": user_email,
-                    "auth_type": "service_account"
-                }
-            except Exception as e:
-                error_msg = str(e)
-                logger.error(f"Google Drive service account test failed: {error_msg}")
-                return {
-                    "status": "error",
-                    "message": f"Service account validation failed: {error_msg}"
-                }
-    
-    except Exception as e:
-        logger.exception("Unexpected error testing Google Drive token")
-        return {
-            "status": "error",
-            "message": f"Unexpected error: {str(e)}"
-        }
-
-@router.get("/google-drive/get-token-info")
-@require_login
-async def get_google_drive_token_info(request: Request):
-    """
-    Get information about the current Google Drive token.
-    Returns the access token if one exists and is valid.
-    Used by the frontend to access the Google Picker API.
-    """
-    try:
-        logger.info("Getting Google Drive token information")
-        
-        # Check if OAuth is enabled and configured
-        if not getattr(settings, 'google_drive_use_oauth', False):
-            logger.warning("OAuth is not enabled, using service account instead")
-            return {
-                "status": "error",
-                "message": "OAuth is not enabled. Service accounts don't support user-facing features like folder picker."
-            }
-            
-        if not (settings.google_drive_client_id and 
-              settings.google_drive_client_secret and
-              settings.google_drive_refresh_token):
-            logger.warning("Google Drive OAuth credentials not fully configured")
-            return {
-                "status": "error", 
-                "message": "Google Drive OAuth credentials are not fully configured"
-            }
-        
-        try:
-            # Get credentials and access token
-            import google.oauth2.credentials
-            from google.auth.transport.requests import Request
-            
-            credentials = google.oauth2.credentials.Credentials(
-                token=None,
-                refresh_token=settings.google_drive_refresh_token,
-                token_uri="https://oauth2.googleapis.com/token",
-                client_id=settings.google_drive_client_id,
-                client_secret=settings.google_drive_client_secret
-            )
-            
-            # Force a refresh to get a fresh access token
-            if not credentials.valid:
-                credentials.refresh(Request())
-            
-            # Get token expiration info
-            expiration_info = {}
-            if hasattr(credentials, 'expiry') and credentials.expiry:
-                now = datetime.now()
-                expiry = credentials.expiry
-                time_left = expiry - now
-                expiration_info = {
-                    "expires_at": expiry.isoformat(),
-                    "expires_in_seconds": max(0, int(time_left.total_seconds())),
-                    "expires_in_human": format_time_remaining(time_left)
-                }
-            
-            # Return the token info
-            logger.info("Successfully retrieved Google Drive access token")
-            
-            return {
-                "status": "success",
-                "message": "Access token successfully retrieved",
-                "access_token": credentials.token,
-                "token_info": expiration_info
-            }
-        except Exception as e:
-            error_msg = str(e)
-            logger.error(f"Failed to get Google Drive token: {error_msg}")
-            
-            # Check if this is a token-related error
-            if "invalid_grant" in error_msg.lower() or "token" in error_msg.lower():
-                return {
-                    "status": "error",
-                    "message": f"OAuth token retrieval failed: {error_msg}",
-                    "needs_reauth": True
-                }
-            return {
-                "status": "error",
-                "message": f"Token retrieval error: {error_msg}"
-            }
-    
-    except Exception as e:
-        logger.exception("Unexpected error getting Google Drive token info")
-        return {
-            "status": "error",
-            "message": f"Unexpected error: {str(e)}"
-        }
-
-def format_time_remaining(time_delta):
-    """Format a timedelta into a human-readable string."""
-    if time_delta.total_seconds() <= 0:
-        return "Expired"
-        
-    days = time_delta.days
-    hours, remainder = divmod(time_delta.seconds, 3600)
-    minutes, seconds = divmod(remainder, 60)
-    
-    parts = []
-    if days > 0:
-        parts.append(f"{days} day{'s' if days != 1 else ''}")
-    if hours > 0:
-        parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
-    if minutes > 0 and days == 0:  # Only show minutes if less than a day
-        parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
-    
-    return ", ".join(parts)
-
-@router.post("/google-drive/save-settings")
-@require_login
-async def save_dropbox_settings(
-    request: Request,
-    client_id: str = Form(None),
-    client_secret: str = Form(None),
-    refresh_token: str = Form(...),
-    folder_id: str = Form(None),
-    use_oauth: str = Form("true")
-):
-    """
-    Save Google Drive settings to the .env file
-    """
-    try:
-        # Get the path to the .env file
-        env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
-        
-        # Convert use_oauth string to boolean
-        use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t")
-        
-        # Define settings to update
-        drive_settings = {
-            "GOOGLE_DRIVE_USE_OAUTH": str(use_oauth_bool).lower()
-        }
-        
-        # Only update these if provided
-        if use_oauth_bool:
-            if refresh_token:
-                drive_settings["GOOGLE_DRIVE_REFRESH_TOKEN"] = refresh_token
-            if client_id:
-                drive_settings["GOOGLE_DRIVE_CLIENT_ID"] = client_id
-            if client_secret:
-                drive_settings["GOOGLE_DRIVE_CLIENT_SECRET"] = client_secret
-        
-        # Always include folder ID if provided
-        if folder_id:
-            drive_settings["GOOGLE_DRIVE_FOLDER_ID"] = folder_id
-            
-        # Try to update the .env file, but don't fail if it doesn't exist (for Docker containers)
-        if os.path.exists(env_path):
-            try:
-                logger.info(f"Updating Google Drive settings in {env_path}")
-                
-                # Read the current .env file
-                with open(env_path, "r") as f:
-                    env_lines = f.readlines()
-                    
-                # Process each line and update or add settings
-                updated = set()
-                new_env_lines = []
-                for line in env_lines:
-                    line = line.rstrip()
-                    is_updated = False
-                    for key, value in drive_settings.items():
-                        if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
-                            if line.startswith("# "):  # Uncomment if commented out
-                                line = line[2:]
-                            new_env_lines.append(f"{key}={value}")
-                            updated.add(key)
-                            is_updated = True
-                            break
-                    if not is_updated:
-                        new_env_lines.append(line)
-                        
-                # Add any settings that weren't updated (they weren't in the file)
-                for key, value in drive_settings.items():
-                    if key not in updated:
-                        new_env_lines.append(f"{key}={value}")
-                        
-                # Write the updated .env file
-                with open(env_path, "w") as f:
-                    f.write("\n".join(new_env_lines) + "\n")
-                    
-                logger.info("Successfully updated Google Drive settings in .env file")
-            except Exception as e:
-                logger.warning(f"Failed to update .env file: {str(e)}, but will continue with in-memory update")
-        else:
-            logger.warning(f".env file not found at {env_path}, skipping file update but continuing with in-memory update")
-            
-        # Update the settings in memory (this always happens)
-        if refresh_token:
-            settings.google_drive_refresh_token = refresh_token
-        if client_id:
-            settings.google_drive_client_id = client_id
-        if client_secret:
-            settings.google_drive_client_secret = client_secret
-        if folder_id:
-            settings.google_drive_folder_id = folder_id
-        
-        # Set OAuth flag
-        settings.google_drive_use_oauth = use_oauth_bool
-            
-        logger.info("Successfully updated Google Drive settings in memory")
-        
-        return {
-            "status": "success",
-            "message": "Google Drive settings have been saved",
-            "in_memory_only": not os.path.exists(env_path)
-        }
-        
-    except Exception as e:
-        logger.exception(f"Unexpected error saving Google Drive settings: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to save Google Drive settings: {str(e)}"
-        )

app/api/onedrive.py

@@ -1,454 +0,0 @@
-"""
-OneDrive API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException, status, Form
-import logging
-import os
-import requests
-import json
-from datetime import datetime, timedelta
-from typing import Optional
-
-from app.auth import require_login
-from app.config import settings
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.post("/onedrive/exchange-token")
-@require_login
-async def exchange_onedrive_token(
-    request: Request,
-    client_id: str = Form(...),
-    client_secret: str = Form(...),
-    redirect_uri: str = Form(...),
-    code: str = Form(...),
-    tenant_id: str = Form(...)
-):
-    """
-    Exchange an authorization code for a refresh token.
-    This is done on the server to avoid exposing client secret in the browser.
-    """
-    try:
-        logger.info(f"Starting OneDrive token exchange process with tenant_id: {tenant_id}")
-        
-        # Prepare the token request
-        token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
-        logger.info(f"Using token URL: {token_url}")
-        
-        payload = {
-            'client_id': client_id,
-            'scope': 'https://graph.microsoft.com/.default offline_access',
-            'code': code,
-            'redirect_uri': redirect_uri,
-            'grant_type': 'authorization_code',
-            'client_secret': client_secret
-        }
-        
-        # Log request details (excluding secret)
-        safe_payload = payload.copy()
-        safe_payload['client_secret'] = '[REDACTED]' 
-        safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
-        logger.info(f"Token exchange request payload: {safe_payload}")
-        
-        # Make the token request
-        logger.info("Sending POST request to Microsoft for token exchange")
-        response = requests.post(token_url, data=payload)
-        
-        # Check if the request was successful
-        logger.info(f"Token exchange response status: {response.status_code}")
-        
-        if response.status_code != 200:
-            # Log the error response for debugging
-            try:
-                error_json = response.json()
-                logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
-                error_detail = error_json
-            except Exception as json_err:
-                logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
-                logger.error(f"Raw response content: {response.content[:500]}")  # Limit log size
-                error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
-            
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail=f"Token exchange failed: {error_detail}"
-            )
-            
-        # Return the token response
-        token_data = response.json()
-        
-        # Validate the token response
-        if "refresh_token" not in token_data:
-            logger.error(f"Microsoft returned success but no refresh token found in response: {token_data.keys()}")
-            raise HTTPException(
-                status_code=status.HTTP_502_BAD_GATEWAY,
-                detail="Microsoft OAuth server returned success but no refresh token was included"
-            )
-        
-        # Calculate token length for logging
-        refresh_token_length = len(token_data.get("refresh_token", ""))
-        access_token_length = len(token_data.get("access_token", ""))
-        
-        logger.info(f"Successfully exchanged authorization code for OneDrive tokens. "
-                   f"Refresh token length: {refresh_token_length}, "
-                   f"Access token length: {access_token_length}")
-        
-        # Return just what's needed by the frontend
-        return {
-            "refresh_token": token_data["refresh_token"],
-            "expires_in": token_data.get("expires_in", 3600)
-        }
-        
-    except HTTPException:
-        # Re-raise HTTP exceptions as they already have appropriate status codes
-        raise
-    except Exception as e:
-        logger.exception(f"Unexpected error during OneDrive token exchange: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to exchange token: {str(e)}"
-        )
-
-@router.get("/onedrive/test-token")
-@require_login
-async def test_onedrive_token(request: Request):
-    """
-    Test if the configured OneDrive token is valid and return expiration information.
-    """
-    try:
-        logger.info("Testing OneDrive token validity")
-        
-        if not settings.onedrive_refresh_token or not settings.onedrive_client_id or not settings.onedrive_client_secret:
-            logger.warning("OneDrive credentials not fully configured")
-            return {
-                "status": "error", 
-                "message": "OneDrive credentials are not fully configured"
-            }
-        
-        # Refresh token to get a new access token and expiration info
-        tenant_id = settings.onedrive_tenant_id or "common"
-        token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
-        
-        refresh_data = {
-            "client_id": settings.onedrive_client_id,
-            "client_secret": settings.onedrive_client_secret,
-            "refresh_token": settings.onedrive_refresh_token,
-            "grant_type": "refresh_token",
-            "scope": "offline_access Files.ReadWrite"
-        }
-        
-        response = requests.post(token_url, data=refresh_data)
-        
-        if response.status_code != 200:
-            logger.error(f"Failed to refresh OneDrive token: {response.text}")
-            return {
-                "status": "error",
-                "message": "Refresh token has expired or is invalid",
-                "needs_reauth": True
-            }
-        
-        token_data = response.json()
-        access_token = token_data.get("access_token")
-        expires_in = token_data.get("expires_in", 3600)  # Default to 1 hour if not specified
-        
-        # Check if we got a new refresh token (Microsoft sometimes issues a new one)
-        new_refresh_token = token_data.get("refresh_token")
-        if new_refresh_token and new_refresh_token != settings.onedrive_refresh_token:
-            logger.info("Received new refresh token from Microsoft - will update configuration")
-            
-            # Update refresh token in memory
-            settings.onedrive_refresh_token = new_refresh_token
-            
-            # Also try to update .env file if it exists
-            try:
-                env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
-                if os.path.exists(env_path):
-                    with open(env_path, "r") as f:
-                        env_lines = f.readlines()
-                    
-                    updated_lines = []
-                    updated = False
-                    
-                    for line in env_lines:
-                        if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
-                            updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
-                            updated = True
-                        else:
-                            updated_lines.append(line)
-                    
-                    if not updated:
-                        updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
-                    
-                    with open(env_path, "w") as f:
-                        f.writelines(updated_lines)
-                    
-                    logger.info("Updated refresh token in .env file")
-                
-            except Exception as e:
-                logger.warning(f"Failed to update refresh token in .env file: {e}")
-        
-        # Test the access token by getting user information
-        user_info_url = "https://graph.microsoft.com/v1.0/me"
-        headers = {"Authorization": f"Bearer {access_token}"}
-        
-        user_response = requests.get(user_info_url, headers=headers)
-        
-        if user_response.status_code != 200:
-            logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
-            return {
-                "status": "error",
-                "message": f"Token validation failed with status {user_response.status_code}: {user_response.text}"
-            }
-        
-        # Get user info
-        user_info = user_response.json()
-        display_name = user_info.get("displayName", "Unknown user")
-        email = user_info.get("userPrincipalName", "Unknown email")
-        
-        # Calculate expiration time
-        now = datetime.now()
-        expiry_time = now + timedelta(seconds=expires_in)
-        
-        # Format expiration info
-        time_left = expiry_time - now
-        token_info = {
-            "expires_at": expiry_time.isoformat(),
-            "expires_in_seconds": expires_in,
-            "expires_in_human": format_time_remaining(time_left),
-            "refresh_token_validity": "Refresh token is valid for 90 days of inactivity"
-        }
-        
-        logger.info(f"Successfully connected to OneDrive as {email}")
-        
-        return {
-            "status": "success",
-            "message": f"OneDrive connection successful",
-            "account": email,
-            "account_name": display_name,
-            "token_info": token_info
-        }
-        
-    except Exception as e:
-        logger.exception(f"Unexpected error testing OneDrive token: {str(e)}")
-        return {
-            "status": "error",
-            "message": f"Connection error: {str(e)}"
-        }
-
-def format_time_remaining(time_delta):
-    """Format a timedelta into a human-readable string."""
-    if time_delta.total_seconds() <= 0:
-        return "Expired"
-        
-    days = time_delta.days
-    hours, remainder = divmod(time_delta.seconds, 3600)
-    minutes, seconds = divmod(remainder, 60)
-    
-    parts = []
-    if days > 0:
-        parts.append(f"{days} day{'s' if days != 1 else ''}")
-    if hours > 0:
-        parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
-    if minutes > 0 and days == 0:  # Only show minutes if less than a day
-        parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
-    
-    return ", ".join(parts)
-
-@router.post("/onedrive/save-settings")
-@require_login
-async def save_onedrive_settings(
-    request: Request,
-    client_id: str = Form(None),
-    client_secret: str = Form(None),
-    refresh_token: str = Form(...),
-    tenant_id: str = Form("common"),
-    folder_path: str = Form(None)
-):
-    """
-    Save OneDrive settings to the .env file
-    """
-    try:
-        # Get the path to the .env file
-        env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
-        
-        if not os.path.exists(env_path):
-            logger.error(f".env file not found at {env_path}")
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail="Could not find .env file to update"
-            )
-            
-        logger.info(f"Updating OneDrive settings in {env_path}")
-        
-        # Read the current .env file
-        with open(env_path, "r") as f:
-            env_lines = f.readlines()
-            
-        # Define settings to update
-        onedrive_settings = {
-            "ONEDRIVE_REFRESH_TOKEN": refresh_token,
-        }
-        
-        # Only update these if provided
-        if client_id:
-            onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
-        if client_secret:
-            onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
-        if tenant_id:
-            onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
-        if folder_path:
-            onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
-            
-        # Process each line and update or add settings
-        updated = set()
-        new_env_lines = []
-        for line in env_lines:
-            line = line.rstrip()
-            is_updated = False
-            for key, value in onedrive_settings.items():
-                if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
-                    if line.startswith("# "):  # Uncomment if commented out
-                        line = line[2:]
-                    new_env_lines.append(f"{key}={value}")
-                    updated.add(key)
-                    is_updated = True
-                    break
-            if not is_updated:
-                new_env_lines.append(line)
-                
-        # Add any settings that weren't updated (they weren't in the file)
-        for key, value in onedrive_settings.items():
-            if key not in updated:
-                new_env_lines.append(f"{key}={value}")
-                
-        # Write the updated .env file
-        with open(env_path, "w") as f:
-            f.write("\n".join(new_env_lines) + "\n")
-            
-        # Update the settings in memory
-        if refresh_token:
-            settings.onedrive_refresh_token = refresh_token
-        if client_id:
-            settings.onedrive_client_id = client_id
-        if client_secret:
-            settings.onedrive_client_secret = client_secret
-        if tenant_id:
-            settings.onedrive_tenant_id = tenant_id
-        if folder_path:
-            settings.onedrive_folder_path = folder_path
-            
-        logger.info("Successfully updated OneDrive settings")
-        
-        return {
-            "status": "success",
-            "message": "OneDrive settings have been saved"
-        }
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.exception(f"Unexpected error saving OneDrive settings: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to save OneDrive settings: {str(e)}"
-        )
-
-@router.post("/onedrive/update-settings")
-@require_login
-async def update_onedrive_settings(
-    request: Request,
-    client_id: str = Form(None),
-    client_secret: str = Form(None),
-    refresh_token: str = Form(...),
-    tenant_id: str = Form("common"),
-    folder_path: str = Form(None)
-):
-    """
-    Update OneDrive settings in memory (without modifying .env file)
-    """
-    try:
-        logger.info("Updating OneDrive settings in memory")
-        
-        # Update settings in memory
-        if refresh_token:
-            settings.onedrive_refresh_token = refresh_token
-            logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory")
-            
-        if client_id:
-            settings.onedrive_client_id = client_id
-            logger.info("Updated ONEDRIVE_CLIENT_ID in memory")
-            
-        if client_secret:
-            settings.onedrive_client_secret = client_secret
-            logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory")
-            
-        if tenant_id:
-            settings.onedrive_tenant_id = tenant_id
-            logger.info("Updated ONEDRIVE_TENANT_ID in memory")
-            
-        if folder_path:
-            settings.onedrive_folder_path = folder_path
-            logger.info("Updated ONEDRIVE_FOLDER_PATH in memory")
-        
-        # Test the token to make sure it works
-        try:
-            from app.tasks.upload_to_onedrive import get_onedrive_token
-            access_token = get_onedrive_token()
-            logger.info("Successfully tested OneDrive token")
-        except Exception as e:
-            logger.error(f"Token test failed after updating settings: {str(e)}")
-            return {
-                "status": "warning",
-                "message": "Settings updated but token test failed: " + str(e)
-            }
-            
-        return {
-            "status": "success",
-            "message": "OneDrive settings have been updated in memory"
-        }
-        
-    except Exception as e:
-        logger.exception(f"Unexpected error updating OneDrive settings: {str(e)}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to update OneDrive settings: {str(e)}"
-        )
-
-@router.get("/onedrive/get-full-config")
-@require_login
-async def get_onedrive_full_config(request: Request):
-    """
-    Get the full OneDrive configuration for sharing with worker nodes
-    """
-    try:
-        # Create a configuration object with all OneDrive settings
-        config = {
-            "client_id": settings.onedrive_client_id or "",
-            "client_secret": settings.onedrive_client_secret or "",
-            "tenant_id": settings.onedrive_tenant_id or "common",
-            "refresh_token": settings.onedrive_refresh_token or "",
-            "folder_path": settings.onedrive_folder_path or "Documents/Uploads"
-        }
-        
-        # Generate environment variable format
-        env_format = "\n".join([
-            f"ONEDRIVE_CLIENT_ID={config['client_id']}",
-            f"ONEDRIVE_CLIENT_SECRET={config['client_secret']}",
-            f"ONEDRIVE_TENANT_ID={config['tenant_id']}",
-            f"ONEDRIVE_REFRESH_TOKEN={config['refresh_token']}",
-            f"ONEDRIVE_FOLDER_PATH={config['folder_path']}"
-        ])
-        
-        return {
-            "status": "success",
-            "config": config,
-            "env_format": env_format
-        }
-    except Exception as e:
-        logger.exception("Error getting OneDrive configuration")
-        return {
-            "status": "error",
-            "message": str(e)
-        }

app/api/openai.py

@@ -1,75 +0,0 @@
-"""
-OpenAI API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException, status
-import logging
-import os
-import requests
-
-from app.auth import require_login
-from app.config import settings
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.get("/openai/test")
-@require_login
-async def test_openai_connection(request: Request):
-    """
-    Test if the configured OpenAI API key is valid.
-    """
-    try:
-        import openai
-        
-        logger.info("Testing OpenAI API key validity")
-        
-        # Check if API key is configured
-        if not settings.openai_api_key:
-            logger.warning("No OpenAI API key configured")
-            return {
-                "status": "error",
-                "message": "No OpenAI API key is configured"
-            }
-        
-        # Configure the client
-        client = openai.OpenAI(api_key=settings.openai_api_key)
-        
-        # Try to make a simple request to validate the key
-        try:
-            # Use a models list endpoint as a simple validation
-            models = client.models.list()
-            
-            # If we got here, the key is valid
-            logger.info("OpenAI API key is valid")
-            return {
-                "status": "success",
-                "message": "OpenAI API key is valid",
-                "models_available": len(models.data) if hasattr(models, "data") else "Unknown"
-            }
-        except Exception as e:
-            error_msg = str(e)
-            logger.error(f"OpenAI API key test failed: {error_msg}")
-            
-            # Determine if this is an authentication error
-            is_auth_error = "auth" in error_msg.lower() or "api key" in error_msg.lower()
-            
-            return {
-                "status": "error",
-                "message": f"API key validation failed: {error_msg}",
-                "is_auth_error": is_auth_error
-            }
-    
-    except ImportError:
-        logger.exception("OpenAI package not installed")
-        return {
-            "status": "error",
-            "message": "OpenAI package not installed"
-        }
-    except Exception as e:
-        logger.exception("Unexpected error testing OpenAI connection")
-        return {
-            "status": "error",
-            "message": f"Unexpected error: {str(e)}"
-        }

app/api/process.py

@@ -1,140 +0,0 @@
-"""
-Document processing API endpoints
-"""
-from fastapi import APIRouter, HTTPException
-import logging
-import os
-
-from app.auth import require_login
-from app.config import settings
-from app.api.common import resolve_file_path
-from app.tasks.process_document import process_document
-from app.tasks.upload_to_dropbox import upload_to_dropbox
-from app.tasks.upload_to_paperless import upload_to_paperless
-from app.tasks.upload_to_nextcloud import upload_to_nextcloud
-from app.tasks.upload_to_google_drive import upload_to_google_drive
-from app.tasks.upload_to_onedrive import upload_to_onedrive
-from app.tasks.send_to_all import send_to_all_destinations
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-@router.post("/process/")
-@require_login
-def process(file_path: str):
-    """API Endpoint to start document processing."""
-    file_path = resolve_file_path(file_path)
-
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-
-    task = process_document.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@router.post("/send_to_dropbox/")
-@require_login
-def send_to_dropbox_endpoint(file_path: str):
-    """Send a document to Dropbox."""
-    file_path = resolve_file_path(file_path, 'processed')
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_dropbox.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@router.post("/send_to_paperless/")
-@require_login
-def send_to_paperless_endpoint(file_path: str):
-    """Send a document to Paperless-ngx."""
-    file_path = resolve_file_path(file_path, 'processed')
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_paperless.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@router.post("/send_to_nextcloud/")
-@require_login
-def send_to_nextcloud_endpoint(file_path: str):
-    """Send a document to NextCloud."""
-    file_path = resolve_file_path(file_path, 'processed')
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_nextcloud.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@router.post("/send_to_google_drive/")
-@require_login
-def send_to_google_drive_endpoint(file_path: str):
-    """Send a document to Google Drive."""
-    file_path = resolve_file_path(file_path, 'processed')
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_google_drive.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@router.post("/send_to_onedrive/")
-@require_login
-def send_to_onedrive_endpoint(file_path: str):
-    """Send a document to OneDrive."""
-    file_path = resolve_file_path(file_path, 'processed')
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_onedrive.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@router.post("/send_to_all_destinations/")
-@require_login
-def send_to_all_destinations_endpoint(file_path: str):
-    """Call the aggregator task that sends this file to all configured destinations."""
-    file_path = resolve_file_path(file_path, 'processed')
-
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-
-    task = send_to_all_destinations.delay(file_path)
-    return {"task_id": task.id, "status": "queued", "file_path": file_path}
-
-@router.post("/processall")
-@require_login
-def process_all_pdfs_in_workdir():
-    """Finds all .pdf files in <workdir> and enqueues them for processing."""
-    target_dir = settings.workdir
-    if not os.path.exists(target_dir):
-        raise HTTPException(
-            status_code=400, detail=f"Directory {target_dir} does not exist."
-        )
-
-    pdf_files = []
-    for filename in os.listdir(target_dir):
-        if filename.lower().endswith(".pdf"):
-            pdf_files.append(filename)
-
-    if not pdf_files:
-        return {"message": "No PDF files found in that directory."}
-
-    task_ids = []
-    for pdf in pdf_files:
-        file_path = os.path.join(target_dir, pdf)
-        task = process_document.delay(file_path)
-        task_ids.append(task.id)
-
-    return {
-        "message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
-        "pdf_files": pdf_files,
-        "task_ids": task_ids
-    }

app/api/user.py

@@ -1,42 +0,0 @@
-"""
-User-related API endpoints
-"""
-from fastapi import APIRouter, Request, HTTPException
-from hashlib import md5
-import logging
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-router = APIRouter()
-
-async def whoami_handler(request: Request):
-    """
-    Returns user info if logged in, else 401.
-    """
-    user = request.session.get("user")
-    if not user:
-        raise HTTPException(status_code=401, detail="Not logged in")
-
-    email = user.get("email")
-    if not email:
-        raise HTTPException(status_code=400, detail="User has no email in session")
-
-    # Generate Gravatar URL from email
-    email_hash = md5(email.strip().lower().encode()).hexdigest()
-    gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
-    
-    # Add the gravatar URL to the user object instead of creating a new response
-    user_response = user.copy()  # Create a copy to avoid modifying the session
-    user_response["picture"] = gravatar_url
-    
-    return user_response
-
-# Register the same handler under two different paths
-@router.get("/whoami")
-async def whoami(request: Request):
-    return await whoami_handler(request)
-
-@router.get("/auth/whoami")
-async def auth_whoami(request: Request):
-    return await whoami_handler(request)

app/auth.py

@@ -1,13 +1,10 @@
 import os
 import inspect
-import hashlib
 from functools import wraps
 
 from authlib.integrations.starlette_client import OAuth
 from fastapi import APIRouter, Request, status
 from starlette.responses import RedirectResponse
-from fastapi.templating import Jinja2Templates
-import pathlib
 
 from app.config import settings
 
@@ -15,15 +12,7 @@ oauth = OAuth()
 
 AUTH_ENABLED = settings.auth_enabled
 
-# Set up templates for authentication
-templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates"
-templates = Jinja2Templates(directory=str(templates_dir))
-
-# Configure OAuth provider if credentials are provided
-OAUTH_CONFIGURED = False
-OAUTH_PROVIDER_NAME = "Single Sign-On"
-
-if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_secret:
+if AUTH_ENABLED:
     oauth.register(
         name="authentik",
         client_id=settings.authentik_client_id,
@@ -31,8 +20,6 @@ if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_s
         server_metadata_url=settings.authentik_config_url,
         client_kwargs={"scope": "openid profile email"},
     )
-    OAUTH_CONFIGURED = True
-    OAUTH_PROVIDER_NAME = settings.oauth_provider_name or "Authentik SSO"
 
 router = APIRouter()
 
@@ -59,123 +46,29 @@ def require_login(func):
     return wrapper
 
 
-def get_gravatar_url(email):
-    """Generate a Gravatar URL for the given email"""
-    email = email.lower().strip()
-    email_hash = hashlib.md5(email.encode('utf-8')).hexdigest()
-    return f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
-
-
 if AUTH_ENABLED:
     @router.get("/login")
     async def login(request: Request):
-        """Show login page with appropriate authentication options"""
-        return templates.TemplateResponse(
-            "login.html", 
-            {
-                "request": request, 
-                "error": request.query_params.get("error"),
-                "message": request.query_params.get("message"),
-                "show_oauth": OAUTH_CONFIGURED,
-                "oauth_provider_name": OAUTH_PROVIDER_NAME,
-                "app_version": settings.version  # Changed from app_version to version
-            }
-        )
-
-    @router.get("/oauth-login")
-    async def oauth_login(request: Request):
-        """Handle OAuth login flow"""
-        if not OAUTH_CONFIGURED:
-            return RedirectResponse(
-                url="/login?error=OAuth+not+configured",
-                status_code=status.HTTP_302_FOUND
-            )
-        
-        redirect_uri = request.url_for("oauth_callback")
+        redirect_uri = request.url_for("auth")
         return await oauth.authentik.authorize_redirect(request, redirect_uri)
 
-    @router.get("/oauth-callback")
-    async def oauth_callback(request: Request):
-        """Handle OAuth callback from provider"""
-        try:
-            token = await oauth.authentik.authorize_access_token(request)
-            userinfo = token.get("userinfo")
-            if not userinfo:
-                return RedirectResponse(
-                    url="/login?error=Failed+to+retrieve+user+information",
-                    status_code=status.HTTP_302_FOUND
-                )
-                
-            # Store user info in session
-            user_data = dict(userinfo)
-            
-            # Add Gravatar picture if no picture is provided
-            if not user_data.get("picture") and user_data.get("email"):
-                user_data["picture"] = get_gravatar_url(user_data["email"])
-            
-            request.session["user"] = user_data
-            
-            # Log the successful authentication
-            print(f"User authenticated via OAuth: {user_data.get('email', 'No email')}")
-            
-            # Redirect to original destination or default
-            redirect_url = request.session.pop("redirect_after_login", "/upload")
-            return RedirectResponse(url=redirect_url)
-        except Exception as e:
-            print(f"OAuth authentication error: {str(e)}")
-            return RedirectResponse(
-                url=f"/login?error=Authentication+failed:+{str(e)}",
-                status_code=status.HTTP_302_FOUND
-            )
-
-    @router.post("/auth")
+    @router.get("/auth")
     async def auth(request: Request):
-        """Handle local username/password authentication"""
-        form_data = await request.form()
-        username = form_data.get("username")
-        password = form_data.get("password")
-        
-        if (username == settings.admin_username and 
-            password == settings.admin_password):
-            # Create user session
-            request.session["user"] = {
-                "id": "admin",
-                "name": "Administrator",
-                "email": f"{username}@local.docuelevate",
-                "preferred_username": username,
-                "picture": "/static/images/default-avatar.svg",
-                "is_admin": True
-            }
-            # Redirect to original destination or default
-            redirect_url = request.session.pop("redirect_after_login", "/upload")
-            return RedirectResponse(url=redirect_url, status_code=302)
-        else:
-            return RedirectResponse(
-                url="/login?error=Invalid+username+or+password", 
-                status_code=302
-            )
+        token = await oauth.authentik.authorize_access_token(request)
+        userinfo = token.get("userinfo")
+        request.session["user"] = dict(userinfo)
+        redirect_url = request.session.pop("redirect_after_login", "/upload")
+        return RedirectResponse(url=redirect_url)
 
     @router.get("/logout")
     async def logout(request: Request):
-        """Handle user logout"""
         request.session.pop("user", None)
-        return RedirectResponse(
-            url="/login?message=You+have+been+logged+out+successfully", 
-            status_code=302
-        )
-
-
-@router.get("/api/auth/whoami")
-@require_login
-async def whoami(request: Request):
-    """API endpoint to get current user information"""
-    user = request.session.get("user")
-    return user or {"error": "Not authenticated"}
+        return RedirectResponse(url="/")
 
 
 @router.get("/private")
 @require_login
 async def private_page(request: Request):
     """A protected endpoint that requires login."""
-    user = request.session.get("user")
-    return {"message": "This is a protected page.", "user": user}
+    user = request.session.get("user")  # e.g. {"email": "...", ...}
+    return {"message": f"This is a protected page. Hello {user['email']}!"}

app/celery_app.py

@@ -18,25 +18,3 @@ celery.conf.task_default_queue = 'document_processor'
 celery.conf.task_routes = {
     "app.tasks.*": {"queue": "document_processor"},
 }
-
-# Task failure notification handler
-from celery.signals import task_failure
-
-@task_failure.connect
-def task_failure_handler(sender=None, task_id=None, exception=None, args=None,
-                        kwargs=None, traceback=None, einfo=None, **kw):
-    """Handler for Celery task failures to send notifications"""
-    if getattr(settings, 'notify_on_task_failure', True):
-        try:
-            # Import here to avoid circular imports
-            from app.utils.notification import notify_celery_failure
-            notify_celery_failure(
-                task_name=sender.name if sender else "Unknown", 
-                task_id=task_id or "N/A",
-                exc=exception, 
-                args=args or [], 
-                kwargs=kwargs or {}
-            )
-        except Exception as e:
-            import logging
-            logging.exception(f"Failed to send task failure notification: {e}")

app/celery_worker.py

@@ -9,9 +9,8 @@ from app.celery_app import celery
 from app import tasks  # <— This imports app/tasks.py so Celery can register tasks
 
 # **Ensure all tasks are imported before Celery starts**
-from app.tasks.process_document import process_document
-from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
-from app.tasks.rotate_pdf_pages import rotate_pdf_pages
+from app.tasks.process_document import process_document  # Updated import
+from app.tasks.process_with_textract import process_with_textract
 from app.tasks.refine_text_with_gpt import refine_text_with_gpt
 from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
 from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
@@ -21,18 +20,8 @@ from app.tasks.convert_to_pdf import convert_to_pdf
 from app.tasks.upload_to_dropbox import upload_to_dropbox
 from app.tasks.upload_to_paperless import upload_to_paperless
 from app.tasks.upload_to_nextcloud import upload_to_nextcloud
-from app.tasks.upload_to_google_drive import upload_to_google_drive
-from app.tasks.upload_to_webdav import upload_to_webdav
-from app.tasks.upload_to_s3 import upload_to_s3
-from app.tasks.upload_to_onedrive import upload_to_onedrive
-from app.tasks.upload_to_ftp import upload_to_ftp
-from app.tasks.upload_to_sftp import upload_to_sftp 
-from app.tasks.upload_to_email import upload_to_email
-
 from app.tasks.imap_tasks import pull_all_inboxes
 from app.tasks.send_to_all import send_to_all_destinations
-from app.tasks.uptime_kuma_tasks import ping_uptime_kuma
-from app.tasks.check_credentials import check_credentials
 
 celery.conf.task_routes = {
     "app.tasks.*": {"queue": "default"},
@@ -45,34 +34,9 @@ def test_task():
 # If you want Celery Beat to run the poll task every minute, add:
 from celery.schedules import crontab
 
-# Run the check_credentials task at startup
-check_credentials.apply_async(countdown=10)  # Run 10 seconds after worker starts
-
 celery.conf.beat_schedule = {
     "poll-inboxes-every-minute": {
         "task": "app.tasks.imap_tasks.pull_all_inboxes",
         "schedule": crontab(minute="*/1"),  # every 1 minute
-        "options": {"expires": 55},  # Ensure tasks don't pile up
-    } if (settings.imap1_host or settings.imap2_host) else None,
-    # Add Uptime Kuma ping task if configured
-    "ping-uptime-kuma": {
-        "task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
-        "schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
-        "options": {"expires": 55},  # Ensure tasks don't pile up
-    } if settings.uptime_kuma_url else None,
-    # Check credentials every 5 minutes
-    "check-credentials-regularly": {
-        "task": "app.tasks.check_credentials.check_credentials",
-        "schedule": crontab(minute="*/5"),  # Every 5 minutes
-        "options": {"expires": 240},  # 4 minutes expiry
     },
-    # Also keep daily check for logs and statistics purposes
-    "check-credentials-daily": {
-        "task": "app.tasks.check_credentials.check_credentials",
-        "schedule": crontab(hour="0", minute="0"),  # Midnight
-        "options": {"expires": 3600},  # 1 hour expiry
-    }
-}
-
-# Remove None entries from beat_schedule
-celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None}
+}

app/config.py

@@ -1,53 +1,41 @@
 #!/usr/bin/env python3
 
 from pydantic_settings import BaseSettings
-from typing import Optional, List, Dict, Any, Union
-from pydantic import Field, validator
-import os
-from datetime import datetime
+from typing import Optional
 
 class Settings(BaseSettings):
+    admin_username: str
+    admin_password: str
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    aws_region: str
     database_url: str
     redis_url: str
+    s3_bucket_name: str
     openai_api_key: str
     openai_base_url: str = "https://api.openai.com/v1"  # Default to OpenAI's endpoint
     openai_model: str = "gpt-4o-mini"  # Default model
     workdir: str
-    debug: bool = False  # Default to False
-    
-    # Making Dropbox optional
-    dropbox_app_key: Optional[str] = None
-    dropbox_app_secret: Optional[str] = None
-    dropbox_folder: Optional[str] = None
-    dropbox_refresh_token: Optional[str] = None
-    
-    # Making Nextcloud optional
-    nextcloud_upload_url: Optional[str] = None
-    nextcloud_username: Optional[str] = None
-    nextcloud_password: Optional[str] = None
-    nextcloud_folder: Optional[str] = None
-    
-    # Making Paperless optional
-    paperless_ngx_api_token: Optional[str] = None
-    paperless_host: Optional[str] = None
-    
+    dropbox_app_key: str
+    dropbox_app_secret: str
+    dropbox_folder: str
+    dropbox_refresh_token: str
+    nextcloud_upload_url: str
+    nextcloud_username: str
+    nextcloud_password: str
+    nextcloud_folder: str
+    paperless_ngx_api_token: str
+    paperless_host: str
     azure_ai_key: str
     azure_region: str
     azure_endpoint: str
     gotenberg_url: str
-    external_hostname: str = "localhost"  # Default to localhost
-
-    # Authentication settings
-    auth_enabled: bool = True  # Default to enabled
-    admin_username: Optional[str] = None
-    admin_password: Optional[str] = None
-    session_secret: Optional[str] = None
-
+    
     # Authentik
     authentik_client_id: Optional[str] = None
     authentik_client_secret: Optional[str] = None
     authentik_config_url: Optional[str] = None
-    oauth_provider_name: Optional[str] = None  # Name to display for the OAuth provider
+    auth_enabled: bool = True  # Default to enabled
 
     # IMAP 1
     imap1_host: Optional[str] = None
@@ -67,169 +55,7 @@ class Settings(BaseSettings):
     imap2_poll_interval_minutes: int = 10
     imap2_delete_after_process: bool = False
 
-    # Google Drive settings
-    google_drive_credentials_json: Optional[str] = ""
-    google_drive_folder_id: Optional[str] = ""
-    google_drive_delegate_to: Optional[str] = ""  # Optional delegated user email
-    
-    # Google Drive OAuth settings
-    google_drive_use_oauth: bool = False  # Default to service account method
-    google_drive_client_id: Optional[str] = ""
-    google_drive_client_secret: Optional[str] = ""
-    google_drive_refresh_token: Optional[str] = ""
-
-    # WebDAV settings
-    webdav_url: Optional[str] = None
-    webdav_username: Optional[str] = None
-    webdav_password: Optional[str] = None
-    webdav_folder: Optional[str] = None
-    webdav_verify_ssl: bool = True
-
-    # FTP settings
-    ftp_host: Optional[str] = None
-    ftp_port: Optional[int] = 21
-    ftp_username: Optional[str] = None
-    ftp_password: Optional[str] = None
-    ftp_folder: Optional[str] = None
-    ftp_use_tls: bool = True  # Default to attempting TLS connection first
-    ftp_allow_plaintext: bool = True  # Default to allowing plaintext fallback
-
-    # SFTP settings
-    sftp_host: Optional[str] = None
-    sftp_port: Optional[int] = 22
-    sftp_username: Optional[str] = None
-    sftp_password: Optional[str] = None
-    sftp_folder: Optional[str] = None
-    sftp_private_key: Optional[str] = None
-    sftp_private_key_passphrase: Optional[str] = None
-
-    # Email settings
-    email_host: Optional[str] = None
-    email_port: Optional[int] = 587
-    email_username: Optional[str] = None
-    email_password: Optional[str] = None
-    email_use_tls: bool = True
-    email_sender: Optional[str] = None  # From address, defaults to email_username if not set
-    email_default_recipient: Optional[str] = None
-
-    # OneDrive settings
-    onedrive_client_id: Optional[str] = None
-    onedrive_client_secret: Optional[str] = None
-    onedrive_tenant_id: Optional[str] = "common"  # Default to "common" for personal accounts
-    onedrive_refresh_token: Optional[str] = None  # Required for personal accounts
-    onedrive_folder_path: Optional[str] = None
-
-    # AWS S3 settings
-    aws_access_key_id: Optional[str] = None
-    aws_secret_access_key: Optional[str] = None
-    aws_region: Optional[str] = "us-east-1"  # Default region
-    s3_bucket_name: Optional[str] = None
-    s3_folder_prefix: Optional[str] = ""  # Optional folder prefix (e.g. "uploads/")
-    s3_storage_class: Optional[str] = "STANDARD"  # Default storage class
-    s3_acl: Optional[str] = "private"  # Default ACL
-
-    # Uptime Kuma settings
-    uptime_kuma_url: Optional[str] = None
-    uptime_kuma_ping_interval: int = 5  # Default ping interval in minutes
-
-    # Feature flags
-    allow_file_delete: bool = True  # Default to allowing file deletion from database
-
-    # Notification settings
-    notification_urls: Union[List[str], str] = Field(
-        default_factory=list,
-        description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)"
-    )
-    notify_on_task_failure: bool = Field(
-        default=True, 
-        description="Send notifications when Celery tasks fail"
-    )
-    notify_on_credential_failure: bool = Field(
-        default=True,
-        description="Send notifications when credential checks fail"
-    )
-    notify_on_startup: bool = Field(
-        default=True,
-        description="Send notifications when application starts"
-    )
-    notify_on_shutdown: bool = Field(
-        default=False,
-        description="Send notifications when application shuts down"
-    )
-    
-    @validator('notification_urls', pre=True)
-    def parse_notification_urls(cls, v):
-        """Parse notification URLs from string or list"""
-        if isinstance(v, str):
-            if ',' in v:
-                return [url.strip() for url in v.split(',') if url.strip()]
-            elif v.strip():
-                return [v.strip()]
-            return []
-        return v
-
-    @validator('session_secret')
-    def validate_session_secret(cls, v, values):
-        """Validate that session_secret is set and has sufficient length when auth is enabled"""
-        if values.get('auth_enabled') and not v:
-            raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True")
-        if values.get('auth_enabled') and v and len(v) < 32:
-            raise ValueError("SESSION_SECRET must be at least 32 characters long")
-        return v
-
-    # Get build date from environment or file
-    @property
-    def build_date(self) -> str:
-        # First try to get build date from environment
-        env_build_date = os.environ.get("BUILD_DATE")
-        if env_build_date:
-            return env_build_date
-        
-        # Then try to get build date from BUILD_DATE file
-        build_date_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE")
-        if os.path.exists(build_date_file):
-            with open(build_date_file, "r") as f:
-                return f.read().strip()
-        
-        # Default to unknown if not found
-        return "Unknown build date"
-
-    # Get version from file or environment
-    @property
-    def version(self) -> str:
-        # First try to get version from environment
-        env_version = os.environ.get("APP_VERSION")
-        if env_version:
-            return env_version
-        
-        # Then try to get version from VERSION file
-        version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION")
-        if os.path.exists(version_file):
-            with open(version_file, "r") as f:
-                return f.read().strip()
-                
-        # Default version if not found
-        return "0.3.2-dev"
-
     class Config:
         env_file = ".env"
-        # Convert string representations of booleans to actual booleans
-        # and strip quotes from string values
-        @classmethod
-        def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
-            # First, strip quotes from the value if it's a string
-            if isinstance(raw_val, str):
-                if (raw_val.startswith('"') and raw_val.endswith('"')) or \
-                   (raw_val.startswith("'") and raw_val.endswith("'")):
-                    raw_val = raw_val[1:-1]
-                raw_val = raw_val.strip()
-                
-            # Convert string representations of booleans to actual booleans
-            if field_name.endswith('_enabled') or field_name == 'debug':
-                if raw_val.lower() in ('false', '0', 'no', 'n', 'f'):
-                    return False
-                if raw_val.lower() in ('true', '1', 'yes', 'y', 't'):
-                    return True
-            return raw_val
 
 settings = Settings()

app/db_migration.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+import sqlite3
+import os
+import logging
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+def run_migrations():
+    """
+    Run database migrations to add missing columns or make other schema changes.
+    """
+    logger.info("Running database migrations...")
+    
+    # Parse the DATABASE_URL to get the SQLite database path
+    db_url = settings.database_url
+    if not db_url.startswith("sqlite:///"):
+        logger.warning(f"Non-SQLite database detected: {db_url}. Migrations may need to be adapted.")
+        return
+    
+    # Extract the database path from the URL
+    db_path = db_url.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        logger.error(f"Database file not found at {db_path}")
+        return
+    
+    # Connect to the database
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        
+        # Check if the processing_logs table exists
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='processing_logs';")
+        if not cursor.fetchone():
+            logger.info("Creating processing_logs table...")
+            cursor.execute("""
+                CREATE TABLE processing_logs (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    file_id INTEGER,
+                    step_name VARCHAR,
+                    status VARCHAR,
+                    message TEXT,
+                    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (file_id) REFERENCES files (id)
+                );
+            """)
+            conn.commit()
+        
+        # Check if the task_id column exists in processing_logs
+        cursor.execute("PRAGMA table_info(processing_logs);")
+        columns = [row[1] for row in cursor.fetchall()]
+        
+        if 'task_id' not in columns:
+            logger.info("Adding task_id column to processing_logs table...")
+            cursor.execute("ALTER TABLE processing_logs ADD COLUMN task_id VARCHAR;")
+            conn.commit()
+            logger.info("Created task_id column in processing_logs")
+            
+            # Create an index on task_id for faster lookups
+            cursor.execute("CREATE INDEX idx_processing_logs_task_id ON processing_logs (task_id);")
+            conn.commit()
+            logger.info("Created index on task_id column")
+        
+        logger.info("Database migrations completed successfully.")
+    except Exception as e:
+        logger.error(f"Error during database migration: {e}")
+        if conn:
+            conn.rollback()
+    finally:
+        if conn:
+            conn.close()
+
+if __name__ == "__main__":
+    # Configure logging
+    logging.basicConfig(level=logging.INFO)
+    run_migrations()

app/frontend.py

@@ -1,10 +1,52 @@
-"""
-Frontend routes for the application.
-This module is now a re-export of the modularized view routers.
-"""
-# Import and re-export the router from the views package
-from app.views import router
+# app/frontend.py
+import os
+from pathlib import Path
+from fastapi import APIRouter, Request, Depends
+from fastapi.templating import Jinja2Templates
+from sqlalchemy.orm import Session
 
-# Keep the original router name for compatibility
-# This allows existing imports in main.py to continue working
+from app.auth import require_login
+from app.database import SessionLocal
 
+router = APIRouter()
+
+templates_dir = Path(__file__).parent.parent / "frontend" / "templates"
+templates = Jinja2Templates(directory=str(templates_dir))
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+@router.get("/files")
+@require_login
+def files_page(request: Request):
+    """
+    Return the 'files.html' template. 
+    The actual file data is fetched via XHR from /api/files in the template.
+    """
+    return templates.TemplateResponse("files.html", {"request": request})
+
+# ... existing routes for /, /upload, /about, etc. ...
+
+
+@router.get("/", include_in_schema=False)
+async def serve_index(request: Request):
+     return templates.TemplateResponse("index.html", {"request": request})
+ 
+@router.get("/about", include_in_schema=False)
+async def serve_about(request: Request):
+     return templates.TemplateResponse("about.html", {"request": request})
+ 
+@router.get("/upload", include_in_schema=False)
+@require_login
+async def serve_upload(request: Request):
+     return templates.TemplateResponse("upload.html", {"request": request})
+  
+@router.get("/favicon.ico", include_in_schema=False)
+def favicon():
+    # If you have a real favicon in `frontend/static/favicon.ico`:
+    favicon_path = Path(__file__).parent.parent / "frontend" / "static" / "favicon.ico"
+    return str(favicon_path)

app/main.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 import os
-import logging
-import pathlib
 
-from fastapi import FastAPI, HTTPException, Request, status
+from fastapi import FastAPI, HTTPException, UploadFile, File, status, Request
+from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from starlette.middleware.sessions import SessionMiddleware
@@ -13,27 +12,26 @@ from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
 from pathlib import Path
 
 from app.database import init_db
+from app.db_migration import run_migrations
 from app.config import settings
-from app.utils.config_validator import check_all_configs
-from app.utils.notification import init_apprise, send_notification, notify_startup, notify_shutdown
+from app.tasks.process_document import process_document  # Updated import
+from app.tasks.upload_to_dropbox import upload_to_dropbox
+from app.tasks.upload_to_paperless import upload_to_paperless
+from app.tasks.upload_to_nextcloud import upload_to_nextcloud
+from app.tasks.send_to_all import send_to_all_destinations
 
-# Import the routers - now using views directly instead of frontend
-from app.views import router as frontend_router
 from app.api import router as api_router
+from app.frontend import router as frontend_router
 from app.auth import router as auth_router
 
-# Explicitly include the files router
-from app.views.files import router as files_router
-
 # Load configuration from .env for the session key
 config = Config(".env")
-# Use settings.session_secret which has proper validation
-# Fallback to raising an error if not set when auth is enabled
-if settings.auth_enabled and not settings.session_secret:
-    raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True. Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'")
-SESSION_SECRET = settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
+SESSION_SECRET = config(
+    "SESSION_SECRET",
+    default="YOUR_DEFAULT_SESSION_SECRET_MUST_BE_32_CHARS_OR_MORE"
+)
 
-app = FastAPI(title="DocuElevate")
+app = FastAPI(title="Document Processing API")
 
 # 1) Session Middleware (for request.session to work)
 app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET)
@@ -43,60 +41,141 @@ app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*")
 
 # 3) (Optional but recommended) Restrict valid hosts:
 app.add_middleware(TrustedHostMiddleware, allowed_hosts=[
-    settings.external_hostname,
+    "docparse.hosterra.net",
     "localhost",
     "127.0.0.1"
 ])
 
-# Mount the static files directory
-static_dir = pathlib.Path(__file__).parents[1] / "frontend" / "static"
-if os.path.exists(static_dir):
-    app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
-else:
-    print(f"WARNING: Static directory not found at {static_dir}. Static files will not be served.")
+# Mount the static folder for CSS/JS:
+frontend_static_dir = Path(__file__).parent.parent / "frontend" / "static"
+app.mount("/static", StaticFiles(directory=frontend_static_dir), name="static")
 
 @app.on_event("startup")
 def on_startup():
     init_db()  # Create tables if they don't exist
+    run_migrations()  # Run migrations to add any missing columns
 
-@app.on_event("startup")
-async def startup_event():
-    """Run startup tasks for the application"""
-    # Force settings dump to log for troubleshooting
-    from app.utils.config_validator import dump_all_settings
-    dump_all_settings()
-    
-    # Validate configuration
-    config_issues = check_all_configs()
-    
-    # Log overall status
-    has_issues = any(config_issues['email']) or any(len(issues) > 0 for provider, issues in config_issues['storage'].items())
-    if has_issues:
-        logging.warning("Application started with configuration issues - some features may be unavailable")
-    else:
-        logging.info("Application started with valid configuration")
-    
-    logging.info("Router organization: Using refactored API routers from app/api/ directory")
-    
-    # Initialize notification system
-    init_apprise()
-    
-    # Send startup notification
-    notify_startup()
+@app.post("/process/")
+def process(file_path: str):
+    """
+    API Endpoint to start document processing.
+    This enqueues document processing which handles the full pipeline.
+    """
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, file_path)
 
-@app.on_event("shutdown")
-async def shutdown_event():
-    """Run shutdown tasks for the application"""
-    logging.info("Application shutting down")
-    
-    # Send shutdown notification
-    notify_shutdown()
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+
+    task = process_document.delay(file_path)  # Updated function call
+    return {"task_id": task.id, "status": "queued"}
+
+@app.post("/send_to_dropbox/")
+def send_to_dropbox(file_path: str):
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_dropbox.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@app.post("/send_to_paperless/")
+def send_to_paperless(file_path: str):
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_paperless.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@app.post("/send_to_nextcloud/")
+def send_to_nextcloud(file_path: str):
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_nextcloud.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@app.post("/send_to_all_destinations/")
+def send_to_all_destinations_endpoint(file_path: str):
+    """
+    Call the aggregator task that sends this file to dropbox, nextcloud, and paperless.
+    """
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+
+    task = send_to_all_destinations.delay(file_path)
+    return {"task_id": task.id, "status": "queued", "file_path": file_path}
+
+@app.post("/processall")
+def process_all_pdfs_in_workdir():
+    """
+    Finds all .pdf files in <workdir> and enqueues them for processing.
+    """
+    target_dir = settings.workdir
+    if not os.path.exists(target_dir):
+        raise HTTPException(
+            status_code=400, detail=f"Directory {target_dir} does not exist."
+        )
+
+    pdf_files = []
+    for filename in os.listdir(target_dir):
+        if filename.lower().endswith(".pdf"):
+            pdf_files.append(filename)
+
+    if not pdf_files:
+        return {"message": "No PDF files found in that directory."}
+
+    task_ids = []
+    for pdf in pdf_files:
+        file_path = os.path.join(target_dir, pdf)
+        task = process_document.delay(file_path)  # Updated function call
+        task_ids.append(task.id)
+
+    return {
+        "message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
+        "pdf_files": pdf_files,
+        "task_ids": task_ids
+    }
+
+@app.post("/ui-upload")
+async def ui_upload(file: UploadFile = File(...)):
+    """Endpoint to accept a user-uploaded file and enqueue it for processing."""
+    workdir = "/workdir"
+    target_path = os.path.join(workdir, file.filename)
+    try:
+        with open(target_path, "wb") as f:
+            content = await file.read()
+            f.write(content)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to save file: {e}"
+        )
+
+    task = process_document.delay(target_path)  # Updated function call
+    return {"task_id": task.id, "status": "queued"}
 
 # Custom 404 - we can still return the Jinja2 template, or the old static file:
+# For a dynamic 404 using the base layout, see "frontend/404.html" usage below:
 @app.exception_handler(404)
 async def custom_404_handler(request: Request, exc: HTTPException):
     # Serve the 404 template directly
-    templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
+    templates = Jinja2Templates(directory=str(frontend_static_dir.parent / "templates"))
     return templates.TemplateResponse(
         "404.html",
         {"request": request},
@@ -105,7 +184,7 @@ async def custom_404_handler(request: Request, exc: HTTPException):
 
 @app.exception_handler(500)
 async def custom_500_handler(request: Request, exc: Exception):
-    templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
+    templates = Jinja2Templates(directory=str(frontend_static_dir.parent / "templates"))
     # Option 1: Keep it simple, just show a funny 500 message:
     return templates.TemplateResponse(
         "500.html",
@@ -117,11 +196,7 @@ async def custom_500_handler(request: Request, exc: Exception):
 def test_500():
     raise RuntimeError("Testing forced 500 error!")
 
-# Include the routers
+# Include the frontend and auth routers
 app.include_router(frontend_router)
-app.include_router(files_router)  # Explicitly include the files router
 app.include_router(auth_router)
 app.include_router(api_router, prefix="/api")
-
-
-

app/routes/license_routes.py

@@ -1,18 +0,0 @@
-from fastapi import APIRouter, HTTPException
-from fastapi.responses import PlainTextResponse, HTMLResponse
-from pathlib import Path
-import os
-
-router = APIRouter()
-
-@router.get("/licenses/lgpl.txt", response_class=PlainTextResponse)
-async def get_lgpl_license():
-    """
-    Serve the LGPL license text file
-    """
-    license_path = Path("frontend/static/licenses/lgpl.txt")
-    if not license_path.exists():
-        raise HTTPException(status_code=404, detail="License file not found")
-    
-    with open(license_path, "r") as f:
-        return f.read()

app/tasks/__init__.py

@@ -1,3 +0,0 @@
-# Import tasks so they can be discovered by Celery
-from app.tasks.process_document import process_document
-from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence

app/tasks/check_credentials.py

@@ -1,267 +0,0 @@
-import logging
-from app.celery_app import celery
-from app.config import settings
-from app.utils.notification import notify_credential_failure
-import time
-import os
-import json
-import asyncio
-import inspect
-
-# Import the test functions from API routes
-from app.api.openai import test_openai_connection
-from app.api.azure import test_azure_connection
-from app.api.dropbox import test_dropbox_token
-from app.api.google_drive import test_google_drive_token
-from app.api.onedrive import test_onedrive_token
-
-# Import config validation utilities
-from app.utils.config_validator import validate_storage_configs, get_provider_status
-
-# Create an enhanced mock Request object for API functions that expect it
-class MockRequest:
-    """Mock request object with session and other attributes needed for API functions"""
-    def __init__(self):
-        self.session = {"user": {"id": "credential_checker", "name": "System Credential Checker"}}
-        self.app = None
-        self.headers = {}
-        self.query_params = {}
-        self.path_params = {}
-
-    async def json(self):
-        return {}
-
-    async def form(self):
-        return {}
-
-logger = logging.getLogger(__name__)
-
-# Path to store failure counts
-FAILURE_STATE_FILE = os.path.join(settings.workdir, 'credential_failures.json')
-
-def get_failure_state():
-    """Read the failure state from file"""
-    try:
-        if os.path.exists(FAILURE_STATE_FILE):
-            with open(FAILURE_STATE_FILE, 'r') as f:
-                return json.load(f)
-    except Exception as e:
-        logger.error(f"Error reading failure state file: {e}")
-    
-    # Default empty state
-    return {}
-
-def save_failure_state(state):
-    """Save failure state to file"""
-    try:
-        with open(FAILURE_STATE_FILE, 'w') as f:
-            json.dump(state, f)
-    except Exception as e:
-        logger.error(f"Error saving failure state file: {e}")
-
-# Helper function to get the inner function without the decorator
-def unwrap_decorated_function(func):
-    """Get the original function from a decorated function"""
-    if hasattr(func, "__wrapped__"):
-        return unwrap_decorated_function(func.__wrapped__)
-    return func
-
-# Create synchronous versions of the test functions that bypass authentication
-def sync_test_openai_connection():
-    """Synchronous wrapper for the OpenAI test function that bypasses auth"""
-    # Get the original function without the @require_login decorator
-    inner_func = unwrap_decorated_function(test_openai_connection)
-    request = MockRequest()
-    if inspect.iscoroutinefunction(inner_func):
-        return asyncio.run(inner_func(request))
-    return inner_func(request)
-
-def sync_test_azure_connection():
-    """Synchronous wrapper for the Azure test function that bypasses auth"""
-    inner_func = unwrap_decorated_function(test_azure_connection)
-    request = MockRequest()
-    if inspect.iscoroutinefunction(inner_func):
-        return asyncio.run(inner_func(request))
-    return inner_func(request)
-
-def sync_test_dropbox_token():
-    """Synchronous wrapper for the Dropbox test function that bypasses auth"""
-    inner_func = unwrap_decorated_function(test_dropbox_token)
-    request = MockRequest()
-    if inspect.iscoroutinefunction(inner_func):
-        return asyncio.run(inner_func(request))
-    return inner_func(request)
-
-def sync_test_google_drive_token():
-    """Synchronous wrapper for the Google Drive test function that bypasses auth"""
-    inner_func = unwrap_decorated_function(test_google_drive_token)
-    request = MockRequest()
-    if inspect.iscoroutinefunction(inner_func):
-        return asyncio.run(inner_func(request))
-    return inner_func(request)
-
-def sync_test_onedrive_token():
-    """Synchronous wrapper for the OneDrive test function that bypasses auth"""
-    inner_func = unwrap_decorated_function(test_onedrive_token)
-    request = MockRequest()
-    if inspect.iscoroutinefunction(inner_func):
-        return asyncio.run(inner_func(request))
-    return inner_func(request)
-
-@celery.task
-def check_credentials():
-    """Check all configured credentials and notify if any are invalid"""
-    logger.info("Starting credential check task")
-    
-    # Load current failure state
-    failure_state = get_failure_state()
-    
-    # Track failures
-    failures = []
-    
-    # Get provider configurations from config_validator
-    provider_status = get_provider_status()
-    storage_configs = validate_storage_configs()
-    
-    # Define services with their test functions and configuration status
-    services = [
-        {
-            "name": "OpenAI", 
-            "check_func": sync_test_openai_connection,
-            "configured": provider_status.get("OpenAI", {}).get("configured", False),
-            "config_issues": []  # OpenAI isn't in storage_configs
-        },
-        {
-            "name": "Azure Document Intelligence", 
-            "check_func": sync_test_azure_connection,
-            "configured": provider_status.get("Azure AI", {}).get("configured", False),
-            "config_issues": []  # Azure isn't in storage_configs
-        },
-        {
-            "name": "Dropbox", 
-            "check_func": sync_test_dropbox_token,
-            "configured": provider_status.get("Dropbox", {}).get("configured", False),
-            "config_issues": storage_configs.get("dropbox", [])
-        },
-        {
-            "name": "Google Drive", 
-            "check_func": sync_test_google_drive_token,
-            "configured": provider_status.get("Google Drive", {}).get("configured", False),
-            "config_issues": storage_configs.get("google_drive", [])
-        },
-        {
-            "name": "OneDrive", 
-            "check_func": sync_test_onedrive_token,
-            "configured": provider_status.get("OneDrive", {}).get("configured", False),
-            "config_issues": storage_configs.get("onedrive", [])
-        }
-    ]
-    
-    # Check each service
-    results = {}
-    current_time = int(time.time())
-    
-    for service in services:
-        service_name = service["name"]
-        logger.info(f"Checking credentials for {service_name}")
-        
-        # Skip services that aren't configured
-        if not service["configured"]:
-            config_issues = service["config_issues"]
-            issue_msg = f"Not properly configured" + (f": {', '.join(config_issues)}" if config_issues else "")
-            logger.info(f"Skipping {service_name}: {issue_msg}")
-            
-            results[service_name] = {
-                "status": "unconfigured",
-                "message": issue_msg
-            }
-            continue
-        
-        try:
-            # Call the synchronized test function and get the result
-            result = service["check_func"]()
-            
-            # All test functions return a dict with "status" field
-            is_valid = result.get("status") == "success"
-            error_message = result.get("message", "Unknown error")
-            
-            # Store the result
-            results[service_name] = {
-                "status": "valid" if is_valid else "invalid",
-                "message": error_message
-            }
-            
-            if not is_valid:
-                failures.append(service_name)
-                
-                # Get current failure count for this service
-                service_state = failure_state.get(service_name, {"count": 0, "last_notified": 0})
-                service_state["count"] = service_state.get("count", 0) + 1
-                
-                # Only notify if we haven't reached the notification threshold (3 failures)
-                # or if this is the first failure after a recovery
-                if service_state["count"] <= 3 or service_state.get("recovered", False):
-                    notify_credential_failure(service_name, error_message)
-                    service_state["last_notified"] = current_time
-                    service_state["recovered"] = False
-                    logger.warning(f"{service_name} credentials check failed ({service_state['count']} times): {error_message}")
-                else:
-                    # We're in cooldown mode
-                    logger.warning(f"{service_name} credentials check failed ({service_state['count']} times): {error_message} - notification suppressed")
-                
-                # Update failure state
-                failure_state[service_name] = service_state
-            else:
-                logger.info(f"{service_name} credentials are valid")
-                
-                # Check if this was previously failing and now recovered
-                if service_name in failure_state and failure_state[service_name].get("count", 0) > 0:
-                    logger.info(f"{service_name} has recovered after {failure_state[service_name]['count']} failures")
-                    
-                    # Mark it as recovered and reset count
-                    failure_state[service_name] = {"count": 0, "recovered": True, "last_notified": 0}
-                elif service_name in failure_state:
-                    # Just make sure recovered flag is cleared if it was there
-                    failure_state[service_name]["recovered"] = True
-                
-        except Exception as e:
-            logger.error(f"Error checking {service_name} credentials: {e}", exc_info=True)
-            failures.append(service_name)
-            error_message = f"Exception during credential check: {str(e)}"
-            
-            # Get current failure count for this service
-            service_state = failure_state.get(service_name, {"count": 0, "last_notified": 0})
-            service_state["count"] = service_state.get("count", 0) + 1
-            
-            # Only notify if we haven't reached the notification threshold or if we just recovered
-            if service_state["count"] <= 3 or service_state.get("recovered", False):
-                notify_credential_failure(service_name, error_message)
-                service_state["last_notified"] = current_time
-                service_state["recovered"] = False
-                
-            # Update failure state
-            failure_state[service_name] = service_state
-            
-            # Store the error result
-            results[service_name] = {
-                "status": "error",
-                "message": error_message
-            }
-    
-    # Save updated failure state
-    save_failure_state(failure_state)
-    
-    # Count only services that were actually checked (configured services)
-    configured_services = [s for s in services if s["configured"]]
-    num_configured = len(configured_services)
-    
-    # Summarize results
-    logger.info(f"Credential check completed. Configured services: {num_configured}, Valid: {num_configured - len(failures)}, Invalid: {len(failures)}")
-    
-    return {
-        "checked": num_configured,
-        "unconfigured": len(services) - num_configured,
-        "failures": len(failures),
-        "results": results,
-        "failure_state": failure_state
-    }

app/tasks/convert_to_pdf.py

@@ -3,10 +3,9 @@ import os
 import requests
 import logging
 import mimetypes
-import json
 from celery import shared_task
 from app.config import settings
-from app.tasks.process_document import process_document
+from app.tasks.process_document import process_document  # Updated import
 
 logger = logging.getLogger(__name__)
 
@@ -15,165 +14,59 @@ def convert_to_pdf(file_path):
     """
     Converts a file to PDF using Gotenberg's API.
     Determines the appropriate Gotenberg endpoint based on the file's MIME type.
-    On success, saves the PDF locally and enqueues it for processing.
+    On success, saves the PDF locally and enqueues it for S3 upload.
     """
     gotenberg_url = getattr(settings, "gotenberg_url", None)
     if not gotenberg_url:
         logger.error("Gotenberg URL is not configured in settings.")
         return
 
-    # Try to guess the MIME type based on file content and extension
+    # Try to guess the MIME type based on file content (using extension-based fallback)
     mime_type, encoding = mimetypes.guess_type(file_path)
-    file_ext = os.path.splitext(file_path)[1].lower()
-    logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}, extension: {file_ext}")
+    logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
 
-    # Determine which Gotenberg endpoint to use
     endpoint = None
-    form_data = {}
-    files = {}
-    
-    # Dictionary mapping file extensions to their handlers
-    OFFICE_EXTENSIONS = {
-        '.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm',  # Word
-        '.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw',  # Excel
-        '.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx',  # PowerPoint
-        '.odt', '.ods', '.odp', '.odg', '.odf',  # OpenOffice/LibreOffice
-        '.rtf', '.txt', '.csv',  # Text formats
-        '.pdf',  # PDF (already in PDF format but can be processed)
-    }
-    
-    IMAGE_EXTENSIONS = {
-        '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg'
-    }
-    
-    HTML_EXTENSIONS = {
-        '.html', '.htm'
-    }
-    
-    # Use LibreOffice endpoint for office documents and images
-    if (mime_type and 'office' in mime_type) or \
-       (mime_type and 'opendocument' in mime_type) or \
-       (mime_type and mime_type.startswith('image/')) or \
-       file_ext in OFFICE_EXTENSIONS or \
-       file_ext in IMAGE_EXTENSIONS:
-        endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
-        files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
-        
-        # Add some quality settings for better PDF output
-        form_data = {
-            'landscape': 'false',
-            'exportBookmarks': 'true',
-            'exportNotes': 'false',
-            'losslessImageCompression': 'true',  # Use lossless compression for images
-            'pdfa': 'PDF/A-2b',  # Produce PDF/A-2b compatible output
-        }
-    
-    # Use Chromium endpoint for HTML documents
-    elif (mime_type and mime_type == 'text/html') or file_ext in HTML_EXTENSIONS:
-        endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
-        # Gotenberg requires the form field to be exactly 'index.html'
-        # The content filename doesn't matter, just the form field key
-        files = {'index.html': ('index.html', open(file_path, 'rb'))}
-        
-        # Add options for better HTML to PDF conversion
-        form_data = {
-            'paperWidth': '8.27',  # A4 width in inches
-            'paperHeight': '11.7',  # A4 height in inches
-            'marginTop': '0.4',
-            'marginBottom': '0.4',
-            'marginLeft': '0.4',
-            'marginRight': '0.4',
-            'printBackground': 'true',
-            'preferCssPageSize': 'false',
-            'waitDelay': '2s',  # Wait for JavaScript to execute
-        }
-    
-    # Use Markdown route for markdown files
-    elif (mime_type and mime_type in ['text/markdown', 'text/x-markdown']) or file_ext in ['.md', '.markdown']:
-        # For Markdown, we need both the markdown file and an HTML wrapper
-        endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown"
-        
-        # Create a simple HTML wrapper for the markdown
-        # IMPORTANT: The filename in the template must match the key used in the files dictionary
-        markdown_filename = os.path.basename(file_path)
-        html_wrapper = f"""<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="UTF-8">
-    <title>Converted Markdown</title>
-    <style>
-        body {{
-            font-family: Arial, sans-serif;
-            line-height: 1.6;
-            margin: 2em;
-            max-width: 50em;
-        }}
-    </style>
-</head>
-<body>
-    {{{{ toHTML "{markdown_filename}" }}}}
-</body>
-</html>"""
-        
-        # Create a temporary HTML wrapper file
-        wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html")
-        with open(wrapper_path, 'w') as f:
-            f.write(html_wrapper)
-        
-        try:
-            files = {
-                'index.html': ('index.html', open(wrapper_path, 'rb')),
-                markdown_filename: (markdown_filename, open(file_path, 'rb'))
-            }
-            
-            form_data = {
-                'paperWidth': '8.27',  # A4 width in inches
-                'paperHeight': '11.7',  # A4 height in inches
-                'marginTop': '0.4',
-                'marginBottom': '0.4',
-                'marginLeft': '0.4',
-                'marginRight': '0.4',
-            }
-        finally:
-            # Clean up the temporary wrapper file after preparing the request
-            if os.path.exists(wrapper_path):
-                os.remove(wrapper_path)
-    
-    # Fallback to LibreOffice for everything else
-    else:
-        endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
-        files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))}
-        logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
+    form_key = "files"  # Default form key for most endpoints
 
-    if not endpoint:
-        logger.error(f"Could not determine Gotenberg endpoint for file type: {mime_type}")
-        return None
+    if mime_type:
+        if mime_type == "text/html":
+            endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
+            # The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
+            form_key = "index.html"
+        elif mime_type.startswith("image/"):
+            # For images, we use the LibreOffice endpoint (which supports image conversion)
+            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        elif mime_type.startswith("text/plain"):
+            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        elif mime_type in ["text/markdown", "text/x-markdown"]:
+            # Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
+            # For now, we'll fallback to LibreOffice.
+            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+        else:
+            # For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
+            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
+    else:
+        # If MIME detection fails, fallback to extension-based detection.
+        ext = os.path.splitext(file_path)[1].lower()
+        if ext in [".html", ".htm"]:
+            endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
+            form_key = "index.html"
+        else:
+            endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
 
     try:
-        logger.info(f"Converting {file_path} using endpoint: {endpoint}")
-        
-        # Send the conversion request to Gotenberg
-        response = requests.post(endpoint, files=files, data=form_data)
-        
+        with open(file_path, "rb") as f:
+            files = {form_key: f}
+            response = requests.post(endpoint, files=files)
+
         if response.status_code == 200:
-            # Save the converted PDF
             converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
             with open(converted_file_path, "wb") as out_file:
                 out_file.write(response.content)
-            
             logger.info(f"Converted file saved as PDF: {converted_file_path}")
-            
-            # Enqueue the PDF for further processing
-            process_document.delay(converted_file_path)
-            
+            process_document.delay(converted_file_path)  # Updated function call
             return converted_file_path
         else:
-            logger.error(
-                f"Conversion failed for {file_path}. "
-                f"Status code: {response.status_code}, "
-                f"Response: {response.text[:500]}..."
-            )
-            return None
+            logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
     except Exception as e:
         logger.exception(f"Error converting {file_path} to PDF: {e}")
-        return None

app/tasks/embed_metadata_into_pdf.py

@@ -2,12 +2,12 @@
 
 import os
 import shutil
-import tempfile
-import PyPDF2  # Replace fitz with PyPDF2
+import fitz  # PyMuPDF for PDF metadata editing
 import json
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.tasks.finalize_document_storage import finalize_document_storage
+from app.utils import task_logger, log_task
 
 # Import the shared Celery instance
 from app.celery_app import celery
@@ -40,6 +40,7 @@ def persist_metadata(metadata, final_pdf_path):
     return json_path
 
 @celery.task(base=BaseTaskWithRetry)
+@log_task("embed_metadata")
 def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata: dict):
     """
     Embeds extracted metadata into the PDF's standard metadata fields.
@@ -52,6 +53,7 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
     After processing, the file is moved to
       <workdir>/processed/<suggested_filename.pdf>
     where <suggested_filename.pdf> is derived from metadata["filename"].
+    The output PDF is saved incrementally while preserving its original encryption.
     Additionally, the metadata is persisted to a JSON file with the same base name.
     """
     # Check for file existence; if not found, try the known shared tmp directory.
@@ -59,46 +61,38 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
         alt_path = os.path.join(settings.workdir, "tmp", os.path.basename(local_file_path))
         if os.path.exists(alt_path):
             local_file_path = alt_path
+            task_logger(f"Using alternative path: {local_file_path}", step_name="embed_metadata")
         else:
-            print(f"[ERROR] Local file {local_file_path} not found, cannot embed metadata.")
+            task_logger(f"Local file {local_file_path} not found, cannot embed metadata.", 
+                      level="error", step_name="embed_metadata")
             return {"error": "File not found"}
 
-    # Work on a safe copy in a secure temporary directory
+    # Work on a safe copy in /tmp
+    tmp_dir = "/tmp"
     original_file = local_file_path
-    # Create a temporary file with the same extension as the original
-    _, ext = os.path.splitext(local_file_path)
-    tmp_file = tempfile.NamedTemporaryFile(mode='wb', suffix=ext, prefix='processed_', delete=False)
-    processed_file = tmp_file.name
-    tmp_file.close()
+    processed_file = os.path.join(tmp_dir, f"processed_{os.path.basename(local_file_path)}")
 
     # Create a safe copy to work on
     shutil.copy(original_file, processed_file)
+    task_logger(f"Created working copy at {processed_file}", step_name="embed_metadata")
 
     try:
-        print(f"[DEBUG] Embedding metadata into {processed_file}...")
+        task_logger(f"Embedding metadata into {processed_file}", step_name="embed_metadata")
 
-        # Open the PDF and modify metadata
-        with open(processed_file, 'rb') as file:
-            pdf_reader = PyPDF2.PdfReader(file)
-            pdf_writer = PyPDF2.PdfWriter()
-            
-            # Copy all pages from the reader to the writer
-            for page in pdf_reader.pages:
-                pdf_writer.add_page(page)
-            
-            # Set PDF metadata
-            pdf_writer.add_metadata({
-                "/Title": metadata.get("filename", "Unknown Document"),
-                "/Author": metadata.get("absender", "Unknown"),
-                "/Subject": metadata.get("document_type", "Unknown"),
-                "/Keywords": ", ".join(metadata.get("tags", []))
-            })
-            
-            # Write the modified PDF
-            with open(processed_file, 'wb') as output_file:
-                pdf_writer.write(output_file)
+        # Open the PDF
+        doc = fitz.open(processed_file)
+        # Set PDF metadata using only the standard keys.
+        doc.set_metadata({
+            "title": metadata.get("filename", "Unknown Document"),
+            "author": metadata.get("absender", "Unknown"),
+            "subject": metadata.get("document_type", "Unknown"),
+            "keywords": ", ".join(metadata.get("tags", []))
+        })
+        # Save incrementally and preserve encryption
+        doc.save(processed_file, incremental=True, encryption=fitz.PDF_ENCRYPT_KEEP)
+        doc.close()
 
-        print(f"[INFO] Metadata embedded successfully in {processed_file}")
+        task_logger("Metadata embedded successfully", step_name="embed_metadata")
 
         # Use the suggested filename from metadata; if not provided, use the original basename.
         suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
@@ -112,35 +106,34 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata:
 
         # Move the processed file using shutil.move to handle cross-device moves.
         shutil.move(processed_file, final_file_path)
+        task_logger(f"Moved processed file to {final_file_path}", step_name="embed_metadata")
+        
         # Ensure the temporary file is deleted if it still exists.
         if os.path.exists(processed_file):
             os.remove(processed_file)
 
         # Persist the metadata into a JSON file with the same base name.
         json_path = persist_metadata(metadata, final_file_path)
-        print(f"[INFO] Metadata persisted to {json_path}")
+        task_logger(f"Metadata persisted to {json_path}", step_name="embed_metadata")
 
         # Trigger the next step: final storage.
-        finalize_document_storage.delay(original_file, final_file_path, metadata)
+        finalize_doc_task = finalize_document_storage.delay(original_file, final_file_path, metadata)
+        task_logger(f"Triggered final document storage with task ID: {finalize_doc_task.id}", 
+                  step_name="embed_metadata")
 
         # After triggering final storage, delete the original file if it is in workdir/tmp.
         workdir_tmp = os.path.join(settings.workdir, "tmp")
         if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
             try:
                 os.remove(original_file)
-                print(f"[INFO] Deleted original file from {original_file}")
+                task_logger(f"Deleted original file from {original_file}", step_name="embed_metadata")
             except Exception as e:
-                print(f"[ERROR] Could not delete original file {original_file}: {e}")
+                task_logger(f"Could not delete original file {original_file}: {e}", 
+                          level="warning", step_name="embed_metadata")
 
         return {"file": final_file_path, "metadata_file": json_path, "status": "Metadata embedded"}
 
     except Exception as e:
-        print(f"[ERROR] Failed to embed metadata into {processed_file}: {e}")
-        # Clean up temporary file in case of error
-        if os.path.exists(processed_file):
-            try:
-                os.remove(processed_file)
-                print(f"[INFO] Cleaned up temporary file {processed_file}")
-            except Exception as cleanup_error:
-                print(f"[ERROR] Could not clean up temporary file {processed_file}: {cleanup_error}")
+        task_logger(f"Failed to embed metadata into {processed_file}: {e}", 
+                  level="error", step_name="embed_metadata")
         return {"error": str(e)}

app/tasks/extract_metadata_with_gpt.py

@@ -2,27 +2,23 @@
 
 import json
 import re
+import os
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
+from app.utils import task_logger, log_task
+from app.database import SessionLocal
+from app.models import FileRecord
 
 # Import the shared Celery instance
 from app.celery_app import celery
 import openai
-import logging
 
-logger = logging.getLogger(__name__)
-
-# Initialize OpenAI client dynamically with better error handling
-try:
-    client = openai.OpenAI(
-        api_key=settings.openai_api_key,
-        base_url=settings.openai_base_url
-    )
-    logger.info("OpenAI client initialized successfully")
-except Exception as e:
-    logger.error(f"Failed to initialize OpenAI client: {e}")
-    client = None
+# Initialize OpenAI client dynamically
+client = openai.OpenAI(
+    api_key=settings.openai_api_key,
+    base_url=settings.openai_base_url
+)
 
 def extract_json_from_text(text):
     """
@@ -42,9 +38,16 @@ def extract_json_from_text(text):
     return None
 
 @celery.task(base=BaseTaskWithRetry)
-def extract_metadata_with_gpt(filename: str, cleaned_text: str):
+@log_task("extract_metadata")
+def extract_metadata_with_gpt(s3_filename: str, cleaned_text: str):
     """Uses OpenAI to classify document metadata."""
-    prompt = f"""
+    task_id = extract_metadata_with_gpt.request.id
+    session = SessionLocal()
+    try:
+        task_logger(f"Starting metadata extraction for {s3_filename}", 
+                  step_name="extract_metadata", task_id=task_id)
+        
+        prompt = f"""
 You are a specialized document analyzer trained to extract structured metadata from documents.
 Your task is to analyze the given text and return a well-structured JSON object.
 
@@ -76,8 +79,7 @@ Extracted text:
 Return only valid JSON with no additional commentary.
 """
 
-    try:
-        print(f"[DEBUG] Sending classification request for {filename}...")
+        task_logger(f"Sending classification request for {s3_filename}", step_name="extract_metadata")
         completion = client.chat.completions.create(
             model=settings.openai_model,
             messages=[
@@ -88,21 +90,35 @@ Return only valid JSON with no additional commentary.
         )
 
         content = completion.choices[0].message.content
-        print(f"[DEBUG] Raw classification response for {filename}: {content}")
+        task_logger(f"Received raw classification response for {s3_filename}", step_name="extract_metadata")
 
         json_text = extract_json_from_text(content)
         if not json_text:
-            print(f"[ERROR] Could not find valid JSON in GPT response for {filename}.")
+            task_logger(f"Could not find valid JSON in GPT response for {s3_filename}", 
+                       level="error", step_name="extract_metadata")
             return {}
 
         metadata = json.loads(json_text)
-        print(f"[DEBUG] Extracted metadata: {metadata}")
+        task_logger(f"Successfully extracted metadata from {s3_filename}", step_name="extract_metadata")
 
         # Trigger the next step: embedding metadata into the PDF
-        embed_metadata_into_pdf.delay(filename, cleaned_text, metadata)
+        embed_task = embed_metadata_into_pdf.delay(s3_filename, cleaned_text, metadata)
+        task_logger(f"Triggered embed_metadata task with ID: {embed_task.id}", step_name="extract_metadata")
 
-        return {"s3_file": filename, "metadata": metadata}
+        # Update database record
+        file_record = session.query(FileRecord).filter(FileRecord.local_filename.like(f'%{s3_filename}')).first()
+        if file_record:
+            # Since we can't store dict directly, you might want to store it as JSON string
+            # or add specific columns for key metadata values
+            task_logger(f"Found file record ID {file_record.id}, updating metadata", step_name="extract_metadata")
+        else:
+            task_logger(f"No file record found for {s3_filename}", level="warning", step_name="extract_metadata")
+
+        return {"file": s3_filename, "metadata": metadata}
 
     except Exception as e:
-        print(f"[ERROR] OpenAI classification failed for {filename}: {e}")
+        task_logger(f"OpenAI classification failed for {s3_filename}: {e}", 
+                  level="error", step_name="extract_metadata")
         return {}
+    finally:
+        session.close()

app/tasks/finalize_document_storage.py

@@ -2,25 +2,29 @@
 
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
-# Import the shared Celery instance
 from app.celery_app import celery
+from app.utils import task_logger, log_task
 
-# 1) Import the aggregator task
+# Import the aggregator task
 from app.tasks.send_to_all import send_to_all_destinations
 
-
 @celery.task(base=BaseTaskWithRetry)
+@log_task("finalize_storage")
 def finalize_document_storage(original_file: str, processed_file: str, metadata: dict):
     """
     Final storage step after embedding metadata.
     We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
     """
-    print(f"[INFO] Finalizing document storage for {processed_file}")
+    task_logger(f"Finalizing document storage for {processed_file}", step_name="finalize_storage")
 
-    # 2) Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
-    send_to_all_destinations.delay(processed_file)
+    # Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
+    send_task = send_to_all_destinations.delay(processed_file)
+    
+    task_logger(f"Triggered send to all destinations with task ID: {send_task.id}", 
+               step_name="finalize_storage", status="success")
 
     return {
         "status": "Completed",
-        "file": processed_file
+        "file": processed_file,
+        "send_task_id": send_task.id
     }

app/tasks/imap_tasks.py

@@ -249,12 +249,8 @@ def fetch_attachments_and_enqueue(email_message):
     """
     Extracts attachments from the email and processes only allowed file types.
     
-    Files are accepted if either:
-    1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
-    2. They have a '.pdf' file extension (regardless of MIME type)
-    
     Allowed file types include:
-      - PDF: application/pdf or *.pdf extension
+      - PDF: application/pdf
       - Microsoft Office files:
           - Word: application/msword, 
             application/vnd.openxmlformats-officedocument.wordprocessingml.document
@@ -267,8 +263,12 @@ def fetch_attachments_and_enqueue(email_message):
           - CSV: text/csv
           - Rich Text Format: application/rtf, text/rtf
     
-    If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
-    any other allowed file is enqueued for conversion to PDF.
+    Attachments not in this list are skipped. Common image MIME types such as
+    image/jpeg, image/png, image/gif, image/bmp, image/tiff, and image/webp are
+    intentionally excluded.
+    
+    If the attachment is a PDF, it is enqueued for upload; any other allowed file
+    is enqueued for conversion to PDF.
     
     Returns True if at least one allowed attachment was processed.
     """
@@ -295,12 +295,8 @@ def fetch_attachments_and_enqueue(email_message):
         if not filename:
             continue
 
-        # Check if it's a PDF file by extension, regardless of MIME type
-        is_pdf_by_extension = filename.lower().endswith('.pdf')
-        
         mime_type = part.get_content_type()
-        # Accept file if it has an allowed MIME type OR it's a PDF by extension
-        if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
+        if mime_type not in ALLOWED_MIME_TYPES:
             logger.info("Skipping attachment %s with MIME type %s",
                         filename, mime_type)
             continue
@@ -309,12 +305,11 @@ def fetch_attachments_and_enqueue(email_message):
         with open(file_path, "wb") as f:
             f.write(part.get_payload(decode=True))
 
-        # If it's a PDF by MIME type or extension, process it directly
-        if mime_type == "application/pdf" or is_pdf_by_extension:
-            process_document.delay(file_path)
-            logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
+        if mime_type == "application/pdf":
+            process_document.delay(file_path)  # Updated function call
+            logger.info("Enqueued PDF for upload: %s", filename)
         elif mime_type in ALLOWED_MIME_TYPES:
-            # Other allowed files are sent for conversion
+            # Enqueue conversion to PDF using the Gotenberg service.
             convert_to_pdf.delay(file_path)
             logger.info("Enqueued file for conversion to PDF: %s", filename)
 
@@ -328,10 +323,6 @@ def email_already_has_label(mail, msg_id, label="Ingested"):
     Returns True if the label is found, False otherwise.
     """
     try:
-        # Convert msg_id to bytes if it's an integer
-        if isinstance(msg_id, int):
-            msg_id = str(msg_id).encode()
-            
         label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
         if label_status == "OK" and label_data and len(label_data) > 0:
             raw_labels = label_data[0][1].decode("utf-8", errors="ignore")

app/tasks/process_document.py

@@ -4,19 +4,20 @@ import os
 import uuid
 import shutil
 import mimetypes
-import PyPDF2  # Replace fitz with PyPDF2
+import fitz  # PyMuPDF for checking embedded text
 
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
-from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
+from app.tasks.process_with_textract import process_with_textract
 from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
 from app.celery_app import celery
 from app.database import SessionLocal
 from app.models import FileRecord
-from app.utils import hash_file
+from app.utils import hash_file, task_logger, log_task
 
 
 @celery.task(base=BaseTaskWithRetry)
+@log_task("process_document")
 def process_document(original_local_file: str):
     """
     Process a document file and trigger appropriate text extraction.
@@ -26,14 +27,17 @@ def process_document(original_local_file: str):
       2. If not found, insert a new DB row and continue with the pipeline:
          - Copy file to /workdir/tmp
          - Check for embedded text. If present, run local GPT extraction
-         - Otherwise, queue Azure Document Intelligence processing
+         - Otherwise, queue Textract-based OCR
     """
+    task_id = process_document.request.id
+    task_logger(f"Processing {original_local_file}", step_name="process_document", task_id=task_id, file_path=original_local_file)
 
     if not os.path.exists(original_local_file):
-        print(f"[ERROR] File {original_local_file} not found.")
+        task_logger(f"File {original_local_file} not found.", level="error", step_name="process_document", task_id=task_id)
         return {"error": "File not found"}
 
     # 0. Compute the file hash and check for duplicates
+    task_logger(f"Computing hash for {original_local_file}", step_name="compute_hash", task_id=task_id)
     filehash = hash_file(original_local_file)
     original_filename = os.path.basename(original_local_file)
     file_size = os.path.getsize(original_local_file)
@@ -43,66 +47,80 @@ def process_document(original_local_file: str):
 
     # Acquire DB session in the task
     with SessionLocal() as db:
-        existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none()
-        if existing:
-            print(f"[INFO] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
+        # Keep all FileRecord operations within this session scope
+        task_logger(f"Checking for duplicate files", step_name="check_duplicates", task_id=task_id)
+        existing_record = db.query(FileRecord).filter(FileRecord.filehash == filehash).one_or_none()
+        if existing_record:
+            task_logger(f"Duplicate file detected (hash={filehash[:10]}...). Skipping processing.",
+                        step_name="process_document", task_id=task_id, file_id=existing_record.id, status="success")
             return {
                 "status": "duplicate_file",
-                "file_id": existing.id,
+                "file_id": existing_record.id,
                 "detail": "File already processed."
             }
+        else:
+            task_logger(f"Creating file record for {original_local_file}", step_name="create_file_record", task_id=task_id)
+            new_record = FileRecord(
+                filehash=filehash,
+                original_filename=original_filename,
+                local_filename="",  # Will fill in after we move it
+                file_size=file_size,
+                mime_type=mime_type,
+            )
+            db.add(new_record)
+            db.commit()
+            db.refresh(new_record)
 
-        # Not a duplicate -> insert a new record
-        new_record = FileRecord(
-            filehash=filehash,
-            original_filename=original_filename,
-            local_filename="",  # Will fill in after we move it
-            file_size=file_size,
-            mime_type=mime_type,
-        )
-        db.add(new_record)
-        db.commit()
-        db.refresh(new_record)
+            # 1. Generate a UUID-based filename and place it in /workdir/tmp
+            task_logger(f"Copying to workdir", step_name="copy_to_workdir", task_id=task_id, file_id=new_record.id)
+            file_ext = os.path.splitext(original_local_file)[1]
+            file_uuid = str(uuid.uuid4())
+            new_filename = f"{file_uuid}{file_ext}"
 
-        # 1. Generate a UUID-based filename and place it in /workdir/tmp
-        file_ext = os.path.splitext(original_local_file)[1]
-        file_uuid = str(uuid.uuid4())
-        new_filename = f"{file_uuid}{file_ext}"
+            tmp_dir = os.path.join(settings.workdir, "tmp")
+            os.makedirs(tmp_dir, exist_ok=True)
+            new_local_path = os.path.join(tmp_dir, new_filename)
 
-        tmp_dir = os.path.join(settings.workdir, "tmp")
-        os.makedirs(tmp_dir, exist_ok=True)
-        new_local_path = os.path.join(tmp_dir, new_filename)
+            # Copy the file instead of moving it
+            shutil.copy(original_local_file, new_local_path)
 
-        # Copy the file instead of moving it
-        shutil.copy(original_local_file, new_local_path)
+            # Update the DB with final local filename
+            new_record.local_filename = new_local_path
+            db.commit()
 
-        # Update the DB with final local filename
-        new_record.local_filename = new_local_path
-        db.commit()
+        # Perform all further interactions with existing_record/new_record here
 
     # 2. Check for embedded text (outside the DB session to avoid long open transactions)
-    with open(new_local_path, 'rb') as file:
-        pdf_reader = PyPDF2.PdfReader(file)
-        has_text = False
-        for page in pdf_reader.pages:
-            if page.extract_text().strip():
-                has_text = True
-                break
+    task_logger(f"Checking for embedded text", step_name="check_embedded_text", task_id=task_id, file_id=new_record.id)
+    pdf_doc = fitz.open(new_local_path)
+    has_text = any(page.get_text() for page in pdf_doc)
+    pdf_doc.close()
 
     if has_text:
-        print(f"[INFO] PDF {original_local_file} contains embedded text. Processing locally.")
+        task_logger(f"PDF {original_local_file} contains embedded text. Processing locally.",
+                    step_name="process_document", task_id=task_id, file_id=new_record.id)
 
         # Extract text locally
         extracted_text = ""
-        with open(new_local_path, 'rb') as file:
-            pdf_reader = PyPDF2.PdfReader(file)
-            for page in pdf_reader.pages:
-                extracted_text += page.extract_text() + "\n"
+        task_logger(f"Extracting text locally", step_name="extract_text_locally", task_id=task_id, file_id=new_record.id)
+        pdf_doc = fitz.open(new_local_path)
+        for page in pdf_doc:
+            extracted_text += page.get_text("text") + "\n"
+        pdf_doc.close()
 
         # Call metadata extraction directly
-        extract_metadata_with_gpt.delay(new_filename, extracted_text)
-        return {"file": new_local_path, "status": "Text extracted locally"}
+        task_logger(f"Text extracted locally. Queuing for metadata extraction.",
+                    step_name="process_document", task_id=task_id, file_id=new_record.id, status="success")
+        metadata_task = extract_metadata_with_gpt.delay(new_filename, extracted_text)
+        task_logger(f"Triggered metadata extraction task: {metadata_task.id}", 
+                   step_name="process_document", task_id=task_id)
+        
+        return {"file": new_local_path, "status": "Text extracted locally", "file_id": new_record.id}
 
-    # 3. If no embedded text, queue Azure Document Intelligence processing
-    process_with_azure_document_intelligence.delay(new_filename)
-    return {"file": new_local_path, "status": "Queued for OCR"}
+    # 3. If no embedded text, queue Textract processing
+    task_logger(f"No embedded text found. Queuing for OCR.",
+                step_name="process_document", task_id=task_id, file_id=new_record.id, status="success")
+    ocr_task = process_with_textract.delay(new_filename)
+    task_logger(f"Triggered OCR task: {ocr_task.id}", step_name="process_document", task_id=task_id)
+    
+    return {"file": new_local_path, "status": "Queued for OCR", "file_id": new_record.id}

app/tasks/process_with_azure_document_intelligence.py

@@ -1,147 +0,0 @@
-import os
-import logging
-import PyPDF2
-from azure.core.credentials import AzureKeyCredential
-from azure.ai.documentintelligence import DocumentIntelligenceClient
-from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
-import azure.core.exceptions
-
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.tasks.rotate_pdf_pages import rotate_pdf_pages
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-# Initialize Azure Document Intelligence client with error handling
-try:
-    document_intelligence_client = DocumentIntelligenceClient(
-        endpoint=settings.azure_endpoint,
-        credential=AzureKeyCredential(settings.azure_ai_key)
-    )
-    logger.info("Azure Document Intelligence client initialized successfully")
-except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e:
-    logger.error(f"Failed to initialize Azure Document Intelligence client: {e}")
-    document_intelligence_client = None
-except Exception as e:
-    logger.error(f"Unexpected error initializing Azure Document Intelligence client: {e}")
-    document_intelligence_client = None
-
-# Azure Document Intelligence service limits for Standard S0 tier
-AZURE_DOC_INTELLIGENCE_LIMITS = {
-    "max_file_size_bytes": 500 * 1024 * 1024,  # 500 MB
-    "max_pages": 2000,
-}
-
-def get_pdf_page_count(file_path):
-    """Get the number of pages in a PDF file."""
-    try:
-        with open(file_path, 'rb') as file:
-            pdf_reader = PyPDF2.PdfReader(file)
-            return len(pdf_reader.pages)
-    except Exception as e:
-        logger.error(f"Error getting PDF page count: {e}")
-        return None
-
-def check_page_rotation(result, filename):
-    """
-    Checks if pages in the document are rotated and logs the rotation information.
-    
-    Args:
-        result: The AnalyzeResult from Azure Document Intelligence API
-        filename: The name of the file being processed
-        
-    Returns:
-        dict: Dictionary mapping page indices (integers) to rotation angles
-    """
-    logger.error(f"Checking rotation for document: {filename}")
-    rotation_data = {}
-    
-    if not hasattr(result, 'pages') or not result.pages:
-        logger.error(f"No page information available for rotation check: {filename}")
-        return rotation_data
-        
-    for i, page in enumerate(result.pages):
-        if hasattr(page, 'angle'):
-            rotation_angle = page.angle
-            if rotation_angle != 0:
-                logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees")
-                # Store page index as integer, not string
-                rotation_data[i] = rotation_angle
-            else:
-                logger.error(f"Page {i+1} has no rotation (0 degrees)")
-        else:
-            logger.error(f"Page {i+1} rotation information not available")
-            
-    return rotation_data
-
-@celery.task(base=BaseTaskWithRetry)
-def process_with_azure_document_intelligence(filename: str):
-    """
-    Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
-    the local temporary file (stored under <workdir>/tmp).
-    
-    Steps:
-      0. Verify the file meets Azure Document Intelligence service limits
-      1. Uploads the document for OCR using Azure Document Intelligence.
-      2. Retrieves the processed PDF with embedded text.
-      3. Saves the OCR-processed PDF locally in the same location as before.
-      4. Checks for page rotation and triggers page rotation if needed.
-      5. Triggers downstream metadata extraction.
-    """
-    try:
-        tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
-        if not os.path.exists(tmp_file_path):
-            raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
-
-        # Check file size against service limits
-        file_size = os.path.getsize(tmp_file_path)
-        if file_size > AZURE_DOC_INTELLIGENCE_LIMITS["max_file_size_bytes"]:
-            error_msg = f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
-            logger.error(error_msg)
-            return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
-
-        # For PDF files, check page count against service limits
-        # "Fail open" approach: only reject if we're sure it exceeds the limit
-        if filename.lower().endswith('.pdf'):
-            page_count = get_pdf_page_count(tmp_file_path)
-            if page_count is not None and page_count > AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"]:
-                error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
-                logger.error(error_msg)
-                return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
-            if page_count is None:
-                logger.warning(f"Could not determine page count for {filename}, proceeding with processing anyway")
-
-        logger.info(f"Processing {filename} with Azure Document Intelligence OCR.")
-
-        # Open and send the document for processing
-        with open(tmp_file_path, "rb") as f:
-            poller = document_intelligence_client.begin_analyze_document(
-                "prebuilt-read", body=f, output=[AnalyzeOutputOption.PDF]
-            )
-        result: AnalyzeResult = poller.result()
-        operation_id = poller.details["operation_id"]
-
-        # Check and log page rotation information
-        rotation_data = check_page_rotation(result, filename)
-
-        # Retrieve the processed searchable PDF
-        response = document_intelligence_client.get_analyze_result_pdf(
-            model_id=result.model_id, result_id=operation_id
-        )
-        searchable_pdf_path = tmp_file_path  # Overwrite the original PDF location
-        with open(searchable_pdf_path, "wb") as writer:
-            writer.writelines(response)
-        logger.info(f"Searchable PDF saved at: {searchable_pdf_path}")
-
-        # Extract raw text content from the result
-        extracted_text = result.content if result.content else ""
-        logger.info(f"Extracted text for {filename}: {len(extracted_text)} characters")
-
-        # Trigger page rotation task if rotation is detected, otherwise proceed to metadata extraction
-        rotate_pdf_pages.delay(filename, extracted_text, rotation_data)
-
-        return {"file": filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text}
-    except Exception as e:
-        logger.error(f"Error processing {filename} with Azure Document Intelligence: {e}")
-        raise

app/tasks/process_with_textract.py

@@ -0,0 +1,101 @@
+import os
+import logging
+from azure.core.credentials import AzureKeyCredential
+from azure.ai.documentintelligence import DocumentIntelligenceClient
+from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
+
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
+from app.celery_app import celery
+from app.utils import task_logger, log_task
+from app.database import SessionLocal
+from app.models import FileRecord
+
+logger = logging.getLogger(__name__)
+
+# Initialize Azure Document Intelligence client
+document_intelligence_client = DocumentIntelligenceClient(
+    endpoint=settings.azure_endpoint,
+    credential=AzureKeyCredential(settings.azure_ai_key)
+)
+
+@celery.task(base=BaseTaskWithRetry)
+@log_task("process_with_textract")
+def process_with_textract(s3_filename: str):
+    """
+    Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
+    the local temporary file (stored under <workdir>/tmp).
+    
+    Steps:
+      1. Uploads the document for OCR using Azure Document Intelligence.
+      2. Retrieves the processed PDF with embedded text.
+      3. Saves the OCR-processed PDF locally in the same location as before.
+      4. Extracts the text content for metadata processing.
+      5. Triggers downstream metadata extraction by calling extract_metadata_with_gpt.
+    """
+    task_id = process_with_textract.request.id
+    tmp_file_path = os.path.join(settings.workdir, "tmp", s3_filename)
+    
+    # Get the file_id from the database
+    file_id = None
+    with SessionLocal() as db:
+        file_record = db.query(FileRecord).filter(
+            FileRecord.local_filename == tmp_file_path
+        ).first()
+        if file_record:
+            file_id = file_record.id
+    
+    task_logger(f"Starting OCR for {s3_filename}", step_name="process_with_textract", 
+               task_id=task_id, file_id=file_id, file_path=tmp_file_path)
+    
+    if not os.path.exists(tmp_file_path):
+        task_logger(f"Local file not found: {tmp_file_path}", level="error", 
+                  step_name="process_with_textract", task_id=task_id, 
+                  file_id=file_id, file_path=tmp_file_path)
+        raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
+
+    try:
+        task_logger(f"Sending document to Azure Document Intelligence", 
+                  step_name="azure_document_intelligence", task_id=task_id, 
+                  file_id=file_id, file_path=tmp_file_path)
+        
+        # Open and send the document for processing
+        with open(tmp_file_path, "rb") as f:
+            poller = document_intelligence_client.begin_analyze_document(
+                "prebuilt-read", body=f, output=[AnalyzeOutputOption.PDF]
+            )
+        result: AnalyzeResult = poller.result()
+        operation_id = poller.details["operation_id"]
+
+        task_logger(f"Azure Document Intelligence processing complete, operation ID: {operation_id}",
+                   step_name="azure_document_intelligence", task_id=task_id)
+
+        # Retrieve the processed searchable PDF
+        task_logger(f"Retrieving searchable PDF", step_name="retrieve_pdf", task_id=task_id)
+        response = document_intelligence_client.get_analyze_result_pdf(
+            model_id=result.model_id, result_id=operation_id
+        )
+        searchable_pdf_path = tmp_file_path  # Overwrite the original PDF location
+        with open(searchable_pdf_path, "wb") as writer:
+            writer.writelines(response)
+        
+        # Extract raw text content from the result
+        extracted_text = result.content if result.content else ""
+        text_length = len(extracted_text)
+        task_logger(f"Extracted {text_length} characters of text", 
+                   step_name="extract_text", task_id=task_id)
+
+        # Trigger downstream metadata extraction
+        task_logger(f"OCR completed. Queueing metadata extraction for {s3_filename}", 
+                   step_name="process_with_textract", task_id=task_id, status="success")
+                   
+        metadata_task = extract_metadata_with_gpt.delay(s3_filename, extracted_text)
+        task_logger(f"Triggered metadata extraction task: {metadata_task.id}", 
+                   step_name="process_with_textract", task_id=task_id)
+
+        return {"file": s3_filename, "searchable_pdf": searchable_pdf_path, "text_length": text_length}
+    except Exception as e:
+        task_logger(f"Error processing with Azure Document Intelligence: {e}", 
+                  level="error", step_name="process_with_textract", task_id=task_id)
+        raise

app/tasks/refine_text_with_gpt.py

@@ -3,6 +3,7 @@
 from app.config import settings
 import openai
 from app.tasks.retry_config import BaseTaskWithRetry
+from app.utils import task_logger, log_task
 
 # Import the shared Celery instance
 from app.celery_app import celery
@@ -14,8 +15,12 @@ client = openai.OpenAI(
 )
 
 @celery.task(base=BaseTaskWithRetry)
-def refine_text_with_gpt(filename: str, raw_text: str):
+@log_task("refine_text")
+def refine_text_with_gpt(s3_filename: str, raw_text: str):
     """Uses OpenAI to clean and refine OCR text."""
+    task_id = refine_text_with_gpt.request.id
+    task_logger(f"Starting text refinement for {s3_filename}", step_name="refine_text", task_id=task_id)
+    
     response = client.chat.completions.create(
         model=settings.openai_model,
         messages=[
@@ -25,10 +30,12 @@ def refine_text_with_gpt(filename: str, raw_text: str):
     )
     
     cleaned_text = response.choices[0].message.content
+    task_logger(f"Text refinement completed for {s3_filename}", step_name="refine_text", task_id=task_id)
 
     # Trigger next task (import locally if needed to avoid circular imports)
     from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
-    extract_metadata_with_gpt.delay(filename, cleaned_text)
+    metadata_task = extract_metadata_with_gpt.delay(s3_filename, cleaned_text)
+    task_logger(f"Triggered metadata extraction task: {metadata_task.id}", step_name="refine_text", task_id=task_id)
 
-    return {"filename": filename, "cleaned_text": cleaned_text}
+    return {"file": s3_filename, "cleaned_text": cleaned_text}
 

app/tasks/rotate_pdf_pages.py

@@ -1,133 +0,0 @@
-import os
-import logging
-import PyPDF2
-import math
-import json
-
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-def determine_rotation_angle(detected_angle):
-    """
-    Determine the optimal rotation angle based on detected angle.
-    
-    Args:
-        detected_angle: The angle detected by Azure Document Intelligence
-        
-    Returns:
-        int: The angle to rotate the page in PyPDF2 (must be multiple of 90 degrees)
-    """
-    # Normalize angle to be between 0 and 360
-    normalized_angle = detected_angle % 360
-    if normalized_angle < 0:
-        normalized_angle += 360
-    
-    # If angle is very small (< 1 degree), don't rotate
-    if abs(normalized_angle) < 1 or abs(normalized_angle - 360) < 1:
-        return 0
-    
-    # For angles close to 90, 180, or 270 degrees (±5°), round to nearest 90° increment
-    for target in [90, 180, 270]:
-        if abs(normalized_angle - target) < 5:
-            # PyPDF2 uses clockwise rotation, so we need to use the complementary angle
-            rotation_value = (360 - target) % 360
-            logger.info(f"Detected angle {detected_angle}° is close to {target}°, will rotate by {rotation_value}°")
-            return rotation_value
-    
-    # For other significant angles, round to nearest 90° increment
-    # (PyPDF2 only supports rotations in 90-degree increments)
-    closest_90_multiple = round(normalized_angle / 90) * 90
-    # Convert to PyPDF2 rotation value (clockwise)
-    rotation_value = (360 - closest_90_multiple) % 360
-    logger.info(f"Detected angle {detected_angle}° rounded to {closest_90_multiple}°, will rotate by {rotation_value}°")
-    return rotation_value
-
-@celery.task(base=BaseTaskWithRetry)
-def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None):
-    """
-    Rotates pages in a PDF document based on detected rotation angles.
-    
-    Args:
-        filename: The name of the file to rotate
-        extracted_text: The extracted text from the document
-        rotation_data: Optional rotation data dictionary {page_index: angle}
-    """
-    try:
-        pdf_path = os.path.join(settings.workdir, "tmp", filename)
-        if not os.path.exists(pdf_path):
-            raise FileNotFoundError(f"PDF file not found: {pdf_path}")
-        
-        # Skip rotation if no rotation data provided
-        if not rotation_data:
-            logger.info(f"No rotation data provided for {filename}, proceeding with metadata extraction")
-            extract_metadata_with_gpt.delay(filename, extracted_text)
-            return {"file": filename, "status": "no_rotation_needed"}
-
-        # Standardize rotation_data keys to integers
-        normalized_rotation_data = {}
-        for key, value in rotation_data.items():
-            try:
-                normalized_rotation_data[int(key)] = float(value)
-            except (ValueError, TypeError):
-                logger.warning(f"Invalid rotation data key-value: {key}:{value}")
-
-        if not any(abs(angle) > 0 for angle in normalized_rotation_data.values()):
-            logger.info(f"No significant rotations detected in {filename}, proceeding with metadata extraction")
-            extract_metadata_with_gpt.delay(filename, extracted_text)
-            return {"file": filename, "status": "no_rotation_needed"}
-        
-        logger.info(f"Rotating {len(normalized_rotation_data)} pages in {filename}")
-        applied_rotations = {}
-        
-        # Load the PDF
-        with open(pdf_path, 'rb') as file:
-            pdf_reader = PyPDF2.PdfReader(file)
-            pdf_writer = PyPDF2.PdfWriter()
-            
-            # Process each page
-            for page_idx in range(len(pdf_reader.pages)):
-                page = pdf_reader.pages[page_idx]
-                
-                # Apply rotation if this page has rotation data
-                if page_idx in normalized_rotation_data and abs(normalized_rotation_data[page_idx]) > 0:
-                    detected_angle = normalized_rotation_data[page_idx]
-                    rotation_angle = determine_rotation_angle(detected_angle)
-                    
-                    if rotation_angle > 0:
-                        # PyPDF2 uses clockwise rotation in 90-degree increments
-                        page.rotate(rotation_angle)
-                        logger.info(f"Page {page_idx+1} rotated by {rotation_angle}° (from detected {detected_angle}°)")
-                        applied_rotations[str(page_idx)] = rotation_angle
-                    else:
-                        logger.info(f"Page {page_idx+1} had detected angle {detected_angle}° but determined it doesn't need rotation")
-                
-                pdf_writer.add_page(page)
-            
-            # Save the rotated PDF
-            with open(pdf_path, 'wb') as output_file:
-                pdf_writer.write(output_file)
-        
-        if applied_rotations:
-            logger.info(f"Successfully rotated PDF: {filename} with rotations: {json.dumps(applied_rotations)}")
-        else:
-            logger.info(f"Detected rotations in {filename} but no rotations were actually applied (angles too small or not multiples of 90°)")
-        
-        # Continue with metadata extraction
-        extract_metadata_with_gpt.delay(filename, extracted_text)
-        
-        return {
-            "file": filename, 
-            "status": "rotated" if applied_rotations else "no_rotation_needed",
-            "detected_rotations": rotation_data,
-            "applied_rotations": applied_rotations
-        }
-        
-    except Exception as e:
-        logger.error(f"Error rotating PDF {filename}: {e}")
-        # Continue with metadata extraction despite rotation failure
-        extract_metadata_with_gpt.delay(filename, extracted_text)
-        return {"file": filename, "status": "rotation_failed", "error": str(e)}

app/tasks/send_to_all.py

@@ -1,218 +1,34 @@
-#!/usr/bin/env python3
+# app/tasks/send_to_all.py
 
-import os
-import logging
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
 from app.tasks.upload_to_dropbox import upload_to_dropbox
 from app.tasks.upload_to_nextcloud import upload_to_nextcloud
 from app.tasks.upload_to_paperless import upload_to_paperless
-from app.tasks.upload_to_google_drive import upload_to_google_drive
-from app.tasks.upload_to_webdav import upload_to_webdav
-from app.tasks.upload_to_ftp import upload_to_ftp
-from app.tasks.upload_to_sftp import upload_to_sftp
-from app.tasks.upload_to_email import upload_to_email
-from app.tasks.upload_to_onedrive import upload_to_onedrive
-from app.tasks.upload_to_s3 import upload_to_s3
-from app.utils.config_validator import get_provider_status
-from app.celery_app import celery
+from app.utils import task_logger, log_task
 
-logger = logging.getLogger(__name__)
-
-def _should_upload_to_dropbox():
-    return (settings.dropbox_app_key and 
-            settings.dropbox_app_secret and 
-            settings.dropbox_refresh_token)
-
-def _should_upload_to_nextcloud():
-    return (settings.nextcloud_upload_url and 
-            settings.nextcloud_username and 
-            settings.nextcloud_password)
-
-def _should_upload_to_paperless():
-    return (settings.paperless_ngx_api_token and 
-            settings.paperless_host)
-
-def _should_upload_to_google_drive():
-    # Check for OAuth configuration
-    if getattr(settings, 'google_drive_use_oauth', False):
-        return (settings.google_drive_client_id and 
-                settings.google_drive_client_secret and 
-                settings.google_drive_refresh_token and
-                settings.google_drive_folder_id)
-    # Or check for service account configuration
-    else:
-        return (settings.google_drive_credentials_json and
-                settings.google_drive_folder_id)
-
-def _should_upload_to_webdav():
-    return (settings.webdav_url and 
-            settings.webdav_username and 
-            settings.webdav_password)
-
-def _should_upload_to_ftp():
-    return (settings.ftp_host and 
-            settings.ftp_username and 
-            settings.ftp_password)
-
-def _should_upload_to_sftp():
-    return (settings.sftp_host and 
-            settings.sftp_username and 
-            (settings.sftp_password or settings.sftp_private_key))
-
-def _should_upload_to_email():
-    return (settings.email_host and 
-            settings.email_username and 
-            settings.email_password and 
-            settings.email_default_recipient)
-
-def _should_upload_to_onedrive():
-    return (settings.onedrive_client_id and 
-            settings.onedrive_client_secret and 
-            settings.onedrive_refresh_token)
-
-def _should_upload_to_s3():
-    return (settings.s3_bucket_name and 
-            settings.aws_access_key_id and 
-            settings.aws_secret_access_key)
-
-def get_configured_services_from_validator():
+@celery.task
+@log_task("send_to_all_destinations")
+def send_to_all_destinations(file_path: str):
     """
-    Use the config validator to determine which services are configured properly.
-    Returns a dictionary with service names as keys and boolean values indicating
-    whether they're properly configured.
+    Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
+    These tasks run in parallel (Celery returns immediately from each .delay()).
     """
-    providers = get_provider_status()
+    task_logger(f"Sending {file_path} to all destinations", step_name="send_to_all")
     
-    service_map = {
-        "Dropbox": "dropbox",
-        "NextCloud": "nextcloud",
-        "Paperless-ngx": "paperless",
-        "Google Drive": "google_drive",
-        "WebDAV": "webdav",
-        "FTP Storage": "ftp",
-        "SFTP Storage": "sftp",
-        "Email": "email",
-        "OneDrive": "onedrive",
-        "S3 Storage": "s3"
-    }
-    
-    result = {}
-    for provider_name, internal_name in service_map.items():
-        if provider_name in providers:
-            result[internal_name] = providers[provider_name].get('configured', False)
-    
-    return result
+    dropbox_task = upload_to_dropbox.delay(file_path)
+    nextcloud_task = upload_to_nextcloud.delay(file_path)
+    paperless_task = upload_to_paperless.delay(file_path)
+
+    task_logger(f"Enqueued file for all destinations: Dropbox (task: {dropbox_task.id}), "
+               f"Nextcloud (task: {nextcloud_task.id}), Paperless (task: {paperless_task.id})",
+               step_name="send_to_all", status="success")
 
-@celery.task(base=BaseTaskWithRetry)
-def send_to_all_destinations(file_path: str, use_validator=True):
-    """
-    Distribute a file to all configured storage destinations.
-    
-    Args:
-        file_path: Path to the file to distribute
-        use_validator: Whether to use the config validator to determine enabled services
-                       (if False, falls back to individual checks)
-    """
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-    
-    logger.info(f"Sending {file_path} to all configured destinations")
-    results = {}
-    
-    # Define service configurations
-    services = [
-        {
-            "name": "dropbox",
-            "should_upload": _should_upload_to_dropbox,
-            "upload_func": upload_to_dropbox,
-        },
-        {
-            "name": "nextcloud",
-            "should_upload": _should_upload_to_nextcloud,
-            "upload_func": upload_to_nextcloud,
-        },
-        {
-            "name": "paperless",
-            "should_upload": _should_upload_to_paperless,
-            "upload_func": upload_to_paperless,
-        },
-        {
-            "name": "google_drive",
-            "should_upload": _should_upload_to_google_drive,
-            "upload_func": upload_to_google_drive,
-        },
-        {
-            "name": "webdav",
-            "should_upload": _should_upload_to_webdav,
-            "upload_func": upload_to_webdav,
-        },
-        {
-            "name": "ftp",
-            "should_upload": _should_upload_to_ftp,
-            "upload_func": upload_to_ftp,
-        },
-        {
-            "name": "sftp",
-            "should_upload": _should_upload_to_sftp,
-            "upload_func": upload_to_sftp,
-        },
-        {
-            "name": "email",
-            "should_upload": _should_upload_to_email,
-            "upload_func": upload_to_email,
-        },
-        {
-            "name": "onedrive",
-            "should_upload": _should_upload_to_onedrive,
-            "upload_func": upload_to_onedrive,
-        },
-        {
-            "name": "s3",
-            "should_upload": _should_upload_to_s3,
-            "upload_func": upload_to_s3,
-        },
-    ]
-    
-    # Optionally get configuration status from validator
-    configured_services = {}
-    if use_validator:
-        try:
-            configured_services = get_configured_services_from_validator()
-            logger.info(f"Configured services according to validator: {configured_services}")
-        except Exception as e:
-            logger.warning(f"Failed to get configuration from validator: {str(e)}")
-            use_validator = False
-    
-    # Process each service
-    for service in services:
-        service_name = service["name"]
-        
-        # Determine if service is configured
-        is_configured = False
-        if use_validator and service_name in configured_services:
-            is_configured = configured_services[service_name]
-            logger.debug(f"{service_name} configuration from validator: {is_configured}")
-        else:
-            try:
-                is_configured = service["should_upload"]()
-                logger.debug(f"{service_name} configuration from function: {is_configured}")
-            except Exception as e:
-                logger.error(f"Error checking configuration for {service_name}: {str(e)}")
-                is_configured = False
-        
-        # Queue the upload task if service is configured
-        if is_configured:
-            logger.info(f"Queueing {file_path} for {service_name} upload")
-            try:
-                task = service["upload_func"].delay(file_path)
-                results[f"{service_name}_task_id"] = task.id
-            except Exception as e:
-                logger.error(f"Failed to queue {service_name} task: {str(e)}")
-                results[f"{service_name}_error"] = str(e)
-    
     return {
-        "status": "Queued",
+        "status": "All upload tasks enqueued",
         "file_path": file_path,
-        "tasks": results
+        "task_ids": {
+            "dropbox": dropbox_task.id,
+            "nextcloud": nextcloud_task.id, 
+            "paperless": paperless_task.id
+        }
     }

app/tasks/upload_to_dropbox.py

@@ -1,48 +1,21 @@
 #!/usr/bin/env python3
 
 import os
-import logging
 import requests
 import dropbox
-from dropbox.exceptions import ApiError, AuthError
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.celery_app import celery
-from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
-
-logger = logging.getLogger(__name__)
-
-def _validate_dropbox_settings():
-    """Validate that all required Dropbox settings are available."""
-    missing = []
-    
-    if not hasattr(settings, 'dropbox_refresh_token') or not settings.dropbox_refresh_token:
-        missing.append("refresh token")
-    
-    if not hasattr(settings, 'dropbox_app_key') or not settings.dropbox_app_key:
-        missing.append("app key")
-    
-    if not hasattr(settings, 'dropbox_app_secret') or not settings.dropbox_app_secret:
-        missing.append("app secret")
-    
-    if missing:
-        logger.error(f"Cannot refresh Dropbox token: Missing {', '.join(missing)}")
-        return False
-    
-    return True
+from app.utils import task_logger, log_task
 
 def get_dropbox_access_token():
     """Refresh the Dropbox access token using the stored refresh token from ENV."""
-    
-    # Check if needed settings are available
-    if not _validate_dropbox_settings():
-        return None
 
     token_url = "https://api.dropbox.com/oauth2/token"
     headers = {"Content-Type": "application/x-www-form-urlencoded"}
     data = {
         "grant_type": "refresh_token",
-        "refresh_token": settings.dropbox_refresh_token,
+        "refresh_token": settings.dropbox_refresh_token,  # Now using ENV
         "client_id": settings.dropbox_app_key,
         "client_secret": settings.dropbox_app_secret,
     }
@@ -53,148 +26,56 @@ def get_dropbox_access_token():
         return response.json()["access_token"]
     else:
         error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
-        logger.error(error_msg)
+        task_logger(error_msg, level="error", step_name="dropbox_auth")
         raise Exception(error_msg)
 
-def get_dropbox_client():
-    """
-    Create and return an authenticated Dropbox client using the configured refresh token.
-    
-    Returns:
-        dropbox.Dropbox: Authenticated Dropbox client instance
-    
-    Raises:
-        ValueError: If required Dropbox configuration is missing
-        AuthError: If authentication with Dropbox fails
-    """
-    app_key = settings.dropbox_app_key
-    app_secret = settings.dropbox_app_secret
-    refresh_token = settings.dropbox_refresh_token
-    
-    # Validate configuration
-    if not app_key or not app_secret:
-        raise ValueError("Dropbox app key or app secret is not configured")
-    
-    if not refresh_token:
-        raise ValueError("Dropbox refresh token is not configured")
-    
-    # Create a Dropbox client with refresh token
-    try:
-        dbx = dropbox.Dropbox(
-            app_key=app_key,
-            app_secret=app_secret,
-            oauth2_refresh_token=refresh_token
-        )
-        
-        # Test the connection
-        dbx.users_get_current_account()
-        logger.info("Successfully authenticated with Dropbox")
-        return dbx
-    
-    except AuthError as auth_error:
-        logger.error(f"Dropbox authentication failed: {str(auth_error)}")
-        raise
-    
-    except Exception as e:
-        logger.error(f"Error creating Dropbox client: {str(e)}")
-        raise
-
 @celery.task(base=BaseTaskWithRetry)
+@log_task("upload_to_dropbox")
 def upload_to_dropbox(file_path: str):
-    """
-    Upload a file to Dropbox.
-    """
+    """Uploads a file to Dropbox using the API."""
+
     if not os.path.exists(file_path):
-        error_msg = f"File not found: {file_path}"
-        logger.error(error_msg)
-        raise FileNotFoundError(error_msg)
-    
-    # Check if Dropbox is properly configured
-    if not (hasattr(settings, 'dropbox_app_key') and settings.dropbox_app_key and
-            hasattr(settings, 'dropbox_app_secret') and settings.dropbox_app_secret and
-            hasattr(settings, 'dropbox_refresh_token') and settings.dropbox_refresh_token):
-        logger.info("Dropbox upload skipped: Missing configuration")
-        return {"status": "Skipped", "reason": "Dropbox settings not configured"}
-    
+        task_logger(f"File not found: {file_path}", level="error", step_name="dropbox_upload")
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename and set target path
     filename = os.path.basename(file_path)
-    
+    dropbox_path = f"{settings.dropbox_folder}/{filename}"
+
     try:
-        # Get the Dropbox client
-        dbx = get_dropbox_client()
+        # Get fresh access token
+        task_logger(f"Getting Dropbox access token", step_name="dropbox_auth")
+        access_token = get_dropbox_access_token()
+        dbx = dropbox.Dropbox(access_token)
+
+        file_size = os.path.getsize(file_path)
+        chunk_size = 4 * 1024 * 1024  # 4MB chunk size
+
+        task_logger(f"Starting upload of {filename} ({file_size} bytes) to Dropbox", step_name="dropbox_upload")
         
-        # Calculate remote path based on local file structure
-        remote_base = settings.dropbox_folder or ""
-        remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
-        
-        # Function to check if file exists in Dropbox
-        def check_exists_in_dropbox(path):
-            try:
-                dbx.files_get_metadata(path)
-                return True
-            except ApiError as e:
-                if e.error.is_path() and e.error.get_path().is_not_found():
-                    return False
-                raise
-        
-        # Get a unique path in case of collision
-        remote_full_path = f"/{remote_path}"  # Dropbox paths should start with /
-        remote_full_path = remote_full_path.replace('//', '/')  # Clean double slashes
-        
-        # Check for potential file collision and get a unique name if needed
-        dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
-        
-        # Upload the file
-        logger.info(f"Uploading {filename} to Dropbox at {dropbox_path}")
-        with open(file_path, 'rb') as file_data:
-            # Use files_upload_session for large files to avoid timeouts
-            file_size = os.path.getsize(file_path)
-            if file_size > 10 * 1024 * 1024:  # 10 MB threshold for chunked upload
-                cursor = None
-                chunk_size = 4 * 1024 * 1024  # 4 MB chunks
-                file_data.seek(0)
-                
-                # Start upload session
-                session_start = dbx.files_upload_session_start(file_data.read(chunk_size))
-                cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
-                
-                # Upload chunks until we reach the end
+        with open(file_path, "rb") as file_data:
+            if file_size <= chunk_size:
+                dbx.files_upload(file_data.read(), dropbox_path)
+            else:
+                task_logger(f"Using chunked upload for {filename}", step_name="dropbox_upload")
+                upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
+                cursor = dropbox.files.UploadSessionCursor(
+                    session_id=upload_session_start_result.session_id,
+                    offset=file_data.tell(),
+                )
+                commit = dropbox.files.CommitInfo(path=dropbox_path)
+
                 while file_data.tell() < file_size:
                     if (file_size - file_data.tell()) <= chunk_size:
-                        # Last chunk
-                        dbx.files_upload_session_finish(
-                            file_data.read(chunk_size),
-                            cursor,
-                            dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite)
-                        )
+                        dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
                     else:
-                        # More chunks to upload
                         dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
                         cursor.offset = file_data.tell()
-            else:
-                # Small file, direct upload
-                file_data.seek(0)
-                dbx.files_upload(
-                    file_data.read(),
-                    dropbox_path,
-                    mode=dropbox.files.WriteMode.overwrite
-                )
-        
-        logger.info(f"Successfully uploaded {filename} to Dropbox at {dropbox_path}")
-        return {
-            "status": "Completed",
-            "file_path": file_path,
-            "dropbox_path": dropbox_path
-        }
-    
-    except AuthError:
-        error_msg = f"[ERROR] Authentication failed while uploading {filename} to Dropbox. Check token."
-        logger.error(error_msg)
-        raise Exception(error_msg)
-    except ApiError as e:
-        error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {e}"
-        logger.error(error_msg)
-        raise Exception(error_msg)
+
+        task_logger(f"Successfully uploaded {filename} to Dropbox at {dropbox_path}", step_name="dropbox_upload", status="success")
+        return {"status": "Completed", "file": file_path}
+
     except Exception as e:
-        error_msg = f"[ERROR] Unexpected error uploading {filename} to Dropbox: {e}"
-        logger.error(error_msg)
+        error_msg = f"Failed to upload {filename} to Dropbox: {str(e)}"
+        task_logger(error_msg, level="error", step_name="dropbox_upload", status="failure")
         raise Exception(error_msg)

app/tasks/upload_to_email.py

@@ -1,254 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import json
-import smtplib
-import socket
-import logging
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-from email.mime.application import MIMEApplication
-from email.mime.image import MIMEImage
-from datetime import datetime
-from pathlib import Path
-from jinja2 import Environment, FileSystemLoader, select_autoescape
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-def get_email_template(template_name="default.html"):
-    """
-    Load email template from one of these locations in order of precedence:
-    1. Custom template from workdir/templates/email/
-    2. Default template from app/templates/email/
-    """
-    # First try to load from workdir (user customizable location)
-    try:
-        workdir_template_path = os.path.join(settings.workdir, "templates", "email")
-        if os.path.exists(workdir_template_path):
-            env = Environment(
-                loader=FileSystemLoader(workdir_template_path),
-                autoescape=select_autoescape(['html', 'xml'])
-            )
-            env.globals['now'] = datetime.now  # Add the now function to the Jinja environment
-            template = env.get_template(template_name)
-            logger.info(f"Using custom email template from workdir: {template_name}")
-            return template
-    except Exception as e:
-        logger.warning(f"Failed to load custom email template: {str(e)}")
-    
-    # Fallback to built-in template
-    try:
-        # Get the app directory path (where this file is)
-        current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        app_template_path = os.path.join(current_dir, "templates", "email")
-        env = Environment(
-            loader=FileSystemLoader(app_template_path),
-            autoescape=select_autoescape(['html', 'xml'])
-        )
-        env.globals['now'] = datetime.now  # Add the now function to the Jinja environment
-        template = env.get_template(template_name)
-        logger.info(f"Using built-in email template: {template_name}")
-        return template
-    except Exception as e:
-        logger.error(f"Failed to load built-in email template: {str(e)}")
-        raise ValueError(f"Could not find any valid email template: {str(e)}")
-
-def extract_metadata_from_file(file_path):
-    """
-    Try to extract metadata from a file using several methods:
-    1. Check for a .json metadata file with the same name
-    2. Extract metadata from PDF if it's embedded
-    
-    Returns a dictionary of metadata or None if not found
-    """
-    metadata = {}
-    
-    # Check for separate metadata JSON file
-    metadata_path = os.path.splitext(file_path)[0] + '.json'
-    if os.path.exists(metadata_path):
-        try:
-            with open(metadata_path, 'r', encoding='utf-8') as f:
-                metadata = json.load(f)
-                logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
-                return metadata
-        except Exception as e:
-            logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
-    
-    # TODO: For PDF files, try to extract embedded metadata using PyPDF2
-    # This would require additional dependencies, so for now we'll just check for external JSON
-    
-    return metadata
-
-def attach_logo(msg):
-    """Attach the DocuElevate logo to the email with proper Content-ID."""
-    try:
-        # Try to find logo in workdir first (for customization)
-        custom_logo_path = os.path.join(settings.workdir, "templates", "email", "logo.png")
-        if os.path.exists(custom_logo_path):
-            logo_path = custom_logo_path
-        else:
-            # Use built-in logo
-            app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-            logo_path = os.path.join(app_dir, "static", "logo.png")
-            # Fallback to logo in frontend/static if app/static doesn't exist
-            if not os.path.exists(logo_path):
-                logo_path = os.path.join(app_dir, "..", "frontend", "static", "logo.png")
-        
-        if os.path.exists(logo_path):
-            with open(logo_path, 'rb') as img:
-                logo_data = img.read()
-                
-            # Determine image MIME type based on extension
-            mimetype = 'image/svg+xml' if logo_path.endswith('.svg') else 'image/png'
-            logo_attach = MIMEImage(logo_data, mimetype)
-            logo_attach.add_header('Content-ID', '<logo>')
-            logo_attach.add_header('Content-Disposition', 'inline', filename='logo.png')
-            msg.attach(logo_attach)
-            logger.info(f"Logo attached from {logo_path}")
-            return True
-        else:
-            logger.warning("Could not find logo file")
-            return False
-            
-    except Exception as e:
-        logger.warning(f"Error attaching logo: {str(e)}")
-        return False
-
-def _prepare_recipients(recipients):
-    """Helper function to prepare email recipients list."""
-    if not recipients:
-        if not settings.email_default_recipient:
-            error_msg = "No recipients specified and no default recipient configured"
-            logger.error(error_msg)
-            return None, error_msg
-        return [settings.email_default_recipient], None
-    elif isinstance(recipients, str):
-        return [recipients], None  # Convert single email to list
-    return recipients, None
-
-def _send_email_with_smtp(msg, filename, recipients):
-    """Helper function to handle SMTP connection and sending."""
-    try:
-        # First try to resolve the hostname
-        socket.gethostbyname(settings.email_host)
-        
-        # Connect to the SMTP server
-        with smtplib.SMTP(settings.email_host, settings.email_port, timeout=30) as server:
-            # Use TLS if specified
-            if settings.email_use_tls:
-                server.starttls()
-            
-            # Login if credentials are provided
-            if settings.email_username and settings.email_password:
-                server.login(settings.email_username, settings.email_password)
-            
-            # Send the email
-            server.send_message(msg)
-
-        logger.info(f"Successfully sent {filename} via email to {', '.join(recipients)}")
-        return None
-    except socket.gaierror as e:
-        error_msg = f"Failed to resolve email host: {settings.email_host} - {str(e)}"
-        logger.error(error_msg)
-        return {"status": "Failed", "reason": error_msg, "error": str(e)}
-    except (ConnectionRefusedError, TimeoutError) as e:
-        error_msg = f"Connection error to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
-        logger.error(error_msg)
-        return {"status": "Failed", "reason": error_msg, "error": str(e)}
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_email(file_path: str, recipients=None, subject=None, message=None, template_name="default.html", include_metadata=True):
-    """
-    Sends a file via email to the specified recipients.
-    If recipients is None, uses the configured default email recipient.
-    """
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
-    filename = os.path.basename(file_path)
-    
-    # Check if email settings are configured
-    if not settings.email_host:
-        error_msg = "Email host is not configured"
-        logger.error(error_msg)
-        return {"status": "Skipped", "reason": error_msg}
-        
-    # Log email configuration for debugging
-    logger.debug(f"Email config - Host: {settings.email_host}, Port: {settings.email_port}, " 
-                f"Username: {settings.email_username}, TLS: {settings.email_use_tls}")
-
-    # Process recipients
-    recipients, error = _prepare_recipients(recipients)
-    if error:
-        return {"status": "Skipped", "reason": error}
-
-    # Use provided subject or create default
-    subject = subject or f"DocuElevate Document: {filename}"
-
-    # Extract document metadata if available
-    metadata = {}
-    if include_metadata:
-        metadata = extract_metadata_from_file(file_path)
-
-    try:
-        # Create the email
-        msg = MIMEMultipart('related')
-        msg['From'] = settings.email_sender or settings.email_username
-        msg['To'] = ", ".join(recipients)
-        msg['Subject'] = subject
-
-        # Create alternative part for HTML content
-        alt_part = MIMEMultipart('alternative')
-        msg.attach(alt_part)
-        
-        # Attach logo to the email
-        has_logo = attach_logo(msg)
-        
-        # Load and render template
-        template = get_email_template(template_name)
-        
-        # Context data for the template
-        context = {
-            "filename": filename,
-            "message": message or f"Attached is the document: {filename}",
-            "app_name": "DocuElevate",
-            "app_url": f"https://{settings.external_hostname}" if settings.external_hostname else None,
-            "custom_message": message,
-            "metadata": metadata,
-            "has_metadata": bool(metadata),
-            "has_logo": has_logo,
-            "current_year": datetime.now().year
-        }
-        
-        # Render HTML body
-        html_content = template.render(**context)
-        alt_part.attach(MIMEText(html_content, 'html'))
-
-        # Attach the file
-        with open(file_path, "rb") as file:
-            attachment = MIMEApplication(file.read(), _subtype="pdf")
-            attachment.add_header('Content-Disposition', f'attachment; filename="{filename}"')
-            msg.attach(attachment)
-
-        # Send the email through SMTP
-        error_result = _send_email_with_smtp(msg, filename, recipients)
-        if error_result:
-            return error_result
-
-        return {
-            "status": "Completed",
-            "file": file_path,
-            "recipients": recipients,
-            "subject": subject,
-            "metadata_included": bool(metadata),
-            "logo_included": has_logo
-        }
-        
-    except Exception as e:
-        error_msg = f"Failed to send {filename} via email: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_to_ftp.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import ftplib
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-import logging
-
-logger = logging.getLogger(__name__)
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_ftp(file_path: str):
-    """Uploads a file to an FTP server in the configured folder."""
-
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
-    filename = os.path.basename(file_path)
-
-    # Check if FTP settings are configured
-    if not settings.ftp_host:
-        error_msg = "FTP host is not configured"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-
-    try:
-        # First attempt FTPS (FTP with TLS)
-        use_tls = getattr(settings, 'ftp_use_tls', True)  # Default to try TLS
-        allow_plaintext = getattr(settings, 'ftp_allow_plaintext', True)  # Default to allow plaintext fallback
-        
-        if use_tls:
-            try:
-                logger.info(f"Attempting FTPS connection to {settings.ftp_host}")
-                ftp = ftplib.FTP_TLS()
-                ftp.connect(
-                    host=settings.ftp_host,
-                    port=settings.ftp_port or 21
-                )
-                
-                # Login with credentials
-                ftp.login(
-                    user=settings.ftp_username, 
-                    passwd=settings.ftp_password
-                )
-                
-                # Enable data protection - encrypt the data channel
-                ftp.prot_p()
-                logger.info("Successfully established FTPS connection with TLS")
-            except Exception as e:
-                if not allow_plaintext:
-                    error_msg = f"FTPS connection failed and plaintext FTP is forbidden: {str(e)}"
-                    logger.error(error_msg)
-                    raise Exception(error_msg)
-                else:
-                    logger.warning(f"FTPS connection failed, falling back to regular FTP: {str(e)}")
-                    # Fall back to regular FTP
-                    ftp = ftplib.FTP()
-                    ftp.connect(
-                        host=settings.ftp_host,
-                        port=settings.ftp_port or 21
-                    )
-                    
-                    # Login with credentials
-                    ftp.login(
-                        user=settings.ftp_username, 
-                        passwd=settings.ftp_password
-                    )
-        else:
-            # Check if plaintext is allowed when TLS is explicitly disabled
-            if not allow_plaintext:
-                error_msg = "Plaintext FTP is forbidden by configuration"
-                logger.error(error_msg)
-                raise Exception(error_msg)
-            
-            # Directly use regular FTP if TLS is explicitly disabled
-            ftp = ftplib.FTP()
-            ftp.connect(
-                host=settings.ftp_host,
-                port=settings.ftp_port or 21
-            )
-            
-            # Login with credentials
-            ftp.login(
-                user=settings.ftp_username, 
-                passwd=settings.ftp_password
-            )
-        
-        # Change to target directory if specified
-        if settings.ftp_folder:
-            try:
-                # Try to navigate to the directory, create if it doesn't exist
-                ftp_folder = settings.ftp_folder
-                # Remove leading slash if present
-                if ftp_folder.startswith('/'):
-                    ftp_folder = ftp_folder[1:]
-                
-                # Try to change to the directory
-                try:
-                    ftp.cwd(ftp_folder)
-                except ftplib.error_perm:
-                    # Create directory structure if it doesn't exist
-                    folders = ftp_folder.split('/')
-                    current_dir = ''
-                    for folder in folders:
-                        if folder:
-                            current_dir += f"/{folder}"
-                            try:
-                                ftp.cwd(current_dir)
-                            except ftplib.error_perm:
-                                ftp.mkd(current_dir)
-                                ftp.cwd(current_dir)
-            except ftplib.Error as e:
-                error_msg = f"Failed to change/create directory on FTP server: {str(e)}"
-                logger.error(error_msg)
-                raise Exception(error_msg)
-        
-        # Upload the file
-        with open(file_path, 'rb') as file_data:
-            ftp.storbinary(f'STOR {filename}', file_data)
-        
-        # Close FTP connection
-        ftp.quit()
-        
-        logger.info(f"Successfully uploaded {filename} to FTP server at {settings.ftp_host}")
-        return {
-            "status": "Completed", 
-            "file": file_path, 
-            "ftp_host": settings.ftp_host,
-            "ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename,
-            "used_tls": isinstance(ftp, ftplib.FTP_TLS)
-        }
-    
-    except Exception as e:
-        error_msg = f"Failed to upload {filename} to FTP server: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_to_google_drive.py

@@ -1,246 +0,0 @@
-"""
-app/tasks/upload_to_google_drive.py
-"""
-
-import os
-import json
-import logging
-from googleapiclient.discovery import build
-from googleapiclient.http import MediaFileUpload
-from google.oauth2.service_account import Credentials
-from google.oauth2.credentials import Credentials as OAuthCredentials
-from google_auth_oauthlib.flow import Flow
-from google.auth.transport.requests import Request
-from google.auth.exceptions import RefreshError
-
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-def get_drive_service_oauth():
-    """
-    Get Google Drive service using OAuth credentials.
-    Uses saved refresh token to get a new access token.
-    """
-    try:
-        # Check for required OAuth settings
-        if not (settings.google_drive_client_id and 
-                settings.google_drive_client_secret and 
-                settings.google_drive_refresh_token):
-            logger.error("Google Drive OAuth credentials not fully configured")
-            return None
-        
-        # Create credentials object from refresh token
-        credentials = OAuthCredentials(
-            None,  # No access token initially, will be refreshed
-            refresh_token=settings.google_drive_refresh_token,
-            token_uri="https://oauth2.googleapis.com/token",
-            client_id=settings.google_drive_client_id,
-            client_secret=settings.google_drive_client_secret,
-            # Use only drive.file scope
-            scopes=['https://www.googleapis.com/auth/drive.file']
-        )
-        
-        # Refresh the access token
-        credentials.refresh(Request())
-        
-        # Build and return the service
-        service = build('drive', 'v3', credentials=credentials)
-        return service
-        
-    except RefreshError as e:
-        logger.error(f"Failed to refresh Google Drive token: {str(e)}")
-        raise
-    except Exception as e:
-        logger.error(f"Failed to authenticate with Google Drive OAuth: {str(e)}")
-        return None
-
-def get_google_drive_service():
-    """
-    Authenticate with Google Drive API using service account credentials
-    and return an authorized service object.
-    """
-    try:
-        # Check if we should use OAuth instead of service account
-        if getattr(settings, 'google_drive_use_oauth', False):
-            return get_drive_service_oauth()
-            
-        # Load service account credentials from settings
-        if not settings.google_drive_credentials_json:
-            logger.error("Google Drive credentials not configured")
-            return None
-            
-        credentials_dict = json.loads(settings.google_drive_credentials_json)
-        credentials = Credentials.from_service_account_info(
-            credentials_dict, 
-            scopes=['https://www.googleapis.com/auth/drive']
-        )
-        
-        # Delegate to user if specified
-        if settings.google_drive_delegate_to:
-            credentials = credentials.with_subject(settings.google_drive_delegate_to)
-            
-        # Build and return the service
-        service = build('drive', 'v3', credentials=credentials)
-        return service
-    
-    except Exception as e:
-        logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
-        return None
-
-def extract_metadata_from_file(file_path):
-    """
-    Try to extract metadata from a file using several methods:
-    1. Check for a .json metadata file with the same name
-    
-    Returns a dictionary of metadata or empty dict if not found
-    """
-    # Check for separate metadata JSON file
-    metadata_path = os.path.splitext(file_path)[0] + '.json'
-    if os.path.exists(metadata_path):
-        try:
-            with open(metadata_path, 'r', encoding='utf-8') as f:
-                metadata = json.load(f)
-                logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
-                return metadata
-        except Exception as e:
-            logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
-    
-    return {}
-
-def truncate_property_value(key, value, max_bytes=100):
-    """
-    Truncate a property value to ensure the key+value stays under the byte limit.
-    Google Drive has a 124 byte limit for property key-value pairs.
-    We reserve ~20-24 bytes for the key and leave ~100 bytes for the value.
-    """
-    # Convert to string if not already
-    str_value = str(value)
-    
-    # Calculate current size of key and value in bytes
-    key_bytes = len(key.encode('utf-8'))
-    value_bytes = len(str_value.encode('utf-8'))
-    total_bytes = key_bytes + value_bytes
-    
-    # If under limit, return original value
-    if total_bytes <= max_bytes:
-        return str_value
-    
-    # Calculate how many bytes we need to trim from value
-    # Leave a small buffer to be safe
-    bytes_to_trim = total_bytes - max_bytes + 4
-    
-    # Iteratively truncate the string until it's under the byte limit
-    while len(str_value.encode('utf-8')) > value_bytes - bytes_to_trim:
-        str_value = str_value[:-1]
-    
-    # Add ellipsis to indicate truncation
-    if str_value != str(value):
-        str_value = str_value[:-3] + "..."
-        
-    return str_value
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_google_drive(file_path: str, include_metadata=True):
-    """Uploads a file to Google Drive in the configured folder with optional metadata."""
-
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename from path
-    filename = os.path.basename(file_path)
-
-    # Extract metadata if available
-    metadata = {}
-    if include_metadata:
-        metadata = extract_metadata_from_file(file_path)
-
-    try:
-        # Get Google Drive service
-        service = get_google_drive_service()
-        if not service:
-            raise Exception("Failed to initialize Google Drive service")
-
-        # Prepare the file metadata
-        file_metadata = {
-            'name': filename,
-        }
-        
-        # If folder ID is specified, set parent folder
-        if settings.google_drive_folder_id:
-            file_metadata['parents'] = [settings.google_drive_folder_id]
-            
-        # Add custom properties if metadata exists
-        if metadata:
-            # Google Drive properties must be strings and can't be nested objects
-            file_metadata['properties'] = {}
-            
-            # Only add a few important top-level metadata fields as properties
-            # Skip nested objects and long values to avoid the 124-byte limit
-            safe_properties = {}
-            for key, value in metadata.items():
-                # Skip nested structures completely - they'll be in the description
-                if isinstance(value, (dict, list)):
-                    continue
-                
-                # Try to add simple values with truncation if needed
-                try:
-                    truncated_value = truncate_property_value(key, value)
-                    safe_properties[key] = truncated_value
-                except Exception as e:
-                    logger.warning(f"Skipping metadata property {key}: {str(e)}")
-            
-            # Only use the safe properties
-            file_metadata['properties'] = safe_properties
-            
-            # Add minimal appProperties
-            file_metadata['appProperties'] = {
-                'docuelevate': 'true'
-            }
-            
-            # Add metadata to file description for better visibility in Google Drive UI
-            # Description has much higher size limits than properties
-            formatted_json = json.dumps(metadata, indent=2)
-            file_metadata['description'] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
-            
-            logger.debug(f"Adding metadata to Google Drive file: {json.dumps(file_metadata['properties'])}")
-        
-        # Upload file with metadata
-        media = MediaFileUpload(
-            file_path, 
-            mimetype='application/pdf',
-            resumable=True
-        )
-        
-        file = service.files().create(
-            body=file_metadata,
-            media_body=media,
-            fields='id,name,webViewLink,properties,appProperties,description'
-        ).execute()
-        
-        # Log success details
-        file_id = file.get('id')
-        web_view_link = file.get('webViewLink')
-        
-        logger.info(f"Successfully uploaded {filename} to Google Drive with ID: {file_id}")
-        logger.info(f"File accessible at: {web_view_link}")
-        
-        result = {
-            "status": "Completed", 
-            "file_path": file_path,
-            "google_drive_file_id": file_id,
-            "google_drive_web_link": web_view_link
-        }
-        
-        # Add metadata info to result if included
-        if metadata:
-            result["metadata_included"] = True
-            
-        return result
-
-    except Exception as e:
-        error_msg = f"Failed to upload {filename} to Google Drive: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_to_nextcloud.py

@@ -1,128 +1,43 @@
 #!/usr/bin/env python3
 
 import os
-import logging
 import requests
-from requests.auth import HTTPBasicAuth
 from app.config import settings
-from app.celery_app import celery
 from app.tasks.retry_config import BaseTaskWithRetry
-from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
-
-logger = logging.getLogger(__name__)
+from app.celery_app import celery
+from app.utils import task_logger, log_task
 
 @celery.task(base=BaseTaskWithRetry)
+@log_task("upload_to_nextcloud")
 def upload_to_nextcloud(file_path: str):
-    """
-    Upload a file to Nextcloud WebDAV.
-    """
+    """Uploads a file to Nextcloud in the configured folder."""
+
     if not os.path.exists(file_path):
-        error_msg = f"File not found: {file_path}"
-        logger.error(error_msg)
-        raise FileNotFoundError(error_msg)
-    
-    # For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
-    # This is what's shown in your env view
-    if not (getattr(settings, 'nextcloud_upload_url', None) and 
-            getattr(settings, 'nextcloud_username', None) and 
-            getattr(settings, 'nextcloud_password', None)):
-        logger.info("Nextcloud upload skipped: Missing configuration")
-        return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
-    
+        task_logger(f"File not found: {file_path}", level="error", step_name="nextcloud_upload")
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
     filename = os.path.basename(file_path)
-    sanitized_filename = sanitize_filename(filename)
+
+    # Construct the full upload URL
+    nextcloud_url = f"{settings.nextcloud_upload_url}/{settings.nextcloud_folder}/{filename}"
     
-    try:
-        # Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
-        webdav_url = settings.nextcloud_upload_url
-        if not webdav_url.endswith('/'):
-            webdav_url += '/'
-            
-        # Calculate remote path based on local file structure
-        remote_base = getattr(settings, 'nextcloud_folder', '') or ""
-        remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
-        full_url = f"{webdav_url}/{remote_path}"
-        
-        # Remove any double slashes (except in http://)
-        full_url = full_url.replace('://', '$PLACEHOLDER$')
-        while '//' in full_url:
-            full_url = full_url.replace('//', '/')
-        full_url = full_url.replace('$PLACEHOLDER$', '://')
-        
-        # Function to check if file exists in Nextcloud
-        def check_exists_in_nextcloud(path):
-            check_url = f"{webdav_url}{os.path.dirname(path)}"
-            try:
-                response = requests.request(
-                    'PROPFIND',
-                    check_url,
-                    auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
-                    headers={'Depth': '1'},
-                    timeout=10
-                )
-                
-                return path in response.text
-            except Exception:
-                # If we can't check, assume it doesn't exist
-                return False
-        
-        # Check for potential file collision and get a unique name if needed
-        remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
-        full_url = f"{webdav_url}/{remote_path}"
-        
-        # Fix double slashes again
-        full_url = full_url.replace('://', '$PLACEHOLDER$')
-        while '//' in full_url:
-            full_url = full_url.replace('//', '/')
-        full_url = full_url.replace('$PLACEHOLDER$', '://')
-        
-        # Create necessary parent folders
-        parent_dirs = os.path.dirname(remote_path)
-        if parent_dirs:
-            current_path = ""
-            for folder in parent_dirs.split('/'):
-                if not folder:
-                    continue
-                current_path += f"{folder}/"
-                mkdir_url = f"{webdav_url}/{current_path}"
-                # Fix double slashes
-                mkdir_url = mkdir_url.replace('://', '$PLACEHOLDER$')
-                while '//' in mkdir_url:
-                    mkdir_url = mkdir_url.replace('//', '/')
-                mkdir_url = mkdir_url.replace('$PLACEHOLDER$', '://')
-                
-                requests.request(
-                    'MKCOL',
-                    mkdir_url,
-                    auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
-                    timeout=10
-                )
-        
-        # Upload the file
-        logger.info(f"Uploading {filename} to Nextcloud at {full_url}")
-        with open(file_path, 'rb') as file_data:
-            response = requests.put(
-                full_url,
-                data=file_data,
-                auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
-                headers={'Content-Type': 'application/octet-stream'},
-                timeout=60  # Longer timeout for larger files
-            )
-        
-        if response.status_code in (201, 204):  # Created or No Content
-            logger.info(f"Successfully uploaded {filename} to Nextcloud at {remote_path}")
-            return {
-                "status": "Completed",
-                "file_path": file_path,
-                "nextcloud_path": remote_path,
-                "response_code": response.status_code
-            }
-        else:
-            error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
-            logger.error(error_msg)
-            raise Exception(error_msg)
-    
-    except Exception as e:
-        error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {str(e)}"
-        logger.error(error_msg)
+    task_logger(f"Starting upload of {filename} to Nextcloud", step_name="nextcloud_upload")
+
+    # Read file content
+    with open(file_path, "rb") as file_data:
+        response = requests.put(
+            nextcloud_url,
+            auth=(settings.nextcloud_username, settings.nextcloud_password),
+            data=file_data
+        )
+
+    # Check if upload was successful
+    if response.status_code in (200, 201):
+        task_logger(f"Successfully uploaded {filename} to Nextcloud at {nextcloud_url}", 
+                   step_name="nextcloud_upload", status="success")
+        return {"status": "Completed", "file": file_path}
+    else:
+        error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
+        task_logger(error_msg, level="error", step_name="nextcloud_upload", status="failure")
         raise Exception(error_msg)

app/tasks/upload_to_onedrive.py

@@ -1,259 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import time
-import logging
-import requests
-import msal
-import urllib.parse
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-def get_onedrive_token():
-    """
-    Get an access token for Microsoft Graph API using the appropriate flow.
-    For personal accounts, uses refresh token flow.
-    For organizational accounts, uses client credentials flow if refresh token isn't provided.
-    """
-    # Check for required settings
-    if not settings.onedrive_client_id or not settings.onedrive_client_secret:
-        raise ValueError("OneDrive client ID and client secret must be configured")
-    
-    # Log more details about the configuration
-    tenant = settings.onedrive_tenant_id or "common"
-    logger.info(f"Using OneDrive tenant: {tenant}")
-    
-    # Define scopes consistently
-    scopes = ["https://graph.microsoft.com/.default"]
-    
-    # Use refresh token flow (works for both personal and org accounts)
-    if settings.onedrive_refresh_token:
-        # Use MSAL's ConfidentialClientApplication instead of PublicClientApplication
-        app = msal.ConfidentialClientApplication(
-            client_id=settings.onedrive_client_id,
-            client_credential=settings.onedrive_client_secret,
-            authority=f"https://login.microsoftonline.com/{tenant}"
-        )
-        
-        # Request new token using refresh token
-        logger.info("Attempting to acquire token using refresh token")
-        token_response = app.acquire_token_by_refresh_token(
-            refresh_token=settings.onedrive_refresh_token,
-            scopes=scopes
-        )
-        
-        if "access_token" not in token_response:
-            error = token_response.get("error", "")
-            error_desc = token_response.get("error_description", "Unknown error")
-            
-            # Log more details about the error
-            logger.error(f"Failed to get access token using refresh token")
-            logger.error(f"Error code: {error}")
-            logger.error(f"Error description: {error_desc}")
-            
-            if error == "invalid_grant":
-                logger.error("The refresh token appears to be expired or revoked")
-                logger.error("A new authorization flow is required to obtain a fresh token")
-                
-            raise ValueError(f"Failed to get access token: {error} - {error_desc}")
-        
-        # Check if we received a new refresh token and update it
-        if "refresh_token" in token_response:
-            new_refresh_token = token_response["refresh_token"]
-            logger.info("Received new refresh token from Microsoft")
-            
-            # Update the refresh token in memory
-            settings.onedrive_refresh_token = new_refresh_token
-            logger.info("Updated refresh token in memory")
-            
-        return token_response["access_token"]
-    
-    # No refresh token - try client credentials (only works for org accounts)
-    elif settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common":
-        authority = f"https://login.microsoftonline.com/{settings.onedrive_tenant_id}"
-        app = msal.ConfidentialClientApplication(
-            client_id=settings.onedrive_client_id,
-            client_credential=settings.onedrive_client_secret,
-            authority=authority
-        )
-        
-        # Acquire token for application
-        token_response = app.acquire_token_for_client(
-            scopes=scopes
-        )
-        
-        if "access_token" not in token_response:
-            error = token_response.get("error", "")
-            error_desc = token_response.get("error_description", "Unknown error")
-            raise ValueError(f"Failed to get access token: {error} - {error_desc}")
-            
-        return token_response["access_token"]
-    
-    else:
-        raise ValueError("For personal Microsoft accounts, ONEDRIVE_REFRESH_TOKEN must be configured")
-
-def create_upload_session(filename, folder_path, access_token):
-    """Creates an upload session for large files in Microsoft Graph API."""
-    # Construct the API endpoint
-    base_url = "https://graph.microsoft.com/v1.0/me/drive"
-    
-    # Format the folder path correctly and properly encode for URL
-    if folder_path:
-        # Remove leading/trailing slashes
-        folder_path = folder_path.strip('/')
-        
-        # URL encode the path components separately
-        path_components = folder_path.split('/')
-        encoded_path = '/'.join(urllib.parse.quote(component) for component in path_components)
-        
-        # Also encode the filename
-        encoded_filename = urllib.parse.quote(filename)
-        item_path = f"/root:/{encoded_path}/{encoded_filename}:/createUploadSession"
-    else:
-        # Just encode the filename
-        encoded_filename = urllib.parse.quote(filename)
-        item_path = f"/root:/{encoded_filename}:/createUploadSession"
-    
-    url = f"{base_url}{item_path}"
-    
-    # Add required request body (can be empty JSON object)
-    request_body = {
-        "item": {
-            "@microsoft.graph.conflictBehavior": "replace"
-        }
-    }
-    
-    headers = {
-        "Authorization": f"Bearer {access_token}",
-        "Content-Type": "application/json"
-    }
-    
-    logger.info(f"Creating upload session for {filename} at path {folder_path}")
-    
-    response = requests.post(url, headers=headers, json=request_body)
-    
-    if response.status_code == 200:
-        upload_url = response.json().get("uploadUrl")
-        logger.info(f"Upload session created successfully for {filename}")
-        return upload_url
-    else:
-        error_msg = f"Failed to create upload session: {response.status_code} - {response.text}"
-        logger.error(error_msg)
-        logger.error(f"Request URL was: {url}")
-        logger.error(f"Request headers: {headers}")
-        logger.error(f"Request body: {request_body}")
-        raise Exception(error_msg)
-
-def upload_large_file(file_path, upload_url):
-    """
-    Upload a large file to OneDrive using the upload session URL.
-    Uses chunked upload for reliability.
-    """
-    # Get file size
-    file_size = os.path.getsize(file_path)
-    
-    # Define chunk size (10 MB)
-    chunk_size = 10 * 1024 * 1024
-    
-    # Open and read file in chunks
-    with open(file_path, 'rb') as f:
-        # Process file in chunks
-        chunk_number = 0
-        while True:
-            chunk = f.read(chunk_size)
-            if not chunk:
-                break
-                
-            # Get the position in the file
-            chunk_start = chunk_number * chunk_size
-            chunk_end = chunk_start + len(chunk) - 1
-            
-            # Prepare content range header
-            content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
-            
-            # Upload chunk
-            headers = {
-                "Content-Length": str(len(chunk)),
-                "Content-Range": content_range
-            }
-            
-            # Try to upload chunk with retries
-            max_retries = 3
-            retry_delay = 2  # seconds
-            
-            for attempt in range(max_retries):
-                try:
-                    response = requests.put(
-                        upload_url,
-                        headers=headers,
-                        data=chunk
-                    )
-                    
-                    # Check if successful
-                    if response.status_code in (201, 202):
-                        # 201 = Created (final chunk), 202 = Accepted (more chunks coming)
-                        break
-                    else:
-                        logger.warning(f"Chunk upload failed (attempt {attempt+1}): {response.status_code}")
-                        if attempt < max_retries - 1:
-                            time.sleep(retry_delay * (attempt + 1))
-                except Exception as e:
-                    logger.warning(f"Chunk upload error (attempt {attempt+1}): {str(e)}")
-                    if attempt < max_retries - 1:
-                        time.sleep(retry_delay * (attempt + 1))
-            
-            if response.status_code not in (201, 202):
-                raise Exception(f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}")
-            
-            # Move to next chunk
-            chunk_number += 1
-            
-    # If we get here, all chunks were uploaded successfully
-    # The last response should contain the file metadata
-    return response.json()
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_onedrive(file_path: str):
-    """Uploads a file to OneDrive in the configured folder."""
-    
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
-    filename = os.path.basename(file_path)
-
-    # Check if OneDrive settings are configured
-    if not settings.onedrive_client_id:
-        error_msg = "OneDrive client ID is not configured"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-
-    try:
-        # Get access token
-        access_token = get_onedrive_token()
-        
-        # Create upload session
-        upload_url = create_upload_session(filename, settings.onedrive_folder_path, access_token)
-        
-        # Upload the file
-        result = upload_large_file(file_path, upload_url)
-        
-        # Log success
-        web_url = result.get("webUrl", "Not available")
-        logger.info(f"Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
-        logger.info(f"File accessible at: {web_url}")
-        
-        return {
-            "status": "Completed",
-            "file_path": file_path,
-            "onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
-            "web_url": web_url
-        }
-        
-    except Exception as e:
-        error_msg = f"Failed to upload {filename} to OneDrive: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_to_paperless.py

@@ -10,6 +10,7 @@ from typing import Dict, Any
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.celery_app import celery
+from app.utils import task_logger, log_task
 
 logger = logging.getLogger(__name__)
 
@@ -46,13 +47,15 @@ def poll_task_for_document_id(task_id: str) -> int:
 
     while attempts < POLL_MAX_ATTEMPTS:
         try:
+            task_logger(f"Polling Paperless for task {task_id}, attempt {attempts+1}/{POLL_MAX_ATTEMPTS}", 
+                      step_name="paperless_poll")
             resp = requests.get(url, headers=_get_headers(), params={"task_id": task_id})
             resp.raise_for_status()
             tasks_data = resp.json()
         except requests.exceptions.RequestException as exc:
-            logger.warning(
-                "Failed to poll for task_id='%s'. Attempt=%d Error=%s",
-                task_id, attempts + 1, exc
+            task_logger(
+                f"Failed to poll for task_id='{task_id}'. Attempt={attempts + 1}/{POLL_MAX_ATTEMPTS} Error={exc}",
+                level="warning", step_name="paperless_poll"
             )
             time.sleep(POLL_INTERVAL_SEC)
             attempts += 1
@@ -67,61 +70,70 @@ def poll_task_for_document_id(task_id: str) -> int:
             if status == "SUCCESS":
                 doc_str = task_info.get("related_document")
                 if doc_str:
+                    task_logger(f"Task {task_id} completed successfully with document ID: {doc_str}", 
+                              step_name="paperless_poll", status="success")
                     return int(doc_str)
                 raise RuntimeError(
                     f"Task {task_id} completed but no doc ID found. Task info: {task_info}"
                 )
             elif status == "FAILURE":
-                raise RuntimeError(f"Task {task_id} failed: {task_info.get('result')}")
+                error_msg = f"Task {task_id} failed: {task_info.get('result')}"
+                task_logger(error_msg, level="error", step_name="paperless_poll", status="failure")
+                raise RuntimeError(error_msg)
+            else:
+                task_logger(f"Task {task_id} status: {status}, waiting {POLL_INTERVAL_SEC}s", 
+                          step_name="paperless_poll")
 
         attempts += 1
         time.sleep(POLL_INTERVAL_SEC)
 
-    raise TimeoutError(
-        f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts."
-    )
+    timeout_msg = f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts."
+    task_logger(timeout_msg, level="error", step_name="paperless_poll", status="failure")
+    raise TimeoutError(timeout_msg)
 
 @celery.task(base=BaseTaskWithRetry)
-def upload_to_paperless(file_path: str):
-    """Uploads a file to Paperless-ngx."""
+@log_task("upload_to_paperless")
+def upload_to_paperless(file_path: str) -> Dict[str, Any]:
+    """
+    Uploads a PDF to Paperless with minimal metadata (filename and date only).
+    
+    1. Extracts the filename and date from the file.
+    2. POSTs the PDF to Paperless => returns a quoted UUID string (task_id).
+    3. Polls /api/tasks/?task_id=<uuid> until SUCCESS or FAILURE => doc_id.
 
+    Returns a dict with status, the paperless_task_id, paperless_document_id, and file_path.
+    """
     if not os.path.exists(file_path):
+        task_logger(f"File not found: {file_path}", level="error", step_name="paperless_upload")
         raise FileNotFoundError(f"File not found: {file_path}")
 
-    # Extract filename
-    filename = os.path.basename(file_path)
-
-    # Check if Paperless settings are configured
-    if not settings.paperless_host or not settings.paperless_ngx_api_token:
-        error_msg = "Paperless-ngx credentials are not fully configured"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
+    base_name = os.path.basename(file_path)
+    task_logger(f"Starting upload of {base_name} to Paperless", step_name="paperless_upload")
 
     # Upload the PDF
     post_url = _paperless_api_url("/api/documents/post_document/")
     with open(file_path, "rb") as f:
         files = {
-            "document": (filename, f, "application/pdf"),
+            "document": (base_name, f, "application/pdf"),
         }
-        data = {"title": filename}  # Title = Filename (no additional metadata)
+        data = {"title": base_name}  # Title = Filename (no additional metadata)
 
         try:
-            logger.debug("Posting document to Paperless: file=%s", filename)
+            task_logger(f"Posting document to Paperless: file={base_name}", step_name="paperless_upload")
             resp = requests.post(post_url, headers=_get_headers(), files=files, data=data)
             resp.raise_for_status()
         except requests.exceptions.RequestException as exc:
-            logger.error(
-                "Failed to upload document '%s' to Paperless. Error: %s. Response=%s",
-                file_path, exc, getattr(exc.response, "text", "<no response>")
-            )
+            error_msg = f"Failed to upload document '{file_path}' to Paperless. Error: {exc}. Response={getattr(exc.response, 'text', '<no response>')}"
+            task_logger(error_msg, level="error", step_name="paperless_upload", status="failure")
             raise
 
         raw_task_id = resp.text.strip().strip('"').strip("'")
-        logger.info(f"Received Paperless task ID: {raw_task_id}")
+        task_logger(f"Received Paperless task ID: {raw_task_id}", step_name="paperless_upload")
 
     # Poll tasks until success/fail => get doc_id
     doc_id = poll_task_for_document_id(raw_task_id)
-    logger.info(f"Document {file_path} successfully ingested => ID={doc_id}")
+    task_logger(f"Document {file_path} successfully ingested => ID={doc_id}", 
+              step_name="paperless_upload", status="success")
 
     return {
         "status": "Completed",

app/tasks/upload_to_s3.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import logging
-import boto3
-from botocore.exceptions import ClientError
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_s3(file_path: str):
-    """Uploads a file to Amazon S3 in the configured bucket and folder."""
-
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
-    filename = os.path.basename(file_path)
-
-    # Check if S3 settings are configured
-    if not settings.s3_bucket_name:
-        error_msg = "S3 bucket name is not configured"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-
-    if not settings.aws_access_key_id or not settings.aws_secret_access_key:
-        error_msg = "AWS credentials are not configured"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-
-    try:
-        # Create S3 client
-        s3_client = boto3.client(
-            's3',
-            region_name=settings.aws_region,
-            aws_access_key_id=settings.aws_access_key_id,
-            aws_secret_access_key=settings.aws_secret_access_key
-        )
-        
-        # Construct the S3 key (path within the bucket)
-        if settings.s3_folder_prefix:
-            # Ensure folder prefix ends with a slash
-            folder_prefix = settings.s3_folder_prefix
-            if not folder_prefix.endswith('/'):
-                folder_prefix += '/'
-            s3_key = f"{folder_prefix}{filename}"
-        else:
-            s3_key = filename
-        
-        # Prepare extra arguments
-        extra_args = {
-            'StorageClass': settings.s3_storage_class
-        }
-        
-        # Add ACL if configured
-        if settings.s3_acl:
-            extra_args['ACL'] = settings.s3_acl
-        
-        # Upload file
-        s3_client.upload_file(
-            file_path, 
-            settings.s3_bucket_name, 
-            s3_key,
-            ExtraArgs=extra_args
-        )
-        
-        # Generate URL to the file (useful for public files)
-        # For private files, this is just a reference and won't be accessible directly
-        s3_url = f"https://{settings.s3_bucket_name}.s3.{settings.aws_region}.amazonaws.com/{s3_key}"
-        
-        logger.info(f"Successfully uploaded {filename} to S3 bucket {settings.s3_bucket_name} at path {s3_key}")
-        return {
-            "status": "Completed",
-            "file": file_path,
-            "s3_bucket": settings.s3_bucket_name,
-            "s3_key": s3_key,
-            "s3_url": s3_url
-        }
-    
-    except ClientError as e:
-        error_msg = f"Failed to upload {filename} to S3: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)
-    
-    except Exception as e:
-        error_msg = f"Error uploading {filename} to S3: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_to_sftp.py

@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import logging
-import paramiko
-from pathlib import Path
-from app.config import settings
-from app.celery_app import celery
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
-
-logger = logging.getLogger(__name__)
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_sftp(file_path: str):
-    """
-    Upload a file to an SFTP server.
-    """
-    if not os.path.exists(file_path):
-        error_msg = f"File not found: {file_path}"
-        logger.error(error_msg)
-        raise FileNotFoundError(error_msg)
-    
-    if not (settings.sftp_host and settings.sftp_port and settings.sftp_username):
-        logger.info("SFTP upload skipped: Missing configuration")
-        return {"status": "Skipped", "reason": "SFTP settings not configured"}
-    
-    filename = os.path.basename(file_path)
-    sanitized_filename = sanitize_filename(filename)
-    
-    # SSH client for SFTP connection
-    ssh = paramiko.SSHClient()
-    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    
-    try:
-        # Setup connection parameters
-        connect_kwargs = {
-            "hostname": settings.sftp_host,
-            "port": settings.sftp_port,
-            "username": settings.sftp_username,
-        }
-        
-        # Check for authentication methods - use key if available, otherwise try password
-        sftp_key_path = getattr(settings, 'sftp_private_key', None)
-        sftp_key_passphrase = getattr(settings, 'sftp_private_key_passphrase', None)
-        
-        if sftp_key_path and os.path.exists(sftp_key_path):
-            logger.info(f"Using SSH key authentication with key: {sftp_key_path}")
-            connect_kwargs["key_filename"] = sftp_key_path
-            if sftp_key_passphrase:
-                connect_kwargs["passphrase"] = sftp_key_passphrase
-        elif settings.sftp_password:
-            logger.info("Using password authentication for SFTP")
-            connect_kwargs["password"] = settings.sftp_password
-        else:
-            error_msg = "No authentication method available for SFTP (no key or password)"
-            logger.error(error_msg)
-            raise Exception(error_msg)
-        
-        # Connect to the server
-        logger.info(f"Connecting to SFTP server at {settings.sftp_host}:{settings.sftp_port}")
-        ssh.connect(**connect_kwargs)
-        
-        # Open SFTP session
-        sftp = ssh.open_sftp()
-        
-        # Calculate remote path based on local file structure
-        remote_base = settings.sftp_folder or ""
-        remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
-        
-        # Ensure the remote path starts with a slash if the base folder does
-        if remote_base.startswith('/') and not remote_path.startswith('/'):
-            remote_path = '/' + remote_path
-        
-        # Function to check if file exists in SFTP server
-        def check_exists_in_sftp(path):
-            try:
-                sftp.stat(path)
-                return True
-            except FileNotFoundError:
-                return False
-        
-        # Check for potential file collision and get a unique name if needed
-        remote_path = get_unique_filename(remote_path, check_exists_in_sftp)
-        
-        # Create parent directories if needed
-        remote_dir = os.path.dirname(remote_path)
-        if remote_dir:
-            try:
-                # Try to create the full directory path
-                current_dir = ""
-                for dir_part in remote_dir.split("/"):
-                    if not dir_part:
-                        continue
-                    current_dir += f"/{dir_part}"
-                    try:
-                        sftp.stat(current_dir)
-                    except FileNotFoundError:
-                        logger.info(f"Creating directory on SFTP server: {current_dir}")
-                        sftp.mkdir(current_dir)
-            except Exception as e:
-                logger.warning(f"Failed to create directory structure {remote_dir}: {str(e)}")
-        
-        # Upload the file
-        logger.info(f"Uploading {filename} to SFTP at {remote_path}")
-        sftp.put(file_path, remote_path)
-        logger.info(f"Successfully uploaded {filename} to SFTP at {remote_path}")
-        
-        # Close connections
-        sftp.close()
-        ssh.close()
-        
-        return {
-            "status": "Completed",
-            "file_path": file_path,
-            "sftp_path": remote_path
-        }
-    
-    except Exception as e:
-        # Make sure connections are closed
-        try:
-            if 'sftp' in locals():
-                sftp.close()
-            ssh.close()
-        except:
-            pass
-            
-        error_msg = f"Failed to upload {filename} to SFTP server: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_to_webdav.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import requests
-from urllib.parse import urljoin
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-import logging
-
-logger = logging.getLogger(__name__)
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_to_webdav(file_path: str):
-    """Uploads a file to a WebDAV server in the configured folder."""
-
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
-    filename = os.path.basename(file_path)
-
-    # Check if WebDAV settings are configured
-    if not settings.webdav_url:
-        error_msg = "WebDAV URL is not configured"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-
-    # Construct the full upload URL
-    webdav_folder = settings.webdav_folder or ""
-    # Ensure folder doesn't have leading slash if we're joining it to the base URL
-    if webdav_folder and webdav_folder.startswith("/"):
-        webdav_folder = webdav_folder[1:]
-    
-    # Join the base URL and folder path
-    target_url = urljoin(settings.webdav_url, webdav_folder)
-    # Ensure URL ends with a slash for proper joining with filename
-    if not target_url.endswith("/"):
-        target_url += "/"
-    
-    # Construct final URL with filename
-    webdav_url = urljoin(target_url, filename)
-
-    # Read file content
-    try:
-        with open(file_path, "rb") as file_data:
-            response = requests.put(
-                webdav_url,
-                auth=(settings.webdav_username, settings.webdav_password),
-                data=file_data,
-                verify=settings.webdav_verify_ssl if hasattr(settings, "webdav_verify_ssl") else True
-            )
-
-        # Check if upload was successful
-        if response.status_code in (200, 201, 204):
-            logger.info(f"Successfully uploaded {filename} to WebDAV at {webdav_url}.")
-            return {"status": "Completed", "file": file_path, "url": webdav_url}
-        else:
-            error_msg = f"Failed to upload {filename} to WebDAV: {response.status_code} - {response.text}"
-            logger.error(error_msg)
-            raise Exception(error_msg)
-    
-    except Exception as e:
-        error_msg = f"Error uploading {filename} to WebDAV: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/upload_with_rclone.py

@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import subprocess
-import json
-import tempfile
-import logging
-from pathlib import Path
-from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
-from app.celery_app import celery
-
-logger = logging.getLogger(__name__)
-
-@celery.task(base=BaseTaskWithRetry)
-def upload_with_rclone(file_path: str, destination: str):
-    """
-    Uploads a file using rclone to the specified destination.
-    
-    Args:
-        file_path: Path to the file to upload
-        destination: Rclone destination in format "remote:path/to/folder"
-                     e.g. "gdrive:Uploads" or "dropbox:Documents/Uploads"
-    """
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-    
-    # Extract filename
-    filename = os.path.basename(file_path)
-    
-    # Check if rclone is installed and config exists
-    rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
-    if not os.path.exists(rclone_config_path):
-        error_msg = f"Rclone configuration not found at {rclone_config_path}"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-    
-    try:
-        # Split destination into remote and path
-        if ":" not in destination:
-            raise ValueError(f"Invalid destination format: {destination}. Expected format: remote:path")
-        
-        remote, remote_path = destination.split(":", 1)
-        
-        # Ensure the remote path exists (create folders if needed)
-        mkdir_cmd = [
-            "rclone", 
-            "mkdir", 
-            "--config", rclone_config_path,
-            destination
-        ]
-        
-        subprocess.run(mkdir_cmd, check=True, capture_output=True)
-        
-        # Construct the upload command
-        upload_cmd = [
-            "rclone", 
-            "copy",
-            "--config", rclone_config_path,
-            file_path,
-            destination,
-            "--progress"
-        ]
-        
-        # Execute the upload command
-        result = subprocess.run(upload_cmd, check=True, capture_output=True, text=True)
-        
-        # Check if upload was successful
-        if result.returncode == 0:
-            # Try to get a public link if possible
-            try:
-                link_cmd = [
-                    "rclone",
-                    "link",
-                    "--config", rclone_config_path,
-                    f"{destination}/{filename}"
-                ]
-                link_result = subprocess.run(link_cmd, capture_output=True, text=True)
-                public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
-            except Exception:
-                public_url = None
-            
-            logger.info(f"Successfully uploaded {filename} to {destination}")
-            return {
-                "status": "Completed",
-                "file": file_path,
-                "destination": destination,
-                "public_url": public_url
-            }
-        else:
-            error_msg = f"Failed to upload {filename} to {destination}: {result.stderr}"
-            logger.error(error_msg)
-            raise Exception(error_msg)
-            
-    except subprocess.CalledProcessError as e:
-        error_msg = f"Rclone error: {e.stderr.decode('utf-8') if hasattr(e.stderr, 'decode') else e.stderr}"
-        logger.error(error_msg)
-        raise Exception(error_msg)
-    
-    except Exception as e:
-        error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)
-
-
-@celery.task(base=BaseTaskWithRetry)
-def send_to_all_rclone_destinations(file_path: str):
-    """
-    Uploads a file to all configured rclone destinations.
-    Destinations are loaded from the rclone configuration file.
-    """
-    if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
-    filename = os.path.basename(file_path)
-    
-    # Path to rclone config
-    rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
-    if not os.path.exists(rclone_config_path):
-        error_msg = f"Rclone configuration not found at {rclone_config_path}"
-        logger.error(error_msg)
-        raise ValueError(error_msg)
-    
-    # Get list of configured destinations from rclone
-    try:
-        remotes_cmd = ["rclone", "listremotes", "--config", rclone_config_path]
-        result = subprocess.run(remotes_cmd, check=True, capture_output=True, text=True)
-        
-        if result.returncode == 0:
-            # Process the list of remotes
-            remotes = [r.strip() for r in result.stdout.splitlines() if r.strip()]
-            
-            # Target directories for each remote (from settings)
-            remote_paths = {}
-            for remote in remotes:
-                remote_name = remote.rstrip(':')
-                path_setting_name = f"rclone_{remote_name}_path"
-                if hasattr(settings, path_setting_name) and getattr(settings, path_setting_name):
-                    remote_paths[remote] = getattr(settings, path_setting_name)
-                else:
-                    # Default to root of remote if not specified
-                    remote_paths[remote] = ""
-            
-            # Queue upload tasks for each configured destination
-            results = {}
-            for remote, path in remote_paths.items():
-                full_destination = f"{remote}{path}"
-                if path and not path.endswith('/'):
-                    full_destination += '/'
-                
-                logger.info(f"Queueing {file_path} for upload to {full_destination}")
-                task = upload_with_rclone.delay(file_path, full_destination)
-                results[f"rclone_{remote.rstrip(':')}_task_id"] = task.id
-            
-            return {
-                "status": "Queued",
-                "file_path": file_path,
-                "tasks": results
-            }
-        else:
-            error_msg = f"Failed to list rclone remotes: {result.stderr}"
-            logger.error(error_msg)
-            raise Exception(error_msg)
-            
-    except Exception as e:
-        error_msg = f"Error setting up rclone uploads for {filename}: {str(e)}"
-        logger.error(error_msg)
-        raise Exception(error_msg)

app/tasks/uptime_kuma_tasks.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python3
-
-import logging
-import requests
-from celery import shared_task
-
-from app.config import settings
-
-logger = logging.getLogger(__name__)
-
-@shared_task
-def ping_uptime_kuma():
-    """
-    Periodic Celery task that pings the configured Uptime Kuma URL
-    to report that the document processor service is running.
-    If no URL is configured, the task does nothing.
-    """
-    if not settings.uptime_kuma_url:
-        logger.debug("Uptime Kuma URL not configured, skipping ping")
-        return
-    
-    try:
-        logger.info(f"Pinging Uptime Kuma at {settings.uptime_kuma_url}")
-        response = requests.get(settings.uptime_kuma_url, timeout=10)
-        response.raise_for_status()
-        logger.info(f"Successfully pinged Uptime Kuma: {response.status_code}")
-        return True
-    except requests.exceptions.RequestException as e:
-        logger.error(f"Failed to ping Uptime Kuma: {e}")
-        return False

app/utils.py

@@ -1,7 +1,177 @@
 # app/utils.py
-# This file is deprecated. Functions have been moved to the utils package.
-# To avoid breaking existing imports, we'll import and re-export the functions
-from app.utils.file_operations import hash_file
-from app.utils.logging import log_task_progress
+import hashlib
+import logging
+import contextlib
+from functools import wraps
+from typing import Optional, Callable
+from celery import Task
+from app.database import SessionLocal
+from app.models import ProcessingLog, FileRecord
 
-# These functions are now available directly from the app.utils package
+logger = logging.getLogger(__name__)
+
+def hash_file(filepath, chunk_size=65536):
+    """
+    Returns the SHA-256 hash of the file at 'filepath'.
+    Reads the file in chunks to handle large files efficiently.
+    """
+    sha256 = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        while True:
+            data = f.read(chunk_size)
+            if not data:
+                break
+            sha256.update(data)
+    return sha256.hexdigest()
+
+
+def log_task_progress(task_id: str, step_name: str, status: str, message: Optional[str] = None, 
+                     file_id: Optional[int] = None, file_path: Optional[str] = None):
+    """
+    Logs the progress of a Celery task to the database.
+    
+    Parameters:
+        task_id (str): The Celery task ID
+        step_name (str): Name of the processing step
+        status (str): Status of the step ("pending", "in_progress", "success", "failure")
+        message (str, optional): Additional message or error details
+        file_id (int, optional): ID of associated FileRecord
+        file_path (str, optional): Path to file - will attempt to find file_id from path
+    """
+    try:
+        with SessionLocal() as db:
+            # If file_path is provided but not file_id, try to look up the file_id
+            if not file_id and file_path:
+                file_record = db.query(FileRecord).filter(
+                    FileRecord.local_filename == file_path
+                ).first()
+                if file_record:
+                    file_id = file_record.id
+                    
+            log_entry = ProcessingLog(
+                task_id=task_id,
+                step_name=step_name,
+                status=status,
+                message=message,
+                file_id=file_id,
+            )
+            db.add(log_entry)
+            db.commit()
+            logger.info(f"Task {task_id} - {step_name}: {status} {message or ''}")
+            return log_entry.id
+    except Exception as e:
+        logger.error(f"Failed to log task progress: {e}")
+        return None
+
+
+@contextlib.contextmanager
+def task_step_logging(task_id: str, step_name: str, file_id: Optional[int] = None, file_path: Optional[str] = None):
+    """
+    Context manager for logging the beginning and end of a task step.
+    
+    Example:
+        with task_step_logging(task.request.id, "extract_text", file_path=pdf_path):
+            # Do the actual work
+            text = extract_text_from_pdf(pdf_path)
+    """
+    log_id = log_task_progress(task_id, step_name, "in_progress", 
+                              "Starting processing step", file_id, file_path)
+    try:
+        yield
+        log_task_progress(task_id, step_name, "success", 
+                         "Successfully completed", file_id, file_path)
+    except Exception as e:
+        log_task_progress(task_id, step_name, "failure", 
+                         f"Error: {str(e)}", file_id, file_path)
+        raise  # Re-raise the exception after logging
+
+
+def log_task(step_name: str):
+    """
+    Decorator for Celery tasks to automatically log progress.
+    
+    Example:
+        @celery.task
+        @log_task("process_pdf")
+        def process_pdf(file_path):
+            # Task implementation
+    """
+    def decorator(func: Callable):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            # Get task_id from Celery's current task
+            task = wrapper.request if hasattr(wrapper, 'request') else None
+            task_id = task.id if task else "unknown_task"
+            
+            # Try to determine file_id or file_path from arguments
+            file_path = None
+            if args and isinstance(args[0], str):
+                file_path = args[0]  # Assume first arg is file path
+                
+            # Log start
+            log_task_progress(task_id, step_name, "pending", "Task queued", file_path=file_path)
+            
+            try:
+                # Log in_progress
+                log_task_progress(task_id, step_name, "in_progress", "Task started", file_path=file_path)
+                
+                # Execute the task
+                result = func(*args, **kwargs)
+                
+                # Log success
+                log_task_progress(task_id, step_name, "success", "Task completed", file_path=file_path)
+                
+                return result
+            except Exception as e:
+                # Log failure
+                log_task_progress(task_id, step_name, "failure", f"Error: {str(e)}", file_path=file_path)
+                raise  # Re-raise the exception
+                
+        return wrapper
+    return decorator
+
+
+def task_logger(message: str, level: str = "info", task_id: str = None, step_name: str = None, 
+               status: str = None, file_path: Optional[str] = None, file_id: Optional[int] = None):
+    """
+    Unified logging function that logs to both console and database.
+    This replaces print() statements in tasks with proper logging.
+    
+    Parameters:
+        message: The log message
+        level: Log level (info, error, warning, debug)
+        task_id: Celery task ID (tries to get from current task if None)
+        step_name: Name of the processing step
+        status: Status for database logging (pending, in_progress, success, failure)
+        file_path: Path to the file being processed
+        file_id: ID of the FileRecord
+    
+    Usage:
+        task_logger("Processing file", task_id=task.request.id, step_name="process_pdf")
+        task_logger("Error processing file", level="error") 
+    """
+    # Get task_id from current task if not provided
+    if task_id is None:
+        from celery._state import get_current_task
+        current_task = get_current_task()
+        task_id = current_task.request.id if current_task else "unknown_task"
+    
+    # Default step name if not provided
+    if step_name is None:
+        step_name = "general"
+    
+    # Default status if not provided
+    if status is None:
+        if level == "error":
+            status = "failure"
+        elif level == "warning":
+            status = "warning"
+        else:
+            status = "in_progress"
+    
+    # Log to console
+    log_method = getattr(logger, level.lower(), logger.info)
+    log_method(f"[{step_name}] {message}")
+    
+    # Log to database
+    return log_task_progress(task_id, step_name, status, message, file_id, file_path)

app/utils/__init__.py

@@ -1,10 +0,0 @@
-"""
-Utility functions and helpers for the document processor application.
-"""
-
-# Import functions to make them available through the package
-from app.utils.file_operations import hash_file
-from app.utils.logging import log_task_progress
-
-# Export all the functions that should be available when importing from app.utils
-__all__ = ['hash_file', 'log_task_progress']

app/utils/config_validator.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python3
-"""
-Configuration validation for the application.
-This file serves as a backward-compatible interface to the config_validator package.
-"""
-
-# Import and re-export all functions from the new package
-from app.utils.config_validator.validators import (
-    validate_email_config, 
-    validate_storage_configs, 
-    validate_notification_config,
-    check_all_configs
-)
-from app.utils.config_validator.masking import mask_sensitive_value
-from app.utils.config_validator.providers import get_provider_status
-from app.utils.config_validator.settings_display import (
-    get_settings_for_display,
-    dump_all_settings
-)
-
-__all__ = [
-    'validate_email_config',
-    'validate_storage_configs',
-    'validate_notification_config',
-    'mask_sensitive_value',
-    'get_provider_status',
-    'get_settings_for_display',
-    'dump_all_settings',
-    'check_all_configs'
-]
-
-

app/utils/config_validator/__init__.py

@@ -1,29 +0,0 @@
-"""
-Configuration validation package for the application.
-"""
-
-from app.utils.config_validator.validators import (
-    validate_email_config,
-    validate_storage_configs,
-    validate_notification_config,
-    validate_auth_config,
-    check_all_configs
-)
-from app.utils.config_validator.masking import mask_sensitive_value
-from app.utils.config_validator.providers import get_provider_status
-from app.utils.config_validator.settings_display import (
-    get_settings_for_display,
-    dump_all_settings
-)
-
-__all__ = [
-    'validate_email_config',
-    'validate_storage_configs',
-    'validate_notification_config',
-    'validate_auth_config',
-    'mask_sensitive_value',
-    'get_provider_status',
-    'get_settings_for_display',
-    'dump_all_settings',
-    'check_all_configs'
-]

app/utils/config_validator/masking.py

@@ -1,12 +0,0 @@
-"""
-Module for masking sensitive information in configuration values
-"""
-
-def mask_sensitive_value(value):
-    """
-    Masks sensitive values like API keys in logs and output
-    """
-    # Return masked value for sensitive data
-    if value and isinstance(value, str) and len(value) > 8:
-        return value[:4] + "*" * (len(value) - 4)
-    return value

app/utils/config_validator/providers.py

@@ -1,298 +0,0 @@
-"""
-Module for handling provider status information
-"""
-
-from app.config import settings
-from app.utils.config_validator.masking import mask_sensitive_value
-
-def get_provider_status():
-    """
-    Returns status information for all configured providers
-    """
-    providers = {}
-    
-    # Add Authentication configuration
-    auth_enabled = getattr(settings, 'auth_enabled', False)
-    using_oidc = bool(getattr(settings, 'authentik_client_id', None) and 
-                     getattr(settings, 'authentik_client_secret', None) and
-                     getattr(settings, 'authentik_config_url', None))
-    
-    auth_method = "OIDC" if using_oidc else "Basic Auth" if auth_enabled else "None"
-    
-    providers["Authentication"] = {
-        "name": "Authentication", 
-        "icon": "fa-solid fa-lock",
-        "configured": bool(auth_enabled and 
-                         (getattr(settings, 'admin_username', None) or 
-                          using_oidc)),
-        "enabled": auth_enabled,
-        "description": "Access control and user authentication",
-        "details": {
-            "method": auth_method,
-            "provider_name": getattr(settings, 'oauth_provider_name', 'Not set') if using_oidc else "N/A",
-            "session_security": "Configured" if getattr(settings, 'session_secret', None) else "Not configured"
-        }
-    }
-    
-    # Add Notification configuration - Make sure this provider is near the top of the list
-    providers["Notifications"] = {
-        "name": "Notifications", 
-        "icon": "fa-solid fa-bell",
-        "configured": bool(getattr(settings, 'notification_urls', None)),
-        "enabled": True,
-        "description": "Send system notifications via various services",
-        "details": {
-            "services": str(len(getattr(settings, 'notification_urls', []))) + " service(s) configured" if getattr(settings, 'notification_urls', None) else "Not configured",
-            "task_failure": getattr(settings, 'notify_on_task_failure', True),
-            "credential_failure": getattr(settings, 'notify_on_credential_failure', True),
-            "startup": getattr(settings, 'notify_on_startup', True),
-            "shutdown": getattr(settings, 'notify_on_shutdown', False)
-        },
-        "testable": True,
-        "test_endpoint": "/api/diagnostic/test-notification"
-    }
-    
-    # Add AI services first
-    providers["OpenAI"] = {
-        "name": "OpenAI", 
-        "icon": "fa-brands fa-openai",
-        "configured": bool(getattr(settings, 'openai_api_key', None) and 
-                           str(getattr(settings, 'openai_api_key', '')).startswith('sk-')),
-        "enabled": True,
-        "description": "AI-powered document analysis and metadata extraction",
-        "details": {
-            "api_key": mask_sensitive_value(getattr(settings, 'openai_api_key', None)),
-            "base_url": getattr(settings, 'openai_base_url', 'https://api.openai.com/v1'),
-            "model": getattr(settings, 'openai_model', 'gpt-4')
-        }
-    }
-    
-    providers["Azure AI"] = {
-        "name": "Azure AI", 
-        "icon": "fa-solid fa-robot",
-        "configured": bool(getattr(settings, 'azure_ai_key', None) and 
-                          getattr(settings, 'azure_endpoint', None)),
-        "enabled": True,
-        "description": "Microsoft Azure Document Intelligence",
-        "details": {
-            "api_key": mask_sensitive_value(getattr(settings, 'azure_ai_key', None)),
-            "endpoint": getattr(settings, 'azure_endpoint', 'Not set'),
-            "region": getattr(settings, 'azure_region', 'Not set')
-        }
-    }
-    
-    # Add Dropbox configuration - alphabetically ordered providers
-    providers["Dropbox"] = {
-        "name": "Dropbox", 
-        "icon": "fa-brands fa-dropbox",
-        "configured": bool(getattr(settings, 'dropbox_app_key', None) and
-                          getattr(settings, 'dropbox_app_secret', None) and
-                          getattr(settings, 'dropbox_refresh_token', None)),
-        "enabled": True,
-        "description": "Upload files to Dropbox cloud storage",
-        "details": {
-            "folder": getattr(settings, 'dropbox_folder', 'Not set'),
-            "app_key": getattr(settings, 'dropbox_app_key', 'Not set'),
-            "app_secret": mask_sensitive_value(getattr(settings, 'dropbox_app_secret', None)),
-            "refresh_token": mask_sensitive_value(getattr(settings, 'dropbox_refresh_token', None))
-        }
-    }
-    
-    # Add Email configuration
-    providers["Email"] = {
-        "name": "Email", 
-        "icon": "fa-solid fa-envelope",
-        "configured": bool(getattr(settings, 'email_host', None) and 
-                         getattr(settings, 'email_default_recipient', None)),
-        "enabled": True,
-        "description": "Send documents via email",
-        "details": {
-            "host": getattr(settings, 'email_host', 'Not set'),
-            "port": getattr(settings, 'email_port', 'Not set'),
-            "username": getattr(settings, 'email_username', 'Not set'),
-            "password": mask_sensitive_value(getattr(settings, 'email_password', None)),
-            "use_tls": getattr(settings, 'email_use_tls', 'Not set'),
-            "sender": getattr(settings, 'email_sender', 'Not set'),
-            "default_recipient": getattr(settings, 'email_default_recipient', 'Not set')
-        }
-    }
-    
-    # Add FTP configuration to providers
-    providers["FTP Storage"] = {
-        "name": "FTP Storage", 
-        "icon": "fa-solid fa-server",
-        "configured": bool(getattr(settings, 'ftp_host', None) and
-                          getattr(settings, 'ftp_username', None) and
-                          getattr(settings, 'ftp_password', None)),
-        "enabled": True,
-        "description": "Upload files to FTP server",
-        "details": {
-            "host": getattr(settings, 'ftp_host', 'Not set'),
-            "port": getattr(settings, 'ftp_port', 'Not set'),
-            "username": getattr(settings, 'ftp_username', 'Not set'),
-            "password": mask_sensitive_value(getattr(settings, 'ftp_password', None)),
-            "folder": getattr(settings, 'ftp_folder', 'Not set'),
-            "tls": getattr(settings, 'ftp_use_tls', True),
-            "allow_plaintext": getattr(settings, 'ftp_allow_plaintext', True)
-        }
-    }
-    
-    # Check Google Drive configuration
-    gdrive_oauth_configured = bool(getattr(settings, 'google_drive_client_id', None) and 
-                                  getattr(settings, 'google_drive_client_secret', None) and
-                                  getattr(settings, 'google_drive_refresh_token', None))
-    
-    gdrive_sa_configured = bool(getattr(settings, 'google_drive_credentials_json', None))
-    
-    # Determine if using OAuth or service account
-    use_oauth = getattr(settings, 'google_drive_use_oauth', False)
-    
-    is_configured = (use_oauth and gdrive_oauth_configured) or (not use_oauth and gdrive_sa_configured)
-    
-    providers["Google Drive"] = {
-        "name": "Google Drive", 
-        "icon": "fa-brands fa-google-drive",
-        "configured": is_configured and bool(getattr(settings, 'google_drive_folder_id', None)),
-        "enabled": True,
-        "description": "Store documents in Google Drive",
-        "details": {
-            "auth_type": "OAuth" if use_oauth else "Service Account",
-            "client_id": getattr(settings, 'google_drive_client_id', 'Not set') if use_oauth else 'N/A',
-            "client_secret": mask_sensitive_value(getattr(settings, 'google_drive_client_secret', None)) if use_oauth else 'N/A',
-            "refresh_token": mask_sensitive_value(getattr(settings, 'google_drive_refresh_token', None)) if use_oauth else 'N/A',
-            "credentials_json": mask_sensitive_value(getattr(settings, 'google_drive_credentials_json', None)) if not use_oauth else 'N/A',
-            "folder_id": getattr(settings, 'google_drive_folder_id', 'Not set'),
-            "delegate": getattr(settings, 'google_drive_delegate_to', 'Not set') if not use_oauth else 'N/A'
-        }
-    }
-    
-    # Check NextCloud configuration
-    nextcloud_url = getattr(settings, 'nextcloud_upload_url', 'Not set')
-    # Extract base URL from WebDAV URL (remove the /remote.php part and everything after it)
-    nextcloud_base_url = nextcloud_url
-    if nextcloud_url != 'Not set' and nextcloud_url is not None and '/remote.php' in nextcloud_url:
-        nextcloud_base_url = nextcloud_url.split('/remote.php')[0]
-        
-    providers["NextCloud"] = {
-        "name": "NextCloud", 
-        "icon": "fa-solid fa-cloud",
-        "configured": bool(getattr(settings, 'nextcloud_upload_url', None) and 
-                          getattr(settings, 'nextcloud_username', None) and 
-                          getattr(settings, 'nextcloud_password', None)),
-        "enabled": True,
-        "description": "Store documents in NextCloud",
-        "details": {
-            "url": getattr(settings, 'nextcloud_upload_url', 'Not set'),
-            "base_url": nextcloud_base_url,
-            "username": getattr(settings, 'nextcloud_username', 'Not set'),
-            "password": mask_sensitive_value(getattr(settings, 'nextcloud_password', None)),
-            "folder": getattr(settings, 'nextcloud_folder', 'Not set')
-        }
-    }
-    
-    # Check OneDrive configuration
-    providers["OneDrive"] = {
-        "name": "OneDrive", 
-        "icon": "fa-brands fa-microsoft",
-        "configured": bool(getattr(settings, 'onedrive_client_id', None) and 
-                          getattr(settings, 'onedrive_client_secret', None) and 
-                          getattr(settings, 'onedrive_refresh_token', None)),
-        "enabled": True,
-        "description": "Store documents in Microsoft OneDrive",
-        "details": {
-            "client_id": getattr(settings, 'onedrive_client_id', 'Not set'),
-            "client_secret": mask_sensitive_value(getattr(settings, 'onedrive_client_secret', None)),
-            "tenant_id": getattr(settings, 'onedrive_tenant_id', 'Not set'),
-            "refresh_token": mask_sensitive_value(getattr(settings, 'onedrive_refresh_token', None)),
-            "folder": getattr(settings, 'onedrive_folder_path', 'Not set')
-        }
-    }
-    
-    # Check Paperless configuration
-    providers["Paperless-ngx"] = {
-        "name": "Paperless-ngx", 
-        "icon": "fa-solid fa-file-lines",
-        "configured": bool(getattr(settings, 'paperless_host', None) and 
-                         getattr(settings, 'paperless_ngx_api_token', None)),
-        "enabled": True,
-        "description": "Document management system for digital archives",
-        "details": {
-            "host": getattr(settings, 'paperless_host', 'Not set'),
-            "api_token": mask_sensitive_value(getattr(settings, 'paperless_ngx_api_token', None))
-        }
-    }
-    
-    # Check S3 configuration
-    providers["S3 Storage"] = {
-        "name": "S3 Storage", 
-        "icon": "fa-brands fa-aws",
-        "configured": bool(getattr(settings, 's3_bucket_name', None) and 
-                          getattr(settings, 'aws_access_key_id', None) and 
-                          getattr(settings, 'aws_secret_access_key', None)),
-        "enabled": True,
-        "description": "Store documents in S3-compatible object storage",
-        "details": {
-            "bucket": getattr(settings, 's3_bucket_name', 'Not set'),
-            "region": getattr(settings, 'aws_region', 'Not set'),
-            "access_key_id": getattr(settings, 'aws_access_key_id', 'Not set'),
-            "secret_access_key": mask_sensitive_value(getattr(settings, 'aws_secret_access_key', None)),
-            "folder_prefix": getattr(settings, 's3_folder_prefix', 'Not set'),
-            "storage_class": getattr(settings, 's3_storage_class', 'Not set'),
-            "acl": getattr(settings, 's3_acl', 'Not set')
-        }
-    }
-    
-    # Check SFTP configuration
-    providers["SFTP Storage"] = {
-        "name": "SFTP Storage", 
-        "icon": "fa-solid fa-lock",
-        "configured": bool(getattr(settings, 'sftp_host', None) and 
-                          getattr(settings, 'sftp_username', None) and 
-                          (getattr(settings, 'sftp_password', None) or 
-                           getattr(settings, 'sftp_private_key', None))),
-        "enabled": True,
-        "description": "Upload files to SFTP server",
-        "details": {
-            "host": getattr(settings, 'sftp_host', 'Not set'),
-            "port": getattr(settings, 'sftp_port', 'Not set'),
-            "username": getattr(settings, 'sftp_username', 'Not set'),
-            "password": mask_sensitive_value(getattr(settings, 'sftp_password', None)),
-            "private_key": getattr(settings, 'sftp_private_key', 'Not set'),
-            "private_key_passphrase": mask_sensitive_value(getattr(settings, 'sftp_private_key_passphrase', None)),
-            "folder": getattr(settings, 'sftp_folder', 'Not set')
-        }
-    }
-    
-    # Add Uptime Kuma configuration
-    providers["Uptime Kuma"] = {
-        "name": "Uptime Kuma", 
-        "icon": "fa-solid fa-heart-pulse",
-        "configured": bool(getattr(settings, 'uptime_kuma_url', None)),
-        "enabled": True,
-        "description": "Server monitoring and status page",
-        "details": {
-            "url": getattr(settings, 'uptime_kuma_url', 'Not set'),
-            "ping_interval": getattr(settings, 'uptime_kuma_ping_interval', 'Not set')
-        }
-    }
-    
-    # Check WebDAV configuration
-    providers["WebDAV"] = {
-        "name": "WebDAV", 
-        "icon": "fa-solid fa-globe",
-        "configured": bool(getattr(settings, 'webdav_url', None) and 
-                          getattr(settings, 'webdav_username', None) and 
-                          getattr(settings, 'webdav_password', None)),
-        "enabled": True,
-        "description": "Store documents on WebDAV servers",
-        "details": {
-            "url": getattr(settings, 'webdav_url', 'Not set'),
-            "username": getattr(settings, 'webdav_username', 'Not set'),
-            "password": mask_sensitive_value(getattr(settings, 'webdav_password', None)),
-            "folder": getattr(settings, 'webdav_folder', 'Not set'),
-            "verify_ssl": getattr(settings, 'webdav_verify_ssl', 'Not set')
-        }
-    }
-    
-    
-    return providers

app/utils/config_validator/settings_display.py

@@ -1,265 +0,0 @@
-"""
-Module for displaying and organizing settings information
-"""
-
-import logging
-from app.config import settings
-from app.utils.config_validator.masking import mask_sensitive_value
-
-logger = logging.getLogger(__name__)
-
-def dump_all_settings():
-    """Log all settings values for diagnostic purposes"""
-    logger.info("--- DUMPING ALL SETTINGS FOR DIAGNOSTIC PURPOSES ---")
-    for key in dir(settings):
-        if not key.startswith('_') and not callable(getattr(settings, key)):
-            value = getattr(settings, key)
-            # Mask sensitive values in logs
-            if key.lower().find('password') >= 0 or key.lower().find('secret') >= 0 or key.lower().find('token') >= 0 or key.lower().find('key') >= 0:
-                if value:
-                    if isinstance(value, str) and len(value) > 10:
-                        visible_start = max(1, len(value) // 3)
-                        visible_end = max(1, len(value) // 4)
-                        value = f"{value[:visible_start]}{'*' * (len(value) - visible_start - visible_end)}{value[-visible_end:]}"
-                    else:
-                        value = f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}" if isinstance(value, str) and len(value) > 4 else "****"
-            
-            # Special handling for notification URLs
-            if key == 'notification_urls' and value:
-                try:
-                    from app.utils.notification import _mask_sensitive_url
-                    if isinstance(value, list):
-                        masked_urls = [_mask_sensitive_url(url) for url in value]
-                        logger.info(f"{key}: {masked_urls}")
-                    else:
-                        logger.info(f"{key}: {_mask_sensitive_url(value)}")
-                    continue  # Skip the default logging
-                except (ImportError, AttributeError):
-                    pass  # Fall back to default logging if _mask_sensitive_url is not available
-                    
-            logger.info(f"{key}: {value}")
-    logger.info("--- END OF SETTINGS DUMP ---")
-
-def get_settings_for_display(show_values=False):
-    """
-    Group settings into logical categories and check if they are configured.
-    Returns a dictionary with categories as keys and lists of setting items as values. 
-    Each setting item is a dict with name, value, and is_configured.
-    
-    If show_values is False, sensitive values are masked.
-    """
-    # First include system info with version in result
-    result = {
-        "System Info": [
-            {
-                "name": "App Version", 
-                "value": settings.version,
-                "is_configured": True
-            },
-            {
-                "name": "Build Date", 
-                "value": settings.build_date,
-                "is_configured": True
-            }
-        ]
-    }
-    
-    # Define categories and their settings
-    categories = {
-        "Core": [
-            "debug", # Explicitly include debug setting
-            "external_hostname",
-            "workdir",
-            "database_url",
-            "redis_url",
-            "gotenberg_url",
-            "allow_file_delete"  # Added allow_file_delete to Core settings
-        ],
-        "Authentication": [
-            "auth_enabled",
-            "session_secret",
-            "admin_username",
-            "admin_password",
-            "authentik_client_id",
-            "authentik_client_secret",
-            "authentik_config_url",
-            "oauth_provider_name"
-        ],
-        "Email": [
-            "email_host",
-            "email_port",
-            "email_username",
-            "email_password",
-            "email_use_tls",
-            "email_sender",
-            "email_default_recipient"
-        ],
-        "IMAP": [
-            "imap1_host",
-            "imap1_port",
-            "imap1_username",
-            "imap1_password",
-            "imap1_ssl",
-            "imap1_poll_interval_minutes",
-            "imap1_delete_after_process",
-            "imap2_host",
-            "imap2_port",
-            "imap2_username",
-            "imap2_password",
-            "imap2_ssl",
-            "imap2_poll_interval_minutes",
-            "imap2_delete_after_process"
-        ],
-        "Dropbox": [
-            "dropbox_app_key",
-            "dropbox_app_secret",
-            "dropbox_folder",
-            "dropbox_refresh_token"
-        ],
-        "NextCloud": [
-            "nextcloud_upload_url",
-            "nextcloud_username",
-            "nextcloud_password",
-            "nextcloud_folder"
-        ],
-        "Paperless": [
-            "paperless_host",
-            "paperless_ngx_api_token"
-        ],
-        "Google Drive": [
-            "google_drive_use_oauth",
-            "google_drive_client_id",
-            "google_drive_client_secret", 
-            "google_drive_refresh_token",
-            "google_drive_credentials_json",
-            "google_drive_folder_id",
-            "google_drive_delegate_to"
-        ],
-        "OneDrive": [
-            "onedrive_client_id",
-            "onedrive_client_secret",
-            "onedrive_tenant_id",
-            "onedrive_refresh_token",
-            "onedrive_folder_path"
-        ],
-        "WebDAV": [
-            "webdav_url",
-            "webdav_username",
-            "webdav_password",
-            "webdav_folder",
-            "webdav_verify_ssl"
-        ],
-        "SFTP": [
-            "sftp_host",
-            "sftp_port",
-            "sftp_username",
-            "sftp_password",
-            "sftp_folder",
-            "sftp_private_key",
-            "sftp_private_key_passphrase"
-        ],
-        "FTP": [
-            "ftp_host",
-            "ftp_port",
-            "ftp_username",
-            "ftp_password",
-            "ftp_folder",
-            "ftp_use_tls",
-            "ftp_allow_plaintext"
-        ],
-        "S3/AWS": [
-            "aws_access_key_id",
-            "aws_secret_access_key",
-            "aws_region",
-            "s3_bucket_name",
-            "s3_folder_prefix",
-            "s3_storage_class",
-            "s3_acl"
-        ],
-        "AI Services": [
-            "openai_api_key",
-            "openai_base_url",
-            "openai_model",
-            "azure_ai_key",
-            "azure_endpoint",
-            "azure_region"
-        ],
-        "Monitoring": [
-            "uptime_kuma_url",
-            "uptime_kuma_ping_interval"
-        ],
-        "Notifications": [
-            "notification_urls",
-            "notify_on_task_failure",
-            "notify_on_credential_failure", 
-            "notify_on_startup",
-            "notify_on_shutdown"
-        ]
-    }
-    
-    # Handle any settings that don't fit into the predefined categories
-    all_settings = set([key for key in dir(settings) 
-                        if not key.startswith('_') and 
-                        not callable(getattr(settings, key)) and
-                        key not in ["model_computed_fields", "model_config", 
-                                    "model_extra", "model_fields",
-                                    "model_fields_set"]])
-    
-    # Ensure 'version' is excluded since we display it separately
-    all_settings.discard("version")
-    
-    categorized_settings = set()
-    for cat_settings in categories.values():
-        categorized_settings.update(cat_settings)
-    
-    uncategorized = all_settings - categorized_settings
-    if uncategorized:
-        categories["Other"] = list(uncategorized)
-    
-    # Build the result
-    for category, setting_keys in categories.items():
-        items = []
-        for key in setting_keys:
-            if hasattr(settings, key):
-                value = getattr(settings, key)
-                
-                # List of patterns that indicate sensitive values
-                sensitive_patterns = [
-                    'password', 'secret', 'token', 'api_key', 'private_key',
-                    'credentials', 'access_key', 'ai_key'
-                ]
-                
-                # Check if this is a sensitive value that should be masked
-                is_sensitive = any(
-                    pattern in key.lower() for pattern in sensitive_patterns
-                )
-                
-                # Special handling for "auth" to avoid matching prefixes like "authentik"
-                if not is_sensitive and "auth" in key.lower():
-                    # Only mark as sensitive if "auth" is a standalone word or at the end
-                    # This avoids matching "authentik" as sensitive
-                    parts = key.lower().split('_')
-                    is_sensitive = any(part == "auth" for part in parts) or key.lower().endswith("auth")
-                
-                # Mask sensitive values regardless of debug mode
-                # Other values are only hidden if debug mode is off AND show_values is False
-                if (is_sensitive or not show_values) and value:
-                    if is_sensitive:
-                        value = mask_sensitive_value(value)
-                
-                # Check if the setting is configured (has a non-None value)
-                # For boolean settings, consider them configured even if False
-                is_configured = value is not None
-                if is_configured and isinstance(value, str):
-                    is_configured = len(value) > 0
-                
-                items.append({
-                    "name": key,
-                    "value": value,
-                    "is_configured": is_configured
-                })
-        
-        if items:  # Only add categories that have items
-            result[category] = items
-    
-    return result

app/utils/config_validator/validators.py

@@ -1,243 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import socket
-import logging
-from app.config import settings
-
-logger = logging.getLogger(__name__)
-
-def validate_email_config():
-    """Validates email configuration settings"""
-    issues = []
-    
-    # Check for required email settings
-    if not getattr(settings, 'email_host', None):
-        issues.append("EMAIL_HOST is not configured")
-    if not getattr(settings, 'email_port', None):
-        issues.append("EMAIL_PORT is not configured")
-    
-    # Test SMTP server connectivity if host is configured
-    if getattr(settings, 'email_host', None) and getattr(settings, 'email_port', None):
-        try:
-            # Attempt to resolve the hostname
-            socket.gethostbyname(settings.email_host)
-        except socket.gaierror:
-            issues.append(f"Cannot resolve email host: {settings.email_host}")
-
-    # Check for authentication settings
-    if not getattr(settings, 'email_username', None):
-        issues.append("EMAIL_USERNAME is not configured")
-    if not getattr(settings, 'email_password', None):
-        issues.append("EMAIL_PASSWORD is not configured")
-    
-    return issues
-
-def validate_auth_config():
-    """Validates authentication configuration settings"""
-    issues = []
-    
-    # If auth is enabled, check for required settings
-    if getattr(settings, 'auth_enabled', False):
-        # Check for session secret
-        if not getattr(settings, 'session_secret', None):
-            issues.append("SESSION_SECRET is not configured but AUTH_ENABLED is True")
-        elif len(getattr(settings, 'session_secret', '')) < 32:
-            issues.append("SESSION_SECRET must be at least 32 characters long")
-        
-        # Check if using simple authentication or OIDC
-        using_simple_auth = bool(getattr(settings, 'admin_username', None) and 
-                                getattr(settings, 'admin_password', None))
-        
-        using_oidc = bool(getattr(settings, 'authentik_client_id', None) and 
-                         getattr(settings, 'authentik_client_secret', None) and
-                         getattr(settings, 'authentik_config_url', None))
-        
-        if not using_simple_auth and not using_oidc:
-            issues.append("Neither simple authentication nor OIDC are properly configured")
-        
-        # If using OIDC, check for provider name
-        if using_oidc and not getattr(settings, 'oauth_provider_name', None):
-            issues.append("OAUTH_PROVIDER_NAME is not configured but OIDC is enabled")
-    
-    return issues
-
-def validate_storage_configs():
-    """Validates configuration for all storage providers"""
-    issues = {}
-    
-    # Validate Dropbox config
-    dropbox_issues = []
-    if not (getattr(settings, 'dropbox_app_key', None) and 
-            getattr(settings, 'dropbox_app_secret', None) and 
-            getattr(settings, 'dropbox_refresh_token', None)):
-        dropbox_issues.append("Dropbox credentials are not fully configured")
-    issues['dropbox'] = dropbox_issues
-    
-    # Validate Nextcloud config
-    nextcloud_issues = []
-    if not (getattr(settings, 'nextcloud_upload_url', None) and 
-            getattr(settings, 'nextcloud_username', None) and 
-            getattr(settings, 'nextcloud_password', None)):
-        nextcloud_issues.append("Nextcloud credentials are not fully configured")
-    issues['nextcloud'] = nextcloud_issues
-    
-    # Validate SFTP config
-    sftp_issues = []
-    if not getattr(settings, 'sftp_host', None):
-        sftp_issues.append("SFTP_HOST is not configured")
-    
-    sftp_key_path = getattr(settings, 'sftp_private_key', None)
-    if sftp_key_path and not os.path.exists(sftp_key_path):
-        sftp_issues.append(f"SFTP_KEY_PATH file not found: {sftp_key_path}")
-    
-    if not sftp_key_path and not getattr(settings, 'sftp_password', None):
-        sftp_issues.append("Neither SFTP_KEY_PATH nor SFTP_PASSWORD is configured")
-    
-    issues['sftp'] = sftp_issues
-    
-    # Validate Email sending
-    email_issues = []
-    if not getattr(settings, 'email_host', None):
-        email_issues.append("EMAIL_HOST is not configured")
-    if not getattr(settings, 'email_default_recipient', None):
-        email_issues.append("EMAIL_DEFAULT_RECIPIENT is not configured")
-    issues['email'] = email_issues
-    
-    # Validate S3
-    s3_issues = []
-    if not getattr(settings, 's3_bucket_name', None):
-        s3_issues.append("S3_BUCKET_NAME is not configured")
-    if not (getattr(settings, 'aws_access_key_id', None) and 
-            getattr(settings, 'aws_secret_access_key', None)):
-        s3_issues.append("AWS credentials are not configured")
-    issues['s3'] = s3_issues
-    
-    # Validate FTP
-    ftp_issues = []
-    if not getattr(settings, 'ftp_host', None):
-        ftp_issues.append("FTP_HOST is not configured")
-    if not getattr(settings, 'ftp_username', None):
-        ftp_issues.append("FTP_USERNAME is not configured")
-    if not getattr(settings, 'ftp_password', None):
-        ftp_issues.append("FTP_PASSWORD is not configured")
-    issues['ftp'] = ftp_issues
-    
-    # Validate WebDAV
-    webdav_issues = []
-    if not getattr(settings, 'webdav_url', None):
-        webdav_issues.append("WEBDAV_URL is not configured")
-    if not getattr(settings, 'webdav_username', None):
-        webdav_issues.append("WEBDAV_USERNAME is not configured")
-    if not getattr(settings, 'webdav_password', None):
-        webdav_issues.append("WEBDAV_PASSWORD is not configured")
-    issues['webdav'] = webdav_issues
-    
-    # Validate Google Drive
-    gdrive_issues = []
-    if not getattr(settings, 'google_drive_credentials_json', None):
-        gdrive_issues.append("GOOGLE_DRIVE_CREDENTIALS_JSON is not configured")
-    if not getattr(settings, 'google_drive_folder_id', None):
-        gdrive_issues.append("GOOGLE_DRIVE_FOLDER_ID is not configured")
-    issues['google_drive'] = gdrive_issues
-    
-    # Validate Paperless
-    paperless_issues = []
-    if not getattr(settings, 'paperless_host', None):
-        paperless_issues.append("PAPERLESS_HOST is not configured")
-    if not getattr(settings, 'paperless_ngx_api_token', None):
-        paperless_issues.append("PAPERLESS_NGX_API_TOKEN is not configured")
-    issues['paperless'] = paperless_issues
-    
-    # Validate OneDrive
-    onedrive_issues = []
-    if not (getattr(settings, 'onedrive_client_id', None) and 
-            getattr(settings, 'onedrive_client_secret', None) and
-            getattr(settings, 'onedrive_refresh_token', None)):
-        onedrive_issues.append("OneDrive credentials are not fully configured")
-    issues['onedrive'] = onedrive_issues
-    
-    # Validate Uptime Kuma
-    uptime_kuma_issues = []
-    if not getattr(settings, 'uptime_kuma_url', None):
-        uptime_kuma_issues.append("UPTIME_KUMA_URL is not configured")
-    issues['uptime_kuma'] = uptime_kuma_issues
-    
-    return issues
-
-def validate_notification_config():
-    """Check notification configuration"""
-    issues = []
-    
-    # Check if any notification URLs are configured
-    if not getattr(settings, 'notification_urls', None):
-        issues.append("No notification URLs configured")
-    else:
-        try:
-            # Try initializing Apprise to validate URLs
-            import apprise
-            a = apprise.Apprise()
-            
-            for url in settings.notification_urls:
-                try:
-                    if not a.add(url):
-                        issues.append(f"Invalid notification URL format: {url}")
-                except Exception as e:
-                    issues.append(f"Error with notification URL: {str(e)}")
-                    
-        except ImportError:
-            issues.append("Apprise module not installed")
-    
-    if not issues:
-        logger.info("Notification configuration valid")
-    else:
-        logger.warning(f"Notification configuration issues: {', '.join(issues)}")
-        
-    return issues
-
-def check_all_configs():
-    """Run all configuration validations and log results"""
-    from app.utils.config_validator.settings_display import dump_all_settings
-    
-    logger.info("Validating application configuration...")
-    
-    # Check if debug is enabled and dump all settings if it is
-    if hasattr(settings, 'debug') and settings.debug:
-        dump_all_settings()
-    
-    # Check auth config
-    auth_issues = validate_auth_config()
-    if auth_issues:
-        logger.warning(f"Authentication configuration issues: {', '.join(auth_issues)}")
-    else:
-        logger.info("Authentication configuration OK")
-    
-    # Check email config
-    email_issues = validate_email_config()
-    if email_issues:
-        logger.warning(f"Email configuration issues: {', '.join(email_issues)}")
-    else:
-        logger.info("Email configuration OK")
-    
-    # Check storage configs
-    storage_issues = validate_storage_configs()
-    for provider, issues in storage_issues.items():
-        if issues:
-            logger.warning(f"{provider.capitalize()} configuration issues: {', '.join(issues)}")
-        else:
-            logger.info(f"{provider.capitalize()} configuration OK")
-    
-    # Check notification configuration
-    notification_issues = validate_notification_config()
-    if notification_issues:
-        logger.warning(f"Notification configuration issues: {', '.join(notification_issues)}")
-    else:
-        logger.info("Notification configuration OK")
-    
-    # Return all identified issues
-    return {
-        'auth': auth_issues,
-        'email': email_issues,
-        'storage': storage_issues,
-        'notification': notification_issues
-    }

app/utils/file_operations.py

@@ -1,15 +0,0 @@
-import hashlib
-
-def hash_file(filepath, chunk_size=65536):
-    """
-    Returns the SHA-256 hash of the file at 'filepath'.
-    Reads the file in chunks to handle large files efficiently.
-    """
-    sha256 = hashlib.sha256()
-    with open(filepath, "rb") as f:
-        while True:
-            data = f.read(chunk_size)
-            if not data:
-                break
-            sha256.update(data)
-    return sha256.hexdigest()

app/utils/filename_utils.py

@@ -1,135 +0,0 @@
-import os
-import re
-import uuid
-import logging
-from datetime import datetime
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-def get_unique_filename(original_path, check_exists_func=None):
-    """
-    Generates a unique filename by appending a timestamp or counter when a collision occurs.
-    
-    Args:
-        original_path (str): The original file path
-        check_exists_func (callable): Function that checks if file exists in target system.
-                                     Takes a path string and returns True if exists, False otherwise.
-                                     If None, will use local filesystem check.
-    
-    Returns:
-        str: A unique filename that doesn't collide with existing files
-    """
-    if check_exists_func is None:
-        check_exists_func = os.path.exists
-    
-    path = Path(original_path)
-    directory = str(path.parent)
-    filename = path.name
-    name, ext = os.path.splitext(filename)
-    
-    # If file doesn't exist, return the original
-    if not check_exists_func(original_path):
-        return original_path
-    
-    # Try timestamp-based suffix first (more user-friendly)
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    new_filename = f"{name}_{timestamp}{ext}"
-    new_path = os.path.join(directory, new_filename)
-    
-    if not check_exists_func(new_path):
-        logger.info(f"Renamed '{filename}' to '{new_filename}' to avoid collision")
-        return new_path
-    
-    # If timestamp-based name also exists, try random UUID
-    uuid_str = str(uuid.uuid4())[:8]  # Use first 8 chars of UUID for brevity
-    new_filename = f"{name}_{uuid_str}{ext}"
-    new_path = os.path.join(directory, new_filename)
-    
-    if not check_exists_func(new_path):
-        logger.info(f"Renamed '{filename}' to '{new_filename}' using UUID to avoid collision")
-        return new_path
-    
-    # If that still exists (very unlikely), use incremental numbering
-    counter = 1
-    while counter < 1000:  # Limit to avoid infinite loop
-        new_filename = f"{name}_{counter}{ext}"
-        new_path = os.path.join(directory, new_filename)
-        if not check_exists_func(new_path):
-            logger.info(f"Renamed '{filename}' to '{new_filename}' using counter to avoid collision")
-            return new_path
-        counter += 1
-    
-    # If we got here, something is weird - just use a full UUID
-    new_filename = f"{name}_{str(uuid.uuid4())}{ext}"
-    new_path = os.path.join(directory, new_filename)
-    logger.warning(f"Had to use full UUID to rename '{filename}' to '{new_filename}'")
-    
-    return new_path
-
-def sanitize_filename(filename):
-    """
-    Sanitize a filename to ensure it's valid across different file systems.
-    
-    Args:
-        filename (str): The filename to sanitize
-        
-    Returns:
-        str: A sanitized filename
-    """
-    # Replace characters that are problematic in various filesystems
-    # Keep only alphanumeric, dash, underscore, period, and space
-    sanitized = re.sub(r'[^\w\-\. ]', '_', filename)
-    
-    # Replace multiple spaces/underscores with single ones
-    sanitized = re.sub(r'__+', '_', sanitized)
-    sanitized = re.sub(r'  +', ' ', sanitized)
-    
-    # Trim leading/trailing spaces and periods which cause issues in Windows
-    sanitized = sanitized.strip('. ')
-    
-    # Ensure the filename isn't empty after sanitization
-    if not sanitized or sanitized == '.':
-        sanitized = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-    
-    return sanitized
-
-def extract_remote_path(file_path, base_dir, remote_base=""):
-    """
-    Extract a remote path for a file by preserving its directory structure
-    relative to the base directory, but with a new remote base path.
-    
-    Modified to skip 'processed' directory in the remote path.
-    """
-    # Normalize paths for consistent handling across platforms
-    file_path = os.path.normpath(file_path)
-    base_dir = os.path.normpath(base_dir)
-    
-    # Get relative path from base directory
-    if file_path.startswith(base_dir):
-        rel_path = os.path.relpath(file_path, base_dir)
-    else:
-        # If not a subdirectory of base_dir, just use the filename
-        rel_path = os.path.basename(file_path)
-    
-    # Skip 'processed' directory if it's in the path
-    path_parts = rel_path.split(os.sep)
-    if 'processed' in path_parts:
-        # Remove 'processed' from the path
-        path_parts.remove('processed')
-        rel_path = os.path.join(*path_parts)
-    
-    # Combine with remote base path
-    if remote_base:
-        if remote_base.startswith('/'):
-            # Handle absolute path for services like Dropbox
-            remote_path = os.path.join(remote_base[1:], rel_path)
-        else:
-            remote_path = os.path.join(remote_base, rel_path)
-    else:
-        remote_path = rel_path
-    
-    # Convert to forward slashes for compatibility with most cloud services
-    remote_path = remote_path.replace(os.sep, '/')
-    
-    return remote_path

app/utils/logging.py

@@ -1,17 +0,0 @@
-from app.database import SessionLocal
-from app.models import ProcessingLog
-
-def log_task_progress(task_id, step_name, status, message=None, file_id=None):
-    """
-    Logs the progress of a Celery task to the database.
-    """
-    with SessionLocal() as db:
-        log_entry = ProcessingLog(
-            task_id=task_id,
-            step_name=step_name,
-            status=status,
-            message=message,
-            file_id=file_id,
-        )
-        db.add(log_entry)
-        db.commit()

app/utils/notification.py

@@ -1,185 +0,0 @@
-import apprise
-import logging
-from typing import List, Optional, Dict, Any, Union
-
-from app.config import settings
-
-logger = logging.getLogger(__name__)
-
-# Global Apprise instance
-_apprise = None
-
-def init_apprise() -> apprise.Apprise:
-    """Initialize the Apprise instance with configured notification services"""
-    global _apprise
-    
-    if _apprise is None:
-        _apprise = apprise.Apprise()
-        
-        # Add all configured notification services
-        if settings.notification_urls:
-            for url in settings.notification_urls:
-                try:
-                    _apprise.add(url)
-                    logger.info(f"Added notification service: {_mask_sensitive_url(url)}")
-                except Exception as e:
-                    logger.error(f"Failed to add notification service: {str(e)}")
-        else:
-            logger.warning("No notification services configured")
-    
-    return _apprise
-
-def _mask_sensitive_url(url: str) -> str:
-    """Mask sensitive parts of notification URLs for logging"""
-    # Simple masking for common URL formats with credentials
-    import re
-    # Match patterns like user:pass@host or token in URL parameters
-    masked = re.sub(r'://([^:]+):([^@]+)@', r'://\1:****@', url)
-    masked = re.sub(r'(discord://)[^/]+/[^/]+', r'\1webhook_id/****', masked)
-    masked = re.sub(r'(tgram://)[^/]+/[^/]+', r'\1bot_token/****', masked)
-    masked = re.sub(r'([?&](token|key|api_key|password|secret)=)([^&]+)', r'\1****', masked)
-    return masked
-
-def send_notification(
-    title: str, 
-    message: str, 
-    notification_type: str = "info",
-    tags: Optional[List[str]] = None,
-    attachments: Optional[List[str]] = None,
-    data: Optional[Dict[str, Any]] = None
-) -> bool:
-    """
-    Send a notification through all configured channels
-    
-    Args:
-        title: The notification title
-        message: The notification body message
-        notification_type: Type of notification (info, success, warning, failure)
-        tags: Optional list of tags for filtering notifications
-        attachments: Optional list of file paths to attach
-        data: Optional additional data for the notification
-        
-    Returns:
-        bool: True if notification was sent successfully to at least one service
-    """
-    if not settings.notification_urls:
-        logger.debug(f"Notification not sent (no services configured): {title}")
-        return False
-        
-    try:
-        apprise_obj = init_apprise()
-        
-        # Set notification type
-        notify_type = apprise.NotifyType.INFO
-        if notification_type == "success":
-            notify_type = apprise.NotifyType.SUCCESS
-        elif notification_type in ("warning", "warn"):
-            notify_type = apprise.NotifyType.WARNING
-        elif notification_type in ("failure", "error", "failed"):
-            notify_type = apprise.NotifyType.FAILURE
-        
-        # Send the notification to each service individually for better error reporting
-        if not apprise_obj.servers:  # Access servers as an attribute, not a method
-            logger.warning("No notification servers available despite having URLs configured")
-            return False
-            
-        total_services = len(apprise_obj.servers)
-        successful_services = 0
-        
-        for server in apprise_obj.servers:  # Iterate through the list directly
-            try:
-                service_name = str(server).split("://")[0] if "://" in str(server) else str(server)
-                service_result = server.notify(
-                    title=title,
-                    body=message,
-                    notify_type=notify_type,
-                    attach=attachments
-                )
-                
-                if service_result:
-                    successful_services += 1
-                    logger.debug(f"Notification sent via {service_name}")
-                else:
-                    logger.warning(f"Failed to send notification via {service_name}")
-            except Exception as e:
-                logger.error(f"Error sending notification via {str(server)}: {str(e)}")
-        
-        overall_result = successful_services > 0
-        
-        if overall_result:
-            logger.debug(f"Notification sent: '{title}' (successful: {successful_services}/{total_services})")
-        else:
-            logger.warning(f"Failed to send notification to ALL services: '{title}' (0/{total_services})")
-        
-        return overall_result
-    
-    except Exception as e:
-        logger.exception(f"Error sending notification: {e}")
-        return False
-
-def notify_celery_failure(task_name: str, task_id: str, exc: Exception, args: list, kwargs: dict) -> bool:
-    """Send a notification about a failed Celery task"""
-    if not settings.notify_on_task_failure:
-        return False
-        
-    title = f"Task Failed: {task_name}"
-    message = f"""
-Task {task_name} ({task_id}) failed with error:
-{type(exc).__name__}: {str(exc)}
-
-Arguments: {args}
-Keyword arguments: {kwargs}
-"""
-    return send_notification(
-        title=title,
-        message=message,
-        notification_type="failure",
-        tags=["celery", "failure", task_name]
-    )
-
-def notify_credential_failure(service_name: str, error: str) -> bool:
-    """Send a notification about a credential failure"""
-    if not settings.notify_on_credential_failure:
-        return False
-        
-    title = f"Credential Failure: {service_name}"
-    message = f"""
-The credentials for {service_name} have failed:
-{error}
-
-Please check and update the credentials in the system settings.
-"""
-    return send_notification(
-        title=title,
-        message=message,
-        notification_type="warning",
-        tags=["credentials", "warning", service_name]
-    )
-
-def notify_startup() -> bool:
-    """Send a notification that the application has started"""
-    if not settings.notify_on_startup:
-        return False
-        
-    title = f"DocuElevate Started"
-    message = f"DocuElevate has been started successfully on {settings.external_hostname}"
-    return send_notification(
-        title=title,
-        message=message,
-        notification_type="success",
-        tags=["system", "startup"]
-    )
-
-def notify_shutdown() -> bool:
-    """Send a notification that the application is shutting down"""
-    if not settings.notify_on_shutdown:
-        return False
-        
-    title = f"DocuElevate Shutting Down"
-    message = f"DocuElevate on {settings.external_hostname} is shutting down"
-    return send_notification(
-        title=title,
-        message=message,
-        notification_type="info",
-        tags=["system", "shutdown"]
-    )

app/views/__init__.py

@@ -1,21 +0,0 @@
-"""
-Aggregated view routers for the application.
-"""
-from fastapi import APIRouter
-
-# Import all the view routers
-from app.views.general import router as general_router
-from app.views.status import router as status_router
-from app.views.onedrive import router as onedrive_router
-from app.views.dropbox import router as dropbox_router
-from app.views.google_drive import router as google_drive_router
-from app.views.license_routes import router as license_router  # Add the license router
-
-# Create a main router that includes all the view routers
-router = APIRouter()
-router.include_router(general_router)
-router.include_router(status_router)
-router.include_router(onedrive_router)
-router.include_router(dropbox_router)
-router.include_router(google_drive_router)
-router.include_router(license_router)  # Include the license router

app/views/base.py

@@ -1,43 +0,0 @@
-"""
-Base setup for views, containing shared functionality and imports.
-"""
-from fastapi import APIRouter, Request, Depends, HTTPException
-from fastapi.templating import Jinja2Templates
-from pathlib import Path
-from sqlalchemy.orm import Session
-import logging
-
-from app.auth import require_login
-from app.database import SessionLocal
-from app.config import settings
-
-# Set up Jinja2 templates
-templates_dir = Path(__file__).parent.parent.parent / "frontend" / "templates"
-templates = Jinja2Templates(directory=str(templates_dir))
-
-# Customize Jinja2Templates to include app_version in all templates
-original_template_response = templates.TemplateResponse
-
-def template_response_with_version(*args, **kwargs):
-    """Wrapper for TemplateResponse to include version in all templates"""
-    # If context dict is provided, add version to it
-    if len(args) >= 2 and isinstance(args[1], dict):
-        args[1].setdefault("version", settings.version)
-    elif "context" in kwargs and isinstance(kwargs["context"], dict):
-        kwargs["context"].setdefault("version", settings.version)
-    return original_template_response(*args, **kwargs)
-
-templates.TemplateResponse = template_response_with_version
-
-# Set up logging
-logger = logging.getLogger(__name__)
-
-def get_db():
-    """
-    Dependency to get a database session.
-    """
-    db = SessionLocal()
-    try:
-        yield db
-    finally:
-        db.close()

app/views/dropbox.py

@@ -1,65 +0,0 @@
-"""
-Dropbox integration views for setup and OAuth callback.
-"""
-from fastapi import Request
-
-from app.views.base import APIRouter, templates, require_login, settings
-
-router = APIRouter()
-
-@router.get("/dropbox-setup")
-@require_login
-async def dropbox_setup_page(request: Request):
-    """
-    Setup page for the Dropbox integration.
-    Shows configuration status and setup instructions.
-    """
-    # Check Dropbox configuration
-    is_configured = bool(settings.dropbox_app_key and 
-                         settings.dropbox_app_secret and 
-                         settings.dropbox_refresh_token)
-    
-    return templates.TemplateResponse(
-        "dropbox.html",
-        {
-            "request": request,
-            "is_configured": is_configured,
-            "app_key_value": settings.dropbox_app_key or "",
-            "app_secret_value": settings.dropbox_app_secret if settings.dropbox_app_secret else "",
-            "refresh_token_value": settings.dropbox_refresh_token if settings.dropbox_refresh_token else "",
-            "folder_path": settings.dropbox_folder or "/Documents/Uploads"  # Default folder path
-        }
-    )
-
-@router.get("/dropbox-callback")
-@require_login
-async def dropbox_callback(request: Request, code: str = None, error: str = None):
-    """
-    Callback endpoint for Dropbox OAuth flow.
-    Automatically exchanges the code for a token and saves it to the configuration.
-    """
-    if error:
-        return templates.TemplateResponse(
-            "dropbox_callback_error.html",
-            {"request": request, "error": error}
-        )
-    
-    if not code:
-        return templates.TemplateResponse(
-            "dropbox_callback_error.html",
-            {"request": request, "error": "No authorization code received from Dropbox"}
-        )
-    
-    # Display the processing page with automatic token exchange
-    # Note: We provide empty strings for app_key_value and app_secret_value
-    # to prevent overriding what's in sessionStorage
-    return templates.TemplateResponse(
-        "dropbox_callback.html",
-        {
-            "request": request, 
-            "code": code,
-            "app_key_value": "",  # The callback will prioritize sessionStorage values
-            "app_secret_value": "",  # The callback will prioritize sessionStorage values
-            "folder_path": ""  # The callback will prioritize sessionStorage values
-        }
-    )

app/views/files.py

@@ -1,39 +0,0 @@
-"""
-File management views for displaying and managing files.
-"""
-from fastapi import Request, Depends
-from sqlalchemy.orm import Session
-
-from app.views.base import APIRouter, templates, require_login, get_db, logger
-
-router = APIRouter()
-
-@router.get("/files")
-@require_login
-def files_page(request: Request, db: Session = Depends(get_db)):
-    """
-    Return the 'files.html' template with files from the database
-    """
-    try:
-        # Import the model here to avoid circular imports
-        from app.models import FileRecord
-        
-        # Fetch all files from the database
-        files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all()
-        
-        # Debug output
-        logger.info(f"Retrieved {len(files)} files from database")
-        
-        return templates.TemplateResponse("files.html", {
-            "request": request,
-            "files": files
-        })
-    except Exception as e:
-        # Log any errors
-        logger.error(f"Error retrieving files: {str(e)}")
-        # Return error message to template
-        return templates.TemplateResponse("files.html", {
-            "request": request,
-            "files": [],
-            "error": str(e)
-        })

app/views/general.py

@@ -1,134 +0,0 @@
-"""
-General routes for the application homepage and basic pages.
-"""
-from fastapi import Request, HTTPException, Depends
-from fastapi.responses import FileResponse
-from pathlib import Path
-from datetime import date
-from sqlalchemy.orm import Session
-
-from app.views.base import APIRouter, templates, require_login, get_db
-from app.utils.config_validator import get_provider_status, validate_storage_configs
-
-router = APIRouter()
-
-@router.get("/", include_in_schema=False)
-async def serve_index(request: Request, db: Session = Depends(get_db)):
-    """Serve the index/home page."""
-    # Get provider information from config validator
-    providers = get_provider_status()
-    
-    # Count configured providers
-    configured_providers = sum(1 for provider in providers.values() if provider['configured'])
-    
-    # Count different types of storage targets
-    storage_issues = validate_storage_configs()
-    configured_storage_targets = sum(1 for provider, issues in storage_issues.items() 
-                                     if not issues and provider in ['dropbox', 'nextcloud', 'sftp', 
-                                                                    's3', 'ftp', 'webdav', 
-                                                                    'google_drive', 'onedrive'])
-    
-    # Query the actual file count from the database
-    processed_files = 0
-    try:
-        # Import the model here to avoid circular imports
-        from app.models import FileRecord
-        processed_files = db.query(FileRecord).count()
-    except Exception as e:
-        # Log error but continue (don't break the page if DB query fails)
-        from app.views.base import logger
-        logger.error(f"Error counting files: {str(e)}")
-    
-    # Create stats object to pass to the template
-    stats = {
-        "processed_files": processed_files,
-        "active_integrations": configured_providers,
-        "storage_targets": configured_storage_targets
-    }
-    
-    return templates.TemplateResponse("index.html", {"request": request, "stats": stats})
-
-@router.get("/about", include_in_schema=False)
-async def serve_about(request: Request):
-    """Serve the about page."""
-    return templates.TemplateResponse("about.html", {"request": request})
-
-@router.get("/privacy", include_in_schema=False)
-async def serve_privacy(request: Request):
-    """Serve the privacy policy page."""
-    # Pass the current date for the "Last Updated" field
-    current_date = date.today().strftime("%B %d, %Y")
-    return templates.TemplateResponse("privacy.html", {"request": request, "current_date": current_date})
-
-@router.get("/imprint", include_in_schema=False)
-async def serve_imprint(request: Request):
-    """Serve the imprint/impressum page."""
-    return templates.TemplateResponse("imprint.html", {"request": request})
-
-@router.get("/upload", include_in_schema=False)
-@require_login
-async def serve_upload(request: Request):
-    """Serve the upload page."""
-    return templates.TemplateResponse("upload.html", {"request": request})
-
-@router.get("/favicon.ico", include_in_schema=False)
-def favicon():
-    """Serve the favicon."""
-    favicon_path = Path(__file__).parent.parent.parent / "frontend" / "static" / "favicon.ico"
-    if not favicon_path.exists():
-        # If favicon doesn't exist, return a 404
-        raise HTTPException(status_code=404, detail="Favicon not found")
-    return FileResponse(favicon_path)
-
-@router.get("/license", include_in_schema=False)
-async def serve_license(request: Request):
-    """Serve the license page."""
-    # Try multiple possible locations for the license file
-    possible_locations = [
-        Path(__file__).parent.parent.parent / "LICENSE",  # Repository root
-        Path("/app/LICENSE"),  # Docker container path
-        Path.home() / "LICENSE",  # Home directory (fallback)
-    ]
-
-    license_text = None
-    
-    # Try to read from any of the possible locations
-    for path in possible_locations:
-        try:
-            with open(path, "r") as f:
-                license_text = f.read()
-                break  # File found and read, exit loop
-        except (FileNotFoundError, PermissionError):
-            continue  # Try next location
-    
-    # If license text is still None, use embedded text
-    if license_text is None:
-        license_text = """
-Apache License
-Version 2.0, January 2004
-http://www.apache.org/licenses/
-
-This software is licensed under the Apache License 2.0.
-The full license text could not be located on this system.
-Please visit http://www.apache.org/licenses/LICENSE-2.0 for the complete license text.
-"""
-
-    return templates.TemplateResponse(
-        "license.html", 
-        {
-            "request": request,
-            "license_text": license_text
-        }
-    )
-
-@router.get("/cookies", include_in_schema=False)
-async def serve_cookies(request: Request):
-    """Serve the cookie policy page."""
-    current_date = date.today().strftime("%B %d, %Y")
-    return templates.TemplateResponse("cookies.html", {"request": request, "current_date": current_date})
-
-@router.get("/terms", include_in_schema=False)
-async def serve_terms(request: Request):
-    """Serve the terms of service page."""
-    current_date = date.today().strftime("%B %d, %Y")
-    return templates.TemplateResponse("terms.html", {"request": request, "current_date": current_date})

app/views/google_drive.py

@@ -1,118 +0,0 @@
-"""
-Google Drive integration views for setup and OAuth callback.
-"""
-from fastapi import Request
-from fastapi.responses import RedirectResponse
-import urllib.parse
-
-from app.views.base import APIRouter, templates, require_login, settings
-
-router = APIRouter()
-
-@router.get("/google-drive-setup")
-@require_login
-async def google_drive_setup_page(request: Request):
-    """
-    Setup page for the Google Drive integration.
-    Shows configuration status and setup instructions.
-    """
-    # Check if using OAuth
-    use_oauth = getattr(settings, 'google_drive_use_oauth', False)
-    
-    # Check Google Drive OAuth configuration
-    oauth_configured = bool(settings.google_drive_client_id and 
-                         settings.google_drive_client_secret and 
-                         settings.google_drive_refresh_token)
-    
-    # Check Google Drive service account configuration
-    sa_configured = bool(settings.google_drive_credentials_json)
-    
-    # Overall configuration status
-    is_configured = (use_oauth and oauth_configured) or (not use_oauth and sa_configured)
-    
-    if settings.google_drive_folder_id:
-        is_configured = is_configured and True
-    else:
-        is_configured = False
-    
-    # Get configuration values to display status (hide sensitive values)
-    return templates.TemplateResponse(
-        "google_drive.html",
-        {
-            "request": request,
-            "is_configured": is_configured,
-            "use_oauth": use_oauth,
-            "oauth_configured": oauth_configured,
-            "sa_configured": sa_configured,
-            "client_id": bool(settings.google_drive_client_id),
-            "client_id_value": settings.google_drive_client_id or "",
-            "client_secret": bool(settings.google_drive_client_secret),
-            "client_secret_value": settings.google_drive_client_secret or "",
-            "refresh_token": bool(settings.google_drive_refresh_token),
-            "refresh_token_value": settings.google_drive_refresh_token or "",
-            "folder_id": settings.google_drive_folder_id or "",
-            "has_credentials_json": bool(settings.google_drive_credentials_json)
-        }
-    )
-
-@router.get("/google-drive-callback")
-@require_login
-async def google_drive_callback(request: Request, code: str = None, error: str = None, state: str = None):
-    """
-    Callback endpoint for Google Drive OAuth flow.
-    Now automatically exchanges the code for a token and saves it to the configuration.
-    """
-    if error:
-        return templates.TemplateResponse(
-            "google_drive_callback_error.html",
-            {"request": request, "error": error}
-        )
-    
-    if not code:
-        return templates.TemplateResponse(
-            "google_drive_callback_error.html",
-            {"request": request, "error": "No authorization code received from Google"}
-        )
-    
-    # Display the processing page with automatic token exchange
-    return templates.TemplateResponse(
-        "google_drive_callback.html",
-        {
-            "request": request, 
-            "code": code,
-            "state": state
-        }
-    )
-
-@router.get("/google-drive-auth-start")
-@require_login
-async def google_drive_auth_start(
-    request: Request, 
-    client_id: str,
-    redirect_uri: str = None
-):
-    """
-    Start the Google Drive OAuth flow by redirecting to Google's authorization page.
-    """
-    if not redirect_uri:
-        redirect_uri = f"{request.url.scheme}://{request.url.netloc}/google-drive-callback"
-    
-    # Create the authorization URL with required scopes
-    # Use only drive.file scope to minimize required permissions
-    scopes = [
-        "https://www.googleapis.com/auth/drive.file"  # Access to files created or opened by the app
-    ]
-    
-    scope_str = urllib.parse.quote(' '.join(scopes))
-    
-    auth_url = (
-        f"https://accounts.google.com/o/oauth2/auth"
-        f"?client_id={client_id}"
-        f"&redirect_uri={urllib.parse.quote(redirect_uri)}"
-        f"&response_type=code"
-        f"&scope={scope_str}"
-        f"&access_type=offline"
-        f"&prompt=consent"  # Force to show consent screen to get refresh token
-    )
-    
-    return RedirectResponse(url=auth_url)

app/views/license_routes.py

@@ -1,27 +0,0 @@
-from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import PlainTextResponse, HTMLResponse
-from pathlib import Path
-import os
-
-from app.views.base import templates, require_login
-
-router = APIRouter()
-
-@router.get("/licenses/lgpl.txt", response_class=PlainTextResponse)
-async def get_lgpl_license():
-    """
-    Serve the LGPL license text file
-    """
-    license_path = Path("frontend/static/licenses/lgpl.txt")
-    if not license_path.exists():
-        raise HTTPException(status_code=404, detail="License file not found")
-    
-    with open(license_path, "r") as f:
-        return f.read()
-
-@router.get("/attribution", response_class=HTMLResponse, include_in_schema=False)
-async def serve_attribution(request: Request):
-    """
-    Serve the third-party attribution page
-    """
-    return templates.TemplateResponse("attribution.html", {"request": request})

app/views/onedrive.py

@@ -1,68 +0,0 @@
-"""
-OneDrive integration views for setup and OAuth callback.
-"""
-from fastapi import Request
-
-from app.views.base import APIRouter, templates, require_login, settings
-
-router = APIRouter()
-
-@router.get("/onedrive-setup")
-@require_login
-async def onedrive_setup_page(request: Request):
-    """
-    Setup page for the OneDrive integration.
-    Shows configuration status and setup instructions.
-    """
-    # Check OneDrive configuration
-    is_configured = bool(settings.onedrive_client_id and 
-                         settings.onedrive_client_secret and 
-                         settings.onedrive_refresh_token)
-    
-    # Get configuration values to display status (hide sensitive values)
-    return templates.TemplateResponse(
-        "onedrive.html",
-        {
-            "request": request,
-            "is_configured": is_configured,
-            "client_id": bool(settings.onedrive_client_id),
-            "client_id_value": settings.onedrive_client_id or "",  # Pass the actual value for the form
-            "client_secret": bool(settings.onedrive_client_secret),
-            "client_secret_value": settings.onedrive_client_secret if settings.onedrive_client_secret else "",
-            "tenant_id": settings.onedrive_tenant_id,
-            "refresh_token": bool(settings.onedrive_refresh_token),
-            "refresh_token_value": settings.onedrive_refresh_token if settings.onedrive_refresh_token else "",
-            "folder_path": settings.onedrive_folder_path or "Documents/Uploads"  # Default folder path
-        }
-    )
-
-@router.get("/onedrive-callback")
-@require_login
-async def onedrive_callback(request: Request, code: str = None, error: str = None):
-    """
-    Callback endpoint for OneDrive OAuth flow.
-    Now automatically exchanges the code for a token and saves it to the configuration.
-    """
-    if error:
-        return templates.TemplateResponse(
-            "onedrive_callback_error.html",
-            {"request": request, "error": error}
-        )
-    
-    if not code:
-        return templates.TemplateResponse(
-            "onedrive_callback_error.html",
-            {"request": request, "error": "No authorization code received from Microsoft"}
-        )
-    
-    # Display the processing page with automatic token exchange
-    return templates.TemplateResponse(
-        "onedrive_callback.html",
-        {
-            "request": request, 
-            "code": code,
-            "client_id_value": settings.onedrive_client_id or "",
-            "client_secret_value": settings.onedrive_client_secret or "",
-            "tenant_id": settings.onedrive_tenant_id or "common"
-        }
-    )

app/views/status.py

@@ -1,130 +0,0 @@
-"""
-Status and configuration views for the application.
-"""
-from fastapi import Request
-from fastapi.responses import JSONResponse
-from datetime import datetime
-import os
-import subprocess
-import logging
-
-from app.views.base import APIRouter, templates, require_login, settings
-
-logger = logging.getLogger(__name__)
-router = APIRouter()
-
-@router.get("/status")
-@require_login
-async def status_dashboard(request: Request):
-    """
-    Status dashboard showing all configured integration targets
-    """
-    from app.utils.config_validator import get_provider_status
-    
-    # Get provider status
-    providers = get_provider_status()
-    
-    # Get build date from settings
-    build_date = getattr(settings, 'build_date', 'Unknown')
-    
-    # Try to get container information
-    container_info = {}
-    try:
-        # Check for Docker environment
-        if os.path.exists('/.dockerenv'):
-            # We're inside a Docker container
-            container_info['is_docker'] = True
-            
-            # Try to get container ID
-            try:
-                with open('/proc/self/cgroup', 'r') as f:
-                    for line in f:
-                        if 'docker' in line:
-                            container_id = line.split('/')[-1].strip()
-                            container_info['id'] = container_id[:12]  # Short ID format
-                            break
-            except Exception:
-                container_info['id'] = 'Unknown'
-            
-            # Try to get Git commit SHA from runtime info
-            try:
-                # First check runtime info directory
-                if os.path.exists('/app/runtime_info/GIT_SHA'):
-                    with open('/app/runtime_info/GIT_SHA', 'r') as f:
-                        git_sha = f.read().strip()
-                # Then try environment variable
-                else:
-                    git_sha = os.environ.get('GIT_COMMIT_SHA', '')
-                
-                # If still not found, try the original file location
-                if not git_sha and os.path.exists('/.git-commit-sha'):
-                    with open('/.git-commit-sha', 'r') as f:
-                        git_sha = f.read().strip()
-                
-                container_info['git_sha'] = git_sha[:7] if git_sha and git_sha != 'unknown' else 'Unknown'
-            except Exception:
-                container_info['git_sha'] = 'Unknown'
-            
-            # Try to get runtime information
-            try:
-                if os.path.exists('/app/runtime_info/RUNTIME_INFO'):
-                    with open('/app/runtime_info/RUNTIME_INFO', 'r') as f:
-                        container_info['runtime_info'] = f.read().strip()
-            except Exception:
-                pass
-        else:
-            container_info['is_docker'] = False
-            
-            # If not in Docker, try to get Git info directly
-            try:
-                git_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], 
-                                                stderr=subprocess.DEVNULL, 
-                                                text=True).strip()[:7]
-                container_info['git_sha'] = git_sha
-            except (subprocess.SubprocessError, FileNotFoundError):
-                container_info['git_sha'] = 'Unknown'
-    except Exception:
-        container_info = {'is_docker': False, 'id': 'Unknown', 'git_sha': 'Unknown'}
-    
-    # Get notification URLs for the notification box
-    notification_urls = getattr(settings, 'notification_urls', [])
-    
-    return templates.TemplateResponse(
-        "status_dashboard.html",
-        {
-            "request": request, 
-            "providers": providers,
-            "app_version": settings.version,
-            "build_date": build_date,
-            "debug_enabled": getattr(settings, 'debug', False),
-            "last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-            "container_info": container_info,
-            "settings": {
-                "notification_urls": notification_urls
-            }
-        }
-    )
-
-@router.get("/env")
-@require_login
-async def env_debug(request: Request):
-    """
-    Debug endpoint to view environment variables and settings
-    Uses actual debug setting from config
-    """
-    # Use the actual debug setting from configuration
-    debug_enabled = settings.debug
-    
-    # Get settings data
-    from app.utils.config_validator import get_settings_for_display
-    settings_data = get_settings_for_display(show_values=debug_enabled)
-    
-    return templates.TemplateResponse(
-        "env_debug.html",
-        {
-            "request": request, 
-            "settings": settings_data,
-            "debug_enabled": debug_enabled,
-            "app_version": settings.version
-        }
-    )

docker-compose.yaml

@@ -1,16 +1,14 @@
+
 services:
   api:
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: christianlouis/document-processor:latest
     container_name: document_api
-    restart: always
 
     # We'll keep the code in /app, but set working_dir to the shared data directory
     working_dir: /workdir
 
     # We'll run uvicorn from the container's /app code
-    command: ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"]
+    command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000 --proxy-headers"]
 
     # Environment variables
     environment:
@@ -28,14 +26,13 @@ services:
 
     # Mount the shared working directory for data
     volumes:
+      # optional: mount your code if you want local dev changes to reflect
+      # - ./app:/app
       - /var/docparse/workdir:/workdir
 
   worker:
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: christianlouis/document-processor:latest
     container_name: document_worker
-    restart: always
 
     # same shared working directory
     working_dir: /workdir
@@ -50,14 +47,16 @@ services:
       - redis
       - gotenberg
 
-    # Mount the shared directory
+    # Mount the shared directory (and optionally your code if you want dev mode)
     volumes:
+      # optional: mount your code if you want local dev changes
+      # - ./app:/app
       - /var/docparse/workdir:/workdir
 
   gotenberg:
     image: gotenberg/gotenberg:latest
     container_name: gotenberg
-    restart: always
+
 
   redis:
     image: redis:alpine

docs/API.md

@@ -1,137 +0,0 @@
-# API Documentation
-
-DocuElevate provides a powerful REST API for programmatic access to all its features. This document serves as a reference for the available endpoints and their usage.
-
-## API Overview
-
-- Base URL: `http://<your-docuelevate-instance>/api`
-- Authentication: OAuth2 (when enabled)
-- Response Format: JSON
-
-## Interactive API Documentation
-
-The most up-to-date and interactive API documentation is available at:
-
-`http://<your-docuelevate-instance>/docs`
-
-This Swagger UI provides a complete reference with the ability to try out API calls directly from your browser.
-
-## Authentication
-
-When authentication is enabled, you must include an authentication token in your requests:
-
-```bash
-curl -X GET "http://<your-docuelevate-instance>/api/files" \
-  -H "Authorization: Bearer <your-token>"
-```
-
-## Common Endpoints
-
-### Document Upload
-
-**POST** `/api/upload`
-
-Upload one or more files for processing.
-
-**Request**: 
-- Multipart form data with file(s)
-
-**Response**:
-```json
-{
-  "success": true,
-  "file_ids": [123, 124],
-  "message": "Files uploaded and queued for processing"
-}
-```
-
-### Get Files
-
-**GET** `/api/files`
-
-Retrieve a list of processed files.
-
-**Parameters**:
-- `limit` (optional): Maximum number of files to return
-- `offset` (optional): Pagination offset
-- `search` (optional): Search term
-
-**Response**:
-```json
-[
-  {
-    "id": 123,
-    "original_filename": "invoice.pdf",
-    "file_size": 1024000,
-    "mime_type": "application/pdf",
-    "created_at": "2023-04-15T12:30:45Z"
-  },
-  ...
-]
-```
-
-### File Metadata
-
-**GET** `/api/files/{file_id}/metadata`
-
-Retrieve metadata for a specific file.
-
-**Response**:
-```json
-{
-  "document_type": "invoice",
-  "date": "2023-04-10",
-  "vendor": "Acme Corp",
-  "amount": "$1,234.56",
-  "extracted_text": "..."
-}
-```
-
-### Process Control
-
-**POST** `/api/files/{file_id}/reprocess`
-
-Reprocess a specific file.
-
-**Response**:
-```json
-{
-  "success": true,
-  "message": "File queued for reprocessing"
-}
-```
-
-**POST** `/send_to_google_drive/`
-
-Send a processed file to Google Drive.
-
-**Parameters**:
-- `file_path`: Path to the file to upload
-
-**Response**:
-```json
-{
-  "task_id": "a1b2c3d4-e5f6-7g8h-9i0j-k1l2m3n4o5p6",
-  "status": "queued"
-}
-```
-
-## Error Handling
-
-Errors follow standard HTTP status codes with descriptive messages:
-
-```json
-{
-  "detail": "File not found",
-  "status_code": 404
-}
-```
-
-## Rate Limiting
-
-The API implements rate limiting to ensure system stability. If you exceed the limits, you'll receive a `429 Too Many Requests` response.
-
-
-## Further Assistance
-
-For additional help with the API, please contact our support team or refer to the [Development Guide](../CONTRIBUTING.md).