added a favicon, updated the README.md file
This commit is contained in:
Christian Krakau-Louis
@@ -1,47 +1,47 @@
|
|||||||
# **Config Variables**
|
# **Config Variables**
|
||||||
DATABASE_URL=sqlite:///./app/database.db
|
DATABASE_URL=sqlite:///./app/database.db
|
||||||
REDIS_URL=redis://redis:6379/0
|
REDIS_URL=redis://redis:6379/0
|
||||||
WORKDIR=/workdir
|
WORKDIR=/workdir
|
||||||
AWS_REGION="eu-central-1"
|
AWS_REGION="eu-central-1"
|
||||||
AZURE_REGION="eastus"
|
AZURE_REGION="eastus"
|
||||||
AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
|
AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
|
||||||
S3_BUCKET_NAME=<your_bucket_name>
|
S3_BUCKET_NAME=<your_bucket_name>
|
||||||
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
|
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
|
||||||
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
|
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
|
||||||
PAPERLESS_NGX_URL=https://paperless.example.com/api/documents/post_document/
|
PAPERLESS_NGX_URL=https://paperless.example.com/api/documents/post_document/
|
||||||
PAPERLESS_HOST=https://paperless.example.com
|
PAPERLESS_HOST=https://paperless.example.com
|
||||||
|
|
||||||
# **Tokens/API Credentials**
|
# **Tokens/API Credentials**
|
||||||
AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY>"
|
AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY>"
|
||||||
AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
|
AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
|
||||||
OPENAI_API_KEY="<OPENAI_API_KEY>"
|
OPENAI_API_KEY="<OPENAI_API_KEY>"
|
||||||
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
|
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
|
||||||
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
|
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
|
||||||
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
|
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
|
||||||
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
|
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
|
||||||
AZURE_AI_KEY=<AZURE_AI_KEY>
|
AZURE_AI_KEY=<AZURE_AI_KEY>
|
||||||
|
|
||||||
# **User Credentials**
|
# **User Credentials**
|
||||||
ADMIN_USERNAME=admin
|
ADMIN_USERNAME=admin
|
||||||
ADMIN_PASSWORD=your_secure_password
|
ADMIN_PASSWORD=your_secure_password
|
||||||
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
|
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
|
||||||
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
|
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
|
||||||
IMAP1_USERNAME=<IMAP1_USERNAME>
|
IMAP1_USERNAME=<IMAP1_USERNAME>
|
||||||
IMAP1_PASSWORD=<IMAP1_PASSWORD>
|
IMAP1_PASSWORD=<IMAP1_PASSWORD>
|
||||||
IMAP2_USERNAME=<IMAP2_USERNAME>
|
IMAP2_USERNAME=<IMAP2_USERNAME>
|
||||||
IMAP2_PASSWORD=<IMAP2_PASSWORD>
|
IMAP2_PASSWORD=<IMAP2_PASSWORD>
|
||||||
|
|
||||||
# **IMAP Settings**
|
# **IMAP Settings**
|
||||||
IMAP1_HOST=mail.example.com
|
IMAP1_HOST=mail.example.com
|
||||||
IMAP1_PORT=993
|
IMAP1_PORT=993
|
||||||
IMAP1_SSL=true
|
IMAP1_SSL=true
|
||||||
IMAP1_POLL_INTERVAL_MINUTES=5
|
IMAP1_POLL_INTERVAL_MINUTES=5
|
||||||
IMAP1_DELETE_AFTER_PROCESS=false
|
IMAP1_DELETE_AFTER_PROCESS=false
|
||||||
|
|
||||||
IMAP2_HOST=imap.gmail.com
|
IMAP2_HOST=imap.gmail.com
|
||||||
IMAP2_PORT=993
|
IMAP2_PORT=993
|
||||||
IMAP2_SSL=true
|
IMAP2_SSL=true
|
||||||
IMAP2_POLL_INTERVAL_MINUTES=10
|
IMAP2_POLL_INTERVAL_MINUTES=10
|
||||||
IMAP2_DELETE_AFTER_PROCESS=false
|
IMAP2_DELETE_AFTER_PROCESS=false
|
||||||
|
|
||||||
GOTENBERG_URL=http://gotenberg:3000
|
GOTENBERG_URL=http://gotenberg:3000
|
||||||
+12
-12
@@ -1,12 +1,12 @@
|
|||||||
# To get started with Dependabot version updates, you'll need to specify which
|
# To get started with Dependabot version updates, you'll need to specify which
|
||||||
# package ecosystems to update and where the package manifests are located.
|
# package ecosystems to update and where the package manifests are located.
|
||||||
# Please see the documentation for all configuration options:
|
# Please see the documentation for all configuration options:
|
||||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||||
|
|
||||||
version: 2
|
version: 2
|
||||||
updates:
|
updates:
|
||||||
- package-ecosystem: "" # See documentation for possible values
|
- package-ecosystem: "" # See documentation for possible values
|
||||||
directory: "/" # Location of package manifests
|
directory: "/" # Location of package manifests
|
||||||
schedule:
|
schedule:
|
||||||
interval: "weekly"
|
interval: "weekly"
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +1,18 @@
|
|||||||
name: Deploy to Production
|
name: Deploy to Production
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_run:
|
workflow_run:
|
||||||
workflows: ["Build and Push Docker Image"]
|
workflows: ["Build and Push Docker Image"]
|
||||||
types:
|
types:
|
||||||
- completed
|
- completed
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
deploy:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Call Deployment Webhook
|
- name: Call Deployment Webhook
|
||||||
run: |
|
run: |
|
||||||
curl -X POST https://docker2.kuechenserver.org/api/stacks/webhooks/960c7d8e-97ec-4175-a8dc-73f037b02349
|
curl -X POST https://docker2.kuechenserver.org/api/stacks/webhooks/960c7d8e-97ec-4175-a8dc-73f037b02349
|
||||||
|
|||||||
@@ -1,54 +1,54 @@
|
|||||||
name: Build and Push Docker Image
|
name: Build and Push Docker Image
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
- develop
|
- develop
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout Code
|
- name: Checkout Code
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v2
|
uses: docker/setup-buildx-action@v2
|
||||||
|
|
||||||
- name: Log in to Docker Hub
|
- name: Log in to Docker Hub
|
||||||
uses: docker/login-action@v2
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKER_USERNAME }}
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|
||||||
- name: Log in to GitHub Container Registry
|
- name: Log in to GitHub Container Registry
|
||||||
uses: docker/login-action@v2
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Build and Push
|
- name: Build and Push
|
||||||
uses: docker/build-push-action@v4
|
uses: docker/build-push-action@v4
|
||||||
with:
|
with:
|
||||||
# Specify target platforms
|
# Specify target platforms
|
||||||
platforms: linux/amd64
|
platforms: linux/amd64
|
||||||
context: .
|
context: .
|
||||||
file: Dockerfile
|
file: Dockerfile
|
||||||
push: true
|
push: true
|
||||||
tags: |
|
tags: |
|
||||||
christianlouis/document-processor:latest
|
christianlouis/document-processor:latest
|
||||||
christianlouis/document-processor:${{ github.sha }}
|
christianlouis/document-processor:${{ github.sha }}
|
||||||
ghcr.io/${{ github.repository_owner }}/document-processor:latest
|
ghcr.io/${{ github.repository_owner }}/document-processor:latest
|
||||||
ghcr.io/${{ github.repository_owner }}/document-processor:${{ github.sha }}
|
ghcr.io/${{ github.repository_owner }}/document-processor:${{ github.sha }}
|
||||||
# Cache options (optional)
|
# Cache options (optional)
|
||||||
cache-from: type=gha
|
cache-from: type=gha
|
||||||
cache-to: type=gha,mode=max
|
cache-to: type=gha,mode=max
|
||||||
|
|||||||
@@ -1,40 +1,40 @@
|
|||||||
name: Run Tests & Linting
|
name: Run Tests & Linting
|
||||||
|
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout Code
|
- name: Checkout Code
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v4
|
uses: actions/setup-python@v4
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
|
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip
|
python -m pip install --upgrade pip
|
||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
pip install pytest flake8 black mypy pylint
|
pip install pytest flake8 black mypy pylint
|
||||||
|
|
||||||
# - name: Run Tests
|
# - name: Run Tests
|
||||||
# run: pytest tests/
|
# run: pytest tests/
|
||||||
|
|
||||||
- name: Run Linter (Flake8)
|
- name: Run Linter (Flake8)
|
||||||
run: flake8 app/
|
run: flake8 app/
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
- name: Run Code Formatter (Black)
|
- name: Run Code Formatter (Black)
|
||||||
run: black --check app/
|
run: black --check app/
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
- name: Run Type Checker (Mypy)
|
- name: Run Type Checker (Mypy)
|
||||||
run: mypy app/
|
run: mypy app/
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|
||||||
- name: Run Linter (Pylint)
|
- name: Run Linter (Pylint)
|
||||||
run: pylint app/
|
run: pylint app/
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
|
|||||||
+171
-171
@@ -1,171 +1,171 @@
|
|||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.py[cod]
|
*.py[cod]
|
||||||
*$py.class
|
*$py.class
|
||||||
|
|
||||||
# C extensions
|
# C extensions
|
||||||
*.so
|
*.so
|
||||||
|
|
||||||
# Distribution / packaging
|
# Distribution / packaging
|
||||||
.Python
|
.Python
|
||||||
build/
|
build/
|
||||||
develop-eggs/
|
develop-eggs/
|
||||||
dist/
|
dist/
|
||||||
downloads/
|
downloads/
|
||||||
eggs/
|
eggs/
|
||||||
.eggs/
|
.eggs/
|
||||||
lib/
|
lib/
|
||||||
lib64/
|
lib64/
|
||||||
parts/
|
parts/
|
||||||
sdist/
|
sdist/
|
||||||
var/
|
var/
|
||||||
wheels/
|
wheels/
|
||||||
share/python-wheels/
|
share/python-wheels/
|
||||||
*.egg-info/
|
*.egg-info/
|
||||||
.installed.cfg
|
.installed.cfg
|
||||||
*.egg
|
*.egg
|
||||||
MANIFEST
|
MANIFEST
|
||||||
.env
|
.env
|
||||||
|
|
||||||
# PyInstaller
|
# PyInstaller
|
||||||
# Usually these files are written by a python script from a template
|
# Usually these files are written by a python script from a template
|
||||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
*.manifest
|
*.manifest
|
||||||
*.spec
|
*.spec
|
||||||
|
|
||||||
# Installer logs
|
# Installer logs
|
||||||
pip-log.txt
|
pip-log.txt
|
||||||
pip-delete-this-directory.txt
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
# Unit test / coverage reports
|
# Unit test / coverage reports
|
||||||
htmlcov/
|
htmlcov/
|
||||||
.tox/
|
.tox/
|
||||||
.nox/
|
.nox/
|
||||||
.coverage
|
.coverage
|
||||||
.coverage.*
|
.coverage.*
|
||||||
.cache
|
.cache
|
||||||
nosetests.xml
|
nosetests.xml
|
||||||
coverage.xml
|
coverage.xml
|
||||||
*.cover
|
*.cover
|
||||||
*.py,cover
|
*.py,cover
|
||||||
.hypothesis/
|
.hypothesis/
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
cover/
|
cover/
|
||||||
|
|
||||||
# Translations
|
# Translations
|
||||||
*.mo
|
*.mo
|
||||||
*.pot
|
*.pot
|
||||||
|
|
||||||
# Django stuff:
|
# Django stuff:
|
||||||
*.log
|
*.log
|
||||||
local_settings.py
|
local_settings.py
|
||||||
db.sqlite3
|
db.sqlite3
|
||||||
db.sqlite3-journal
|
db.sqlite3-journal
|
||||||
|
|
||||||
# Flask stuff:
|
# Flask stuff:
|
||||||
instance/
|
instance/
|
||||||
.webassets-cache
|
.webassets-cache
|
||||||
|
|
||||||
# Scrapy stuff:
|
# Scrapy stuff:
|
||||||
.scrapy
|
.scrapy
|
||||||
|
|
||||||
# Sphinx documentation
|
# Sphinx documentation
|
||||||
docs/_build/
|
docs/_build/
|
||||||
|
|
||||||
# PyBuilder
|
# PyBuilder
|
||||||
.pybuilder/
|
.pybuilder/
|
||||||
target/
|
target/
|
||||||
|
|
||||||
# Jupyter Notebook
|
# Jupyter Notebook
|
||||||
.ipynb_checkpoints
|
.ipynb_checkpoints
|
||||||
|
|
||||||
# IPython
|
# IPython
|
||||||
profile_default/
|
profile_default/
|
||||||
ipython_config.py
|
ipython_config.py
|
||||||
|
|
||||||
# pyenv
|
# pyenv
|
||||||
# For a library or package, you might want to ignore these files since the code is
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
# intended to run in multiple environments; otherwise, check them in:
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
# .python-version
|
# .python-version
|
||||||
|
|
||||||
# pipenv
|
# pipenv
|
||||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
# install all needed dependencies.
|
# install all needed dependencies.
|
||||||
#Pipfile.lock
|
#Pipfile.lock
|
||||||
|
|
||||||
# UV
|
# UV
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
# commonly ignored for libraries.
|
# commonly ignored for libraries.
|
||||||
#uv.lock
|
#uv.lock
|
||||||
|
|
||||||
# poetry
|
# poetry
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
# commonly ignored for libraries.
|
# commonly ignored for libraries.
|
||||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
#poetry.lock
|
#poetry.lock
|
||||||
|
|
||||||
# pdm
|
# pdm
|
||||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
#pdm.lock
|
#pdm.lock
|
||||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||||
# in version control.
|
# in version control.
|
||||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||||
.pdm.toml
|
.pdm.toml
|
||||||
.pdm-python
|
.pdm-python
|
||||||
.pdm-build/
|
.pdm-build/
|
||||||
|
|
||||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
__pypackages__/
|
__pypackages__/
|
||||||
|
|
||||||
# Celery stuff
|
# Celery stuff
|
||||||
celerybeat-schedule
|
celerybeat-schedule
|
||||||
celerybeat.pid
|
celerybeat.pid
|
||||||
|
|
||||||
# SageMath parsed files
|
# SageMath parsed files
|
||||||
*.sage.py
|
*.sage.py
|
||||||
|
|
||||||
# Environments
|
# Environments
|
||||||
.venv
|
.venv
|
||||||
env/
|
env/
|
||||||
venv/
|
venv/
|
||||||
ENV/
|
ENV/
|
||||||
env.bak/
|
env.bak/
|
||||||
venv.bak/
|
venv.bak/
|
||||||
|
|
||||||
# Spyder project settings
|
# Spyder project settings
|
||||||
.spyderproject
|
.spyderproject
|
||||||
.spyproject
|
.spyproject
|
||||||
|
|
||||||
# Rope project settings
|
# Rope project settings
|
||||||
.ropeproject
|
.ropeproject
|
||||||
|
|
||||||
# mkdocs documentation
|
# mkdocs documentation
|
||||||
/site
|
/site
|
||||||
|
|
||||||
# mypy
|
# mypy
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
.dmypy.json
|
.dmypy.json
|
||||||
dmypy.json
|
dmypy.json
|
||||||
|
|
||||||
# Pyre type checker
|
# Pyre type checker
|
||||||
.pyre/
|
.pyre/
|
||||||
|
|
||||||
# pytype static type analyzer
|
# pytype static type analyzer
|
||||||
.pytype/
|
.pytype/
|
||||||
|
|
||||||
# Cython debug symbols
|
# Cython debug symbols
|
||||||
cython_debug/
|
cython_debug/
|
||||||
|
|
||||||
# PyCharm
|
# PyCharm
|
||||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
#.idea/
|
#.idea/
|
||||||
|
|
||||||
# PyPI configuration file
|
# PyPI configuration file
|
||||||
.pypirc
|
.pypirc
|
||||||
|
|||||||
+30
-30
@@ -1,30 +1,30 @@
|
|||||||
|
|
||||||
# Stage 1: Build dependencies
|
# Stage 1: Build dependencies
|
||||||
FROM python:3.11 AS builder
|
FROM python:3.11 AS builder
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt /app/
|
COPY requirements.txt /app/
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
# Stage 2: Final image
|
# Stage 2: Final image
|
||||||
FROM python:3.11-slim
|
FROM python:3.11-slim
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Copy installed dependencies
|
# Copy installed dependencies
|
||||||
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
||||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||||
|
|
||||||
# Copy application files correctly
|
# Copy application files correctly
|
||||||
COPY ./app /app/app
|
COPY ./app /app/app
|
||||||
COPY ./frontend /app/frontend
|
COPY ./frontend /app/frontend
|
||||||
|
|
||||||
# Set Python path explicitly
|
# Set Python path explicitly
|
||||||
ENV PYTHONPATH=/app
|
ENV PYTHONPATH=/app
|
||||||
|
|
||||||
# Expose API port
|
# Expose API port
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||||
|
|||||||
@@ -1,201 +1,201 @@
|
|||||||
Apache License
|
Apache License
|
||||||
Version 2.0, January 2004
|
Version 2.0, January 2004
|
||||||
http://www.apache.org/licenses/
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
1. Definitions.
|
1. Definitions.
|
||||||
|
|
||||||
"License" shall mean the terms and conditions for use, reproduction,
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
and distribution as defined by Sections 1 through 9 of this document.
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
"Licensor" shall mean the copyright owner or entity authorized by
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
the copyright owner that is granting the License.
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
"Legal Entity" shall mean the union of the acting entity and all
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
other entities that control, are controlled by, or are under common
|
other entities that control, are controlled by, or are under common
|
||||||
control with that entity. For the purposes of this definition,
|
control with that entity. For the purposes of this definition,
|
||||||
"control" means (i) the power, direct or indirect, to cause the
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
direction or management of such entity, whether by contract or
|
direction or management of such entity, whether by contract or
|
||||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
"You" (or "Your") shall mean an individual or Legal Entity
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
exercising permissions granted by this License.
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
"Source" form shall mean the preferred form for making modifications,
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
including but not limited to software source code, documentation
|
including but not limited to software source code, documentation
|
||||||
source, and configuration files.
|
source, and configuration files.
|
||||||
|
|
||||||
"Object" form shall mean any form resulting from mechanical
|
"Object" form shall mean any form resulting from mechanical
|
||||||
transformation or translation of a Source form, including but
|
transformation or translation of a Source form, including but
|
||||||
not limited to compiled object code, generated documentation,
|
not limited to compiled object code, generated documentation,
|
||||||
and conversions to other media types.
|
and conversions to other media types.
|
||||||
|
|
||||||
"Work" shall mean the work of authorship, whether in Source or
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
Object form, made available under the License, as indicated by a
|
Object form, made available under the License, as indicated by a
|
||||||
copyright notice that is included in or attached to the work
|
copyright notice that is included in or attached to the work
|
||||||
(an example is provided in the Appendix below).
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
"Derivative Works" shall mean any work, whether in Source or Object
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
form, that is based on (or derived from) the Work and for which the
|
form, that is based on (or derived from) the Work and for which the
|
||||||
editorial revisions, annotations, elaborations, or other modifications
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
represent, as a whole, an original work of authorship. For the purposes
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
of this License, Derivative Works shall not include works that remain
|
of this License, Derivative Works shall not include works that remain
|
||||||
separable from, or merely link (or bind by name) to the interfaces of,
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
the Work and Derivative Works thereof.
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
"Contribution" shall mean any work of authorship, including
|
"Contribution" shall mean any work of authorship, including
|
||||||
the original version of the Work and any modifications or additions
|
the original version of the Work and any modifications or additions
|
||||||
to that Work or Derivative Works thereof, that is intentionally
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
or by an individual or Legal Entity authorized to submit on behalf of
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
the copyright owner. For the purposes of this definition, "submitted"
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
means any form of electronic, verbal, or written communication sent
|
means any form of electronic, verbal, or written communication sent
|
||||||
to the Licensor or its representatives, including but not limited to
|
to the Licensor or its representatives, including but not limited to
|
||||||
communication on electronic mailing lists, source code control systems,
|
communication on electronic mailing lists, source code control systems,
|
||||||
and issue tracking systems that are managed by, or on behalf of, the
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
Licensor for the purpose of discussing and improving the Work, but
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
excluding communication that is conspicuously marked or otherwise
|
excluding communication that is conspicuously marked or otherwise
|
||||||
designated in writing by the copyright owner as "Not a Contribution."
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
on behalf of whom a Contribution has been received by Licensor and
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
subsequently incorporated within the Work.
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
copyright license to reproduce, prepare Derivative Works of,
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
publicly display, publicly perform, sublicense, and distribute the
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
Work and such Derivative Works in Source or Object form.
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
3. Grant of Patent License. Subject to the terms and conditions of
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
(except as stated in this section) patent license to make, have made,
|
(except as stated in this section) patent license to make, have made,
|
||||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
where such license applies only to those patent claims licensable
|
where such license applies only to those patent claims licensable
|
||||||
by such Contributor that are necessarily infringed by their
|
by such Contributor that are necessarily infringed by their
|
||||||
Contribution(s) alone or by combination of their Contribution(s)
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
with the Work to which such Contribution(s) was submitted. If You
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
institute patent litigation against any entity (including a
|
institute patent litigation against any entity (including a
|
||||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
or a Contribution incorporated within the Work constitutes direct
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
or contributory patent infringement, then any patent licenses
|
or contributory patent infringement, then any patent licenses
|
||||||
granted to You under this License for that Work shall terminate
|
granted to You under this License for that Work shall terminate
|
||||||
as of the date such litigation is filed.
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
4. Redistribution. You may reproduce and distribute copies of the
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
Work or Derivative Works thereof in any medium, with or without
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
modifications, and in Source or Object form, provided that You
|
modifications, and in Source or Object form, provided that You
|
||||||
meet the following conditions:
|
meet the following conditions:
|
||||||
|
|
||||||
(a) You must give any other recipients of the Work or
|
(a) You must give any other recipients of the Work or
|
||||||
Derivative Works a copy of this License; and
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
(b) You must cause any modified files to carry prominent notices
|
(b) You must cause any modified files to carry prominent notices
|
||||||
stating that You changed the files; and
|
stating that You changed the files; and
|
||||||
|
|
||||||
(c) You must retain, in the Source form of any Derivative Works
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
that You distribute, all copyright, patent, trademark, and
|
that You distribute, all copyright, patent, trademark, and
|
||||||
attribution notices from the Source form of the Work,
|
attribution notices from the Source form of the Work,
|
||||||
excluding those notices that do not pertain to any part of
|
excluding those notices that do not pertain to any part of
|
||||||
the Derivative Works; and
|
the Derivative Works; and
|
||||||
|
|
||||||
(d) If the Work includes a "NOTICE" text file as part of its
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
distribution, then any Derivative Works that You distribute must
|
distribution, then any Derivative Works that You distribute must
|
||||||
include a readable copy of the attribution notices contained
|
include a readable copy of the attribution notices contained
|
||||||
within such NOTICE file, excluding those notices that do not
|
within such NOTICE file, excluding those notices that do not
|
||||||
pertain to any part of the Derivative Works, in at least one
|
pertain to any part of the Derivative Works, in at least one
|
||||||
of the following places: within a NOTICE text file distributed
|
of the following places: within a NOTICE text file distributed
|
||||||
as part of the Derivative Works; within the Source form or
|
as part of the Derivative Works; within the Source form or
|
||||||
documentation, if provided along with the Derivative Works; or,
|
documentation, if provided along with the Derivative Works; or,
|
||||||
within a display generated by the Derivative Works, if and
|
within a display generated by the Derivative Works, if and
|
||||||
wherever such third-party notices normally appear. The contents
|
wherever such third-party notices normally appear. The contents
|
||||||
of the NOTICE file are for informational purposes only and
|
of the NOTICE file are for informational purposes only and
|
||||||
do not modify the License. You may add Your own attribution
|
do not modify the License. You may add Your own attribution
|
||||||
notices within Derivative Works that You distribute, alongside
|
notices within Derivative Works that You distribute, alongside
|
||||||
or as an addendum to the NOTICE text from the Work, provided
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
that such additional attribution notices cannot be construed
|
that such additional attribution notices cannot be construed
|
||||||
as modifying the License.
|
as modifying the License.
|
||||||
|
|
||||||
You may add Your own copyright statement to Your modifications and
|
You may add Your own copyright statement to Your modifications and
|
||||||
may provide additional or different license terms and conditions
|
may provide additional or different license terms and conditions
|
||||||
for use, reproduction, or distribution of Your modifications, or
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
for any such Derivative Works as a whole, provided Your use,
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
reproduction, and distribution of the Work otherwise complies with
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
the conditions stated in this License.
|
the conditions stated in this License.
|
||||||
|
|
||||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
any Contribution intentionally submitted for inclusion in the Work
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
by You to the Licensor shall be under the terms and conditions of
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
this License, without any additional terms or conditions.
|
this License, without any additional terms or conditions.
|
||||||
Notwithstanding the above, nothing herein shall supersede or modify
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
the terms of any separate license agreement you may have executed
|
the terms of any separate license agreement you may have executed
|
||||||
with Licensor regarding such Contributions.
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
6. Trademarks. This License does not grant permission to use the trade
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
names, trademarks, service marks, or product names of the Licensor,
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
except as required for reasonable and customary use in describing the
|
except as required for reasonable and customary use in describing the
|
||||||
origin of the Work and reproducing the content of the NOTICE file.
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
agreed to in writing, Licensor provides the Work (and each
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
implied, including, without limitation, any warranties or conditions
|
implied, including, without limitation, any warranties or conditions
|
||||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
appropriateness of using or redistributing the Work and assume any
|
appropriateness of using or redistributing the Work and assume any
|
||||||
risks associated with Your exercise of permissions under this License.
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
8. Limitation of Liability. In no event and under no legal theory,
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
whether in tort (including negligence), contract, or otherwise,
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
unless required by applicable law (such as deliberate and grossly
|
unless required by applicable law (such as deliberate and grossly
|
||||||
negligent acts) or agreed to in writing, shall any Contributor be
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
liable to You for damages, including any direct, indirect, special,
|
liable to You for damages, including any direct, indirect, special,
|
||||||
incidental, or consequential damages of any character arising as a
|
incidental, or consequential damages of any character arising as a
|
||||||
result of this License or out of the use or inability to use the
|
result of this License or out of the use or inability to use the
|
||||||
Work (including but not limited to damages for loss of goodwill,
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
work stoppage, computer failure or malfunction, or any and all
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
other commercial damages or losses), even if such Contributor
|
other commercial damages or losses), even if such Contributor
|
||||||
has been advised of the possibility of such damages.
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
9. Accepting Warranty or Additional Liability. While redistributing
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
the Work or Derivative Works thereof, You may choose to offer,
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
or other liability obligations and/or rights consistent with this
|
or other liability obligations and/or rights consistent with this
|
||||||
License. However, in accepting such obligations, You may act only
|
License. However, in accepting such obligations, You may act only
|
||||||
on Your own behalf and on Your sole responsibility, not on behalf
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
of any other Contributor, and only if You agree to indemnify,
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
defend, and hold each Contributor harmless for any liability
|
defend, and hold each Contributor harmless for any liability
|
||||||
incurred by, or claims asserted against, such Contributor by reason
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
of your accepting any such warranty or additional liability.
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
APPENDIX: How to apply the Apache License to your work.
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
To apply the Apache License to your work, attach the following
|
To apply the Apache License to your work, attach the following
|
||||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
replaced with your own identifying information. (Don't include
|
replaced with your own identifying information. (Don't include
|
||||||
the brackets!) The text should be enclosed in the appropriate
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
comment syntax for the file format. We also recommend that a
|
comment syntax for the file format. We also recommend that a
|
||||||
file or class name and description of purpose be included on the
|
file or class name and description of purpose be included on the
|
||||||
same "printed page" as the copyright notice for easier
|
same "printed page" as the copyright notice for easier
|
||||||
identification within third-party archives.
|
identification within third-party archives.
|
||||||
|
|
||||||
Copyright [yyyy] [name of copyright owner]
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
You may obtain a copy of the License at
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
Unless required by applicable law or agreed to in writing, software
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
See the License for the specific language governing permissions and
|
See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|||||||
@@ -1,144 +1,199 @@
|
|||||||
# Document Processing System
|
# Document Processing System
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
This project automates the handling, extraction, and processing of documents using a combination of services such as OpenAI, Dropbox, Nextcloud, and Paperless NGX. The system extracts metadata, processes document contents, and stores the results efficiently. It is designed for flexibility and configurability through environment variables, making it easily customizable for different workflows.
|
This project automates the handling, extraction, and processing of documents using a variety of services, including:
|
||||||
|
|
||||||
## Features
|
- **OpenAI** for metadata extraction and text refinement.
|
||||||
|
- **Dropbox** and **Nextcloud** for file storage and uploads.
|
||||||
- **Document Upload & Storage**: Upload and manage documents via Dropbox and Nextcloud.
|
- **Paperless NGX** for document indexing and management.
|
||||||
- **OCR Processing**: Extract text from scanned documents.
|
- **Azure Document Intelligence** (optional) for OCR on PDFs (replacing Textract).
|
||||||
- **Metadata Extraction**: Automatically extract key information using OpenAI's API.
|
- **Gotenberg** for file-to-PDF conversions.
|
||||||
- **Document Management**: Store processed documents and metadata in Paperless NGX for easy retrieval.
|
- **AWS S3** (currently implemented but may be removed in the future).
|
||||||
- **IMAP Integration**: Fetch documents from multiple IMAP email accounts for processing.
|
|
||||||
|
It is designed for flexibility and configurability through environment variables, making it easily customizable for different workflows. The system can fetch documents from multiple IMAP mailboxes, process them (OCR, metadata extraction, PDF conversion), and store them in the desired destinations.
|
||||||
## Environment Variables
|
|
||||||
|
## Features
|
||||||
The project is configured via the `.env` file, where credentials and settings for different services are defined. Below is a breakdown of key configuration variables:
|
|
||||||
|
- **Document Upload & Storage**:
|
||||||
### General Configuration
|
- Manual uploads (via API) to S3, or direct uploads to Dropbox/Nextcloud/Paperless.
|
||||||
|
- **OCR Processing (Azure)**:
|
||||||
| **Variable** | **Description** | **How to Obtain** |
|
- Extract text from scanned PDFs using Azure Document Intelligence.
|
||||||
|-------------|----------------|-------------------|
|
- **Metadata Extraction (OpenAI)**:
|
||||||
| `DATABASE_URL` | Path to the SQLite database. | Example: `sqlite:///./app/database.db` |
|
- Use GPT to classify, label, or otherwise enrich the text with structured metadata.
|
||||||
| `REDIS_URL` | URL for Redis connection. | Example: `redis://redis:6379/0` |
|
- **PDF Conversion (Gotenberg)**:
|
||||||
| `WORKDIR` | Working directory for the application. | Example: `/workdir` |
|
- Convert non-PDF attachments (e.g., Word docs, images) into PDFs.
|
||||||
| `NEXTCLOUD_UPLOAD_URL` | Nextcloud WebDAV upload URL. | Example: `https://nextcloud.example.com/remote.php/dav/files/<USERNAME>` |
|
- **Document Management (Paperless NGX)**:
|
||||||
| `NEXTCLOUD_FOLDER` | Folder in Nextcloud for file uploads. | Example: `/Documents/Uploads` |
|
- Store processed documents and metadata in a Paperless NGX instance.
|
||||||
| `PAPERLESS_NGX_URL` | Paperless NGX API endpoint. | Example: `https://paperless.example.com/api/documents/post_document/` |
|
- **IMAP Integration**:
|
||||||
| `PAPERLESS_HOST` | Root URL for Paperless NGX. | Example: `https://paperless.example.com` |
|
- Fetch documents from multiple mailboxes (including Gmail) and automatically enqueue them for processing.
|
||||||
|
|
||||||
### Tokens/API Credentials
|
## Environment Variables
|
||||||
|
|
||||||
| **Variable** | **Description** | **How to Obtain** |
|
The `.env` file drives all configuration. This table breaks down key variables—some are optional, depending on which services you actually use.
|
||||||
|-------------|----------------|-------------------|
|
|
||||||
| `OPENAI_API_KEY` | API key for OpenAI services. | Get from [OpenAI platform](https://platform.openai.com/account/api-keys). |
|
### Core Settings
|
||||||
| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. | Obtain from your Paperless NGX instance. |
|
|
||||||
| `DROPBOX_APP_KEY` | Dropbox API key. | Generate from the [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create). |
|
| **Variable** | **Description** | **Example** |
|
||||||
| `DROPBOX_APP_SECRET` | Dropbox API secret. | Available in the Dropbox Developer Console. |
|
|------------------------|----------------------------------------------------------|--------------------------------|
|
||||||
| `DROPBOX_REFRESH_TOKEN` | Dropbox OAuth refresh token. | Obtain by following Dropbox's OAuth flow. |
|
| `DATABASE_URL` | Path/URL to the SQLite database (or other SQL backend). | `sqlite:///./app/database.db` |
|
||||||
|
| `REDIS_URL` | URL for Redis, used by Celery for broker & result store. | `redis://redis:6379/0` |
|
||||||
### User Credentials
|
| `WORKDIR` | Working directory for the application. | `/workdir` |
|
||||||
|
| `GOTENBERG_URL` | Gotenberg PDF processing URL. | `http://gotenberg:3000` |
|
||||||
| **Variable** | **Description** |
|
|
||||||
|-------------|----------------|
|
### IMAP Configuration (Multiple Mailboxes)
|
||||||
| `ADMIN_USERNAME` | Admin username for system access. |
|
|
||||||
| `ADMIN_PASSWORD` | Admin password for system access. |
|
| **Variable** | **Description** | **Example** |
|
||||||
| `NEXTCLOUD_USERNAME` | Username for Nextcloud authentication. |
|
|-------------------------------|--------------------------------------------------------------|-------------------|
|
||||||
| `NEXTCLOUD_PASSWORD` | Password for Nextcloud authentication. |
|
| `IMAP1_HOST` | Hostname for first IMAP server. | `mail.example.com`|
|
||||||
|
| `IMAP1_PORT` | Port number (usually `993`). | `993` |
|
||||||
### IMAP Configuration
|
| `IMAP1_USERNAME` | IMAP login (first mailbox). | `user@example.com`|
|
||||||
|
| `IMAP1_PASSWORD` | IMAP password (first mailbox). | `*******` |
|
||||||
| **Variable** | **Description** |
|
| `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` |
|
||||||
|-------------|----------------|
|
| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` |
|
||||||
| `IMAP1_USERNAME` | IMAP username for the first email account. |
|
| `IMAP1_DELETE_AFTER_PROCESS` | Delete emails after processing (`true`/`false`). | `false` |
|
||||||
| `IMAP1_PASSWORD` | IMAP password for the first email account. |
|
| `IMAP2_HOST` | Hostname for second IMAP server (optional). | `imap.gmail.com` |
|
||||||
| `IMAP1_HOST` | Hostname of the first IMAP server. |
|
| `IMAP2_PORT` | Port number for second mailbox. | `993` |
|
||||||
| `IMAP1_PORT` | IMAP server port (typically `993`). |
|
| `IMAP2_USERNAME` | IMAP login for second mailbox. | `you@gmail.com` |
|
||||||
| `IMAP1_SSL` | Enable SSL (`true` or `false`). |
|
| `IMAP2_PASSWORD` | IMAP password for second mailbox. | `*******` |
|
||||||
| `IMAP1_POLL_INTERVAL_MINUTES` | Polling interval for IMAP server. |
|
| `IMAP2_SSL` | Use SSL for second mailbox (`true`/`false`). | `true` |
|
||||||
| `IMAP1_DELETE_AFTER_PROCESS` | Delete emails after processing (`true` or `false`). |
|
| `IMAP2_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll second mailbox. | `10` |
|
||||||
|
| `IMAP2_DELETE_AFTER_PROCESS` | Delete emails after processing (`true`/`false`) for mailbox.| `false` |
|
||||||
### Additional Services
|
|
||||||
|
### OpenAI & Azure Document Intelligence
|
||||||
| **Variable** | **Description** |
|
|
||||||
|-------------|----------------|
|
| **Variable** | **Description** | **How to Obtain** |
|
||||||
| `GOTENBERG_URL` | URL for Gotenberg PDF processing. |
|
|-----------------------|--------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|
|
||||||
|
| `OPENAI_API_KEY` | API key for OpenAI services (used for metadata extraction/refinement). | [OpenAI platform](https://platform.openai.com/account/api-keys) |
|
||||||
## Running as a Docker Container
|
| `AZURE_AI_KEY` | Azure Document Intelligence key (for OCR). | [Azure Portal](https://portal.azure.com/) |
|
||||||
|
| `AZURE_REGION` | Azure region of your Document Intelligence instance. | e.g. `eastus`, `westeurope` |
|
||||||
This project includes a `docker-compose.yml` file that allows for easy deployment using Docker. The following services are defined:
|
| `AZURE_ENDPOINT` | Endpoint URL for Document Intelligence. | e.g. `https://<yourendpoint>.cognitiveservices.azure.com/` |
|
||||||
|
|
||||||
- **API Service**: Runs the document processing API using `uvicorn`.
|
### Paperless NGX
|
||||||
- **Worker Service**: Runs the Celery worker for handling document processing tasks.
|
|
||||||
- **Redis**: Used as a message broker for Celery.
|
| **Variable** | **Description** |
|
||||||
- **Gotenberg**: Provides PDF processing capabilities.
|
|-------------------------------|-----------------------------------------------------|
|
||||||
|
| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. |
|
||||||
### Running the Application with Docker Compose
|
| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). |
|
||||||
|
|
||||||
1. **Ensure Docker and Docker Compose are installed**.
|
### Dropbox
|
||||||
2. **Clone the repository and navigate to the directory**:
|
|
||||||
```bash
|
| **Variable** | **Description** | **How to Obtain** |
|
||||||
git clone <repository_url>
|
|-------------------------|--------------------------------------------------|------------------------------------------------------------------------------------|
|
||||||
cd <repository_name>
|
| `DROPBOX_APP_KEY` | Dropbox API app key. | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create) |
|
||||||
```
|
| `DROPBOX_APP_SECRET` | Dropbox API app secret. | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create) |
|
||||||
3. **Create and configure the `.env` file**.
|
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. | Follow Dropbox OAuth flow to retrieve |
|
||||||
4. **Start the services**:
|
| `DROPBOX_FOLDER` | Default folder path for Dropbox uploads. | e.g. `"/Documents/Uploads"` |
|
||||||
```bash
|
|
||||||
docker-compose up -d
|
### Nextcloud
|
||||||
```
|
|
||||||
5. The API will be available at `http://localhost:8000`.
|
| **Variable** | **Description** |
|
||||||
|
|-------------------------|---------------------------------------------------------------|
|
||||||
### Services in `docker-compose.yml`
|
| `NEXTCLOUD_UPLOAD_URL` | Nextcloud WebDAV URL (e.g. `https://nc.example.com/remote.php/dav/files/<USERNAME>`). |
|
||||||
|
| `NEXTCLOUD_USERNAME` | Nextcloud login username. |
|
||||||
```yaml
|
| `NEXTCLOUD_PASSWORD` | Nextcloud login password. |
|
||||||
services:
|
| `NEXTCLOUD_FOLDER` | Destination folder in Nextcloud (e.g. `"/Documents/Uploads"`). |
|
||||||
api:
|
|
||||||
image: christianlouis/document-processor:latest
|
### AWS S3
|
||||||
container_name: document_api
|
|
||||||
working_dir: /workdir
|
| **Variable** | **Description** |
|
||||||
command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
|
|------------------------|---------------------------------------------------------------|
|
||||||
environment:
|
| `AWS_ACCESS_KEY_ID` | AWS Access Key (used for S3 upload). |
|
||||||
- PYTHONPATH=/app
|
| `AWS_SECRET_ACCESS_KEY`| AWS Secret Key (used for S3 upload). |
|
||||||
env_file:
|
| `AWS_REGION` | AWS region for S3. |
|
||||||
- .env
|
| `S3_BUCKET_NAME` | Default S3 bucket name if using S3 upload. |
|
||||||
ports:
|
|
||||||
- "8000:8000"
|
### General Admin Credentials
|
||||||
depends_on:
|
|
||||||
- redis
|
| **Variable** | **Description** |
|
||||||
- worker
|
|--------------------|---------------------------------------------|
|
||||||
volumes:
|
| `ADMIN_USERNAME` | Admin username for system access. |
|
||||||
- /var/docparse/workdir:/workdir
|
| `ADMIN_PASSWORD` | Admin password for system access. |
|
||||||
|
|
||||||
worker:
|
## Running as a Docker Container
|
||||||
image: christianlouis/document-processor:latest
|
|
||||||
container_name: document_worker
|
This project uses Celery (with Redis) for asynchronous task management and Gotenberg for PDF conversion. The `docker-compose.yml` file defines these services:
|
||||||
working_dir: /workdir
|
|
||||||
command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
|
- **API Service**: Runs the FastAPI application via `uvicorn`.
|
||||||
env_file:
|
- **Worker Service**: Runs the Celery worker for processing tasks (PDF conversions, OCR, etc.).
|
||||||
- .env
|
- **Redis**: Provides the message broker & result backend for Celery.
|
||||||
environment:
|
- **Gotenberg**: Offers PDF conversion capabilities.
|
||||||
- PYTHONPATH=/app
|
|
||||||
depends_on:
|
### Running the Application with Docker Compose
|
||||||
- redis
|
|
||||||
- gotenberg
|
1. **Install Docker and Docker Compose** on your system.
|
||||||
volumes:
|
2. **Clone the repository** and navigate into it:
|
||||||
- /var/docparse/workdir:/workdir
|
```bash
|
||||||
|
git clone <repository_url>
|
||||||
gotenberg:
|
cd <repository_name>
|
||||||
image: gotenberg/gotenberg:latest
|
```
|
||||||
container_name: gotenberg
|
3. **Create and configure the `.env` file**:
|
||||||
|
- Fill in the variables from the tables above.
|
||||||
redis:
|
- (At minimum, you need `DATABASE_URL`, `REDIS_URL`, `WORKDIR`, plus whichever service creds you plan to use.)
|
||||||
image: redis:alpine
|
4. **Launch the services**:
|
||||||
container_name: document_redis
|
```bash
|
||||||
restart: always
|
docker-compose up -d
|
||||||
```
|
```
|
||||||
|
5. The API will be available at **`http://localhost:8000`**.
|
||||||
## To-Do List
|
|
||||||
|
### Services in `docker-compose.yml`
|
||||||
- Refactor AWS-related code to Azure.
|
|
||||||
- Remove unnecessary environment variables.
|
Below is the default structure (simplified):
|
||||||
- Make upload targets configurable.
|
|
||||||
- Remove S3 upload functionality.
|
```yaml
|
||||||
|
services:
|
||||||
|
api:
|
||||||
|
image: christianlouis/document-processor:latest
|
||||||
|
container_name: document_api
|
||||||
|
working_dir: /workdir
|
||||||
|
command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
|
||||||
|
environment:
|
||||||
|
- PYTHONPATH=/app
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
ports:
|
||||||
|
- "8000:8000"
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
- worker
|
||||||
|
volumes:
|
||||||
|
- /var/docparse/workdir:/workdir
|
||||||
|
|
||||||
|
worker:
|
||||||
|
image: christianlouis/document-processor:latest
|
||||||
|
container_name: document_worker
|
||||||
|
working_dir: /workdir
|
||||||
|
command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
environment:
|
||||||
|
- PYTHONPATH=/app
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
- gotenberg
|
||||||
|
volumes:
|
||||||
|
- /var/docparse/workdir:/workdir
|
||||||
|
|
||||||
|
gotenberg:
|
||||||
|
image: gotenberg/gotenberg:latest
|
||||||
|
container_name: gotenberg
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:alpine
|
||||||
|
container_name: document_redis
|
||||||
|
restart: always
|
||||||
|
```
|
||||||
|
|
||||||
|
## To-Do List
|
||||||
|
|
||||||
|
- **Refactor AWS-related code** to rely on Azure or remove if no longer needed.
|
||||||
|
- **Remove unnecessary environment variables** once final service usage is determined.
|
||||||
|
- **Make upload targets configurable** (e.g., easily choose only Dropbox, Nextcloud, or Paperless).
|
||||||
|
- **Potentially remove or consolidate S3 upload code** if Azure is the preferred cloud option.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Questions or Issues?**
|
||||||
|
- Feel free to open an issue or pull request.
|
||||||
|
- For local testing or development, use `docker-compose up` and watch the logs via `docker-compose logs -f`.
|
||||||
|
- Ensure your `.env` aligns with the environment variables listed above. If you see unexpected errors, check for typos or missing values.
|
||||||
+20
-20
@@ -1,20 +1,20 @@
|
|||||||
# app/celery_app.py
|
# app/celery_app.py
|
||||||
|
|
||||||
from celery import Celery
|
from celery import Celery
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
celery = Celery(
|
celery = Celery(
|
||||||
"document_processor",
|
"document_processor",
|
||||||
broker=settings.redis_url,
|
broker=settings.redis_url,
|
||||||
backend=settings.redis_url,
|
backend=settings.redis_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Optionally add this line to retain connection retry behavior at startup:
|
# Optionally add this line to retain connection retry behavior at startup:
|
||||||
celery.conf.broker_connection_retry_on_startup = True
|
celery.conf.broker_connection_retry_on_startup = True
|
||||||
|
|
||||||
# Set the default queue and routing so that tasks are enqueued on "document_processor"
|
# Set the default queue and routing so that tasks are enqueued on "document_processor"
|
||||||
celery.conf.task_default_queue = 'document_processor'
|
celery.conf.task_default_queue = 'document_processor'
|
||||||
celery.conf.task_routes = {
|
celery.conf.task_routes = {
|
||||||
"app.tasks.*": {"queue": "document_processor"},
|
"app.tasks.*": {"queue": "document_processor"},
|
||||||
}
|
}
|
||||||
|
|||||||
+41
-41
@@ -1,42 +1,42 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
# Import the shared Celery instance
|
# Import the shared Celery instance
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
# Ensure tasks are loaded
|
# Ensure tasks are loaded
|
||||||
from app import tasks # <— This imports app/tasks.py so Celery can register tasks
|
from app import tasks # <— This imports app/tasks.py so Celery can register tasks
|
||||||
|
|
||||||
# **Ensure all tasks are imported before Celery starts**
|
# **Ensure all tasks are imported before Celery starts**
|
||||||
from app.tasks.upload_to_s3 import upload_to_s3
|
from app.tasks.upload_to_s3 import upload_to_s3
|
||||||
from app.tasks.process_with_textract import process_with_textract
|
from app.tasks.process_with_textract import process_with_textract
|
||||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt
|
from app.tasks.refine_text_with_gpt import refine_text_with_gpt
|
||||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||||
|
|
||||||
# Import new send tasks
|
# Import new send tasks
|
||||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||||
from app.tasks.imap_tasks import pull_all_inboxes
|
from app.tasks.imap_tasks import pull_all_inboxes
|
||||||
from app.tasks.send_to_all import send_to_all_destinations
|
from app.tasks.send_to_all import send_to_all_destinations
|
||||||
|
|
||||||
celery.conf.task_routes = {
|
celery.conf.task_routes = {
|
||||||
"app.tasks.*": {"queue": "default"},
|
"app.tasks.*": {"queue": "default"},
|
||||||
}
|
}
|
||||||
|
|
||||||
@celery.task
|
@celery.task
|
||||||
def test_task():
|
def test_task():
|
||||||
return "Celery is working!"
|
return "Celery is working!"
|
||||||
|
|
||||||
# If you want Celery Beat to run the poll task every minute, add:
|
# If you want Celery Beat to run the poll task every minute, add:
|
||||||
from celery.schedules import crontab
|
from celery.schedules import crontab
|
||||||
|
|
||||||
celery.conf.beat_schedule = {
|
celery.conf.beat_schedule = {
|
||||||
"poll-inboxes-every-minute": {
|
"poll-inboxes-every-minute": {
|
||||||
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
||||||
"schedule": crontab(minute="*/1"), # every 1 minute
|
"schedule": crontab(minute="*/1"), # every 1 minute
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
+53
-53
@@ -1,53 +1,53 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from pydantic_settings import BaseSettings
|
from pydantic_settings import BaseSettings
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
admin_username: str
|
admin_username: str
|
||||||
admin_password: str
|
admin_password: str
|
||||||
aws_access_key_id: str
|
aws_access_key_id: str
|
||||||
aws_secret_access_key: str
|
aws_secret_access_key: str
|
||||||
aws_region: str
|
aws_region: str
|
||||||
database_url: str
|
database_url: str
|
||||||
redis_url: str
|
redis_url: str
|
||||||
s3_bucket_name: str
|
s3_bucket_name: str
|
||||||
openai_api_key: str
|
openai_api_key: str
|
||||||
workdir: str
|
workdir: str
|
||||||
dropbox_app_key: str
|
dropbox_app_key: str
|
||||||
dropbox_app_secret: str
|
dropbox_app_secret: str
|
||||||
dropbox_folder: str
|
dropbox_folder: str
|
||||||
dropbox_refresh_token: str
|
dropbox_refresh_token: str
|
||||||
nextcloud_upload_url: str
|
nextcloud_upload_url: str
|
||||||
nextcloud_username: str
|
nextcloud_username: str
|
||||||
nextcloud_password: str
|
nextcloud_password: str
|
||||||
nextcloud_folder: str
|
nextcloud_folder: str
|
||||||
paperless_ngx_api_token: str
|
paperless_ngx_api_token: str
|
||||||
paperless_host: str
|
paperless_host: str
|
||||||
azure_ai_key: str
|
azure_ai_key: str
|
||||||
azure_region: str
|
azure_region: str
|
||||||
azure_endpoint: str
|
azure_endpoint: str
|
||||||
gotenberg_url: str
|
gotenberg_url: str
|
||||||
|
|
||||||
# IMAP 1
|
# IMAP 1
|
||||||
imap1_host: Optional[str] = None
|
imap1_host: Optional[str] = None
|
||||||
imap1_port: Optional[int] = 993
|
imap1_port: Optional[int] = 993
|
||||||
imap1_username: Optional[str] = None
|
imap1_username: Optional[str] = None
|
||||||
imap1_password: Optional[str] = None
|
imap1_password: Optional[str] = None
|
||||||
imap1_ssl: bool = True
|
imap1_ssl: bool = True
|
||||||
imap1_poll_interval_minutes: int = 5
|
imap1_poll_interval_minutes: int = 5
|
||||||
imap1_delete_after_process: bool = False
|
imap1_delete_after_process: bool = False
|
||||||
|
|
||||||
# IMAP 2
|
# IMAP 2
|
||||||
imap2_host: Optional[str] = None
|
imap2_host: Optional[str] = None
|
||||||
imap2_port: Optional[int] = 993
|
imap2_port: Optional[int] = 993
|
||||||
imap2_username: Optional[str] = None
|
imap2_username: Optional[str] = None
|
||||||
imap2_password: Optional[str] = None
|
imap2_password: Optional[str] = None
|
||||||
imap2_ssl: bool = True
|
imap2_ssl: bool = True
|
||||||
imap2_poll_interval_minutes: int = 10
|
imap2_poll_interval_minutes: int = 10
|
||||||
imap2_delete_after_process: bool = False
|
imap2_delete_after_process: bool = False
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
env_file = ".env"
|
env_file = ".env"
|
||||||
|
|
||||||
settings = Settings()
|
settings = Settings()
|
||||||
|
|||||||
+17
-17
@@ -1,17 +1,17 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from sqlalchemy import create_engine, Column, String, Integer
|
from sqlalchemy import create_engine, Column, String, Integer
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
from .config import settings
|
from .config import settings
|
||||||
|
|
||||||
Base = declarative_base()
|
Base = declarative_base()
|
||||||
engine = create_engine(settings.database_url, connect_args={"check_same_thread": False})
|
engine = create_engine(settings.database_url, connect_args={"check_same_thread": False})
|
||||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
|
||||||
def get_db():
|
def get_db():
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
yield db
|
yield db
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|||||||
+24
-19
@@ -1,19 +1,24 @@
|
|||||||
# app/frontend.py (new file or inline in main.py)
|
# app/frontend.py (new file or inline in main.py)
|
||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
import os
|
import os
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
# 1) Serve the folder that contains index.html, etc.
|
# 1) Serve the folder that contains index.html, etc.
|
||||||
# e.g. "frontend" is relative to your project root
|
# e.g. "frontend" is relative to your project root
|
||||||
frontend_folder = os.path.join(os.path.dirname(__file__), "..", "frontend")
|
frontend_folder = os.path.join(os.path.dirname(__file__), "..", "frontend")
|
||||||
|
|
||||||
# If you just want to serve the entire folder as static:
|
# If you just want to serve the entire folder as static:
|
||||||
router.mount("/static", StaticFiles(directory=frontend_folder), name="static")
|
router.mount("/static", StaticFiles(directory=frontend_folder), name="static")
|
||||||
|
|
||||||
# 2) For the root route ("/"), return the index.html
|
# 2) For the root route ("/"), return the index.html
|
||||||
@router.get("/ui", response_class=FileResponse)
|
@router.get("/ui", response_class=FileResponse)
|
||||||
def serve_ui():
|
def serve_ui():
|
||||||
return os.path.join(frontend_folder, "index.html")
|
return os.path.join(frontend_folder, "index.html")
|
||||||
|
|
||||||
|
# 3) Serve favicon.ico from the frontend folder
|
||||||
|
@router.get("/favicon.ico", response_class=FileResponse)
|
||||||
|
def favicon():
|
||||||
|
return os.path.join(frontend_folder, "favicon.ico")
|
||||||
+132
-132
@@ -1,133 +1,133 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from fastapi import FastAPI, HTTPException, UploadFile, File
|
from fastapi import FastAPI, HTTPException, UploadFile, File
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.upload_to_s3 import upload_to_s3
|
from app.tasks.upload_to_s3 import upload_to_s3
|
||||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||||
from app.tasks.send_to_all import send_to_all_destinations
|
from app.tasks.send_to_all import send_to_all_destinations
|
||||||
from app.frontend import router as frontend_router
|
from app.frontend import router as frontend_router
|
||||||
|
|
||||||
app = FastAPI(title="Document Processing API")
|
app = FastAPI(title="Document Processing API")
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
def root():
|
def root():
|
||||||
return {"message": "Document Processing API"}
|
return {"message": "Document Processing API"}
|
||||||
|
|
||||||
@app.post("/process/")
|
@app.post("/process/")
|
||||||
def process(file_path: str):
|
def process(file_path: str):
|
||||||
"""
|
"""
|
||||||
API Endpoint to start document processing.
|
API Endpoint to start document processing.
|
||||||
This enqueues the first task (upload_to_s3), which handles the full pipeline.
|
This enqueues the first task (upload_to_s3), which handles the full pipeline.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# If file_path is not absolute, treat it as relative to settings.workdir.
|
# If file_path is not absolute, treat it as relative to settings.workdir.
|
||||||
if not os.path.isabs(file_path):
|
if not os.path.isabs(file_path):
|
||||||
file_path = os.path.join(settings.workdir, file_path)
|
file_path = os.path.join(settings.workdir, file_path)
|
||||||
|
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||||
|
|
||||||
task = upload_to_s3.delay(file_path)
|
task = upload_to_s3.delay(file_path)
|
||||||
return {"task_id": task.id, "status": "queued"}
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
@app.post("/send_to_dropbox/")
|
@app.post("/send_to_dropbox/")
|
||||||
def send_to_dropbox(file_path: str):
|
def send_to_dropbox(file_path: str):
|
||||||
if not os.path.isabs(file_path):
|
if not os.path.isabs(file_path):
|
||||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||||
task = upload_to_dropbox.delay(file_path)
|
task = upload_to_dropbox.delay(file_path)
|
||||||
return {"task_id": task.id, "status": "queued"}
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
@app.post("/send_to_paperless/")
|
@app.post("/send_to_paperless/")
|
||||||
def send_to_paperless(file_path: str):
|
def send_to_paperless(file_path: str):
|
||||||
if not os.path.isabs(file_path):
|
if not os.path.isabs(file_path):
|
||||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||||
task = upload_to_paperless.delay(file_path)
|
task = upload_to_paperless.delay(file_path)
|
||||||
return {"task_id": task.id, "status": "queued"}
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
@app.post("/send_to_nextcloud/")
|
@app.post("/send_to_nextcloud/")
|
||||||
def send_to_nextcloud(file_path: str):
|
def send_to_nextcloud(file_path: str):
|
||||||
if not os.path.isabs(file_path):
|
if not os.path.isabs(file_path):
|
||||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||||
task = upload_to_nextcloud.delay(file_path)
|
task = upload_to_nextcloud.delay(file_path)
|
||||||
return {"task_id": task.id, "status": "queued"}
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
|
|
||||||
@app.post("/send_to_all_destinations/")
|
@app.post("/send_to_all_destinations/")
|
||||||
def send_to_all_destinations_endpoint(file_path: str):
|
def send_to_all_destinations_endpoint(file_path: str):
|
||||||
"""
|
"""
|
||||||
Call the aggregator task that sends this file to dropbox, nextcloud, and paperless.
|
Call the aggregator task that sends this file to dropbox, nextcloud, and paperless.
|
||||||
"""
|
"""
|
||||||
if not os.path.isabs(file_path):
|
if not os.path.isabs(file_path):
|
||||||
# If not absolute, assume it's in processed subdir
|
# If not absolute, assume it's in processed subdir
|
||||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||||
|
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=f"File {file_path} not found."
|
detail=f"File {file_path} not found."
|
||||||
)
|
)
|
||||||
|
|
||||||
task = send_to_all_destinations.delay(file_path)
|
task = send_to_all_destinations.delay(file_path)
|
||||||
return {"task_id": task.id, "status": "queued", "file_path": file_path}
|
return {"task_id": task.id, "status": "queued", "file_path": file_path}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/processall")
|
@app.post("/processall")
|
||||||
def process_all_pdfs_in_workdir():
|
def process_all_pdfs_in_workdir():
|
||||||
"""
|
"""
|
||||||
Finds all .pdf files in <workdir>/processed
|
Finds all .pdf files in <workdir>/processed
|
||||||
and enqueues them for upload_to_s3.
|
and enqueues them for upload_to_s3.
|
||||||
"""
|
"""
|
||||||
target_dir = settings.workdir
|
target_dir = settings.workdir
|
||||||
if not os.path.exists(target_dir):
|
if not os.path.exists(target_dir):
|
||||||
raise HTTPException(status_code=400, detail=f"Directory {target_dir} does not exist.")
|
raise HTTPException(status_code=400, detail=f"Directory {target_dir} does not exist.")
|
||||||
|
|
||||||
pdf_files = []
|
pdf_files = []
|
||||||
for filename in os.listdir(target_dir):
|
for filename in os.listdir(target_dir):
|
||||||
if filename.lower().endswith(".pdf"):
|
if filename.lower().endswith(".pdf"):
|
||||||
pdf_files.append(filename)
|
pdf_files.append(filename)
|
||||||
|
|
||||||
if not pdf_files:
|
if not pdf_files:
|
||||||
return {"message": "No PDF files found in processed directory."}
|
return {"message": "No PDF files found in processed directory."}
|
||||||
|
|
||||||
task_ids = []
|
task_ids = []
|
||||||
for pdf in pdf_files:
|
for pdf in pdf_files:
|
||||||
file_path = os.path.join(target_dir, pdf)
|
file_path = os.path.join(target_dir, pdf)
|
||||||
# Enqueue upload_to_s3
|
# Enqueue upload_to_s3
|
||||||
from app.tasks.upload_to_s3 import upload_to_s3
|
from app.tasks.upload_to_s3 import upload_to_s3
|
||||||
task = upload_to_s3.delay(file_path)
|
task = upload_to_s3.delay(file_path)
|
||||||
task_ids.append(task.id)
|
task_ids.append(task.id)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
|
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
|
||||||
"pdf_files": pdf_files,
|
"pdf_files": pdf_files,
|
||||||
"task_ids": task_ids
|
"task_ids": task_ids
|
||||||
}
|
}
|
||||||
|
|
||||||
app.include_router(frontend_router)
|
app.include_router(frontend_router)
|
||||||
|
|
||||||
@app.post("/ui-upload")
|
@app.post("/ui-upload")
|
||||||
async def ui_upload(file: UploadFile = File(...)):
|
async def ui_upload(file: UploadFile = File(...)):
|
||||||
# You can store this file in your 'workdir' (like how /process does) or a tmp dir
|
# You can store this file in your 'workdir' (like how /process does) or a tmp dir
|
||||||
workdir = "/workdir"
|
workdir = "/workdir"
|
||||||
target_path = os.path.join(workdir, file.filename)
|
target_path = os.path.join(workdir, file.filename)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(target_path, "wb") as f:
|
with open(target_path, "wb") as f:
|
||||||
content = await file.read()
|
content = await file.read()
|
||||||
f.write(content)
|
f.write(content)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
|
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
|
||||||
|
|
||||||
# Now you can call your existing Celery flow:
|
# Now you can call your existing Celery flow:
|
||||||
task = upload_to_s3.delay(target_path)
|
task = upload_to_s3.delay(target_path)
|
||||||
return {"task_id": task.id, "status": "queued"}
|
return {"task_id": task.id, "status": "queued"}
|
||||||
+14
-14
@@ -1,14 +1,14 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from .database import Base
|
from .database import Base
|
||||||
from sqlalchemy import Column, String, Integer
|
from sqlalchemy import Column, String, Integer
|
||||||
|
|
||||||
class DocumentMetadata(Base):
|
class DocumentMetadata(Base):
|
||||||
__tablename__ = "documents"
|
__tablename__ = "documents"
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, index=True)
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
filename = Column(String, unique=True, index=True)
|
filename = Column(String, unique=True, index=True)
|
||||||
sender = Column(String)
|
sender = Column(String)
|
||||||
recipient = Column(String)
|
recipient = Column(String)
|
||||||
tags = Column(String)
|
tags = Column(String)
|
||||||
summary = Column(String)
|
summary = Column(String)
|
||||||
|
|||||||
+72
-72
@@ -1,72 +1,72 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
import logging
|
import logging
|
||||||
import mimetypes
|
import mimetypes
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.upload_to_s3 import upload_to_s3
|
from app.tasks.upload_to_s3 import upload_to_s3
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def convert_to_pdf(file_path):
|
def convert_to_pdf(file_path):
|
||||||
"""
|
"""
|
||||||
Converts a file to PDF using Gotenberg's API.
|
Converts a file to PDF using Gotenberg's API.
|
||||||
Determines the appropriate Gotenberg endpoint based on the file's MIME type.
|
Determines the appropriate Gotenberg endpoint based on the file's MIME type.
|
||||||
On success, saves the PDF locally and enqueues it for S3 upload.
|
On success, saves the PDF locally and enqueues it for S3 upload.
|
||||||
"""
|
"""
|
||||||
gotenberg_url = getattr(settings, "gotenberg_url", None)
|
gotenberg_url = getattr(settings, "gotenberg_url", None)
|
||||||
if not gotenberg_url:
|
if not gotenberg_url:
|
||||||
logger.error("Gotenberg URL is not configured in settings.")
|
logger.error("Gotenberg URL is not configured in settings.")
|
||||||
return
|
return
|
||||||
|
|
||||||
# Try to guess the MIME type based on file content (using extension-based fallback)
|
# Try to guess the MIME type based on file content (using extension-based fallback)
|
||||||
mime_type, encoding = mimetypes.guess_type(file_path)
|
mime_type, encoding = mimetypes.guess_type(file_path)
|
||||||
logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
|
logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
|
||||||
|
|
||||||
endpoint = None
|
endpoint = None
|
||||||
form_key = "files" # Default form key for most endpoints
|
form_key = "files" # Default form key for most endpoints
|
||||||
|
|
||||||
if mime_type:
|
if mime_type:
|
||||||
if mime_type == "text/html":
|
if mime_type == "text/html":
|
||||||
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
||||||
# The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
|
# The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
|
||||||
form_key = "index.html"
|
form_key = "index.html"
|
||||||
elif mime_type.startswith("image/"):
|
elif mime_type.startswith("image/"):
|
||||||
# For images, we use the LibreOffice endpoint (which supports image conversion)
|
# For images, we use the LibreOffice endpoint (which supports image conversion)
|
||||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||||
elif mime_type.startswith("text/plain"):
|
elif mime_type.startswith("text/plain"):
|
||||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||||
elif mime_type in ["text/markdown", "text/x-markdown"]:
|
elif mime_type in ["text/markdown", "text/x-markdown"]:
|
||||||
# Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
|
# Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
|
||||||
# For now, we'll fallback to LibreOffice.
|
# For now, we'll fallback to LibreOffice.
|
||||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||||
else:
|
else:
|
||||||
# For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
|
# For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
|
||||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||||
else:
|
else:
|
||||||
# If MIME detection fails, fallback to extension-based detection.
|
# If MIME detection fails, fallback to extension-based detection.
|
||||||
ext = os.path.splitext(file_path)[1].lower()
|
ext = os.path.splitext(file_path)[1].lower()
|
||||||
if ext in [".html", ".htm"]:
|
if ext in [".html", ".htm"]:
|
||||||
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
||||||
form_key = "index.html"
|
form_key = "index.html"
|
||||||
else:
|
else:
|
||||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(file_path, "rb") as f:
|
with open(file_path, "rb") as f:
|
||||||
files = {form_key: f}
|
files = {form_key: f}
|
||||||
response = requests.post(endpoint, files=files)
|
response = requests.post(endpoint, files=files)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
|
converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
|
||||||
with open(converted_file_path, "wb") as out_file:
|
with open(converted_file_path, "wb") as out_file:
|
||||||
out_file.write(response.content)
|
out_file.write(response.content)
|
||||||
logger.info(f"Converted file saved as PDF: {converted_file_path}")
|
logger.info(f"Converted file saved as PDF: {converted_file_path}")
|
||||||
upload_to_s3.delay(converted_file_path)
|
upload_to_s3.delay(converted_file_path)
|
||||||
return converted_file_path
|
return converted_file_path
|
||||||
else:
|
else:
|
||||||
logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
|
logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Error converting {file_path} to PDF: {e}")
|
logger.exception(f"Error converting {file_path} to PDF: {e}")
|
||||||
|
|||||||
@@ -1,128 +1,128 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import fitz # PyMuPDF for PDF metadata editing
|
import fitz # PyMuPDF for PDF metadata editing
|
||||||
import json
|
import json
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.tasks.finalize_document_storage import finalize_document_storage
|
from app.tasks.finalize_document_storage import finalize_document_storage
|
||||||
|
|
||||||
# Import the shared Celery instance
|
# Import the shared Celery instance
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
def unique_filepath(directory, base_filename, extension=".pdf"):
|
def unique_filepath(directory, base_filename, extension=".pdf"):
|
||||||
"""
|
"""
|
||||||
Returns a unique filepath in the specified directory.
|
Returns a unique filepath in the specified directory.
|
||||||
If 'base_filename.pdf' exists, it will append an underscore and counter.
|
If 'base_filename.pdf' exists, it will append an underscore and counter.
|
||||||
"""
|
"""
|
||||||
candidate = os.path.join(directory, base_filename + extension)
|
candidate = os.path.join(directory, base_filename + extension)
|
||||||
if not os.path.exists(candidate):
|
if not os.path.exists(candidate):
|
||||||
return candidate
|
return candidate
|
||||||
counter = 1
|
counter = 1
|
||||||
while True:
|
while True:
|
||||||
candidate = os.path.join(directory, f"{base_filename}_{counter}{extension}")
|
candidate = os.path.join(directory, f"{base_filename}_{counter}{extension}")
|
||||||
if not os.path.exists(candidate):
|
if not os.path.exists(candidate):
|
||||||
return candidate
|
return candidate
|
||||||
counter += 1
|
counter += 1
|
||||||
|
|
||||||
def persist_metadata(metadata, final_pdf_path):
|
def persist_metadata(metadata, final_pdf_path):
|
||||||
"""
|
"""
|
||||||
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
|
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
|
||||||
For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
|
For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
|
||||||
the metadata will be saved as "<workdir>/processed/MyFile.json".
|
the metadata will be saved as "<workdir>/processed/MyFile.json".
|
||||||
"""
|
"""
|
||||||
base, _ = os.path.splitext(final_pdf_path)
|
base, _ = os.path.splitext(final_pdf_path)
|
||||||
json_path = base + ".json"
|
json_path = base + ".json"
|
||||||
with open(json_path, "w", encoding="utf-8") as f:
|
with open(json_path, "w", encoding="utf-8") as f:
|
||||||
json.dump(metadata, f, ensure_ascii=False, indent=2)
|
json.dump(metadata, f, ensure_ascii=False, indent=2)
|
||||||
return json_path
|
return json_path
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata: dict):
|
def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata: dict):
|
||||||
"""
|
"""
|
||||||
Embeds extracted metadata into the PDF's standard metadata fields.
|
Embeds extracted metadata into the PDF's standard metadata fields.
|
||||||
The mapping is as follows:
|
The mapping is as follows:
|
||||||
- title: uses the extracted metadata "filename"
|
- title: uses the extracted metadata "filename"
|
||||||
- author: uses "absender" (or "Unknown" if missing)
|
- author: uses "absender" (or "Unknown" if missing)
|
||||||
- subject: uses "document_type" (or "Unknown")
|
- subject: uses "document_type" (or "Unknown")
|
||||||
- keywords: a comma‐separated list from the "tags" field
|
- keywords: a comma‐separated list from the "tags" field
|
||||||
|
|
||||||
After processing, the file is moved to
|
After processing, the file is moved to
|
||||||
<workdir>/processed/<suggested_filename.pdf>
|
<workdir>/processed/<suggested_filename.pdf>
|
||||||
where <suggested_filename.pdf> is derived from metadata["filename"].
|
where <suggested_filename.pdf> is derived from metadata["filename"].
|
||||||
The output PDF is saved incrementally while preserving its original encryption.
|
The output PDF is saved incrementally while preserving its original encryption.
|
||||||
Additionally, the metadata is persisted to a JSON file with the same base name.
|
Additionally, the metadata is persisted to a JSON file with the same base name.
|
||||||
"""
|
"""
|
||||||
# Check for file existence; if not found, try the known shared tmp directory.
|
# Check for file existence; if not found, try the known shared tmp directory.
|
||||||
if not os.path.exists(local_file_path):
|
if not os.path.exists(local_file_path):
|
||||||
alt_path = os.path.join(settings.workdir, "tmp", os.path.basename(local_file_path))
|
alt_path = os.path.join(settings.workdir, "tmp", os.path.basename(local_file_path))
|
||||||
if os.path.exists(alt_path):
|
if os.path.exists(alt_path):
|
||||||
local_file_path = alt_path
|
local_file_path = alt_path
|
||||||
else:
|
else:
|
||||||
print(f"[ERROR] Local file {local_file_path} not found, cannot embed metadata.")
|
print(f"[ERROR] Local file {local_file_path} not found, cannot embed metadata.")
|
||||||
return {"error": "File not found"}
|
return {"error": "File not found"}
|
||||||
|
|
||||||
# Work on a safe copy in /tmp
|
# Work on a safe copy in /tmp
|
||||||
tmp_dir = "/tmp"
|
tmp_dir = "/tmp"
|
||||||
original_file = local_file_path
|
original_file = local_file_path
|
||||||
processed_file = os.path.join(tmp_dir, f"processed_{os.path.basename(local_file_path)}")
|
processed_file = os.path.join(tmp_dir, f"processed_{os.path.basename(local_file_path)}")
|
||||||
|
|
||||||
# Create a safe copy to work on
|
# Create a safe copy to work on
|
||||||
shutil.copy(original_file, processed_file)
|
shutil.copy(original_file, processed_file)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
print(f"[DEBUG] Embedding metadata into {processed_file}...")
|
print(f"[DEBUG] Embedding metadata into {processed_file}...")
|
||||||
|
|
||||||
# Open the PDF
|
# Open the PDF
|
||||||
doc = fitz.open(processed_file)
|
doc = fitz.open(processed_file)
|
||||||
# Set PDF metadata using only the standard keys.
|
# Set PDF metadata using only the standard keys.
|
||||||
doc.set_metadata({
|
doc.set_metadata({
|
||||||
"title": metadata.get("filename", "Unknown Document"),
|
"title": metadata.get("filename", "Unknown Document"),
|
||||||
"author": metadata.get("absender", "Unknown"),
|
"author": metadata.get("absender", "Unknown"),
|
||||||
"subject": metadata.get("document_type", "Unknown"),
|
"subject": metadata.get("document_type", "Unknown"),
|
||||||
"keywords": ", ".join(metadata.get("tags", []))
|
"keywords": ", ".join(metadata.get("tags", []))
|
||||||
})
|
})
|
||||||
# Save incrementally and preserve encryption
|
# Save incrementally and preserve encryption
|
||||||
doc.save(processed_file, incremental=True, encryption=fitz.PDF_ENCRYPT_KEEP)
|
doc.save(processed_file, incremental=True, encryption=fitz.PDF_ENCRYPT_KEEP)
|
||||||
doc.close()
|
doc.close()
|
||||||
|
|
||||||
print(f"[INFO] Metadata embedded successfully in {processed_file}")
|
print(f"[INFO] Metadata embedded successfully in {processed_file}")
|
||||||
|
|
||||||
# Use the suggested filename from metadata; if not provided, use the original basename.
|
# Use the suggested filename from metadata; if not provided, use the original basename.
|
||||||
suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
|
suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
|
||||||
# Remove any extension and then add .pdf
|
# Remove any extension and then add .pdf
|
||||||
suggested_filename = os.path.splitext(suggested_filename)[0]
|
suggested_filename = os.path.splitext(suggested_filename)[0]
|
||||||
# Define the final directory based on settings.workdir and ensure it exists.
|
# Define the final directory based on settings.workdir and ensure it exists.
|
||||||
final_dir = os.path.join(settings.workdir, "processed")
|
final_dir = os.path.join(settings.workdir, "processed")
|
||||||
os.makedirs(final_dir, exist_ok=True)
|
os.makedirs(final_dir, exist_ok=True)
|
||||||
# Get a unique filepath in case of collisions.
|
# Get a unique filepath in case of collisions.
|
||||||
final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
|
final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
|
||||||
|
|
||||||
# Move the processed file using shutil.move to handle cross-device moves.
|
# Move the processed file using shutil.move to handle cross-device moves.
|
||||||
shutil.move(processed_file, final_file_path)
|
shutil.move(processed_file, final_file_path)
|
||||||
# Ensure the temporary file is deleted if it still exists.
|
# Ensure the temporary file is deleted if it still exists.
|
||||||
if os.path.exists(processed_file):
|
if os.path.exists(processed_file):
|
||||||
os.remove(processed_file)
|
os.remove(processed_file)
|
||||||
|
|
||||||
# Persist the metadata into a JSON file with the same base name.
|
# Persist the metadata into a JSON file with the same base name.
|
||||||
json_path = persist_metadata(metadata, final_file_path)
|
json_path = persist_metadata(metadata, final_file_path)
|
||||||
print(f"[INFO] Metadata persisted to {json_path}")
|
print(f"[INFO] Metadata persisted to {json_path}")
|
||||||
|
|
||||||
# Trigger the next step: final storage.
|
# Trigger the next step: final storage.
|
||||||
finalize_document_storage.delay(original_file, final_file_path, metadata)
|
finalize_document_storage.delay(original_file, final_file_path, metadata)
|
||||||
|
|
||||||
# After triggering final storage, delete the original file if it is in workdir/tmp.
|
# After triggering final storage, delete the original file if it is in workdir/tmp.
|
||||||
workdir_tmp = os.path.join(settings.workdir, "tmp")
|
workdir_tmp = os.path.join(settings.workdir, "tmp")
|
||||||
if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
|
if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
|
||||||
try:
|
try:
|
||||||
os.remove(original_file)
|
os.remove(original_file)
|
||||||
print(f"[INFO] Deleted original file from {original_file}")
|
print(f"[INFO] Deleted original file from {original_file}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[ERROR] Could not delete original file {original_file}: {e}")
|
print(f"[ERROR] Could not delete original file {original_file}: {e}")
|
||||||
|
|
||||||
return {"file": final_file_path, "metadata_file": json_path, "status": "Metadata embedded"}
|
return {"file": final_file_path, "metadata_file": json_path, "status": "Metadata embedded"}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[ERROR] Failed to embed metadata into {processed_file}: {e}")
|
print(f"[ERROR] Failed to embed metadata into {processed_file}: {e}")
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|||||||
@@ -1,97 +1,97 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||||
|
|
||||||
# Import the shared Celery instance
|
# Import the shared Celery instance
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
client = OpenAI(api_key=settings.openai_api_key)
|
client = OpenAI(api_key=settings.openai_api_key)
|
||||||
|
|
||||||
def extract_json_from_text(text):
|
def extract_json_from_text(text):
|
||||||
"""
|
"""
|
||||||
Try to extract a JSON object from the text.
|
Try to extract a JSON object from the text.
|
||||||
- First, check for a JSON block inside triple backticks.
|
- First, check for a JSON block inside triple backticks.
|
||||||
- If not found, try to extract text from the first '{' to the last '}'.
|
- If not found, try to extract text from the first '{' to the last '}'.
|
||||||
"""
|
"""
|
||||||
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
||||||
match = re.search(pattern, text, re.DOTALL)
|
match = re.search(pattern, text, re.DOTALL)
|
||||||
if match:
|
if match:
|
||||||
return match.group(1)
|
return match.group(1)
|
||||||
else:
|
else:
|
||||||
start = text.find("{")
|
start = text.find("{")
|
||||||
end = text.rfind("}")
|
end = text.rfind("}")
|
||||||
if start != -1 and end != -1 and end > start:
|
if start != -1 and end != -1 and end > start:
|
||||||
return text[start:end+1]
|
return text[start:end+1]
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def extract_metadata_with_gpt(s3_filename: str, cleaned_text: str):
|
def extract_metadata_with_gpt(s3_filename: str, cleaned_text: str):
|
||||||
"""Uses OpenAI GPT-4o-mini to classify document metadata."""
|
"""Uses OpenAI GPT-4o-mini to classify document metadata."""
|
||||||
|
|
||||||
prompt = f"""
|
prompt = f"""
|
||||||
You are a specialized document analyzer trained to extract structured metadata from documents.
|
You are a specialized document analyzer trained to extract structured metadata from documents.
|
||||||
Your task is to analyze the given text and return a well-structured JSON object.
|
Your task is to analyze the given text and return a well-structured JSON object.
|
||||||
|
|
||||||
Extract and return the following fields:
|
Extract and return the following fields:
|
||||||
1. **filename**: Machine-readable filename (YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).
|
1. **filename**: Machine-readable filename (YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).
|
||||||
2. **empfaenger**: The recipient, or "Unknown" if not found.
|
2. **empfaenger**: The recipient, or "Unknown" if not found.
|
||||||
3. **absender**: The sender, or "Unknown" if not found.
|
3. **absender**: The sender, or "Unknown" if not found.
|
||||||
4. **correspondent**: The entity or company that issued the document (shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").
|
4. **correspondent**: The entity or company that issued the document (shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").
|
||||||
5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].
|
5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].
|
||||||
6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, Private_Korrespondenz, Sonstige_Informationen].
|
6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, Private_Korrespondenz, Sonstige_Informationen].
|
||||||
7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).
|
7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).
|
||||||
8. **tags**: A list of up to 4 relevant thematic keywords.
|
8. **tags**: A list of up to 4 relevant thematic keywords.
|
||||||
9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").
|
9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").
|
||||||
10. **title**: A human-readable title summarizing the document content.
|
10. **title**: A human-readable title summarizing the document content.
|
||||||
11. **confidence_score**: A numeric value (0-100) indicating the confidence level of the extracted metadata.
|
11. **confidence_score**: A numeric value (0-100) indicating the confidence level of the extracted metadata.
|
||||||
12. **reference_number**: Extracted invoice/order/reference number if available.
|
12. **reference_number**: Extracted invoice/order/reference number if available.
|
||||||
13. **monetary_amounts**: A list of key monetary values detected in the document.
|
13. **monetary_amounts**: A list of key monetary values detected in the document.
|
||||||
|
|
||||||
### Important Rules:
|
### Important Rules:
|
||||||
- **OCR Correction**: Assume the text has been corrected for OCR errors.
|
- **OCR Correction**: Assume the text has been corrected for OCR errors.
|
||||||
- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.
|
- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.
|
||||||
- **Title**: Concise, no addresses, and contains key identifying features.
|
- **Title**: Concise, no addresses, and contains key identifying features.
|
||||||
- **Date Selection**: Use the most relevant date if multiple are found.
|
- **Date Selection**: Use the most relevant date if multiple are found.
|
||||||
- **Output Language**: Maintain the document's original language.
|
- **Output Language**: Maintain the document's original language.
|
||||||
|
|
||||||
Extracted text:
|
Extracted text:
|
||||||
{cleaned_text}
|
{cleaned_text}
|
||||||
|
|
||||||
Return only valid JSON with no additional commentary.
|
Return only valid JSON with no additional commentary.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
print(f"[DEBUG] Sending classification request for {s3_filename}...")
|
print(f"[DEBUG] Sending classification request for {s3_filename}...")
|
||||||
completion = client.chat.completions.create(
|
completion = client.chat.completions.create(
|
||||||
model="gpt-4o-mini",
|
model="gpt-4o-mini",
|
||||||
messages=[
|
messages=[
|
||||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||||
{"role": "user", "content": prompt}
|
{"role": "user", "content": prompt}
|
||||||
],
|
],
|
||||||
temperature=0
|
temperature=0
|
||||||
)
|
)
|
||||||
|
|
||||||
content = completion.choices[0].message.content
|
content = completion.choices[0].message.content
|
||||||
print(f"[DEBUG] Raw classification response for {s3_filename}: {content}")
|
print(f"[DEBUG] Raw classification response for {s3_filename}: {content}")
|
||||||
|
|
||||||
json_text = extract_json_from_text(content)
|
json_text = extract_json_from_text(content)
|
||||||
if not json_text:
|
if not json_text:
|
||||||
print(f"[ERROR] Could not find valid JSON in GPT response for {s3_filename}.")
|
print(f"[ERROR] Could not find valid JSON in GPT response for {s3_filename}.")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
metadata = json.loads(json_text)
|
metadata = json.loads(json_text)
|
||||||
print(f"[DEBUG] Extracted metadata: {metadata}")
|
print(f"[DEBUG] Extracted metadata: {metadata}")
|
||||||
|
|
||||||
# Trigger the next step: embedding metadata into the PDF
|
# Trigger the next step: embedding metadata into the PDF
|
||||||
embed_metadata_into_pdf.delay(s3_filename, cleaned_text, metadata)
|
embed_metadata_into_pdf.delay(s3_filename, cleaned_text, metadata)
|
||||||
|
|
||||||
return {"s3_file": s3_filename, "metadata": metadata}
|
return {"s3_file": s3_filename, "metadata": metadata}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[ERROR] OpenAI classification failed for {s3_filename}: {e}")
|
print(f"[ERROR] OpenAI classification failed for {s3_filename}: {e}")
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
@@ -1,26 +1,26 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
# Import the shared Celery instance
|
# Import the shared Celery instance
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
# 1) Import the aggregator task
|
# 1) Import the aggregator task
|
||||||
from app.tasks.send_to_all import send_to_all_destinations
|
from app.tasks.send_to_all import send_to_all_destinations
|
||||||
|
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def finalize_document_storage(original_file: str, processed_file: str, metadata: dict):
|
def finalize_document_storage(original_file: str, processed_file: str, metadata: dict):
|
||||||
"""
|
"""
|
||||||
Final storage step after embedding metadata.
|
Final storage step after embedding metadata.
|
||||||
We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
|
We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
|
||||||
"""
|
"""
|
||||||
print(f"[INFO] Finalizing document storage for {processed_file}")
|
print(f"[INFO] Finalizing document storage for {processed_file}")
|
||||||
|
|
||||||
# 2) Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
|
# 2) Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
|
||||||
send_to_all_destinations.delay(processed_file)
|
send_to_all_destinations.delay(processed_file)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": "Completed",
|
"status": "Completed",
|
||||||
"file": processed_file
|
"file": processed_file
|
||||||
}
|
}
|
||||||
|
|||||||
+412
-412
@@ -1,412 +1,412 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import email
|
import email
|
||||||
import imaplib
|
import imaplib
|
||||||
import logging
|
import logging
|
||||||
import redis
|
import redis
|
||||||
import re
|
import re
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.upload_to_s3 import upload_to_s3
|
from app.tasks.upload_to_s3 import upload_to_s3
|
||||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Initialize Redis connection using Celery's Redis settings
|
# Initialize Redis connection using Celery's Redis settings
|
||||||
redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
|
redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
|
||||||
|
|
||||||
LOCK_KEY = "imap_lock" # Unique key for locking
|
LOCK_KEY = "imap_lock" # Unique key for locking
|
||||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||||
|
|
||||||
# Local cache file for tracking processed emails
|
# Local cache file for tracking processed emails
|
||||||
CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
|
CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
|
||||||
|
|
||||||
|
|
||||||
def acquire_lock():
|
def acquire_lock():
|
||||||
"""Attempt to acquire a Redis-based lock. If acquired, set an expiration."""
|
"""Attempt to acquire a Redis-based lock. If acquired, set an expiration."""
|
||||||
lock_acquired = redis_client.setnx(LOCK_KEY, "locked")
|
lock_acquired = redis_client.setnx(LOCK_KEY, "locked")
|
||||||
if lock_acquired:
|
if lock_acquired:
|
||||||
redis_client.expire(LOCK_KEY, LOCK_EXPIRE)
|
redis_client.expire(LOCK_KEY, LOCK_EXPIRE)
|
||||||
logger.info("Lock acquired for IMAP processing.")
|
logger.info("Lock acquired for IMAP processing.")
|
||||||
return True
|
return True
|
||||||
logger.warning("Lock already held. Skipping this cycle.")
|
logger.warning("Lock already held. Skipping this cycle.")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def release_lock():
|
def release_lock():
|
||||||
"""Release the lock by deleting the Redis key."""
|
"""Release the lock by deleting the Redis key."""
|
||||||
redis_client.delete(LOCK_KEY)
|
redis_client.delete(LOCK_KEY)
|
||||||
logger.info("Lock released.")
|
logger.info("Lock released.")
|
||||||
|
|
||||||
|
|
||||||
def load_processed_emails():
|
def load_processed_emails():
|
||||||
"""Load the list of already processed emails from a local JSON file."""
|
"""Load the list of already processed emails from a local JSON file."""
|
||||||
if os.path.exists(CACHE_FILE):
|
if os.path.exists(CACHE_FILE):
|
||||||
try:
|
try:
|
||||||
with open(CACHE_FILE, "r") as f:
|
with open(CACHE_FILE, "r") as f:
|
||||||
processed_emails = json.load(f)
|
processed_emails = json.load(f)
|
||||||
processed_emails = cleanup_old_entries(processed_emails)
|
processed_emails = cleanup_old_entries(processed_emails)
|
||||||
return processed_emails
|
return processed_emails
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
logger.warning("Failed to decode JSON, resetting processed emails cache.")
|
logger.warning("Failed to decode JSON, resetting processed emails cache.")
|
||||||
return {}
|
return {}
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def save_processed_emails(processed_emails):
|
def save_processed_emails(processed_emails):
|
||||||
"""Save the processed email IDs to a local JSON file."""
|
"""Save the processed email IDs to a local JSON file."""
|
||||||
with open(CACHE_FILE, "w") as f:
|
with open(CACHE_FILE, "w") as f:
|
||||||
json.dump(processed_emails, f, indent=4)
|
json.dump(processed_emails, f, indent=4)
|
||||||
|
|
||||||
|
|
||||||
def cleanup_old_entries(processed_emails):
|
def cleanup_old_entries(processed_emails):
|
||||||
"""Remove entries older than 7 days from the cache to avoid infinite growth."""
|
"""Remove entries older than 7 days from the cache to avoid infinite growth."""
|
||||||
seven_days_ago = datetime.now(timezone.utc) - timedelta(days=7)
|
seven_days_ago = datetime.now(timezone.utc) - timedelta(days=7)
|
||||||
valid_emails = {}
|
valid_emails = {}
|
||||||
for msg_id, date_str in processed_emails.items():
|
for msg_id, date_str in processed_emails.items():
|
||||||
naive_dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
|
naive_dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
|
||||||
aware_dt = naive_dt.replace(tzinfo=timezone.utc)
|
aware_dt = naive_dt.replace(tzinfo=timezone.utc)
|
||||||
if aware_dt > seven_days_ago:
|
if aware_dt > seven_days_ago:
|
||||||
valid_emails[msg_id] = date_str
|
valid_emails[msg_id] = date_str
|
||||||
return valid_emails
|
return valid_emails
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def pull_all_inboxes():
|
def pull_all_inboxes():
|
||||||
"""
|
"""
|
||||||
Periodic Celery task that checks all configured IMAP mailboxes
|
Periodic Celery task that checks all configured IMAP mailboxes
|
||||||
and fetches attachments from new emails.
|
and fetches attachments from new emails.
|
||||||
Ensures only one instance runs at a time using Redis-based locking.
|
Ensures only one instance runs at a time using Redis-based locking.
|
||||||
"""
|
"""
|
||||||
if not acquire_lock():
|
if not acquire_lock():
|
||||||
logger.info("Skipping execution: Another instance is running.")
|
logger.info("Skipping execution: Another instance is running.")
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logger.info("Starting pull_all_inboxes")
|
logger.info("Starting pull_all_inboxes")
|
||||||
|
|
||||||
# Mailbox #1 (non-Gmail)
|
# Mailbox #1 (non-Gmail)
|
||||||
check_and_pull_mailbox(
|
check_and_pull_mailbox(
|
||||||
mailbox_key="imap1",
|
mailbox_key="imap1",
|
||||||
host=settings.imap1_host,
|
host=settings.imap1_host,
|
||||||
port=settings.imap1_port,
|
port=settings.imap1_port,
|
||||||
username=settings.imap1_username,
|
username=settings.imap1_username,
|
||||||
password=settings.imap1_password,
|
password=settings.imap1_password,
|
||||||
use_ssl=settings.imap1_ssl,
|
use_ssl=settings.imap1_ssl,
|
||||||
delete_after_process=settings.imap1_delete_after_process,
|
delete_after_process=settings.imap1_delete_after_process,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Mailbox #2 (Gmail)
|
# Mailbox #2 (Gmail)
|
||||||
check_and_pull_mailbox(
|
check_and_pull_mailbox(
|
||||||
mailbox_key="imap2",
|
mailbox_key="imap2",
|
||||||
host=settings.imap2_host,
|
host=settings.imap2_host,
|
||||||
port=settings.imap2_port,
|
port=settings.imap2_port,
|
||||||
username=settings.imap2_username,
|
username=settings.imap2_username,
|
||||||
password=settings.imap2_password,
|
password=settings.imap2_password,
|
||||||
use_ssl=settings.imap2_ssl,
|
use_ssl=settings.imap2_ssl,
|
||||||
delete_after_process=settings.imap2_delete_after_process,
|
delete_after_process=settings.imap2_delete_after_process,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Finished pull_all_inboxes")
|
logger.info("Finished pull_all_inboxes")
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
release_lock()
|
release_lock()
|
||||||
|
|
||||||
|
|
||||||
def check_and_pull_mailbox(
|
def check_and_pull_mailbox(
|
||||||
mailbox_key: str,
|
mailbox_key: str,
|
||||||
host: str | None,
|
host: str | None,
|
||||||
port: int | None,
|
port: int | None,
|
||||||
username: str | None,
|
username: str | None,
|
||||||
password: str | None,
|
password: str | None,
|
||||||
use_ssl: bool,
|
use_ssl: bool,
|
||||||
delete_after_process: bool,
|
delete_after_process: bool,
|
||||||
):
|
):
|
||||||
"""Validates config and invokes pulling from the mailbox if valid."""
|
"""Validates config and invokes pulling from the mailbox if valid."""
|
||||||
if not (host and port and username and password):
|
if not (host and port and username and password):
|
||||||
logger.warning(f"Mailbox {mailbox_key} is missing config, skipping.")
|
logger.warning(f"Mailbox {mailbox_key} is missing config, skipping.")
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.info(f"Checking mailbox: {mailbox_key}")
|
logger.info(f"Checking mailbox: {mailbox_key}")
|
||||||
pull_inbox(
|
pull_inbox(
|
||||||
mailbox_key=mailbox_key,
|
mailbox_key=mailbox_key,
|
||||||
host=host,
|
host=host,
|
||||||
port=port,
|
port=port,
|
||||||
username=username,
|
username=username,
|
||||||
password=password,
|
password=password,
|
||||||
use_ssl=use_ssl,
|
use_ssl=use_ssl,
|
||||||
delete_after_process=delete_after_process,
|
delete_after_process=delete_after_process,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
|
def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
|
||||||
delete_after_process):
|
delete_after_process):
|
||||||
"""
|
"""
|
||||||
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
||||||
and processes attachments while preserving the original unread status.
|
and processes attachments while preserving the original unread status.
|
||||||
|
|
||||||
For Gmail:
|
For Gmail:
|
||||||
- Attempts to select the localized All Mail folder.
|
- Attempts to select the localized All Mail folder.
|
||||||
- Runs an X-GM-RAW query: "in:anywhere in:unread newer_than:3d has:attachment".
|
- Runs an X-GM-RAW query: "in:anywhere in:unread newer_than:3d has:attachment".
|
||||||
|
|
||||||
For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
|
For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
|
||||||
"""
|
"""
|
||||||
logger.info("Connecting to %s at %s:%s (SSL=%s)",
|
logger.info("Connecting to %s at %s:%s (SSL=%s)",
|
||||||
mailbox_key, host, port, use_ssl)
|
mailbox_key, host, port, use_ssl)
|
||||||
processed_emails = load_processed_emails()
|
processed_emails = load_processed_emails()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
mail = imaplib.IMAP4_SSL(host, port) if use_ssl else imaplib.IMAP4(host, port)
|
mail = imaplib.IMAP4_SSL(host, port) if use_ssl else imaplib.IMAP4(host, port)
|
||||||
mail.login(username, password)
|
mail.login(username, password)
|
||||||
|
|
||||||
is_gmail_host = "gmail" in host.lower()
|
is_gmail_host = "gmail" in host.lower()
|
||||||
if is_gmail_host:
|
if is_gmail_host:
|
||||||
# For Gmail, try to select the localized All Mail folder.
|
# For Gmail, try to select the localized All Mail folder.
|
||||||
all_mail_folder = find_all_mail_folder(mail)
|
all_mail_folder = find_all_mail_folder(mail)
|
||||||
if all_mail_folder:
|
if all_mail_folder:
|
||||||
logger.info("Using Gmail All Mail folder: %s", all_mail_folder)
|
logger.info("Using Gmail All Mail folder: %s", all_mail_folder)
|
||||||
mail.select(f'"{all_mail_folder}"')
|
mail.select(f'"{all_mail_folder}"')
|
||||||
else:
|
else:
|
||||||
logger.warning("Gmail All Mail folder not found, falling back to INBOX.")
|
logger.warning("Gmail All Mail folder not found, falling back to INBOX.")
|
||||||
mail.select("INBOX")
|
mail.select("INBOX")
|
||||||
# Use the X-GM-RAW query for Gmail.
|
# Use the X-GM-RAW query for Gmail.
|
||||||
raw_query = "in:anywhere in:unread newer_than:3d has:attachment"
|
raw_query = "in:anywhere in:unread newer_than:3d has:attachment"
|
||||||
status, search_data = mail.search(None, "X-GM-RAW", f'"{raw_query}"')
|
status, search_data = mail.search(None, "X-GM-RAW", f'"{raw_query}"')
|
||||||
else:
|
else:
|
||||||
# For non-Gmail, select INBOX and use SINCE/UNSEEN query.
|
# For non-Gmail, select INBOX and use SINCE/UNSEEN query.
|
||||||
mail.select("INBOX")
|
mail.select("INBOX")
|
||||||
since_date = (datetime.now(timezone.utc) - timedelta(days=3)
|
since_date = (datetime.now(timezone.utc) - timedelta(days=3)
|
||||||
).strftime("%d-%b-%Y")
|
).strftime("%d-%b-%Y")
|
||||||
status, search_data = mail.search(None, f'(SINCE {since_date} UNSEEN)')
|
status, search_data = mail.search(None, f'(SINCE {since_date} UNSEEN)')
|
||||||
|
|
||||||
if status != "OK":
|
if status != "OK":
|
||||||
logger.warning("Search failed on mailbox %s. Status=%s",
|
logger.warning("Search failed on mailbox %s. Status=%s",
|
||||||
mailbox_key, status)
|
mailbox_key, status)
|
||||||
mail.close()
|
mail.close()
|
||||||
mail.logout()
|
mail.logout()
|
||||||
return
|
return
|
||||||
|
|
||||||
msg_numbers = search_data[0].split()
|
msg_numbers = search_data[0].split()
|
||||||
logger.info("Found %d unread emails in %s.", len(msg_numbers), mailbox_key)
|
logger.info("Found %d unread emails in %s.", len(msg_numbers), mailbox_key)
|
||||||
|
|
||||||
for num in msg_numbers:
|
for num in msg_numbers:
|
||||||
status, msg_data = mail.fetch(num, "(RFC822)")
|
status, msg_data = mail.fetch(num, "(RFC822)")
|
||||||
if status != "OK":
|
if status != "OK":
|
||||||
logger.warning("Failed to fetch message %s in %s. Status=%s",
|
logger.warning("Failed to fetch message %s in %s. Status=%s",
|
||||||
num, mailbox_key, status)
|
num, mailbox_key, status)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
raw_email = msg_data[0][1]
|
raw_email = msg_data[0][1]
|
||||||
email_message = email.message_from_bytes(raw_email)
|
email_message = email.message_from_bytes(raw_email)
|
||||||
msg_id = email_message.get("Message-ID")
|
msg_id = email_message.get("Message-ID")
|
||||||
|
|
||||||
if not msg_id:
|
if not msg_id:
|
||||||
logger.warning("Skipping email without Message-ID in %s", mailbox_key)
|
logger.warning("Skipping email without Message-ID in %s", mailbox_key)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if msg_id in processed_emails:
|
if msg_id in processed_emails:
|
||||||
logger.info("Skipping already processed email %s in %s", msg_id, mailbox_key)
|
logger.info("Skipping already processed email %s in %s", msg_id, mailbox_key)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# For Gmail, check if the email already has the "Ingested" label.
|
# For Gmail, check if the email already has the "Ingested" label.
|
||||||
if is_gmail_host:
|
if is_gmail_host:
|
||||||
if email_already_has_label(mail, num, "Ingested"):
|
if email_already_has_label(mail, num, "Ingested"):
|
||||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.",
|
logger.info("Skipping email %s in %s, already labeled 'Ingested'.",
|
||||||
msg_id, mailbox_key)
|
msg_id, mailbox_key)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Process attachments (and convert non-PDF files).
|
# Process attachments (and convert non-PDF files).
|
||||||
# We call the function without assigning its return value since it is not used.
|
# We call the function without assigning its return value since it is not used.
|
||||||
fetch_attachments_and_enqueue(email_message)
|
fetch_attachments_and_enqueue(email_message)
|
||||||
|
|
||||||
if is_gmail_host:
|
if is_gmail_host:
|
||||||
mark_as_processed_with_star(mail, num)
|
mark_as_processed_with_star(mail, num)
|
||||||
mark_as_processed_with_label(mail, num, label="Ingested")
|
mark_as_processed_with_label(mail, num, label="Ingested")
|
||||||
|
|
||||||
processed_emails[msg_id] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
processed_emails[msg_id] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
save_processed_emails(processed_emails)
|
save_processed_emails(processed_emails)
|
||||||
|
|
||||||
if delete_after_process:
|
if delete_after_process:
|
||||||
logger.info("Deleting message %s from %s", num.decode(), mailbox_key)
|
logger.info("Deleting message %s from %s", num.decode(), mailbox_key)
|
||||||
mail.store(num, "+FLAGS", "\\Deleted")
|
mail.store(num, "+FLAGS", "\\Deleted")
|
||||||
else:
|
else:
|
||||||
mail.store(num, "-FLAGS", "\\Seen")
|
mail.store(num, "-FLAGS", "\\Seen")
|
||||||
|
|
||||||
if delete_after_process:
|
if delete_after_process:
|
||||||
mail.expunge()
|
mail.expunge()
|
||||||
|
|
||||||
mail.close()
|
mail.close()
|
||||||
mail.logout()
|
mail.logout()
|
||||||
logger.info("Finished processing mailbox %s", mailbox_key)
|
logger.info("Finished processing mailbox %s", mailbox_key)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
|
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
|
||||||
|
|
||||||
|
|
||||||
def fetch_attachments_and_enqueue(email_message):
|
def fetch_attachments_and_enqueue(email_message):
|
||||||
"""
|
"""
|
||||||
Extracts attachments from the email and processes only allowed file types.
|
Extracts attachments from the email and processes only allowed file types.
|
||||||
|
|
||||||
Allowed file types include:
|
Allowed file types include:
|
||||||
- PDF: application/pdf
|
- PDF: application/pdf
|
||||||
- Microsoft Office files:
|
- Microsoft Office files:
|
||||||
- Word: application/msword,
|
- Word: application/msword,
|
||||||
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
||||||
- Excel: application/vnd.ms-excel,
|
- Excel: application/vnd.ms-excel,
|
||||||
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
||||||
- PowerPoint: application/vnd.ms-powerpoint,
|
- PowerPoint: application/vnd.ms-powerpoint,
|
||||||
application/vnd.openxmlformats-officedocument.presentationml.presentation
|
application/vnd.openxmlformats-officedocument.presentationml.presentation
|
||||||
- Other meaningful attachments:
|
- Other meaningful attachments:
|
||||||
- Plain text: text/plain
|
- Plain text: text/plain
|
||||||
- CSV: text/csv
|
- CSV: text/csv
|
||||||
- Rich Text Format: application/rtf, text/rtf
|
- Rich Text Format: application/rtf, text/rtf
|
||||||
|
|
||||||
Attachments not in this list are skipped. Common image MIME types such as
|
Attachments not in this list are skipped. Common image MIME types such as
|
||||||
image/jpeg, image/png, image/gif, image/bmp, image/tiff, and image/webp are
|
image/jpeg, image/png, image/gif, image/bmp, image/tiff, and image/webp are
|
||||||
intentionally excluded.
|
intentionally excluded.
|
||||||
|
|
||||||
If the attachment is a PDF, it is enqueued for upload; any other allowed file
|
If the attachment is a PDF, it is enqueued for upload; any other allowed file
|
||||||
is enqueued for conversion to PDF.
|
is enqueued for conversion to PDF.
|
||||||
|
|
||||||
Returns True if at least one allowed attachment was processed.
|
Returns True if at least one allowed attachment was processed.
|
||||||
"""
|
"""
|
||||||
ALLOWED_MIME_TYPES = {
|
ALLOWED_MIME_TYPES = {
|
||||||
"application/pdf",
|
"application/pdf",
|
||||||
"application/msword",
|
"application/msword",
|
||||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
"application/vnd.ms-excel",
|
"application/vnd.ms-excel",
|
||||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
"application/vnd.ms-powerpoint",
|
"application/vnd.ms-powerpoint",
|
||||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||||
"text/plain",
|
"text/plain",
|
||||||
"text/csv",
|
"text/csv",
|
||||||
"application/rtf",
|
"application/rtf",
|
||||||
"text/rtf",
|
"text/rtf",
|
||||||
}
|
}
|
||||||
|
|
||||||
has_attachment = False
|
has_attachment = False
|
||||||
for part in email_message.walk():
|
for part in email_message.walk():
|
||||||
if part.get_content_maintype() == "multipart":
|
if part.get_content_maintype() == "multipart":
|
||||||
continue
|
continue
|
||||||
|
|
||||||
filename = part.get_filename()
|
filename = part.get_filename()
|
||||||
if not filename:
|
if not filename:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
mime_type = part.get_content_type()
|
mime_type = part.get_content_type()
|
||||||
if mime_type not in ALLOWED_MIME_TYPES:
|
if mime_type not in ALLOWED_MIME_TYPES:
|
||||||
logger.info("Skipping attachment %s with MIME type %s",
|
logger.info("Skipping attachment %s with MIME type %s",
|
||||||
filename, mime_type)
|
filename, mime_type)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
file_path = os.path.join(settings.workdir, filename)
|
file_path = os.path.join(settings.workdir, filename)
|
||||||
with open(file_path, "wb") as f:
|
with open(file_path, "wb") as f:
|
||||||
f.write(part.get_payload(decode=True))
|
f.write(part.get_payload(decode=True))
|
||||||
|
|
||||||
if mime_type == "application/pdf":
|
if mime_type == "application/pdf":
|
||||||
upload_to_s3.delay(file_path)
|
upload_to_s3.delay(file_path)
|
||||||
logger.info("Enqueued PDF for upload: %s", filename)
|
logger.info("Enqueued PDF for upload: %s", filename)
|
||||||
elif mime_type in ALLOWED_MIME_TYPES:
|
elif mime_type in ALLOWED_MIME_TYPES:
|
||||||
# Enqueue conversion to PDF using the Gotenberg service.
|
# Enqueue conversion to PDF using the Gotenberg service.
|
||||||
convert_to_pdf.delay(file_path)
|
convert_to_pdf.delay(file_path)
|
||||||
logger.info("Enqueued file for conversion to PDF: %s", filename)
|
logger.info("Enqueued file for conversion to PDF: %s", filename)
|
||||||
|
|
||||||
has_attachment = True
|
has_attachment = True
|
||||||
return has_attachment
|
return has_attachment
|
||||||
|
|
||||||
|
|
||||||
def email_already_has_label(mail, msg_id, label="Ingested"):
|
def email_already_has_label(mail, msg_id, label="Ingested"):
|
||||||
"""
|
"""
|
||||||
Checks if the given message (msg_id) has the specified Gmail label.
|
Checks if the given message (msg_id) has the specified Gmail label.
|
||||||
Returns True if the label is found, False otherwise.
|
Returns True if the label is found, False otherwise.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
|
label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
|
||||||
if label_status == "OK" and label_data and len(label_data) > 0:
|
if label_status == "OK" and label_data and len(label_data) > 0:
|
||||||
raw_labels = label_data[0][1].decode("utf-8", errors="ignore")
|
raw_labels = label_data[0][1].decode("utf-8", errors="ignore")
|
||||||
if label in raw_labels:
|
if label in raw_labels:
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Failed to fetch labels for msg_id=%s: %s", msg_id, e)
|
logger.error("Failed to fetch labels for msg_id=%s: %s", msg_id, e)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def mark_as_processed_with_star(mail, msg_id):
|
def mark_as_processed_with_star(mail, msg_id):
|
||||||
"""Stars the email in Gmail."""
|
"""Stars the email in Gmail."""
|
||||||
try:
|
try:
|
||||||
mail.store(msg_id, "+FLAGS", "\\Flagged")
|
mail.store(msg_id, "+FLAGS", "\\Flagged")
|
||||||
logger.info("Email %s starred in Gmail.", msg_id)
|
logger.info("Email %s starred in Gmail.", msg_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Failed to star email %s: %s", msg_id, e)
|
logger.error("Failed to star email %s: %s", msg_id, e)
|
||||||
|
|
||||||
|
|
||||||
def mark_as_processed_with_label(mail, msg_id, label="Ingested"):
|
def mark_as_processed_with_label(mail, msg_id, label="Ingested"):
|
||||||
"""Adds a custom label to the email in Gmail."""
|
"""Adds a custom label to the email in Gmail."""
|
||||||
try:
|
try:
|
||||||
mail.store(msg_id, "+X-GM-LABELS", label)
|
mail.store(msg_id, "+X-GM-LABELS", label)
|
||||||
logger.info("Email %s labeled '%s' in Gmail.", msg_id, label)
|
logger.info("Email %s labeled '%s' in Gmail.", msg_id, label)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Failed to label email %s with %s: %s", msg_id, label, e)
|
logger.error("Failed to label email %s with %s: %s", msg_id, label, e)
|
||||||
|
|
||||||
|
|
||||||
def find_all_mail_folder(mail):
|
def find_all_mail_folder(mail):
|
||||||
"""
|
"""
|
||||||
Attempts to select the Gmail All Mail folder using known localized names.
|
Attempts to select the Gmail All Mail folder using known localized names.
|
||||||
Falls back to using XLIST if needed.
|
Falls back to using XLIST if needed.
|
||||||
Returns the folder name if found, otherwise None.
|
Returns the folder name if found, otherwise None.
|
||||||
"""
|
"""
|
||||||
COMMON_ALL_MAIL_NAMES = [
|
COMMON_ALL_MAIL_NAMES = [
|
||||||
"[Gmail]/Alle Nachrichten",
|
"[Gmail]/Alle Nachrichten",
|
||||||
"[Gmail]/All Mail",
|
"[Gmail]/All Mail",
|
||||||
"[Gmail]/Todos",
|
"[Gmail]/Todos",
|
||||||
"[Gmail]/Tutte le mail",
|
"[Gmail]/Tutte le mail",
|
||||||
"[Gmail]/Tous les messages",
|
"[Gmail]/Tous les messages",
|
||||||
]
|
]
|
||||||
for candidate in COMMON_ALL_MAIL_NAMES:
|
for candidate in COMMON_ALL_MAIL_NAMES:
|
||||||
status, _ = mail.select(f'"{candidate}"', readonly=True)
|
status, _ = mail.select(f'"{candidate}"', readonly=True)
|
||||||
if status == "OK":
|
if status == "OK":
|
||||||
return candidate
|
return candidate
|
||||||
|
|
||||||
capabilities = get_capabilities(mail)
|
capabilities = get_capabilities(mail)
|
||||||
if "XLIST" in capabilities:
|
if "XLIST" in capabilities:
|
||||||
candidate = find_all_mail_xlist(mail)
|
candidate = find_all_mail_xlist(mail)
|
||||||
if candidate:
|
if candidate:
|
||||||
return candidate
|
return candidate
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_capabilities(mail):
|
def get_capabilities(mail):
|
||||||
"""Returns a list of capabilities supported by the IMAP server."""
|
"""Returns a list of capabilities supported by the IMAP server."""
|
||||||
typ, data = mail.capability()
|
typ, data = mail.capability()
|
||||||
if typ == "OK" and data:
|
if typ == "OK" and data:
|
||||||
caps = data[0].decode("utf-8", errors="ignore").upper().split()
|
caps = data[0].decode("utf-8", errors="ignore").upper().split()
|
||||||
return caps
|
return caps
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def find_all_mail_xlist(mail):
|
def find_all_mail_xlist(mail):
|
||||||
"""
|
"""
|
||||||
Uses XLIST to discover the mailbox flagged as All Mail.
|
Uses XLIST to discover the mailbox flagged as All Mail.
|
||||||
Returns the folder name if found, otherwise None.
|
Returns the folder name if found, otherwise None.
|
||||||
"""
|
"""
|
||||||
tag = mail._new_tag().decode("ascii")
|
tag = mail._new_tag().decode("ascii")
|
||||||
command_str = f"{tag} XLIST \"\" \"*\""
|
command_str = f"{tag} XLIST \"\" \"*\""
|
||||||
mail.send((command_str + "\r\n").encode("utf-8"))
|
mail.send((command_str + "\r\n").encode("utf-8"))
|
||||||
|
|
||||||
all_mail_folder = None
|
all_mail_folder = None
|
||||||
while True:
|
while True:
|
||||||
line = mail.readline()
|
line = mail.readline()
|
||||||
if not line:
|
if not line:
|
||||||
break
|
break
|
||||||
line_str = line.decode("utf-8", errors="ignore").strip()
|
line_str = line.decode("utf-8", errors="ignore").strip()
|
||||||
if line_str.upper().startswith("* XLIST ") and "\\ALLMAIL" in line_str.upper():
|
if line_str.upper().startswith("* XLIST ") and "\\ALLMAIL" in line_str.upper():
|
||||||
match = re.search(r'"([^"]+)"$', line_str)
|
match = re.search(r'"([^"]+)"$', line_str)
|
||||||
if match:
|
if match:
|
||||||
candidate = match.group(1)
|
candidate = match.group(1)
|
||||||
logger.info("Found All Mail folder via XLIST: %s", candidate)
|
logger.info("Found All Mail folder via XLIST: %s", candidate)
|
||||||
all_mail_folder = candidate
|
all_mail_folder = candidate
|
||||||
if line_str.startswith(tag):
|
if line_str.startswith(tag):
|
||||||
break
|
break
|
||||||
return all_mail_folder
|
return all_mail_folder
|
||||||
|
|||||||
@@ -1,67 +1,67 @@
|
|||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
from azure.core.credentials import AzureKeyCredential
|
from azure.core.credentials import AzureKeyCredential
|
||||||
from azure.ai.documentintelligence import DocumentIntelligenceClient
|
from azure.ai.documentintelligence import DocumentIntelligenceClient
|
||||||
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
|
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Initialize Azure Document Intelligence client
|
# Initialize Azure Document Intelligence client
|
||||||
document_intelligence_client = DocumentIntelligenceClient(
|
document_intelligence_client = DocumentIntelligenceClient(
|
||||||
endpoint=settings.azure_endpoint,
|
endpoint=settings.azure_endpoint,
|
||||||
credential=AzureKeyCredential(settings.azure_ai_key)
|
credential=AzureKeyCredential(settings.azure_ai_key)
|
||||||
)
|
)
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def process_with_textract(s3_filename: str):
|
def process_with_textract(s3_filename: str):
|
||||||
"""
|
"""
|
||||||
Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
|
Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
|
||||||
the local temporary file (stored under <workdir>/tmp).
|
the local temporary file (stored under <workdir>/tmp).
|
||||||
|
|
||||||
Steps:
|
Steps:
|
||||||
1. Uploads the document for OCR using Azure Document Intelligence.
|
1. Uploads the document for OCR using Azure Document Intelligence.
|
||||||
2. Retrieves the processed PDF with embedded text.
|
2. Retrieves the processed PDF with embedded text.
|
||||||
3. Saves the OCR-processed PDF locally in the same location as before.
|
3. Saves the OCR-processed PDF locally in the same location as before.
|
||||||
4. Extracts the text content for metadata processing.
|
4. Extracts the text content for metadata processing.
|
||||||
5. Triggers downstream metadata extraction by calling extract_metadata_with_gpt.
|
5. Triggers downstream metadata extraction by calling extract_metadata_with_gpt.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
tmp_file_path = os.path.join(settings.workdir, "tmp", s3_filename)
|
tmp_file_path = os.path.join(settings.workdir, "tmp", s3_filename)
|
||||||
if not os.path.exists(tmp_file_path):
|
if not os.path.exists(tmp_file_path):
|
||||||
raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
|
raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
|
||||||
|
|
||||||
logger.info(f"Processing {s3_filename} with Azure Document Intelligence OCR.")
|
logger.info(f"Processing {s3_filename} with Azure Document Intelligence OCR.")
|
||||||
|
|
||||||
# Open and send the document for processing
|
# Open and send the document for processing
|
||||||
with open(tmp_file_path, "rb") as f:
|
with open(tmp_file_path, "rb") as f:
|
||||||
poller = document_intelligence_client.begin_analyze_document(
|
poller = document_intelligence_client.begin_analyze_document(
|
||||||
"prebuilt-read", body=f, output=[AnalyzeOutputOption.PDF]
|
"prebuilt-read", body=f, output=[AnalyzeOutputOption.PDF]
|
||||||
)
|
)
|
||||||
result: AnalyzeResult = poller.result()
|
result: AnalyzeResult = poller.result()
|
||||||
operation_id = poller.details["operation_id"]
|
operation_id = poller.details["operation_id"]
|
||||||
|
|
||||||
# Retrieve the processed searchable PDF
|
# Retrieve the processed searchable PDF
|
||||||
response = document_intelligence_client.get_analyze_result_pdf(
|
response = document_intelligence_client.get_analyze_result_pdf(
|
||||||
model_id=result.model_id, result_id=operation_id
|
model_id=result.model_id, result_id=operation_id
|
||||||
)
|
)
|
||||||
searchable_pdf_path = tmp_file_path # Overwrite the original PDF location
|
searchable_pdf_path = tmp_file_path # Overwrite the original PDF location
|
||||||
with open(searchable_pdf_path, "wb") as writer:
|
with open(searchable_pdf_path, "wb") as writer:
|
||||||
writer.writelines(response)
|
writer.writelines(response)
|
||||||
logger.info(f"Searchable PDF saved at: {searchable_pdf_path}")
|
logger.info(f"Searchable PDF saved at: {searchable_pdf_path}")
|
||||||
|
|
||||||
# Extract raw text content from the result
|
# Extract raw text content from the result
|
||||||
extracted_text = result.content if result.content else ""
|
extracted_text = result.content if result.content else ""
|
||||||
logger.info(f"Extracted text for {s3_filename}: {len(extracted_text)} characters")
|
logger.info(f"Extracted text for {s3_filename}: {len(extracted_text)} characters")
|
||||||
|
|
||||||
# Trigger downstream metadata extraction
|
# Trigger downstream metadata extraction
|
||||||
extract_metadata_with_gpt.delay(s3_filename, extracted_text)
|
extract_metadata_with_gpt.delay(s3_filename, extracted_text)
|
||||||
|
|
||||||
return {"s3_file": s3_filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text}
|
return {"s3_file": s3_filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error processing {s3_filename} with Azure Document Intelligence: {e}")
|
logger.error(f"Error processing {s3_filename} with Azure Document Intelligence: {e}")
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -1,34 +1,34 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
|
|
||||||
# Import the shared Celery instance
|
# Import the shared Celery instance
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
client = OpenAI(api_key=settings.openai_api_key)
|
client = OpenAI(api_key=settings.openai_api_key)
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def refine_text_with_gpt(s3_filename: str, raw_text: str):
|
def refine_text_with_gpt(s3_filename: str, raw_text: str):
|
||||||
"""Uses GPT to clean and refine OCR text."""
|
"""Uses GPT to clean and refine OCR text."""
|
||||||
|
|
||||||
# Use the Chat Completions endpoint with 'messages'
|
# Use the Chat Completions endpoint with 'messages'
|
||||||
response = client.chat.completions.create(
|
response = client.chat.completions.create(
|
||||||
model="gpt-4",
|
model="gpt-4",
|
||||||
messages=[
|
messages=[
|
||||||
{"role": "system", "content": "Clean and format the following text. The idea is that the text you see comes from an OCR system and your task is to eliminate OCR errors. Keep the original language when doing so."},
|
{"role": "system", "content": "Clean and format the following text. The idea is that the text you see comes from an OCR system and your task is to eliminate OCR errors. Keep the original language when doing so."},
|
||||||
{"role": "user", "content": raw_text}
|
{"role": "user", "content": raw_text}
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
cleaned_text = response.choices[0].message.content
|
cleaned_text = response.choices[0].message.content
|
||||||
|
|
||||||
# Trigger next task (import locally if needed to avoid circular imports)
|
# Trigger next task (import locally if needed to avoid circular imports)
|
||||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||||
extract_metadata_with_gpt.delay(s3_filename, cleaned_text)
|
extract_metadata_with_gpt.delay(s3_filename, cleaned_text)
|
||||||
|
|
||||||
return {"s3_file": s3_filename, "cleaned_text": cleaned_text}
|
return {"s3_file": s3_filename, "cleaned_text": cleaned_text}
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from celery import Task
|
from celery import Task
|
||||||
|
|
||||||
class BaseTaskWithRetry(Task):
|
class BaseTaskWithRetry(Task):
|
||||||
autoretry_for = (Exception,)
|
autoretry_for = (Exception,)
|
||||||
retry_kwargs = {"max_retries": 3, "countdown": 10} # 3 retries, 10s delay
|
retry_kwargs = {"max_retries": 3, "countdown": 10} # 3 retries, 10s delay
|
||||||
retry_backoff = True # Exponential backoff
|
retry_backoff = True # Exponential backoff
|
||||||
|
|
||||||
|
|||||||
+21
-21
@@ -1,21 +1,21 @@
|
|||||||
# app/tasks/send_to_all.py
|
# app/tasks/send_to_all.py
|
||||||
|
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||||
|
|
||||||
@celery.task
|
@celery.task
|
||||||
def send_to_all_destinations(file_path: str):
|
def send_to_all_destinations(file_path: str):
|
||||||
"""
|
"""
|
||||||
Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
|
Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
|
||||||
These tasks run in parallel (Celery returns immediately from each .delay()).
|
These tasks run in parallel (Celery returns immediately from each .delay()).
|
||||||
"""
|
"""
|
||||||
upload_to_dropbox.delay(file_path)
|
upload_to_dropbox.delay(file_path)
|
||||||
upload_to_nextcloud.delay(file_path)
|
upload_to_nextcloud.delay(file_path)
|
||||||
upload_to_paperless.delay(file_path)
|
upload_to_paperless.delay(file_path)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": "All upload tasks enqueued",
|
"status": "All upload tasks enqueued",
|
||||||
"file_path": file_path
|
"file_path": file_path
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,74 +1,74 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
import dropbox
|
import dropbox
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
def get_dropbox_access_token():
|
def get_dropbox_access_token():
|
||||||
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
|
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
|
||||||
|
|
||||||
token_url = "https://api.dropbox.com/oauth2/token"
|
token_url = "https://api.dropbox.com/oauth2/token"
|
||||||
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||||
data = {
|
data = {
|
||||||
"grant_type": "refresh_token",
|
"grant_type": "refresh_token",
|
||||||
"refresh_token": settings.dropbox_refresh_token, # Now using ENV
|
"refresh_token": settings.dropbox_refresh_token, # Now using ENV
|
||||||
"client_id": settings.dropbox_app_key,
|
"client_id": settings.dropbox_app_key,
|
||||||
"client_secret": settings.dropbox_app_secret,
|
"client_secret": settings.dropbox_app_secret,
|
||||||
}
|
}
|
||||||
|
|
||||||
response = requests.post(token_url, headers=headers, data=data)
|
response = requests.post(token_url, headers=headers, data=data)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
return response.json()["access_token"]
|
return response.json()["access_token"]
|
||||||
else:
|
else:
|
||||||
error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
|
error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
|
||||||
print(f"[ERROR] {error_msg}")
|
print(f"[ERROR] {error_msg}")
|
||||||
raise Exception(error_msg)
|
raise Exception(error_msg)
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def upload_to_dropbox(file_path: str):
|
def upload_to_dropbox(file_path: str):
|
||||||
"""Uploads a file to Dropbox using the API."""
|
"""Uploads a file to Dropbox using the API."""
|
||||||
|
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise FileNotFoundError(f"File not found: {file_path}")
|
raise FileNotFoundError(f"File not found: {file_path}")
|
||||||
|
|
||||||
# Extract filename and set target path
|
# Extract filename and set target path
|
||||||
filename = os.path.basename(file_path)
|
filename = os.path.basename(file_path)
|
||||||
dropbox_path = f"{settings.dropbox_folder}/{filename}"
|
dropbox_path = f"{settings.dropbox_folder}/{filename}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Get fresh access token
|
# Get fresh access token
|
||||||
access_token = get_dropbox_access_token()
|
access_token = get_dropbox_access_token()
|
||||||
dbx = dropbox.Dropbox(access_token)
|
dbx = dropbox.Dropbox(access_token)
|
||||||
|
|
||||||
file_size = os.path.getsize(file_path)
|
file_size = os.path.getsize(file_path)
|
||||||
chunk_size = 4 * 1024 * 1024 # 4MB chunk size
|
chunk_size = 4 * 1024 * 1024 # 4MB chunk size
|
||||||
|
|
||||||
with open(file_path, "rb") as file_data:
|
with open(file_path, "rb") as file_data:
|
||||||
if file_size <= chunk_size:
|
if file_size <= chunk_size:
|
||||||
dbx.files_upload(file_data.read(), dropbox_path)
|
dbx.files_upload(file_data.read(), dropbox_path)
|
||||||
else:
|
else:
|
||||||
upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
|
upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
|
||||||
cursor = dropbox.files.UploadSessionCursor(
|
cursor = dropbox.files.UploadSessionCursor(
|
||||||
session_id=upload_session_start_result.session_id,
|
session_id=upload_session_start_result.session_id,
|
||||||
offset=file_data.tell(),
|
offset=file_data.tell(),
|
||||||
)
|
)
|
||||||
commit = dropbox.files.CommitInfo(path=dropbox_path)
|
commit = dropbox.files.CommitInfo(path=dropbox_path)
|
||||||
|
|
||||||
while file_data.tell() < file_size:
|
while file_data.tell() < file_size:
|
||||||
if (file_size - file_data.tell()) <= chunk_size:
|
if (file_size - file_data.tell()) <= chunk_size:
|
||||||
dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
|
dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
|
||||||
else:
|
else:
|
||||||
dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
|
dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
|
||||||
cursor.offset = file_data.tell()
|
cursor.offset = file_data.tell()
|
||||||
|
|
||||||
print(f"[INFO] Successfully uploaded {filename} to Dropbox at {dropbox_path}.")
|
print(f"[INFO] Successfully uploaded {filename} to Dropbox at {dropbox_path}.")
|
||||||
return {"status": "Completed", "file": file_path}
|
return {"status": "Completed", "file": file_path}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {str(e)}"
|
error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {str(e)}"
|
||||||
print(error_msg)
|
print(error_msg)
|
||||||
raise Exception(error_msg)
|
raise Exception(error_msg)
|
||||||
|
|||||||
@@ -1,37 +1,37 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def upload_to_nextcloud(file_path: str):
|
def upload_to_nextcloud(file_path: str):
|
||||||
"""Uploads a file to Nextcloud in the configured folder."""
|
"""Uploads a file to Nextcloud in the configured folder."""
|
||||||
|
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise FileNotFoundError(f"File not found: {file_path}")
|
raise FileNotFoundError(f"File not found: {file_path}")
|
||||||
|
|
||||||
# Extract filename
|
# Extract filename
|
||||||
filename = os.path.basename(file_path)
|
filename = os.path.basename(file_path)
|
||||||
|
|
||||||
# Construct the full upload URL
|
# Construct the full upload URL
|
||||||
nextcloud_url = f"{settings.nextcloud_upload_url}/{settings.nextcloud_folder}/{filename}"
|
nextcloud_url = f"{settings.nextcloud_upload_url}/{settings.nextcloud_folder}/{filename}"
|
||||||
|
|
||||||
# Read file content
|
# Read file content
|
||||||
with open(file_path, "rb") as file_data:
|
with open(file_path, "rb") as file_data:
|
||||||
response = requests.put(
|
response = requests.put(
|
||||||
nextcloud_url,
|
nextcloud_url,
|
||||||
auth=(settings.nextcloud_username, settings.nextcloud_password),
|
auth=(settings.nextcloud_username, settings.nextcloud_password),
|
||||||
data=file_data
|
data=file_data
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check if upload was successful
|
# Check if upload was successful
|
||||||
if response.status_code in (200, 201):
|
if response.status_code in (200, 201):
|
||||||
print(f"[INFO] Successfully uploaded {filename} to Nextcloud at {nextcloud_url}.")
|
print(f"[INFO] Successfully uploaded {filename} to Nextcloud at {nextcloud_url}.")
|
||||||
return {"status": "Completed", "file": file_path}
|
return {"status": "Completed", "file": file_path}
|
||||||
else:
|
else:
|
||||||
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
|
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
|
||||||
print(error_msg)
|
print(error_msg)
|
||||||
raise Exception(error_msg)
|
raise Exception(error_msg)
|
||||||
|
|||||||
+131
-131
@@ -1,131 +1,131 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import requests
|
import requests
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, Any
|
from typing import Dict, Any
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
POLL_MAX_ATTEMPTS = 10
|
POLL_MAX_ATTEMPTS = 10
|
||||||
POLL_INTERVAL_SEC = 3
|
POLL_INTERVAL_SEC = 3
|
||||||
|
|
||||||
def _get_headers():
|
def _get_headers():
|
||||||
"""Returns HTTP headers for Paperless-ngx API calls."""
|
"""Returns HTTP headers for Paperless-ngx API calls."""
|
||||||
return {
|
return {
|
||||||
"Authorization": f"Token {settings.paperless_ngx_api_token}"
|
"Authorization": f"Token {settings.paperless_ngx_api_token}"
|
||||||
}
|
}
|
||||||
|
|
||||||
def _paperless_api_url(path: str) -> str:
|
def _paperless_api_url(path: str) -> str:
|
||||||
"""
|
"""
|
||||||
Constructs a full Paperless-ngx API URL using `settings.paperless_host`.
|
Constructs a full Paperless-ngx API URL using `settings.paperless_host`.
|
||||||
Ensures the path is appended with a leading slash if missing.
|
Ensures the path is appended with a leading slash if missing.
|
||||||
"""
|
"""
|
||||||
host = settings.paperless_host.rstrip("/")
|
host = settings.paperless_host.rstrip("/")
|
||||||
if not path.startswith("/"):
|
if not path.startswith("/"):
|
||||||
path = "/" + path
|
path = "/" + path
|
||||||
return f"{host}{path}"
|
return f"{host}{path}"
|
||||||
|
|
||||||
def poll_task_for_document_id(task_id: str) -> int:
|
def poll_task_for_document_id(task_id: str) -> int:
|
||||||
"""
|
"""
|
||||||
Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
|
Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
|
||||||
or until we run out of attempts.
|
or until we run out of attempts.
|
||||||
|
|
||||||
On SUCCESS: returns the int document_id from 'related_document'.
|
On SUCCESS: returns the int document_id from 'related_document'.
|
||||||
On FAILURE: raises RuntimeError with the task's 'result' message.
|
On FAILURE: raises RuntimeError with the task's 'result' message.
|
||||||
If times out, raises TimeoutError.
|
If times out, raises TimeoutError.
|
||||||
"""
|
"""
|
||||||
url = _paperless_api_url("/api/tasks/")
|
url = _paperless_api_url("/api/tasks/")
|
||||||
attempts = 0
|
attempts = 0
|
||||||
|
|
||||||
while attempts < POLL_MAX_ATTEMPTS:
|
while attempts < POLL_MAX_ATTEMPTS:
|
||||||
try:
|
try:
|
||||||
resp = requests.get(url, headers=_get_headers(), params={"task_id": task_id})
|
resp = requests.get(url, headers=_get_headers(), params={"task_id": task_id})
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
tasks_data = resp.json()
|
tasks_data = resp.json()
|
||||||
except requests.exceptions.RequestException as exc:
|
except requests.exceptions.RequestException as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Failed to poll for task_id='%s'. Attempt=%d Error=%s",
|
"Failed to poll for task_id='%s'. Attempt=%d Error=%s",
|
||||||
task_id, attempts + 1, exc
|
task_id, attempts + 1, exc
|
||||||
)
|
)
|
||||||
time.sleep(POLL_INTERVAL_SEC)
|
time.sleep(POLL_INTERVAL_SEC)
|
||||||
attempts += 1
|
attempts += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if isinstance(tasks_data, dict) and "results" in tasks_data:
|
if isinstance(tasks_data, dict) and "results" in tasks_data:
|
||||||
tasks_data = tasks_data["results"]
|
tasks_data = tasks_data["results"]
|
||||||
|
|
||||||
if tasks_data:
|
if tasks_data:
|
||||||
task_info = tasks_data[0]
|
task_info = tasks_data[0]
|
||||||
status = task_info.get("status")
|
status = task_info.get("status")
|
||||||
if status == "SUCCESS":
|
if status == "SUCCESS":
|
||||||
doc_str = task_info.get("related_document")
|
doc_str = task_info.get("related_document")
|
||||||
if doc_str:
|
if doc_str:
|
||||||
return int(doc_str)
|
return int(doc_str)
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Task {task_id} completed but no doc ID found. Task info: {task_info}"
|
f"Task {task_id} completed but no doc ID found. Task info: {task_info}"
|
||||||
)
|
)
|
||||||
elif status == "FAILURE":
|
elif status == "FAILURE":
|
||||||
raise RuntimeError(f"Task {task_id} failed: {task_info.get('result')}")
|
raise RuntimeError(f"Task {task_id} failed: {task_info.get('result')}")
|
||||||
|
|
||||||
attempts += 1
|
attempts += 1
|
||||||
time.sleep(POLL_INTERVAL_SEC)
|
time.sleep(POLL_INTERVAL_SEC)
|
||||||
|
|
||||||
raise TimeoutError(
|
raise TimeoutError(
|
||||||
f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts."
|
f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts."
|
||||||
)
|
)
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def upload_to_paperless(file_path: str) -> Dict[str, Any]:
|
def upload_to_paperless(file_path: str) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Uploads a PDF to Paperless with minimal metadata (filename and date only).
|
Uploads a PDF to Paperless with minimal metadata (filename and date only).
|
||||||
|
|
||||||
1. Extracts the filename and date from the file.
|
1. Extracts the filename and date from the file.
|
||||||
2. POSTs the PDF to Paperless => returns a quoted UUID string (task_id).
|
2. POSTs the PDF to Paperless => returns a quoted UUID string (task_id).
|
||||||
3. Polls /api/tasks/?task_id=<uuid> until SUCCESS or FAILURE => doc_id.
|
3. Polls /api/tasks/?task_id=<uuid> until SUCCESS or FAILURE => doc_id.
|
||||||
|
|
||||||
Returns a dict with status, the paperless_task_id, paperless_document_id, and file_path.
|
Returns a dict with status, the paperless_task_id, paperless_document_id, and file_path.
|
||||||
"""
|
"""
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
raise FileNotFoundError(f"File not found: {file_path}")
|
raise FileNotFoundError(f"File not found: {file_path}")
|
||||||
|
|
||||||
base_name = os.path.basename(file_path)
|
base_name = os.path.basename(file_path)
|
||||||
|
|
||||||
# Upload the PDF
|
# Upload the PDF
|
||||||
post_url = _paperless_api_url("/api/documents/post_document/")
|
post_url = _paperless_api_url("/api/documents/post_document/")
|
||||||
with open(file_path, "rb") as f:
|
with open(file_path, "rb") as f:
|
||||||
files = {
|
files = {
|
||||||
"document": (base_name, f, "application/pdf"),
|
"document": (base_name, f, "application/pdf"),
|
||||||
}
|
}
|
||||||
data = {"title": base_name} # Title = Filename (no additional metadata)
|
data = {"title": base_name} # Title = Filename (no additional metadata)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logger.debug("Posting document to Paperless: file=%s", base_name)
|
logger.debug("Posting document to Paperless: file=%s", base_name)
|
||||||
resp = requests.post(post_url, headers=_get_headers(), files=files, data=data)
|
resp = requests.post(post_url, headers=_get_headers(), files=files, data=data)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
except requests.exceptions.RequestException as exc:
|
except requests.exceptions.RequestException as exc:
|
||||||
logger.error(
|
logger.error(
|
||||||
"Failed to upload document '%s' to Paperless. Error: %s. Response=%s",
|
"Failed to upload document '%s' to Paperless. Error: %s. Response=%s",
|
||||||
file_path, exc, getattr(exc.response, "text", "<no response>")
|
file_path, exc, getattr(exc.response, "text", "<no response>")
|
||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
raw_task_id = resp.text.strip().strip('"').strip("'")
|
raw_task_id = resp.text.strip().strip('"').strip("'")
|
||||||
logger.info(f"Received Paperless task ID: {raw_task_id}")
|
logger.info(f"Received Paperless task ID: {raw_task_id}")
|
||||||
|
|
||||||
# Poll tasks until success/fail => get doc_id
|
# Poll tasks until success/fail => get doc_id
|
||||||
doc_id = poll_task_for_document_id(raw_task_id)
|
doc_id = poll_task_for_document_id(raw_task_id)
|
||||||
logger.info(f"Document {file_path} successfully ingested => ID={doc_id}")
|
logger.info(f"Document {file_path} successfully ingested => ID={doc_id}")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": "Completed",
|
"status": "Completed",
|
||||||
"paperless_task_id": raw_task_id,
|
"paperless_task_id": raw_task_id,
|
||||||
"paperless_document_id": doc_id,
|
"paperless_document_id": doc_id,
|
||||||
"file_path": file_path
|
"file_path": file_path
|
||||||
}
|
}
|
||||||
|
|||||||
+87
-87
@@ -1,87 +1,87 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
import boto3
|
import boto3
|
||||||
import shutil
|
import shutil
|
||||||
import fitz # PyMuPDF for checking embedded text
|
import fitz # PyMuPDF for checking embedded text
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.retry_config import BaseTaskWithRetry
|
from app.tasks.retry_config import BaseTaskWithRetry
|
||||||
from app.tasks.process_with_textract import process_with_textract
|
from app.tasks.process_with_textract import process_with_textract
|
||||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||||
|
|
||||||
# Import the shared Celery instance
|
# Import the shared Celery instance
|
||||||
from app.celery_app import celery
|
from app.celery_app import celery
|
||||||
|
|
||||||
# Initialize S3 client
|
# Initialize S3 client
|
||||||
s3_client = boto3.client(
|
s3_client = boto3.client(
|
||||||
"s3",
|
"s3",
|
||||||
aws_access_key_id=settings.aws_access_key_id,
|
aws_access_key_id=settings.aws_access_key_id,
|
||||||
aws_secret_access_key=settings.aws_secret_access_key,
|
aws_secret_access_key=settings.aws_secret_access_key,
|
||||||
region_name=settings.aws_region,
|
region_name=settings.aws_region,
|
||||||
)
|
)
|
||||||
|
|
||||||
@celery.task(base=BaseTaskWithRetry)
|
@celery.task(base=BaseTaskWithRetry)
|
||||||
def upload_to_s3(original_local_file: str):
|
def upload_to_s3(original_local_file: str):
|
||||||
"""
|
"""
|
||||||
Uploads a file to S3 with a UUID-based filename and triggers processing.
|
Uploads a file to S3 with a UUID-based filename and triggers processing.
|
||||||
- If the PDF already contains embedded text, skip Textract and extract text locally.
|
- If the PDF already contains embedded text, skip Textract and extract text locally.
|
||||||
- Otherwise, upload to S3 and process with Textract.
|
- Otherwise, upload to S3 and process with Textract.
|
||||||
"""
|
"""
|
||||||
bucket_name = settings.s3_bucket_name
|
bucket_name = settings.s3_bucket_name
|
||||||
if not bucket_name:
|
if not bucket_name:
|
||||||
print("[ERROR] S3 bucket name not set.")
|
print("[ERROR] S3 bucket name not set.")
|
||||||
return {"error": "Missing S3 bucket name"}
|
return {"error": "Missing S3 bucket name"}
|
||||||
|
|
||||||
if not os.path.exists(original_local_file):
|
if not os.path.exists(original_local_file):
|
||||||
print(f"[ERROR] File {original_local_file} not found.")
|
print(f"[ERROR] File {original_local_file} not found.")
|
||||||
return {"error": "File not found"}
|
return {"error": "File not found"}
|
||||||
|
|
||||||
# Generate UUID and create a new filename
|
# Generate UUID and create a new filename
|
||||||
file_ext = os.path.splitext(original_local_file)[1] # Preserve original file extension
|
file_ext = os.path.splitext(original_local_file)[1] # Preserve original file extension
|
||||||
file_uuid = str(uuid.uuid4())
|
file_uuid = str(uuid.uuid4())
|
||||||
new_filename = f"{file_uuid}{file_ext}"
|
new_filename = f"{file_uuid}{file_ext}"
|
||||||
|
|
||||||
# Construct the new local path using settings.workdir and a 'tmp' subdirectory
|
# Construct the new local path using settings.workdir and a 'tmp' subdirectory
|
||||||
tmp_dir = os.path.join(settings.workdir, "tmp")
|
tmp_dir = os.path.join(settings.workdir, "tmp")
|
||||||
new_local_path = os.path.join(tmp_dir, new_filename)
|
new_local_path = os.path.join(tmp_dir, new_filename)
|
||||||
|
|
||||||
# Ensure the target tmp directory exists
|
# Ensure the target tmp directory exists
|
||||||
os.makedirs(tmp_dir, exist_ok=True)
|
os.makedirs(tmp_dir, exist_ok=True)
|
||||||
|
|
||||||
# Copy the file instead of moving it
|
# Copy the file instead of moving it
|
||||||
shutil.copy(original_local_file, new_local_path)
|
shutil.copy(original_local_file, new_local_path)
|
||||||
|
|
||||||
# Check for embedded text
|
# Check for embedded text
|
||||||
pdf_doc = fitz.open(new_local_path)
|
pdf_doc = fitz.open(new_local_path)
|
||||||
has_text = any(page.get_text() for page in pdf_doc)
|
has_text = any(page.get_text() for page in pdf_doc)
|
||||||
pdf_doc.close()
|
pdf_doc.close()
|
||||||
|
|
||||||
if has_text:
|
if has_text:
|
||||||
print(f"[INFO] PDF {original_local_file} contains embedded text. Skipping Textract.")
|
print(f"[INFO] PDF {original_local_file} contains embedded text. Skipping Textract.")
|
||||||
|
|
||||||
# Extract text locally
|
# Extract text locally
|
||||||
extracted_text = ""
|
extracted_text = ""
|
||||||
pdf_doc = fitz.open(new_local_path)
|
pdf_doc = fitz.open(new_local_path)
|
||||||
for page in pdf_doc:
|
for page in pdf_doc:
|
||||||
extracted_text += page.get_text("text") + "\n"
|
extracted_text += page.get_text("text") + "\n"
|
||||||
pdf_doc.close()
|
pdf_doc.close()
|
||||||
|
|
||||||
# Call metadata extraction directly
|
# Call metadata extraction directly
|
||||||
extract_metadata_with_gpt.delay(new_filename, extracted_text)
|
extract_metadata_with_gpt.delay(new_filename, extracted_text)
|
||||||
|
|
||||||
return {"file": new_local_path, "status": "Text extracted locally"}
|
return {"file": new_local_path, "status": "Text extracted locally"}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
print(f"[INFO] Uploading {new_local_path} to s3://{bucket_name}/{new_filename}...")
|
print(f"[INFO] Uploading {new_local_path} to s3://{bucket_name}/{new_filename}...")
|
||||||
s3_client.upload_file(new_local_path, bucket_name, new_filename)
|
s3_client.upload_file(new_local_path, bucket_name, new_filename)
|
||||||
print(f"[INFO] File uploaded successfully: {new_filename}")
|
print(f"[INFO] File uploaded successfully: {new_filename}")
|
||||||
|
|
||||||
# Trigger Textract processing if no embedded text was found
|
# Trigger Textract processing if no embedded text was found
|
||||||
process_with_textract.delay(new_filename)
|
process_with_textract.delay(new_filename)
|
||||||
|
|
||||||
return {"file": new_local_path, "s3_key": new_filename, "status": "Uploaded to S3 for OCR"}
|
return {"file": new_local_path, "s3_key": new_filename, "status": "Uploaded to S3 for OCR"}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[ERROR] Failed to upload {new_local_path} to S3: {e}")
|
print(f"[ERROR] Failed to upload {new_local_path} to S3: {e}")
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|||||||
+64
-64
@@ -1,64 +1,64 @@
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
api:
|
api:
|
||||||
image: christianlouis/document-processor:latest
|
image: christianlouis/document-processor:latest
|
||||||
container_name: document_api
|
container_name: document_api
|
||||||
|
|
||||||
# We'll keep the code in /app, but set working_dir to the shared data directory
|
# We'll keep the code in /app, but set working_dir to the shared data directory
|
||||||
working_dir: /workdir
|
working_dir: /workdir
|
||||||
|
|
||||||
# We'll run uvicorn from the container's /app code
|
# We'll run uvicorn from the container's /app code
|
||||||
command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
|
command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
|
||||||
|
|
||||||
# Environment variables
|
# Environment variables
|
||||||
environment:
|
environment:
|
||||||
- PYTHONPATH=/app
|
- PYTHONPATH=/app
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
|
|
||||||
# Expose container's 8000 -> Host's 8000
|
# Expose container's 8000 -> Host's 8000
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "8000:8000"
|
||||||
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- worker
|
- worker
|
||||||
|
|
||||||
# Mount the shared working directory for data
|
# Mount the shared working directory for data
|
||||||
volumes:
|
volumes:
|
||||||
# optional: mount your code if you want local dev changes to reflect
|
# optional: mount your code if you want local dev changes to reflect
|
||||||
# - ./app:/app
|
# - ./app:/app
|
||||||
- /var/docparse/workdir:/workdir
|
- /var/docparse/workdir:/workdir
|
||||||
|
|
||||||
worker:
|
worker:
|
||||||
image: christianlouis/document-processor:latest
|
image: christianlouis/document-processor:latest
|
||||||
container_name: document_worker
|
container_name: document_worker
|
||||||
|
|
||||||
# same shared working directory
|
# same shared working directory
|
||||||
working_dir: /workdir
|
working_dir: /workdir
|
||||||
|
|
||||||
command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
|
command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
environment:
|
environment:
|
||||||
- PYTHONPATH=/app
|
- PYTHONPATH=/app
|
||||||
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- gotenberg
|
- gotenberg
|
||||||
|
|
||||||
# Mount the shared directory (and optionally your code if you want dev mode)
|
# Mount the shared directory (and optionally your code if you want dev mode)
|
||||||
volumes:
|
volumes:
|
||||||
# optional: mount your code if you want local dev changes
|
# optional: mount your code if you want local dev changes
|
||||||
# - ./app:/app
|
# - ./app:/app
|
||||||
- /var/docparse/workdir:/workdir
|
- /var/docparse/workdir:/workdir
|
||||||
|
|
||||||
gotenberg:
|
gotenberg:
|
||||||
image: gotenberg/gotenberg:latest
|
image: gotenberg/gotenberg:latest
|
||||||
container_name: gotenberg
|
container_name: gotenberg
|
||||||
|
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:alpine
|
image: redis:alpine
|
||||||
container_name: document_redis
|
container_name: document_redis
|
||||||
restart: always
|
restart: always
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
+89
-89
@@ -1,89 +1,89 @@
|
|||||||
<!-- File: frontend/index.html -->
|
<!-- File: frontend/index.html -->
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<title>Document Processor - Upload</title>
|
<title>Document Processor - Upload</title>
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
<script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
|
||||||
<script>
|
<script>
|
||||||
// Optional: Additional JS for drag-and-drop, or you can keep it inline.
|
// Optional: Additional JS for drag-and-drop, or you can keep it inline.
|
||||||
</script>
|
</script>
|
||||||
</head>
|
</head>
|
||||||
<body class="bg-gray-50 h-screen flex flex-col items-center justify-center">
|
<body class="bg-gray-50 h-screen flex flex-col items-center justify-center">
|
||||||
<h1 class="text-3xl font-bold mb-8">Upload a File</h1>
|
<h1 class="text-3xl font-bold mb-8">Upload a File</h1>
|
||||||
|
|
||||||
<div
|
<div
|
||||||
id="dropZone"
|
id="dropZone"
|
||||||
class="border-4 border-dashed border-gray-300 rounded-lg p-8 bg-white text-center w-1/2"
|
class="border-4 border-dashed border-gray-300 rounded-lg p-8 bg-white text-center w-1/2"
|
||||||
ondrop="handleDrop(event)"
|
ondrop="handleDrop(event)"
|
||||||
ondragover="handleDragOver(event)"
|
ondragover="handleDragOver(event)"
|
||||||
>
|
>
|
||||||
<p class="text-gray-500">
|
<p class="text-gray-500">
|
||||||
Drag & drop a file here, or click to select a file.
|
Drag & drop a file here, or click to select a file.
|
||||||
</p>
|
</p>
|
||||||
<input
|
<input
|
||||||
id="fileInput"
|
id="fileInput"
|
||||||
type="file"
|
type="file"
|
||||||
class="hidden"
|
class="hidden"
|
||||||
onchange="handleFileSelect(event)"
|
onchange="handleFileSelect(event)"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div id="statusMessage" class="mt-4 text-gray-700"></div>
|
<div id="statusMessage" class="mt-4 text-gray-700"></div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
const dropZone = document.getElementById("dropZone");
|
const dropZone = document.getElementById("dropZone");
|
||||||
const fileInput = document.getElementById("fileInput");
|
const fileInput = document.getElementById("fileInput");
|
||||||
const statusMessage = document.getElementById("statusMessage");
|
const statusMessage = document.getElementById("statusMessage");
|
||||||
|
|
||||||
dropZone.addEventListener("click", () => {
|
dropZone.addEventListener("click", () => {
|
||||||
fileInput.click();
|
fileInput.click();
|
||||||
});
|
});
|
||||||
|
|
||||||
function handleDragOver(e) {
|
function handleDragOver(e) {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
e.dataTransfer.dropEffect = "copy";
|
e.dataTransfer.dropEffect = "copy";
|
||||||
dropZone.classList.add("bg-gray-100");
|
dropZone.classList.add("bg-gray-100");
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleDrop(e) {
|
function handleDrop(e) {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
dropZone.classList.remove("bg-gray-100");
|
dropZone.classList.remove("bg-gray-100");
|
||||||
if (e.dataTransfer.files.length) {
|
if (e.dataTransfer.files.length) {
|
||||||
uploadFile(e.dataTransfer.files[0]);
|
uploadFile(e.dataTransfer.files[0]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleFileSelect(e) {
|
function handleFileSelect(e) {
|
||||||
if (e.target.files.length) {
|
if (e.target.files.length) {
|
||||||
uploadFile(e.target.files[0]);
|
uploadFile(e.target.files[0]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadFile(file) {
|
async function uploadFile(file) {
|
||||||
statusMessage.textContent = `Uploading ${file.name}...`;
|
statusMessage.textContent = `Uploading ${file.name}...`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// We'll POST the file to /upload
|
// We'll POST the file to /upload
|
||||||
let formData = new FormData();
|
let formData = new FormData();
|
||||||
formData.append("file", file);
|
formData.append("file", file);
|
||||||
|
|
||||||
const response = await fetch("/ui-upload", {
|
const response = await fetch("/ui-upload", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: formData,
|
body: formData,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Upload failed with status ${response.status}`);
|
throw new Error(`Upload failed with status ${response.status}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
statusMessage.textContent = `File ${file.name} uploaded. Task ID: ${result.task_id}`;
|
statusMessage.textContent = `File ${file.name} uploaded. Task ID: ${result.task_id}`;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
statusMessage.textContent = `Error: ${err}`;
|
statusMessage.textContent = `Error: ${err}`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
+14
-14
@@ -1,15 +1,15 @@
|
|||||||
fastapi[all]
|
fastapi[all]
|
||||||
uvicorn
|
uvicorn
|
||||||
celery
|
celery
|
||||||
redis
|
redis
|
||||||
sqlalchemy
|
sqlalchemy
|
||||||
pydantic
|
pydantic
|
||||||
boto3
|
boto3
|
||||||
pikepdf
|
pikepdf
|
||||||
openai
|
openai
|
||||||
# Add this line explicitly
|
# Add this line explicitly
|
||||||
#pymupdf==1.23.5
|
#pymupdf==1.23.5
|
||||||
pymupdf
|
pymupdf
|
||||||
requests
|
requests
|
||||||
dropbox
|
dropbox
|
||||||
azure-ai-documentintelligence
|
azure-ai-documentintelligence
|
||||||
Reference in New Issue
Block a user