Compare commits
29 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 16449e75a2 | |||
| d82000ab7f | |||
| 2b092592cb | |||
| 116b04ddc1 | |||
| 6e4824cb74 | |||
| f1b55de6bc | |||
| 6fc00b8de1 | |||
| 427db102d8 | |||
| e46f9b9a21 | |||
| 10a6c6255a | |||
| 77d184efd8 | |||
| 1ca7f562ef | |||
| 3bccbf883d | |||
| b788b553c0 | |||
| 4b46c4baf8 | |||
| 8e744c076d | |||
| cc494c6937 | |||
| e2fa96318f | |||
| 416c3c4758 | |||
| 1c7ef28c5d | |||
| 1a0218799b | |||
| 048f28a671 | |||
| b818f07c80 | |||
| 06507ed8bf | |||
| 044a9a86d6 | |||
| 62d4ca6367 | |||
| 871f788f0b | |||
| 18f5596b01 | |||
| afb8b367ee |
+14
-14
@@ -27,9 +27,9 @@ jobs:
|
||||
name: Ruff Lint & Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
@@ -48,9 +48,9 @@ jobs:
|
||||
name: Alembic Migration Chain Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
- name: Validate migration chain
|
||||
@@ -60,9 +60,9 @@ jobs:
|
||||
name: HTML Accessibility Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
@@ -78,9 +78,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
@@ -93,9 +93,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
@@ -116,10 +116,10 @@ jobs:
|
||||
ports: ["5672:5672", "15672:15672"]
|
||||
options: --health-cmd "rabbitmq-diagnostics -q ping" --health-interval 10s --health-timeout 5s --health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4.2.2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
@@ -154,7 +154,7 @@ jobs:
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v4.2.2
|
||||
- name: Generate Build Metadata
|
||||
run: |
|
||||
chmod +x scripts/generate_build_metadata.sh
|
||||
@@ -212,7 +212,7 @@ jobs:
|
||||
echo "tag=main-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "image=ghcr.io/${{ github.repository_owner }}/docuelevate:main-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
- name: Checkout k8s-cluster-state
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v4.2.2
|
||||
with:
|
||||
repository: christianlouis/k8s-cluster-state
|
||||
token: ${{ secrets.GH_PAT }}
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
build-mode: none
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v4.2.2
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
|
||||
@@ -23,13 +23,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
@@ -28,13 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout PR branch
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v4.2.2
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v5.4.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
|
||||
@@ -200,3 +200,5 @@ cython_debug/
|
||||
# Build metadata files - generated at build time
|
||||
GIT_SHA
|
||||
RUNTIME_INFO
|
||||
node_modules
|
||||
frontend/node_modules
|
||||
|
||||
@@ -28,3 +28,7 @@
|
||||
**Vulnerability:** The `/process-url` endpoint used `httpx.AsyncClient(follow_redirects=True)` after validating the initial user-provided URL against SSRF protections. However, it did not validate the target URLs of any subsequent HTTP redirects, allowing an attacker to provide a safe URL that redirects to an internal/private IP, bypassing the security check.
|
||||
**Learning:** Initial URL validation is insufficient when the HTTP client is configured to follow redirects automatically. The client must be explicitly configured to validate every redirect target.
|
||||
**Prevention:** When using `httpx.AsyncClient(follow_redirects=True)` for user-provided URLs, always implement a redirect validator hook function (e.g., using `event_hooks={'response': [validate_redirect]}`) that resolves the `Location` header and passes it through the same SSRF validation logic before the redirect is followed.
|
||||
## 2026-05-27 - DOM-based XSS in upload.js
|
||||
**Vulnerability:** The `upload.js` frontend file directly injected the user-controlled `file.name` into a string literal passed to `row.innerHTML`. This allowed a malicious filename (e.g., `<img src=x onerror=alert(1)>`) to execute arbitrary JavaScript in the victim's browser context (DOM-based XSS).
|
||||
**Learning:** Even internal or local files uploaded by a user can contain malicious filenames. Any data dynamically interpolated into an HTML string that is subsequently rendered via `innerHTML` must be strictly sanitized.
|
||||
**Prevention:** Implement and apply an HTML escaping function (e.g., `_escapeHtml`) to replace dangerous characters (`&`, `<`, `>`, `"`, `'`) with their corresponding HTML entities before injecting user-controlled data into the DOM.
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
2026-04-07T09:34:53Z
|
||||
2026-05-23T00:38:15Z
|
||||
|
||||
@@ -10,6 +10,96 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
<!-- version list -->
|
||||
|
||||
## v0.173.2 (2026-05-23)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Resolve dependabot npm alerts
|
||||
([`6fc00b8`](https://github.com/christianlouis/DocuElevate/commit/6fc00b8de10b50b4b2f92f6fadbcf7ebbee7136f))
|
||||
|
||||
### Documentation
|
||||
|
||||
- Extend product roadmap and milestones
|
||||
([`e46f9b9`](https://github.com/christianlouis/DocuElevate/commit/e46f9b9a21e5838f883eca4370445e0f1b57e6c9))
|
||||
|
||||
- **changelog**: Update changelog [skip ci]
|
||||
([`427db10`](https://github.com/christianlouis/DocuElevate/commit/427db102d85fa676dcf01118fd3757fb300b979d))
|
||||
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Documentation
|
||||
|
||||
- Extend product roadmap and milestones
|
||||
([`e46f9b9`](https://github.com/christianlouis/DocuElevate/commit/e46f9b9a21e5838f883eca4370445e0f1b57e6c9))
|
||||
|
||||
|
||||
## v0.173.1 (2026-05-22)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Register Evernote task settings
|
||||
([`1ca7f56`](https://github.com/christianlouis/DocuElevate/commit/1ca7f562ef284d2fcceca84b38661abe9038912a))
|
||||
|
||||
|
||||
## v0.173.0 (2026-05-22)
|
||||
|
||||
### Code Style
|
||||
|
||||
- Apply ruff auto-fix ([#862](https://github.com/christianlouis/DocuElevate/pull/862),
|
||||
[`4b46c4b`](https://github.com/christianlouis/DocuElevate/commit/4b46c4baf890f3179ac060569ce2e076cf0551b0))
|
||||
|
||||
### Features
|
||||
|
||||
- **storage**: Add Evernote destination
|
||||
([#862](https://github.com/christianlouis/DocuElevate/pull/862),
|
||||
[`4b46c4b`](https://github.com/christianlouis/DocuElevate/commit/4b46c4baf890f3179ac060569ce2e076cf0551b0))
|
||||
|
||||
|
||||
## v0.172.12 (2026-05-17)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Validate webhook targets before delivery
|
||||
([#846](https://github.com/christianlouis/DocuElevate/pull/846),
|
||||
[`e2fa963`](https://github.com/christianlouis/DocuElevate/commit/e2fa96318f5bd45607baa0fe08a0bf14e1ca83d4))
|
||||
|
||||
### Testing
|
||||
|
||||
- Cover webhook SSRF validation ([#846](https://github.com/christianlouis/DocuElevate/pull/846),
|
||||
[`e2fa963`](https://github.com/christianlouis/DocuElevate/commit/e2fa96318f5bd45607baa0fe08a0bf14e1ca83d4))
|
||||
|
||||
|
||||
## v0.172.11 (2026-05-17)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Escape search result template values
|
||||
([#853](https://github.com/christianlouis/DocuElevate/pull/853),
|
||||
[`1a02187`](https://github.com/christianlouis/DocuElevate/commit/1a0218799b9a1eb4154e2f4fbb2572cb3922106a))
|
||||
|
||||
### Documentation
|
||||
|
||||
- **changelog**: Update changelog [skip ci]
|
||||
([`048f28a`](https://github.com/christianlouis/DocuElevate/commit/048f28a6717fa7f5cf4b235f9142e625e80e5d59))
|
||||
|
||||
|
||||
## Unreleased
|
||||
|
||||
|
||||
## v0.172.10 (2026-05-17)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **url-upload**: Handle unsafe redirects as client errors
|
||||
([`871f788`](https://github.com/christianlouis/DocuElevate/commit/871f788f0bd782ba8ad3a7d70e5cd4ccd24f749b))
|
||||
|
||||
### Documentation
|
||||
|
||||
- **changelog**: Update changelog [skip ci]
|
||||
([`58b14ae`](https://github.com/christianlouis/DocuElevate/commit/58b14ae769b85e25290126256de936743609af06))
|
||||
|
||||
|
||||
## Unreleased
|
||||
|
||||
|
||||
|
||||
+115
-69
@@ -1,6 +1,6 @@
|
||||
# DocuElevate Milestones
|
||||
|
||||
**Last Updated:** 2026-02-08
|
||||
**Last Updated:** 2026-05-23
|
||||
|
||||
This document outlines the release milestones, versioning strategy, and detailed feature breakdown for DocuElevate.
|
||||
|
||||
@@ -19,17 +19,15 @@ DocuElevate follows [Semantic Versioning 2.0.0](https://semver.org/):
|
||||
|
||||
---
|
||||
|
||||
## Current Release: v0.5.0 (February 2026)
|
||||
## Current State (Continuous Releases)
|
||||
|
||||
### Status: Stable
|
||||
- Production-ready document processing
|
||||
- Multi-provider storage support
|
||||
- **Database-backed settings management with encryption**
|
||||
- **Setup wizard for first-time configuration**
|
||||
- **Admin UI for runtime configuration**
|
||||
- **Automated semantic versioning and releases**
|
||||
- OAuth2 authentication with admin group support
|
||||
- Basic web UI and REST API
|
||||
DocuElevate ships continuously via automated semantic versioning. Use **GitHub Releases** for the latest build artifacts and **GitHub Milestones** (below) for roadmap tracking.
|
||||
|
||||
### Last Shipped Milestone: v0.5.0 (Released February 8, 2026)
|
||||
- Database-backed settings management with encryption
|
||||
- Setup wizard for first-time configuration
|
||||
- Admin UI for runtime configuration
|
||||
- Release automation via semantic-release
|
||||
|
||||
### Important Note on Versioning
|
||||
As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
@@ -130,87 +128,78 @@ As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
|
||||
## Upcoming Milestones
|
||||
|
||||
### v0.6.0 - Enhanced Search & UI Improvements (April 2026)
|
||||
**Target Date:** April 1, 2026
|
||||
### v0.6.0 - Clarity: Enhanced Search & UI (Target: July 31, 2026)
|
||||
**Target Date:** July 31, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** User Experience, Search, Performance
|
||||
**Theme:** Search, Discovery, Modern UX
|
||||
**Epic:** #863
|
||||
|
||||
#### Goals
|
||||
- Implement full-text search across documents
|
||||
- Responsive mobile interface
|
||||
- Dark mode support
|
||||
- Document preview in browser
|
||||
- Performance optimizations
|
||||
- Improved error handling and user feedback
|
||||
- Hybrid discovery: keyword + semantic search, fast filtering, saved searches
|
||||
- Preview-first UX (open, skim, and act quickly)
|
||||
- Modern UX polish (accessibility, responsiveness, performance)
|
||||
|
||||
#### Deliverables
|
||||
- Full-text search API and UI
|
||||
- Advanced filtering capabilities
|
||||
- Responsive CSS framework integration
|
||||
- Dark mode toggle
|
||||
- In-browser document viewer
|
||||
- Loading states and progress indicators
|
||||
- Performance benchmarks
|
||||
- Mobile-optimized interface
|
||||
- Semantic search foundation (vectorization + ranking signals)
|
||||
- Saved searches / smart views
|
||||
- In-browser preview + “quick actions” (tag, route, export)
|
||||
- Bulk operations and pagination improvements
|
||||
- UX polish (dark mode/accessibility where applicable)
|
||||
|
||||
#### Breaking Changes
|
||||
- API response format changes for search endpoints (documented)
|
||||
- Potential pagination/search response changes (must be versioned and documented)
|
||||
|
||||
#### Migration Path
|
||||
- Search endpoint changes will be versioned (/api/v1/search → /api/v2/search)
|
||||
- Old endpoints deprecated but functional for 2 releases
|
||||
- Version endpoints where needed and keep previous versions working for at least 2 minor milestones
|
||||
|
||||
---
|
||||
|
||||
### v0.4.5 - Workflow Automation (June 2026)
|
||||
**Target Date:** June 1, 2026
|
||||
### v0.7.0 - Conductor: Workflow Automation & Integrations (Target: September 30, 2026)
|
||||
**Target Date:** September 30, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Automation, Integration, Webhooks
|
||||
**Epic:** #864
|
||||
|
||||
#### Goals
|
||||
- Custom processing pipelines
|
||||
- Conditional routing based on document type
|
||||
- Webhook support for external integrations
|
||||
- Rule-based classification
|
||||
- Scheduled batch processing
|
||||
- First-class workflow model (steps, state, retries) that matches what the system actually executes
|
||||
- Workflow-aware UI status, retries, and observability
|
||||
- Webhooks + event-driven automation foundations
|
||||
|
||||
#### Deliverables
|
||||
- Pipeline configuration UI
|
||||
- Webhook management interface
|
||||
- Rule engine for document routing
|
||||
- Batch processing scheduler
|
||||
- Integration examples and templates
|
||||
- Webhook payload documentation
|
||||
- Workflow object model and storage
|
||||
- Workflow-aware file detail view + status dashboard
|
||||
- Scheduling primitives (recurring jobs / delayed runs)
|
||||
- Webhook system (outbound events + inbound triggers)
|
||||
- Integration templates and documentation
|
||||
|
||||
---
|
||||
|
||||
### v0.7.0 - Advanced AI & Multi-language (August 2026)
|
||||
**Target Date:** August 1, 2026
|
||||
### v0.8.0 - Signal: AI Quality, RAG, and Multi-language (Target: November 30, 2026)
|
||||
**Target Date:** November 30, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** AI Enhancement, Internationalization
|
||||
**Theme:** AI Quality, Retrieval, Internationalization
|
||||
**Epic:** #865
|
||||
|
||||
#### Goals
|
||||
- Custom AI model support
|
||||
- Multi-language OCR
|
||||
- Document similarity detection
|
||||
- Duplicate detection
|
||||
- UI internationalization (i18n)
|
||||
- API localization
|
||||
- “Chat with Library” foundations (retrieval + UI)
|
||||
- Local AI options for privacy-sensitive setups
|
||||
- Measurable AI quality (confidence + human review loop)
|
||||
- Expand multilingual capability across OCR + UI
|
||||
|
||||
#### Deliverables
|
||||
- Custom model integration API
|
||||
- Multi-language OCR configuration
|
||||
- Similarity algorithm implementation
|
||||
- Duplicate detection service
|
||||
- Translation framework (10+ languages)
|
||||
- Localized documentation
|
||||
- Vector DB integration and embeddings pipeline
|
||||
- Chat UI foundations and retrieval API
|
||||
- Confidence scoring + human review/edit loop for extracted fields
|
||||
- Multi-language OCR configuration improvements
|
||||
- Expanded i18n coverage + localized docs
|
||||
|
||||
---
|
||||
|
||||
### v1.0.0 - Enterprise Edition (November 2026)
|
||||
**Target Date:** November 1, 2026
|
||||
### v1.0.0 - Summit: Enterprise Edition (Target: March 31, 2027)
|
||||
**Target Date:** March 31, 2027
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Enterprise Features, Scalability, Multi-tenancy
|
||||
**Epic:** #866
|
||||
|
||||
This is our first major release, marking production-ready enterprise capabilities.
|
||||
|
||||
@@ -244,6 +233,60 @@ This is our first major release, marking production-ready enterprise capabilitie
|
||||
|
||||
---
|
||||
|
||||
### v2.0.0 - Horizon: Platform Expansion (Target: September 30, 2027)
|
||||
**Target Date:** September 30, 2027
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Ecosystem, Platform, Distribution
|
||||
**Epic:** #867
|
||||
|
||||
#### Goals
|
||||
- Make DocuElevate extensible by design (plugins + templates)
|
||||
- Expand integrations and developer experience
|
||||
- Harden multi-surface experiences (web, mobile, extension, CLI) as a cohesive product
|
||||
|
||||
#### Deliverables
|
||||
- Plugin system foundations and public extension points
|
||||
- Template library for pipelines/workflows + “starter kits”
|
||||
- Integration hub patterns (webhooks, events, connectors)
|
||||
- SDK + documentation for extensions
|
||||
|
||||
---
|
||||
|
||||
### v2.1.0 - Sentinel: Governance & Policy (Target: March 31, 2028)
|
||||
**Target Date:** March 31, 2028
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Governance, Compliance, Policy-driven Automation
|
||||
**Epic:** #868
|
||||
|
||||
#### Goals
|
||||
- Make governance first-class (retention, legal hold, PII workflows)
|
||||
- Provide tamper-evident auditing and admin controls
|
||||
- Introduce policy-driven approvals for sensitive automation
|
||||
|
||||
#### Deliverables
|
||||
- Retention policies + legal hold primitives
|
||||
- PII detection + redaction workflows
|
||||
- Tamper-evident audit trails + admin activity feed
|
||||
- Policy-as-code concepts for workflows (with approval gates)
|
||||
|
||||
---
|
||||
|
||||
### v3.0.0 - Constellation: Integration Hub & Agent Platform (Target: September 30, 2028)
|
||||
**Target Date:** September 30, 2028
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Ecosystem, Agents, Interoperability
|
||||
**Epic:** #869
|
||||
|
||||
#### Goals
|
||||
- Make DocuElevate the “system of record” for document intelligence in an organization
|
||||
- Support external automation ecosystems (Zapier/Make/n8n) and agent runtimes
|
||||
- Provide a clean interoperability layer for modern AI tools
|
||||
|
||||
#### Deliverables
|
||||
- DocuElevate MCP server (search, retrieve, summarize, route) and documentation
|
||||
- Connector marketplace concepts (curated + community)
|
||||
- Event stream + webhooks at scale (delivery guarantees, retries, signing)
|
||||
|
||||
## Release Process
|
||||
|
||||
### Automated Semantic Versioning (v0.6.0+)
|
||||
@@ -255,15 +298,15 @@ Starting with v0.6.0, releases are fully automated using `python-semantic-releas
|
||||
4. **Automatic Updates**:
|
||||
- Updates `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates Git tag (e.g., `v0.6.0`)
|
||||
- Creates Git tag (e.g., `v0.173.1`)
|
||||
- Creates GitHub Release with notes
|
||||
- Triggers Docker image builds
|
||||
5. **No Manual Steps**: VERSION and CHANGELOG are never edited manually
|
||||
|
||||
### Version Bump Rules
|
||||
- `feat:` commits → Minor version (0.5.0 → 0.6.0)
|
||||
- `fix:`, `perf:` → Patch version (0.5.0 → 0.5.1)
|
||||
- `feat!:`, `BREAKING CHANGE:` → Major version (0.5.0 → 1.0.0)
|
||||
- `feat:` commits → Minor version (e.g., 0.173.1 → 0.174.0)
|
||||
- `fix:`, `perf:` → Patch version (e.g., 0.173.1 → 0.173.2)
|
||||
- `feat!:`, `BREAKING CHANGE:` → Major version (e.g., 0.173.1 → 1.0.0)
|
||||
- Other types (docs, chore, etc.) → No version bump
|
||||
|
||||
### Pre-release Checklist (Automated)
|
||||
@@ -297,10 +340,13 @@ Starting with v0.6.0, releases are fully automated using `python-semantic-releas
|
||||
| v0.3.2 | 2026-02-06 | Security Updates | Released |
|
||||
| v0.3.3 | 2026-02-08 | Drag-and-Drop Upload | Released |
|
||||
| v0.5.0 | 2026-02-08 | **Settings & Encryption** | **Released** |
|
||||
| v0.6.0 | 2026-04 | Search & UX | Planned |
|
||||
| v0.7.0 | 2026-08 | Advanced AI | Planned |
|
||||
| v1.0.0 | 2026-11 | Enterprise | Planned |
|
||||
| v2.0.0 | 2027-Q3 | Platform Expansion | Future |
|
||||
| v0.6.0 | 2026-07-31 | **Clarity:** Search & UX | Planned |
|
||||
| v0.7.0 | 2026-09-30 | **Conductor:** Workflows & Integrations | Planned |
|
||||
| v0.8.0 | 2026-11-30 | **Signal:** AI Quality, RAG, Multi-language | Planned |
|
||||
| v1.0.0 | 2027-03-31 | **Summit:** Enterprise | Planned |
|
||||
| v2.0.0 | 2027-09-30 | **Horizon:** Platform Expansion | Future |
|
||||
| v2.1.0 | 2028-03-31 | **Sentinel:** Governance & Policy | Future |
|
||||
| v3.0.0 | 2028-09-30 | **Constellation:** Integration Hub & Agents | Future |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ DocuElevate is an intelligent document processing system that automates the inge
|
||||
|
||||
- **AI-Powered Metadata Extraction** — pluggable AI providers including OpenAI, Anthropic Claude, Google Gemini, Ollama (local), OpenRouter, Portkey, and Azure OpenAI via LiteLLM
|
||||
- **Multi-Engine OCR** — Azure Document Intelligence, Tesseract, EasyOCR, Mistral OCR, Google Cloud Document AI, and AWS Textract with configurable merge strategies
|
||||
- **12 Storage Destinations** — Dropbox, Google Drive, OneDrive, Amazon S3, Nextcloud, WebDAV, FTP, SFTP, iCloud Drive, Email (SMTP), Paperless-ngx, and Rclone
|
||||
- **13 Storage Destinations** — Dropbox, Google Drive, OneDrive, Amazon S3, Nextcloud, WebDAV, FTP, SFTP, iCloud Drive, Email (SMTP), Paperless-ngx, Evernote, and Rclone
|
||||
- **Multi-Channel Ingestion** — web upload, browser extension, mobile app, CLI, REST API, IMAP email, and watched folders (local, cloud, FTP/SFTP)
|
||||
- **Processing Pipelines** — customizable multi-step workflows with conditional routing rules
|
||||
- **Full-Text Search** — powered by Meilisearch for instant document discovery
|
||||
@@ -106,6 +106,7 @@ Processed documents are distributed to any combination of configured destination
|
||||
| **iCloud Drive** | Apple cloud |
|
||||
| **Email (SMTP)** | Send as attachment |
|
||||
| **Paperless-ngx** | Document management system |
|
||||
| **Evernote** | Notes with PDF attachments |
|
||||
| **Rclone** | 70+ cloud providers via Rclone |
|
||||
|
||||
## Features
|
||||
@@ -245,6 +246,7 @@ See the [Kubernetes Deployment Guide](docs/KubernetesDeployment.md) for full det
|
||||
| [Google Drive](docs/GoogleDriveSetup.md) | Google Drive service account / OAuth |
|
||||
| [OneDrive](docs/OneDriveSetup.md) | Microsoft OneDrive setup |
|
||||
| [Amazon S3](docs/AmazonS3Setup.md) | S3 bucket configuration |
|
||||
| [Evernote](docs/EvernoteSetup.md) | Evernote note creation |
|
||||
| [Authentication](docs/AuthenticationSetup.md) | OAuth2, OIDC, and social login |
|
||||
| [Notifications](docs/NotificationsSetup.md) | Notification backend setup |
|
||||
|
||||
@@ -327,6 +329,7 @@ The following is a summary of the licenses used by our direct dependencies:
|
||||
| pypdf | BSD |
|
||||
| Requests | Apache 2.0 |
|
||||
| Dropbox SDK | MIT |
|
||||
| Evernote SDK | BSD |
|
||||
| Azure AI Document Intelligence | MIT |
|
||||
| Authlib | BSD |
|
||||
| Starlette | BSD |
|
||||
|
||||
+103
-133
@@ -1,169 +1,139 @@
|
||||
# DocuElevate Roadmap
|
||||
|
||||
**Last Updated:** 2026-02-08
|
||||
**Version:** 1.0
|
||||
**Last Updated:** 2026-05-23
|
||||
**Version:** 2.0
|
||||
|
||||
## Vision
|
||||
|
||||
DocuElevate aims to be the premier open-source intelligent document processing platform, providing seamless integration with cloud storage providers, advanced AI-powered metadata extraction, and enterprise-grade security and scalability.
|
||||
|
||||
## How to Read This Roadmap
|
||||
|
||||
DocuElevate ships frequently (automated semantic versioning), so this roadmap is organized around **milestone outcomes** and **themes**, not exact build numbers.
|
||||
|
||||
- **P0** = required for the milestone to feel “done”
|
||||
- **P1** = strongly desired; may slip if needed
|
||||
- **P2** = nice-to-have / opportunistic
|
||||
|
||||
For the detailed milestone breakdown and target dates, see [MILESTONES.md](MILESTONES.md).
|
||||
|
||||
## Release Naming
|
||||
|
||||
Each major milestone release carries a codename to anchor key project moments. These names appear in the status dashboard, build metadata, and changelog. For details, see [docs/ReleaseNaming.md](docs/ReleaseNaming.md).
|
||||
|
||||
| Version Range | Codename | Theme |
|
||||
|---------------|---------------|--------------------------------------------------|
|
||||
| 0.5.x | **Foundation** | Core platform, multi-provider storage, AI, UI |
|
||||
| 0.6.x | **Clarity** | Enhanced search, filtering, UI/UX improvements |
|
||||
| 0.7.x | **Conductor** | Workflow automation, pipelines, rule-based logic |
|
||||
| 1.0.x | **Summit** | Enterprise features, multi-tenancy, RBAC |
|
||||
| 1.1.x | **Bridge** | Collaboration, sharing, analytics |
|
||||
| 2.0.x | **Horizon** | On-premise AI, platform expansion |
|
||||
| Milestone | Codename | Theme |
|
||||
|----------|------------------|-------|
|
||||
| v0.6.0 | **Clarity** | Search, discovery, and modern UX |
|
||||
| v0.7.0 | **Conductor** | Workflows, orchestration, and integrations |
|
||||
| v0.8.0 | **Signal** | AI quality, multilingual, and “Chat with Library” foundations |
|
||||
| v1.0.0 | **Summit** | Enterprise readiness (multi-tenancy, RBAC, scaling) |
|
||||
| v2.0.0 | **Horizon** | Platform expansion and ecosystem maturity |
|
||||
| v2.1.0+ | **Sentinel** | Governance, compliance, and policy-driven automation |
|
||||
| v3.0.0 | **Constellation**| Integration hub, agents, and interoperability |
|
||||
|
||||
## Current Status (v0.5.0 "Foundation")
|
||||
## Current Product Capabilities (Today)
|
||||
|
||||
### Core Features ✅
|
||||
- Multi-provider document storage (Dropbox, Google Drive, OneDrive, Nextcloud, S3, etc.)
|
||||
- IMAP email integration for document ingestion
|
||||
- OCR processing via Azure Document Intelligence
|
||||
- AI-powered metadata extraction via OpenAI
|
||||
- PDF conversion via Gotenberg
|
||||
- Web UI for document upload and management
|
||||
- **Database-backed settings management with admin UI**
|
||||
- **Fernet encryption for sensitive configuration**
|
||||
- **Setup wizard for first-time installation**
|
||||
- REST API with OpenAPI documentation
|
||||
- Celery-based async task processing
|
||||
- OAuth2 authentication via Authentik with admin group support
|
||||
- Multi-channel ingestion (web upload, IMAP email, watched folders, mobile, CLI, API)
|
||||
- Multi-engine OCR + AI extraction with configurable providers
|
||||
- Customizable processing pipelines and routing rules
|
||||
- Full-text search and document discovery
|
||||
- Multi-destination distribution (cloud providers, DMS, protocols, email)
|
||||
- Admin UI for configuration (database-backed settings, encryption, setup wizard)
|
||||
- Production hardening building blocks (CI/CD, security docs, deployment guides)
|
||||
|
||||
## Short-term Goals (Q1-Q2 2026) - v0.4.x to v0.5.x "Foundation"
|
||||
## Feature Landscape (Themes)
|
||||
|
||||
### Quality & Stability 🎯
|
||||
- **Test Coverage** (High Priority)
|
||||
- [ ] Achieve 80% code coverage for core modules
|
||||
- [ ] Add integration tests for all storage providers
|
||||
- [ ] Add end-to-end workflow tests
|
||||
- [ ] Performance benchmarks and load testing
|
||||
### 1) Search & Discovery
|
||||
- **P0:** hybrid search (keyword + semantic), fast faceted filtering, saved searches
|
||||
- **P1:** “explain results” (why a document matched), query suggestions, pinned results
|
||||
- **P2:** entity search (people/companies/amounts/dates) and graph-style exploration
|
||||
|
||||
- **Code Quality** (High Priority)
|
||||
- [ ] Enable strict linting in CI/CD
|
||||
- [ ] Refactor large modules for better maintainability
|
||||
- [ ] Add comprehensive type hints
|
||||
- [ ] Improve error handling and user feedback
|
||||
### 2) AI Quality & Trust
|
||||
- **P0:** confidence scoring, human review/edit loop, extraction evaluation harness
|
||||
- **P1:** per-document-type schemas/templates, active learning (feedback improves extraction)
|
||||
- **P2:** multi-model routing (choose model by cost/latency/accuracy per step)
|
||||
|
||||
- **Security** (Critical Priority)
|
||||
- [x] Fix known vulnerabilities in dependencies
|
||||
- [ ] Implement rate limiting on API endpoints
|
||||
- [ ] Add CSRF protection
|
||||
- [ ] Security audit by external party
|
||||
- [ ] Implement API key rotation
|
||||
- [ ] Add audit logging for sensitive operations
|
||||
### 3) Workflow Automation & Orchestration
|
||||
- **P0:** first-class workflow model (steps, state, retries), workflow-aware UI status
|
||||
- **P1:** visual workflow builder, scheduling, webhooks, and event-driven triggers
|
||||
- **P2:** agentic workflows (“autopilot” suggestions with approval gates)
|
||||
|
||||
- **Release Automation** (Completed ✅)
|
||||
- [x] Implement semantic-release for automated versioning
|
||||
- [x] Add conventional commit validation
|
||||
- [x] Automate CHANGELOG generation
|
||||
- [x] Integrate Docker builds with releases
|
||||
### 4) Integrations & Ecosystem (Including MCP)
|
||||
- **P0:** stable webhooks + outbound actions (Slack/Teams, email, DMS), bi-directional sync where supported
|
||||
- **P1:** “Integration Hub” (Zapier/Make/n8n style), connector templates, secrets handling patterns
|
||||
- **P2:** **MCP**: ship a DocuElevate MCP server (search, retrieve, summarize, route) + allow MCP tools as pipeline steps
|
||||
|
||||
### Features - v0.4.0
|
||||
- **Enhanced Search & Filtering** → _preparing for v0.6.0 "Clarity"_
|
||||
- [ ] Full-text search across documents
|
||||
- [ ] Advanced filtering by metadata, tags, date ranges
|
||||
- [ ] Saved search queries
|
||||
- [ ] Bulk operations on search results
|
||||
### 5) Governance, Compliance, and Security
|
||||
- **P0:** audit trails, tamper-evident logs, API key lifecycle/rotation, admin activity feed
|
||||
- **P1:** retention policies, legal hold, PII detection + redaction, data residency controls
|
||||
- **P2:** compliance packs (SOC2/GDPR/HIPAA), BYOK/KMS integration paths
|
||||
|
||||
- **Improved UI/UX**
|
||||
- [ ] Responsive mobile interface
|
||||
- [ ] Dark mode support
|
||||
- [ ] Document preview in browser
|
||||
- [ ] Drag-and-drop file upload
|
||||
- [ ] Progress indicators for long-running tasks
|
||||
- [ ] Real-time notifications via WebSocket
|
||||
### 6) Enterprise & Scale
|
||||
- **P0:** multi-tenancy, RBAC, horizontal scaling reference architecture
|
||||
- **P1:** SCIM provisioning, SAML/Okta/Azure AD hardening, quotas/billing at org level
|
||||
- **P2:** multi-region deployment patterns and disaster recovery playbooks
|
||||
|
||||
### Features - v0.5.0 "Foundation"
|
||||
- **Workflow Automation** → _evolving into v0.7.0 "Conductor"_
|
||||
- [ ] Custom processing pipelines
|
||||
- [ ] Conditional routing based on document type
|
||||
- [ ] Scheduled batch processing
|
||||
- [ ] Webhook support for external integrations
|
||||
- [ ] Rule-based document classification
|
||||
## Release Plan (Extended)
|
||||
|
||||
- **Advanced AI Features**
|
||||
- [ ] Custom AI models for specialized document types
|
||||
- [ ] Multi-language OCR support
|
||||
- [ ] Document similarity detection
|
||||
- [ ] Automatic duplicate detection
|
||||
- [ ] Intelligent document splitting
|
||||
This plan extends the existing milestones with a clearer thematic arc and a forward-looking “beyond v2.0” horizon. Each milestone links to an epic issue that owns scope and sub-issues.
|
||||
|
||||
## Medium-term Goals (Q3-Q4 2026) - v1.0.x "Summit"
|
||||
### v0.6.0 — Clarity (Search & UX)
|
||||
- **Outcome:** users can reliably find, preview, and act on documents in seconds
|
||||
- **P0:** semantic search + hybrid ranking, saved searches, fast filters, preview-first UX
|
||||
- **P1:** bulk operations, query suggestions, accessibility/dark mode polish
|
||||
- **Tracking:** GitHub milestone `v0.6.0 - Enhanced Search & UI` (epic #863)
|
||||
|
||||
### Enterprise Features - v1.0.0 "Summit"
|
||||
- **Multi-tenancy**
|
||||
- [ ] Organization/team management
|
||||
- [ ] Role-based access control (RBAC)
|
||||
- [ ] Per-tenant configuration
|
||||
- [ ] Resource quotas and limits
|
||||
- [ ] Audit logs per organization
|
||||
### v0.7.0 — Conductor (Workflows & Integrations)
|
||||
- **Outcome:** workflows are explicit, inspectable, and automatable end-to-end
|
||||
- **P0:** workflow object model + workflow-aware UI status, retries, pipeline definitions
|
||||
- **P1:** workflow builder, scheduling, inbound/outbound webhooks
|
||||
- **P2:** integration templates + “connector marketplace” concepts
|
||||
- **Tracking:** GitHub milestone `v0.7.0 - Workflow Automation` (epic #864)
|
||||
|
||||
- **Scalability**
|
||||
- [ ] Horizontal scaling support
|
||||
- [ ] Distributed task processing
|
||||
- [ ] Caching layer (Redis/Memcached)
|
||||
- [ ] Database connection pooling
|
||||
- [ ] Message queue optimization
|
||||
### v0.8.0 — Signal (AI Quality + “Chat with Library” Foundations)
|
||||
- **Outcome:** AI features are measurable, reviewable, and safe to trust
|
||||
- **P0:** vector DB + embeddings pipeline, chat UI foundations, local AI options
|
||||
- **P1:** confidence scoring and review loop, extraction evaluation harness
|
||||
- **P2:** multilingual UX + localization expansion
|
||||
- **Tracking:** GitHub milestone `v0.8.0 - Advanced AI & Multi-language` (epic #865)
|
||||
|
||||
- **Advanced Integrations**
|
||||
- [ ] Microsoft SharePoint integration
|
||||
- [ ] Slack/Teams bot integration
|
||||
- [ ] Zapier/Make.com integration
|
||||
- [ ] Custom webhook receivers
|
||||
- [ ] GraphQL API
|
||||
### v1.0.0 — Summit (Enterprise Readiness)
|
||||
- **Outcome:** teams can run DocuElevate with strong isolation, access control, and scale
|
||||
- **P0:** multi-tenancy, RBAC, audit logging, scaling guidance
|
||||
- **P1:** SSO hardening (SAML/LDAP), org-level quotas and billing hooks
|
||||
- **P2:** enterprise admin experience (policies, approvals, reporting)
|
||||
- **Tracking:** GitHub milestone `v1.0.0 - Enterprise Edition` (epic #866)
|
||||
|
||||
### Features - v1.1.0 "Bridge"
|
||||
- **Collaboration**
|
||||
- [ ] Document sharing with expiring links
|
||||
- [ ] Comments and annotations
|
||||
- [ ] Version history and rollback
|
||||
- [ ] Real-time collaborative editing metadata
|
||||
- [ ] Activity feed
|
||||
### v2.0.0 — Horizon (Platform Expansion)
|
||||
- **Outcome:** DocuElevate becomes an extensible platform with a thriving ecosystem
|
||||
- **P0:** plugin system foundations, SDK + templates, deeper integrations
|
||||
- **P1:** marketplace patterns, app distribution, mobile/extension maturity
|
||||
- **P2:** multi-workspace experiences (personal + org)
|
||||
- **Tracking:** GitHub milestone `v2.0.0 - Platform Expansion` (epic #867)
|
||||
|
||||
- **Reporting & Analytics**
|
||||
- [ ] Processing statistics dashboard
|
||||
- [ ] Storage usage analytics
|
||||
- [ ] AI confidence scores and accuracy tracking
|
||||
- [ ] Cost analysis per provider
|
||||
- [ ] Export reports (PDF, CSV, Excel)
|
||||
### v2.1.0+ — Sentinel (Governance & Policy)
|
||||
- **Outcome:** governance becomes a first-class layer (policy-driven automation)
|
||||
- **P0:** retention + legal hold, PII detection/redaction, tamper-evident audit trails
|
||||
- **P1:** BYOK/KMS integration patterns, advanced access policies, compliance reporting
|
||||
- **P2:** “policy as code” for workflows + approvals (change management)
|
||||
- **Tracking:** GitHub milestone `v2.1.0 - Governance & Policy (Sentinel)` (epic #868)
|
||||
|
||||
## Long-term Goals (2027+) - v2.0+ "Horizon"
|
||||
### v3.0.0 — Constellation (Integration Hub & Agent Platform)
|
||||
- **Outcome:** DocuElevate plugs into modern automation and AI ecosystems as a first-class system of record
|
||||
- **P0:** MCP server, durable event stream + production-grade webhooks
|
||||
- **P1:** connector templates + curated catalog, agent-friendly permissioning and auditing
|
||||
- **P2:** bring-your-own-agent patterns (sandboxing, scoped credentials)
|
||||
- **Tracking:** GitHub milestone `v3.0.0 - Integration Hub & Agent Platform (Constellation)` (epic #869)
|
||||
|
||||
### Strategic Initiatives
|
||||
- **On-Premise AI Models**
|
||||
- [ ] Self-hosted OCR (Tesseract, EasyOCR)
|
||||
- [ ] Local LLM integration (Ollama, LLaMA)
|
||||
- [ ] GPU acceleration support
|
||||
- [ ] Model fine-tuning interface
|
||||
- [ ] Hybrid cloud/on-premise processing
|
||||
## Research Bets (Optional / Experimental)
|
||||
|
||||
- **Advanced Document Management**
|
||||
- [ ] Document lifecycle management
|
||||
- [ ] Retention policies and auto-deletion
|
||||
- [ ] Compliance templates (GDPR, HIPAA, SOC2)
|
||||
- [ ] Digital signature support
|
||||
- [ ] Encryption at rest and in transit
|
||||
These are longer-horizon bets that should only be productized if they prove real user value.
|
||||
|
||||
- **Platform Expansion**
|
||||
- [ ] Desktop applications (Electron)
|
||||
- [ ] Mobile apps (iOS/Android)
|
||||
- [ ] Browser extensions
|
||||
- [ ] Command-line interface (CLI)
|
||||
- [ ] VS Code extension for developers
|
||||
|
||||
### Research & Innovation
|
||||
- [ ] Machine learning for custom document types
|
||||
- [ ] Blockchain for document provenance
|
||||
- [ ] Federated learning for privacy-preserving AI
|
||||
- [ ] Edge computing support
|
||||
- [ ] Quantum-resistant encryption
|
||||
- Knowledge graph over extracted entities (contracts ↔ vendors ↔ invoices)
|
||||
- Auto-generated “case files” (collections) from intent (“tax 2025”, “project alpha”)
|
||||
- Privacy-preserving learning (federated patterns) to improve extraction quality
|
||||
- Document provenance (signing, attestations) and tamper detection
|
||||
|
||||
## Community & Ecosystem
|
||||
|
||||
|
||||
+6
-6
@@ -1,10 +1,10 @@
|
||||
DocuElevate Build Information
|
||||
==============================
|
||||
Version: 0.172.9
|
||||
Build Date: 2026-04-07T09:34:53Z
|
||||
Git Commit: 3bd8a52ea201b33d6071c9b3a7fdace582e65fd5
|
||||
Git Short SHA: 3bd8a52
|
||||
Version: 0.173.2
|
||||
Build Date: 2026-05-23T00:38:15Z
|
||||
Git Commit: 6fc00b8de10b50b4b2f92f6fadbcf7ebbee7136f
|
||||
Git Short SHA: 6fc00b8
|
||||
Git Branch: main
|
||||
Commit Date: 2026-04-07T11:34:28+02:00
|
||||
Build Timestamp: 2026-04-07T09:34:53Z
|
||||
Commit Date: 2026-05-23T02:37:54+02:00
|
||||
Build Timestamp: 2026-05-23T00:38:15Z
|
||||
==============================
|
||||
|
||||
+13
-18
@@ -28,6 +28,10 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class UnsafeRedirectError(httpx.RequestError):
|
||||
"""Raised when a redirect target fails URL safety checks."""
|
||||
|
||||
|
||||
class URLUploadRequest(BaseModel):
|
||||
"""Request model for URL-based file upload"""
|
||||
|
||||
@@ -120,9 +124,10 @@ async def verify_redirect(response: httpx.Response) -> None:
|
||||
try:
|
||||
validate_url_safety(new_url)
|
||||
except HTTPException as e:
|
||||
# Map the validation error to an httpx exception so it can be handled
|
||||
# properly by the caller, avoiding raw HTTPExceptions escaping the client scope
|
||||
raise httpx.RequestError(f"Redirect to unsafe URL blocked: {e.detail}", request=response.request) from e
|
||||
raise UnsafeRedirectError(
|
||||
f"Redirect to unsafe URL blocked: {e.detail}",
|
||||
request=response.request,
|
||||
) from e
|
||||
|
||||
|
||||
@router.post("/process-url")
|
||||
@@ -170,19 +175,6 @@ async def process_url(
|
||||
if not safe_filename:
|
||||
safe_filename = "download"
|
||||
|
||||
# Hook to validate redirects and prevent SSRF
|
||||
async def validate_redirect(response: httpx.Response):
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("Location")
|
||||
if location:
|
||||
# Resolve relative URLs
|
||||
next_url = urllib.parse.urljoin(str(response.url), location)
|
||||
try:
|
||||
validate_url_safety(next_url)
|
||||
except HTTPException as e:
|
||||
# Reraise as a RequestError so httpx aborts the request
|
||||
raise httpx.RequestError(f"Unsafe redirect target: {e.detail}", request=response.request)
|
||||
|
||||
# Download file with security measures
|
||||
# Initialize target_path to None to prevent UnboundLocalError in exception handlers
|
||||
# that may execute before target_path is assigned during error cases
|
||||
@@ -194,11 +186,10 @@ async def process_url(
|
||||
async with httpx.AsyncClient(
|
||||
timeout=settings.http_request_timeout,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [validate_redirect]},
|
||||
event_hooks={"response": [verify_redirect]},
|
||||
headers={
|
||||
"User-Agent": "DocuElevate/1.0", # Identify ourselves
|
||||
},
|
||||
event_hooks={"response": [verify_redirect]},
|
||||
) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
response.raise_for_status()
|
||||
@@ -285,6 +276,10 @@ async def process_url(
|
||||
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
|
||||
|
||||
except UnsafeRedirectError as e:
|
||||
logger.warning(f"Unsafe redirect blocked while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
except httpx.RequestError as e:
|
||||
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
|
||||
|
||||
@@ -46,6 +46,7 @@ from app.tasks.translate_to_default_language import translate_to_default_languag
|
||||
# Import new send tasks
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
|
||||
from app.tasks.upload_to_email import upload_to_email # noqa: F401
|
||||
from app.tasks.upload_to_evernote import upload_to_evernote # noqa: F401
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401
|
||||
from app.tasks.upload_to_icloud import upload_to_icloud # noqa: F401
|
||||
|
||||
@@ -154,6 +154,17 @@ class Settings(BaseSettings):
|
||||
# "language": "Language", "correspondent": "Correspondent"}
|
||||
paperless_custom_fields_mapping: Optional[str] = None
|
||||
|
||||
# Evernote destination settings
|
||||
evernote_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Enable Evernote as an upload destination. Set to False to disable uploads even when credentials are configured.",
|
||||
)
|
||||
evernote_auth_token: Optional[str] = None
|
||||
evernote_sandbox: bool = False
|
||||
evernote_notebook_guid: Optional[str] = None
|
||||
evernote_default_tags: Optional[str] = None
|
||||
evernote_include_metadata: bool = True
|
||||
|
||||
azure_ai_key: str
|
||||
azure_region: str
|
||||
azure_endpoint: str
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.models import FileRecord, IntegrationDirection, UserIntegration
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
from app.tasks.upload_to_evernote import upload_to_evernote
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_icloud import upload_to_icloud
|
||||
@@ -100,6 +101,11 @@ def _should_upload_to_email():
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_evernote():
|
||||
token = getattr(settings, "evernote_auth_token", None)
|
||||
return bool(getattr(settings, "evernote_enabled", True) and isinstance(token, str) and token.strip())
|
||||
|
||||
|
||||
def _should_upload_to_onedrive():
|
||||
return bool(
|
||||
getattr(settings, "onedrive_enabled", True)
|
||||
@@ -151,6 +157,7 @@ def get_configured_services_from_validator():
|
||||
"FTP Storage": "ftp",
|
||||
"SFTP Storage": "sftp",
|
||||
"Email": "email",
|
||||
"Evernote": "evernote",
|
||||
"OneDrive": "onedrive",
|
||||
"S3 Storage": "s3",
|
||||
"SharePoint": "sharepoint",
|
||||
@@ -254,6 +261,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
|
||||
"should_upload": _should_upload_to_email,
|
||||
"upload_func": upload_to_email,
|
||||
},
|
||||
{
|
||||
"name": "evernote",
|
||||
"should_upload": _should_upload_to_evernote,
|
||||
"upload_func": upload_to_evernote,
|
||||
},
|
||||
{
|
||||
"name": "onedrive",
|
||||
"should_upload": _should_upload_to_onedrive,
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
from html import escape
|
||||
from typing import Any
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import UploadTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNKNOWN_PLACEHOLDERS = {"", "Unknown", "unknown", "N/A", "n/a", "None", "none", "null"}
|
||||
_MAX_EVERNOTE_TITLE_LENGTH = 255
|
||||
|
||||
|
||||
def _get_evernote_sdk():
|
||||
"""Import the Evernote SDK lazily so the missing dependency error is actionable."""
|
||||
try:
|
||||
from evernote.edam.notestore import NoteStore
|
||||
from evernote.edam.type import ttypes as Types
|
||||
from evernote.edam.userstore import UserStore
|
||||
from thrift.protocol import TBinaryProtocol
|
||||
from thrift.transport import THttpClient
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Evernote upload requires the evernote3 package. Install requirements.txt again.") from exc
|
||||
return NoteStore, Types, UserStore, TBinaryProtocol, THttpClient
|
||||
|
||||
|
||||
def _build_thrift_client(client_cls, url: str, binary_protocol, http_transport):
|
||||
transport = http_transport.THttpClient(url)
|
||||
protocol = binary_protocol.TBinaryProtocol(transport)
|
||||
return client_cls(protocol)
|
||||
|
||||
|
||||
def _get_note_store(auth_token: str):
|
||||
NoteStore, Types, UserStore, TBinaryProtocol, THttpClient = _get_evernote_sdk()
|
||||
|
||||
base_url = (
|
||||
"https://sandbox.evernote.com" if getattr(settings, "evernote_sandbox", False) else "https://www.evernote.com"
|
||||
)
|
||||
user_store = _build_thrift_client(UserStore.Client, f"{base_url}/edam/user", TBinaryProtocol, THttpClient)
|
||||
user = user_store.getUser(auth_token)
|
||||
shard_id = getattr(user, "shardId", None)
|
||||
if not shard_id:
|
||||
raise RuntimeError("Evernote user response did not include a shard ID")
|
||||
|
||||
note_store_url = f"{base_url}/shard/{shard_id}/notestore"
|
||||
note_store = _build_thrift_client(NoteStore.Client, note_store_url, TBinaryProtocol, THttpClient)
|
||||
return note_store, Types
|
||||
|
||||
|
||||
def _load_metadata(file_path: str) -> dict[str, Any]:
|
||||
"""Load extracted DocuElevate metadata from the companion JSON file, when present."""
|
||||
json_path = os.path.splitext(file_path)[0] + ".json"
|
||||
if not os.path.exists(json_path):
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as metadata_file:
|
||||
data = json.load(metadata_file)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Failed to load Evernote metadata from %s: %s", json_path, exc)
|
||||
return {}
|
||||
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _normalize_metadata_value(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
normalized = ", ".join(str(item) for item in value if item is not None)
|
||||
elif isinstance(value, dict):
|
||||
normalized = json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
else:
|
||||
normalized = str(value)
|
||||
|
||||
normalized = normalized.strip()
|
||||
return "" if normalized in _UNKNOWN_PLACEHOLDERS else normalized
|
||||
|
||||
|
||||
def _metadata_rows(metadata: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
rows = []
|
||||
for key in sorted(metadata):
|
||||
value = _normalize_metadata_value(metadata[key])
|
||||
if value:
|
||||
rows.append((key, value))
|
||||
return rows
|
||||
|
||||
|
||||
def _extract_tags(metadata: dict[str, Any]) -> list[str]:
|
||||
tags: list[str] = []
|
||||
|
||||
def add_tag(value: Any) -> None:
|
||||
normalized = _normalize_metadata_value(value)
|
||||
if normalized and normalized not in tags:
|
||||
tags.append(normalized)
|
||||
|
||||
default_tags = getattr(settings, "evernote_default_tags", None)
|
||||
if default_tags:
|
||||
for tag in str(default_tags).split(","):
|
||||
add_tag(tag)
|
||||
|
||||
metadata_tags = metadata.get("tags")
|
||||
if isinstance(metadata_tags, str):
|
||||
for tag in metadata_tags.split(","):
|
||||
add_tag(tag)
|
||||
elif isinstance(metadata_tags, (list, tuple, set)):
|
||||
for tag in metadata_tags:
|
||||
add_tag(tag)
|
||||
|
||||
return tags
|
||||
|
||||
|
||||
def _note_title(file_path: str, metadata: dict[str, Any]) -> str:
|
||||
title = (
|
||||
_normalize_metadata_value(metadata.get("title"))
|
||||
or _normalize_metadata_value(metadata.get("filename"))
|
||||
or os.path.basename(file_path)
|
||||
)
|
||||
return title[:_MAX_EVERNOTE_TITLE_LENGTH]
|
||||
|
||||
|
||||
def _build_enml(metadata: dict[str, Any], resource_hash: str, resource_mime: str, include_metadata: bool) -> str:
|
||||
body_parts = ['<?xml version="1.0" encoding="UTF-8"?>']
|
||||
body_parts.append('<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">')
|
||||
body_parts.append("<en-note>")
|
||||
|
||||
if include_metadata:
|
||||
rows = _metadata_rows(metadata)
|
||||
if rows:
|
||||
body_parts.append("<div><b>DocuElevate metadata</b></div>")
|
||||
for key, value in rows:
|
||||
body_parts.append(f"<div><b>{escape(key)}:</b> {escape(value)}</div>")
|
||||
body_parts.append("<br/>")
|
||||
|
||||
body_parts.append(f'<en-media type="{escape(resource_mime)}" hash="{resource_hash}"/>')
|
||||
body_parts.append("</en-note>")
|
||||
return "".join(body_parts)
|
||||
|
||||
|
||||
def _create_evernote_note(file_path: str, metadata: dict[str, Any], task_id: str):
|
||||
auth_token = getattr(settings, "evernote_auth_token", None)
|
||||
if not auth_token:
|
||||
raise ValueError("Evernote auth token is not configured (EVERNOTE_AUTH_TOKEN)")
|
||||
|
||||
note_store, Types = _get_note_store(auth_token)
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
with open(file_path, "rb") as pdf_file:
|
||||
resource_body = pdf_file.read()
|
||||
|
||||
body_hash = hashlib.md5(resource_body).digest() # noqa: S324 - Evernote API requires MD5 resource hashes.
|
||||
body_hash_hex = hashlib.md5(resource_body).hexdigest() # noqa: S324 - Evernote ENML references MD5 hashes.
|
||||
resource_mime = mimetypes.guess_type(filename)[0] or "application/pdf"
|
||||
|
||||
data = Types.Data()
|
||||
data.size = len(resource_body)
|
||||
data.bodyHash = body_hash
|
||||
data.body = resource_body
|
||||
|
||||
resource = Types.Resource()
|
||||
resource.mime = resource_mime
|
||||
resource.data = data
|
||||
resource.attributes = Types.ResourceAttributes(fileName=filename)
|
||||
|
||||
note = Types.Note()
|
||||
note.title = _note_title(file_path, metadata)
|
||||
note.content = _build_enml(
|
||||
metadata,
|
||||
body_hash_hex,
|
||||
resource_mime,
|
||||
include_metadata=getattr(settings, "evernote_include_metadata", True),
|
||||
)
|
||||
note.resources = [resource]
|
||||
|
||||
notebook_guid = getattr(settings, "evernote_notebook_guid", None)
|
||||
if notebook_guid:
|
||||
note.notebookGuid = notebook_guid
|
||||
|
||||
tag_names = _extract_tags(metadata)
|
||||
if tag_names:
|
||||
note.tagNames = tag_names
|
||||
|
||||
created_note = note_store.createNote(auth_token, note)
|
||||
|
||||
logger.info("[%s] Created Evernote note %s for %s", task_id, getattr(created_note, "guid", None), file_path)
|
||||
return created_note
|
||||
|
||||
|
||||
@celery.task(base=UploadTaskWithRetry, bind=True)
|
||||
def upload_to_evernote(self, file_path: str, file_id: int = None):
|
||||
"""
|
||||
Upload a document to Evernote by creating a note with metadata and a PDF attachment.
|
||||
|
||||
Args:
|
||||
file_path: Path to the PDF file to upload
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
filename = os.path.basename(file_path)
|
||||
logger.info("[%s] Starting Evernote upload: %s", task_id, file_path)
|
||||
log_task_progress(
|
||||
task_id, "upload_to_evernote", "in_progress", f"Uploading to Evernote: {filename}", file_id=file_id
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error("[%s] %s", task_id, error_msg)
|
||||
log_task_progress(task_id, "upload_to_evernote", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
if not getattr(settings, "evernote_auth_token", None):
|
||||
error_msg = "Evernote auth token is not configured (EVERNOTE_AUTH_TOKEN)"
|
||||
logger.error("[%s] %s", task_id, error_msg)
|
||||
log_task_progress(task_id, "upload_to_evernote", "failure", error_msg, file_id=file_id)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
metadata = _load_metadata(file_path)
|
||||
created_note = _create_evernote_note(file_path, metadata, task_id)
|
||||
except Exception as exc:
|
||||
error_msg = f"Failed to upload to Evernote: {exc}"
|
||||
logger.error("[%s] %s", task_id, error_msg)
|
||||
log_task_progress(task_id, "upload_to_evernote", "failure", error_msg, file_id=file_id)
|
||||
raise
|
||||
|
||||
note_guid = getattr(created_note, "guid", None)
|
||||
log_task_progress(task_id, "upload_to_evernote", "success", f"Uploaded to Evernote: {note_guid}", file_id=file_id)
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"evernote_note_guid": note_guid,
|
||||
"evernote_title": getattr(created_note, "title", None),
|
||||
"evernote_notebook_guid": getattr(created_note, "notebookGuid", None),
|
||||
}
|
||||
@@ -174,6 +174,21 @@ def get_provider_status() -> dict[str, dict[str, object]]:
|
||||
},
|
||||
}
|
||||
|
||||
providers["Evernote"] = {
|
||||
"name": "Evernote",
|
||||
"icon": "fa-brands fa-evernote",
|
||||
"configured": bool(getattr(settings, "evernote_auth_token", None)),
|
||||
"enabled": getattr(settings, "evernote_enabled", True),
|
||||
"description": "Create Evernote notes with document metadata and PDF attachments",
|
||||
"details": {
|
||||
"auth_token": mask_sensitive_value(getattr(settings, "evernote_auth_token", None)),
|
||||
"sandbox": getattr(settings, "evernote_sandbox", False),
|
||||
"notebook_guid": getattr(settings, "evernote_notebook_guid", "Not set"),
|
||||
"default_tags": getattr(settings, "evernote_default_tags", "Not set"),
|
||||
"include_metadata": getattr(settings, "evernote_include_metadata", True),
|
||||
},
|
||||
}
|
||||
|
||||
# Add FTP configuration to providers
|
||||
providers["FTP Storage"] = {
|
||||
"name": "FTP Storage",
|
||||
|
||||
@@ -145,6 +145,12 @@ def validate_storage_configs() -> dict[str, list[str]]:
|
||||
email_issues.append("DEST_EMAIL_DEFAULT_RECIPIENT is not configured")
|
||||
issues["email"] = email_issues
|
||||
|
||||
# Validate Evernote
|
||||
evernote_issues = []
|
||||
if not getattr(settings, "evernote_auth_token", None):
|
||||
evernote_issues.append("EVERNOTE_AUTH_TOKEN is not configured")
|
||||
issues["evernote"] = evernote_issues
|
||||
|
||||
# Validate S3
|
||||
s3_issues = []
|
||||
if not getattr(settings, "s3_bucket_name", None):
|
||||
|
||||
@@ -1089,6 +1089,55 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - Evernote
|
||||
"evernote_enabled": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Enable Evernote as an upload destination. When disabled, no documents will be sent to Evernote even if credentials are configured.",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"evernote_auth_token": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Evernote developer token for note creation",
|
||||
"type": "string",
|
||||
"sensitive": True,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"evernote_sandbox": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Use the Evernote sandbox environment instead of production",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"evernote_notebook_guid": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Optional Evernote notebook GUID for uploaded notes",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"evernote_default_tags": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Comma-separated Evernote tags to apply to uploaded notes",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"evernote_include_metadata": {
|
||||
"category": "Storage Providers",
|
||||
"description": "Include extracted document metadata in Evernote note content",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Storage Providers - Google Drive
|
||||
"google_drive_enabled": {
|
||||
"category": "Storage Providers",
|
||||
|
||||
@@ -12,11 +12,13 @@ import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models import InAppNotification, UserNotificationPreference, UserNotificationTarget
|
||||
from app.utils.network import is_private_ip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,6 +30,11 @@ USER_EVENT_LABELS: dict[str, str] = {
|
||||
EVENT_DOCUMENT_PROCESSED: "Document Processed",
|
||||
EVENT_DOCUMENT_FAILED: "Document Processing Failed",
|
||||
}
|
||||
METADATA_ENDPOINTS = {
|
||||
"169.254.169.254",
|
||||
"169.254.169.253",
|
||||
"metadata.google.internal",
|
||||
}
|
||||
|
||||
|
||||
def create_in_app_notification(
|
||||
@@ -128,6 +135,20 @@ def _send_webhook_notification(target_config: dict[str, Any], event_type: str, t
|
||||
logger.warning("Webhook notification target missing url")
|
||||
return False
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
if parsed_url.scheme not in {"http", "https"}:
|
||||
logger.warning("Webhook notification to %s blocked: invalid scheme %s", url, parsed_url.scheme)
|
||||
return False
|
||||
|
||||
hostname = parsed_url.hostname
|
||||
if not hostname:
|
||||
logger.warning("Webhook notification to %s blocked: missing hostname", url)
|
||||
return False
|
||||
|
||||
if hostname in METADATA_ENDPOINTS or is_private_ip(hostname):
|
||||
logger.warning("Webhook notification to %s blocked: private or metadata endpoint", url)
|
||||
return False
|
||||
|
||||
payload = {
|
||||
"event": event_type,
|
||||
"title": title,
|
||||
|
||||
@@ -18,11 +18,13 @@ import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models import WebhookConfig
|
||||
from app.utils.network import is_private_ip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -40,6 +42,11 @@ VALID_EVENTS: frozenset[str] = frozenset(
|
||||
|
||||
#: Timeout (seconds) for outgoing webhook HTTP requests.
|
||||
WEBHOOK_TIMEOUT = 10
|
||||
METADATA_ENDPOINTS = {
|
||||
"169.254.169.254",
|
||||
"169.254.169.253",
|
||||
"metadata.google.internal",
|
||||
}
|
||||
|
||||
|
||||
def compute_signature(payload_bytes: bytes, secret: str) -> str:
|
||||
@@ -67,6 +74,20 @@ def deliver_webhook(url: str, payload: dict[str, Any], secret: str | None = None
|
||||
Returns:
|
||||
``True`` when the remote server responds with a 2xx status.
|
||||
"""
|
||||
parsed_url = urlparse(url)
|
||||
if parsed_url.scheme not in {"http", "https"}:
|
||||
logger.warning("Webhook to %s blocked: invalid scheme %s", url, parsed_url.scheme)
|
||||
return False
|
||||
|
||||
hostname = parsed_url.hostname
|
||||
if not hostname:
|
||||
logger.warning("Webhook to %s blocked: missing hostname", url)
|
||||
return False
|
||||
|
||||
if hostname in METADATA_ENDPOINTS or is_private_ip(hostname):
|
||||
logger.warning("Webhook to %s blocked: private or metadata endpoint", url)
|
||||
return False
|
||||
|
||||
body = json.dumps(payload, default=str, sort_keys=True)
|
||||
body_bytes = body.encode("utf-8")
|
||||
|
||||
|
||||
@@ -717,6 +717,7 @@ def _compute_processing_flow(logs, pipeline_steps=None):
|
||||
"upload_to_ftp": "FTP Storage",
|
||||
"upload_to_sftp": "SFTP Storage",
|
||||
"upload_to_email": "Email",
|
||||
"upload_to_evernote": "Evernote",
|
||||
"queue_dropbox": "Dropbox",
|
||||
"queue_nextcloud": "Nextcloud",
|
||||
"queue_paperless": "Paperless-ngx",
|
||||
@@ -727,6 +728,7 @@ def _compute_processing_flow(logs, pipeline_steps=None):
|
||||
"queue_ftp": "FTP Storage",
|
||||
"queue_sftp": "SFTP Storage",
|
||||
"queue_email": "Email",
|
||||
"queue_evernote": "Evernote",
|
||||
}
|
||||
|
||||
# Create a map of step names to their log entries
|
||||
|
||||
@@ -1330,6 +1330,19 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS
|
||||
| `DEST_EMAIL_SENDER` | From address for delivered documents (e.g., `"DocuElevate Delivery <docuelevate@example.com>"`). |
|
||||
| `DEST_EMAIL_DEFAULT_RECIPIENT` | Fallback recipient email when none is specified for a delivery task. |
|
||||
|
||||
### Evernote
|
||||
|
||||
| **Variable** | **Description** |
|
||||
|-------------------------------|---------------------------------------------------------------------|
|
||||
| `EVERNOTE_ENABLED` | Set to `false` to disable Evernote uploads without removing credentials. Default: `true` |
|
||||
| `EVERNOTE_AUTH_TOKEN` | Evernote developer token or OAuth access token used to create notes. |
|
||||
| `EVERNOTE_SANDBOX` | Use Evernote sandbox API endpoints. Default: `false` |
|
||||
| `EVERNOTE_NOTEBOOK_GUID` | Optional target notebook GUID. If omitted, Evernote uses the default notebook. |
|
||||
| `EVERNOTE_DEFAULT_TAGS` | Optional comma-separated tags applied to every created note. |
|
||||
| `EVERNOTE_INCLUDE_METADATA` | Include extracted metadata in the Evernote note body. Default: `true` |
|
||||
|
||||
For detailed setup instructions, see the [Evernote Setup Guide](EvernoteSetup.md).
|
||||
|
||||
### OneDrive / Microsoft Graph
|
||||
|
||||
| **Variable** | **Description** |
|
||||
|
||||
@@ -13,6 +13,7 @@ DocuElevate is designed to be highly configurable through environment variables,
|
||||
- [Dropbox Setup](DropboxSetup.md) - How to set up Dropbox integration
|
||||
- [OneDrive Setup](OneDriveSetup.md) - How to set up Microsoft OneDrive/Graph integration
|
||||
- [Amazon S3 Setup](AmazonS3Setup.md) - How to set up Amazon S3 integration
|
||||
- [Evernote Setup](EvernoteSetup.md) - How to set up Evernote note creation
|
||||
- [Authentication Setup](AuthenticationSetup.md) - How to set up user authentication
|
||||
- [Notifications Setup](NotificationsSetup.md) - How to set up system notifications
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# Setting up Evernote Integration
|
||||
|
||||
This guide explains how to configure DocuElevate to create Evernote notes for processed documents.
|
||||
|
||||
## Overview
|
||||
|
||||
The Evernote destination creates one note per processed document. The note contains:
|
||||
|
||||
- A visible metadata section populated from DocuElevate's extracted metadata JSON
|
||||
- The processed PDF attached as an Evernote resource
|
||||
- Optional tags from `EVERNOTE_DEFAULT_TAGS` plus extracted document tags
|
||||
|
||||
## Required Configuration
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `EVERNOTE_ENABLED` | Set to `false` to disable Evernote uploads without removing credentials. Default: `true` |
|
||||
| `EVERNOTE_AUTH_TOKEN` | Evernote developer token or OAuth access token with note creation permissions |
|
||||
|
||||
## Optional Configuration
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `EVERNOTE_SANDBOX` | Use Evernote sandbox API endpoints. Default: `false` |
|
||||
| `EVERNOTE_NOTEBOOK_GUID` | Target notebook GUID. If omitted, Evernote uses the account default notebook |
|
||||
| `EVERNOTE_DEFAULT_TAGS` | Comma-separated tags to apply to every created note, for example `docuelevate,archive` |
|
||||
| `EVERNOTE_INCLUDE_METADATA` | Include extracted metadata in the note body. Default: `true` |
|
||||
|
||||
## Example
|
||||
|
||||
```dotenv
|
||||
EVERNOTE_ENABLED=true
|
||||
EVERNOTE_AUTH_TOKEN=your-evernote-token
|
||||
EVERNOTE_NOTEBOOK_GUID=optional-notebook-guid
|
||||
EVERNOTE_DEFAULT_TAGS=docuelevate,processed
|
||||
EVERNOTE_INCLUDE_METADATA=true
|
||||
```
|
||||
|
||||
## Metadata and Attachments
|
||||
|
||||
DocuElevate reads the companion metadata file next to the processed PDF, for example `invoice.pdf` and `invoice.json`. Non-empty metadata fields are rendered into the Evernote note body. Values such as `Unknown`, empty strings, and null values are skipped.
|
||||
|
||||
If the metadata contains a `tags` field, those tags are applied to the note together with any tags configured in `EVERNOTE_DEFAULT_TAGS`.
|
||||
|
||||
The processed PDF is attached directly to the note using Evernote's resource model, so it appears as a normal Evernote attachment.
|
||||
|
||||
## Notes
|
||||
|
||||
- Evernote tokens can expire or be revoked. If uploads start failing with authentication errors, generate or refresh the token and update `EVERNOTE_AUTH_TOKEN`.
|
||||
- If `EVERNOTE_NOTEBOOK_GUID` points to a missing or inaccessible notebook, Evernote will reject the note creation request.
|
||||
- Evernote enforces account upload quotas and per-note size limits. Large PDFs may fail if they exceed those limits.
|
||||
@@ -24,6 +24,7 @@ Welcome to the DocuElevate documentation. This directory contains comprehensive
|
||||
- [OneDrive Setup](OneDriveSetup.md) - How to set up Microsoft OneDrive/Graph integration
|
||||
- [SharePoint Setup](SharePointSetup.md) - How to set up Microsoft SharePoint Online integration
|
||||
- [Amazon S3 Setup](AmazonS3Setup.md) - How to set up Amazon S3 integration
|
||||
- [Evernote Setup](EvernoteSetup.md) - How to set up Evernote note creation
|
||||
- [Authentication Setup](AuthenticationSetup.md) - How to set up user authentication
|
||||
- [Notifications Setup](NotificationsSetup.md) - How to set up system notifications
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ Settings are organized into logical categories for easy navigation:
|
||||
- **Authentication**: Login settings, session secrets, OAuth configuration, admin group
|
||||
- **AI Services**: AI provider selection, model configuration, embeddings, and credentials (OpenAI, Azure, Anthropic, Gemini, Ollama, OpenRouter, Portkey, LiteLLM)
|
||||
- **OCR Engines**: OCR provider selection and configuration (Tesseract, EasyOCR, Mistral, Google DocAI, AWS Textract)
|
||||
- **Storage Providers**: Dropbox, Google Drive, OneDrive, SharePoint, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless
|
||||
- **Storage Providers**: Dropbox, Google Drive, OneDrive, SharePoint, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless, Evernote
|
||||
- **Email**: SMTP configuration for sending emails
|
||||
- **IMAP**: Email ingestion configuration (supports two mailbox accounts)
|
||||
- **Monitoring**: Uptime Kuma integration
|
||||
|
||||
@@ -125,6 +125,7 @@ Several cloud storage integrations include their own guided configuration pages
|
||||
| Google Drive | `/google-drive-setup` | [GoogleDriveSetup.md](GoogleDriveSetup.md) |
|
||||
| OneDrive / SharePoint | `/onedrive-setup` | [OneDriveSetup.md](OneDriveSetup.md) |
|
||||
| Amazon S3 | Configured via settings | [AmazonS3Setup.md](AmazonS3Setup.md) |
|
||||
| Evernote | Configured via settings | [EvernoteSetup.md](EvernoteSetup.md) |
|
||||
|
||||
These pages are accessed **after** the main Setup Wizard is complete and are independent wizard flows specific to each integration.
|
||||
|
||||
|
||||
Generated
+7
-7
@@ -473,9 +473,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.11",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
|
||||
"integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==",
|
||||
"version": "3.3.12",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz",
|
||||
"integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -569,9 +569,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.8",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz",
|
||||
"integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==",
|
||||
"version": "8.5.15",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
|
||||
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -589,7 +589,7 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.11",
|
||||
"nanoid": "^3.3.12",
|
||||
"picocolors": "^1.1.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
|
||||
@@ -293,13 +293,25 @@ function processFiles(files, progressContainer, statusMessage) {
|
||||
updateStatus();
|
||||
}
|
||||
|
||||
function _escapeHtml(unsafe) {
|
||||
if (!unsafe) return '';
|
||||
return String(unsafe)
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
// Pre-create one progress row per file.
|
||||
const queueItems = fileArray.map((file) => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'flex flex-col mb-2';
|
||||
const safeFileName = _escapeHtml(file.name);
|
||||
|
||||
row.innerHTML = `
|
||||
<div class="flex justify-between">
|
||||
<span class="text-sm truncate" title="${file.name}">${file.name}</span>
|
||||
<span class="text-sm truncate" title="${safeFileName}">${safeFileName}</span>
|
||||
<span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 h-2 rounded-full mt-1">
|
||||
|
||||
@@ -1538,6 +1538,28 @@
|
||||
});
|
||||
}
|
||||
|
||||
function escapeHtml(str) {
|
||||
if (!str) return '';
|
||||
return String(str)
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
function sanitizeHighlight(html) {
|
||||
if (!html) return '';
|
||||
let safe = String(html)
|
||||
.replace(/<mark>/gi, '\x00MARK_OPEN\x00')
|
||||
.replace(/<\/mark>/gi, '\x00MARK_CLOSE\x00');
|
||||
safe = escapeHtml(safe);
|
||||
safe = safe
|
||||
.replace(/\x00MARK_OPEN\x00/g, '<mark>')
|
||||
.replace(/\x00MARK_CLOSE\x00/g, '</mark>');
|
||||
return safe;
|
||||
}
|
||||
|
||||
function renderSearchResults(data, q) {
|
||||
const panel = document.getElementById('search-results-panel');
|
||||
const list = document.getElementById('search-results-list');
|
||||
@@ -1555,25 +1577,31 @@
|
||||
|
||||
list.innerHTML = results.map(hit => {
|
||||
const fmt = hit._formatted || {};
|
||||
const title = fmt.document_title || hit.document_title || hit.original_filename || __i18n.untitled;
|
||||
const filename = fmt.original_filename || hit.original_filename || '';
|
||||
const snippet = fmt.ocr_text || '';
|
||||
const tags = Array.isArray(hit.tags) ? hit.tags.join(', ') : (hit.tags || '');
|
||||
const docType = hit.document_type || '';
|
||||
const titleRaw = fmt.document_title || hit.document_title || hit.original_filename || __i18n.untitled;
|
||||
const filenameRaw = fmt.original_filename || hit.original_filename || '';
|
||||
const snippetRaw = fmt.ocr_text || '';
|
||||
const tagsRaw = Array.isArray(hit.tags) ? hit.tags.join(', ') : (hit.tags || '');
|
||||
const docTypeRaw = hit.document_type || '';
|
||||
|
||||
const safeTitle = fmt.document_title ? sanitizeHighlight(titleRaw) : escapeHtml(titleRaw);
|
||||
const safeFilename = escapeHtml(filenameRaw);
|
||||
const safeSnippet = sanitizeHighlight(snippetRaw);
|
||||
const safeTags = escapeHtml(tagsRaw);
|
||||
const safeDocType = escapeHtml(docTypeRaw);
|
||||
|
||||
return `<div style="padding: 0.75rem 1rem; border-bottom: 1px solid #f3f4f6; display: flex; gap: 0.75rem; align-items: flex-start;">
|
||||
<div style="flex-shrink: 0; color: #3b82f6; font-size: 1.25rem; padding-top: 0.1rem;">
|
||||
<i class="fas fa-file-pdf"></i>
|
||||
</div>
|
||||
<div style="flex: 1; min-width: 0;">
|
||||
<div style="font-weight: 600; font-size: 0.9rem; color: #111827;">${title}</div>
|
||||
${filename ? `<div style="font-size: 0.8rem; color: #6b7280; margin-top: 0.15rem;">${filename}</div>` : ''}
|
||||
${docType ? `<span style="display: inline-block; margin-top: 0.25rem; padding: 0.1rem 0.5rem; background: #eff6ff; color: #1d4ed8; border-radius: 9999px; font-size: 0.75rem;">${docType}</span>` : ''}
|
||||
${tags ? `<span style="display: inline-block; margin-top: 0.25rem; margin-left: 0.25rem; padding: 0.1rem 0.5rem; background: #f0fdf4; color: #15803d; border-radius: 9999px; font-size: 0.75rem;">${tags}</span>` : ''}
|
||||
${snippet ? `<div style="margin-top: 0.4rem; font-size: 0.8rem; color: #374151; white-space: pre-wrap; word-break: break-word;">…${snippet}…</div>` : ''}
|
||||
<div style="font-weight: 600; font-size: 0.9rem; color: #111827;">${safeTitle}</div>
|
||||
${safeFilename ? `<div style="font-size: 0.8rem; color: #6b7280; margin-top: 0.15rem;">${safeFilename}</div>` : ''}
|
||||
${safeDocType ? `<span style="display: inline-block; margin-top: 0.25rem; padding: 0.1rem 0.5rem; background: #eff6ff; color: #1d4ed8; border-radius: 9999px; font-size: 0.75rem;">${safeDocType}</span>` : ''}
|
||||
${safeTags ? `<span style="display: inline-block; margin-top: 0.25rem; margin-left: 0.25rem; padding: 0.1rem 0.5rem; background: #f0fdf4; color: #15803d; border-radius: 9999px; font-size: 0.75rem;">${safeTags}</span>` : ''}
|
||||
${safeSnippet ? `<div style="margin-top: 0.4rem; font-size: 0.8rem; color: #374151; white-space: pre-wrap; word-break: break-word;">…${safeSnippet}…</div>` : ''}
|
||||
</div>
|
||||
<div style="flex-shrink: 0;">
|
||||
<a href="/files/${hit.file_id}" style="padding: 0.25rem 0.6rem; background: #f3f4f6; color: #374151; border-radius: 0.25rem; font-size: 0.8rem; text-decoration: none; white-space: nowrap;" title="${__i18n.viewFile}">
|
||||
<a href="/files/${escapeHtml(hit.file_id)}" style="padding: 0.25rem 0.6rem; background: #f3f4f6; color: #374151; border-radius: 0.25rem; font-size: 0.8rem; text-decoration: none; white-space: nowrap;" title="${__i18n.viewFile}">
|
||||
<i class="fas fa-external-link-alt"></i>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -55,6 +55,7 @@ nav:
|
||||
- Google Drive: GoogleDriveSetup
|
||||
- OneDrive: OneDriveSetup
|
||||
- Amazon S3: AmazonS3Setup
|
||||
- Evernote: EvernoteSetup
|
||||
- Authentication: AuthenticationSetup
|
||||
- Notifications: NotificationsSetup
|
||||
- Security:
|
||||
|
||||
Generated
+346
-335
File diff suppressed because it is too large
Load Diff
@@ -58,6 +58,13 @@
|
||||
"eslint-config-expo": "~10.0.0",
|
||||
"typescript": "^5.3.0"
|
||||
},
|
||||
"overrides": {
|
||||
"postcss": "8.5.15",
|
||||
"uuid": "11.1.1",
|
||||
"brace-expansion@^1.1.7": "1.1.13",
|
||||
"brace-expansion@^2.0.2": "2.1.0",
|
||||
"brace-expansion@^5.0.2": "5.0.6"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.19.4"
|
||||
},
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=45", "wheel"]
|
||||
requires = ["setuptools>=82.0.1", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
|
||||
+66
-63
@@ -1,63 +1,66 @@
|
||||
fastapi[all] # Web framework with all extras
|
||||
uvicorn # ASGI server
|
||||
celery # Task queue
|
||||
redis # Message broker for Celery
|
||||
sqlalchemy # Database ORM
|
||||
pydantic # Data validation
|
||||
cryptography>=41.0.0 # Encryption for sensitive settings in database
|
||||
openai # GPT integration for metadata extraction
|
||||
pypdf>=3.9.0 # PDF processing for text extraction, metadata editing and rotation (upgraded from PyPDF2 to fix CVE-2023-36464)
|
||||
requests # HTTP client
|
||||
click>=8.0.0 # CLI framework for docuelevate command
|
||||
puremagic>=1.25,<2.0 # File type detection (pure Python)
|
||||
filetype>=1.2.0,<2.0 # File type detection fallback (pure Python)
|
||||
dropbox>=11.36.0 # Dropbox integration
|
||||
azure-ai-documentintelligence # Azure OCR service
|
||||
authlib>=1.6.5 # Authentication - fixed security vulnerabilities (GHSA-xxx)
|
||||
python-dotenv # Environment variables
|
||||
starlette>=0.49.1 # ASGI toolkit (used by FastAPI) - fixed DoS vulnerability
|
||||
alembic # Database migrations
|
||||
slowapi>=0.1.9 # Rate limiting middleware for FastAPI
|
||||
|
||||
# Google Drive API
|
||||
google-api-python-client>=2.79.0
|
||||
google-auth>=2.22.0
|
||||
google-auth-oauthlib>=1.0.0
|
||||
|
||||
# OneDrive/Microsoft Graph API
|
||||
msgraph-core>=1.0.0
|
||||
msal>=1.20.0
|
||||
|
||||
# AWS S3
|
||||
boto3>=1.28.0
|
||||
|
||||
# SFTP
|
||||
paramiko>=3.4.0 # SSH/SFTP implementation for Python (LGPL license)
|
||||
|
||||
# iCloud Drive
|
||||
pyicloud>=2.4.0 # Unofficial Apple iCloud API client (MIT license)
|
||||
|
||||
# Safe XML parsing (protection against XML bomb / XXE attacks)
|
||||
defusedxml>=0.7.1
|
||||
|
||||
# Notification service
|
||||
apprise>=1.4.0
|
||||
|
||||
# AI provider aggregator - enables Anthropic, Gemini, Ollama, and 100+ LLM providers
|
||||
litellm>=1.0.0,<2.0.0
|
||||
|
||||
# Self-hosted OCR engines (optional – only required when the provider is enabled)
|
||||
pytesseract>=0.3.10 # Python wrapper for Tesseract OCR
|
||||
pdf2image>=1.17.0 # Convert PDF pages to images (used by Tesseract and EasyOCR providers)
|
||||
ocrmypdf>=16.0.0,<18.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
|
||||
meilisearch>=0.31.0 # Full-text search engine client
|
||||
stripe>=7.0.0,<16.0.0 # Stripe billing SDK (MIT license)
|
||||
|
||||
# Error and performance monitoring
|
||||
sentry-sdk[fastapi,celery,sqlalchemy]>=2.20.0,<3.0.0
|
||||
|
||||
# GraphQL API
|
||||
strawberry-graphql[fastapi]>=0.243.0,<1.0.0
|
||||
|
||||
aiofiles>=24.1.0 # Asynchronous file I/O support
|
||||
segno>=1.6.0 # Pure-Python QR code generator (server-side rendering, no Pillow dependency)
|
||||
fastapi[all]!=0.136.3 # Web framework with all extras
|
||||
uvicorn # ASGI server
|
||||
celery # Task queue
|
||||
redis # Message broker for Celery
|
||||
sqlalchemy # Database ORM
|
||||
pydantic # Data validation
|
||||
cryptography>=41.0.0 # Encryption for sensitive settings in database
|
||||
openai # GPT integration for metadata extraction
|
||||
pypdf>=3.9.0 # PDF processing for text extraction, metadata editing and rotation (upgraded from PyPDF2 to fix CVE-2023-36464)
|
||||
requests # HTTP client
|
||||
click>=8.0.0 # CLI framework for docuelevate command
|
||||
puremagic>=1.25,<2.0 # File type detection (pure Python)
|
||||
filetype>=1.2.0,<2.0 # File type detection fallback (pure Python)
|
||||
dropbox>=11.36.0 # Dropbox integration
|
||||
azure-ai-documentintelligence # Azure OCR service
|
||||
authlib>=1.6.5 # Authentication - fixed security vulnerabilities (GHSA-xxx)
|
||||
python-dotenv # Environment variables
|
||||
starlette>=0.49.1 # ASGI toolkit (used by FastAPI) - fixed DoS vulnerability
|
||||
alembic # Database migrations
|
||||
slowapi>=0.1.9 # Rate limiting middleware for FastAPI
|
||||
|
||||
# Google Drive API
|
||||
google-api-python-client>=2.79.0
|
||||
google-auth>=2.22.0
|
||||
google-auth-oauthlib>=1.0.0
|
||||
|
||||
# OneDrive/Microsoft Graph API
|
||||
msgraph-core>=1.0.0
|
||||
msal>=1.20.0
|
||||
|
||||
# AWS S3
|
||||
boto3>=1.28.0
|
||||
|
||||
# SFTP
|
||||
paramiko>=3.4.0 # SSH/SFTP implementation for Python (LGPL license)
|
||||
|
||||
# iCloud Drive
|
||||
pyicloud>=2.4.0 # Unofficial Apple iCloud API client (MIT license)
|
||||
|
||||
# Evernote
|
||||
evernote3>=1.25.14 # Evernote Cloud API SDK for Python 3 (BSD license)
|
||||
|
||||
# Safe XML parsing (protection against XML bomb / XXE attacks)
|
||||
defusedxml>=0.7.1
|
||||
|
||||
# Notification service
|
||||
apprise>=1.4.0
|
||||
|
||||
# AI provider aggregator - enables Anthropic, Gemini, Ollama, and 100+ LLM providers
|
||||
litellm>=1.0.0,<2.0.0
|
||||
|
||||
# Self-hosted OCR engines (optional – only required when the provider is enabled)
|
||||
pytesseract>=0.3.10 # Python wrapper for Tesseract OCR
|
||||
pdf2image>=1.17.0 # Convert PDF pages to images (used by Tesseract and EasyOCR providers)
|
||||
ocrmypdf>=16.0.0,<18.0.0 # Post-processing: embeds searchable text layers into PDFs via Tesseract
|
||||
meilisearch>=0.31.0 # Full-text search engine client
|
||||
stripe>=7.0.0,<16.0.0 # Stripe billing SDK (MIT license)
|
||||
|
||||
# Error and performance monitoring
|
||||
sentry-sdk[fastapi,celery,sqlalchemy]>=2.20.0,<3.0.0
|
||||
|
||||
# GraphQL API
|
||||
strawberry-graphql[fastapi]>=0.243.0,<1.0.0
|
||||
|
||||
aiofiles>=24.1.0 # Asynchronous file I/O support
|
||||
segno>=1.6.0 # Pure-Python QR code generator (server-side rendering, no Pillow dependency)
|
||||
|
||||
@@ -35,6 +35,7 @@ class TestValidateStorageConfigs:
|
||||
"google_drive",
|
||||
"onedrive",
|
||||
"email",
|
||||
"evernote",
|
||||
"paperless",
|
||||
"uptime_kuma",
|
||||
]
|
||||
@@ -83,6 +84,13 @@ class TestValidateStorageConfigs:
|
||||
assert "DEST_EMAIL_HOST is not configured" in result["email"]
|
||||
assert "DEST_EMAIL_DEFAULT_RECIPIENT is not configured" in result["email"]
|
||||
|
||||
def test_evernote_missing_token(self):
|
||||
"""Test validation when Evernote destination auth token is missing."""
|
||||
with patch("app.utils.config_validator.validators.settings") as mock_settings:
|
||||
mock_settings.evernote_auth_token = None
|
||||
result = validate_storage_configs()
|
||||
assert "EVERNOTE_AUTH_TOKEN is not configured" in result["evernote"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestValidateEmailConfig:
|
||||
|
||||
@@ -663,6 +663,7 @@ def _all_should_upload_false():
|
||||
"ftp",
|
||||
"sftp",
|
||||
"email",
|
||||
"evernote",
|
||||
"onedrive",
|
||||
"s3",
|
||||
"sharepoint",
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
from app.tasks.send_to_all import (
|
||||
_should_upload_to_dropbox,
|
||||
_should_upload_to_email,
|
||||
_should_upload_to_evernote,
|
||||
_should_upload_to_ftp,
|
||||
_should_upload_to_google_drive,
|
||||
_should_upload_to_icloud,
|
||||
@@ -128,6 +129,14 @@ class TestShouldUploadFunctions:
|
||||
|
||||
assert _should_upload_to_email() is True
|
||||
|
||||
@patch("app.tasks.send_to_all.settings")
|
||||
def test_should_upload_to_evernote_configured(self, mock_settings):
|
||||
"""Test Evernote upload check."""
|
||||
mock_settings.evernote_enabled = True
|
||||
mock_settings.evernote_auth_token = "token"
|
||||
|
||||
assert _should_upload_to_evernote() is True
|
||||
|
||||
@patch("app.tasks.send_to_all.settings")
|
||||
def test_should_upload_to_onedrive_configured(self, mock_settings):
|
||||
"""Test OneDrive upload check."""
|
||||
@@ -248,6 +257,14 @@ class TestShouldUploadEnabledFlag:
|
||||
|
||||
assert _should_upload_to_email() is False
|
||||
|
||||
@patch("app.tasks.send_to_all.settings")
|
||||
def test_evernote_disabled_with_credentials(self, mock_settings):
|
||||
"""Test Evernote upload is blocked when disabled even with valid credentials."""
|
||||
mock_settings.evernote_enabled = False
|
||||
mock_settings.evernote_auth_token = "token"
|
||||
|
||||
assert _should_upload_to_evernote() is False
|
||||
|
||||
@patch("app.tasks.send_to_all.settings")
|
||||
def test_onedrive_disabled_with_credentials(self, mock_settings):
|
||||
"""Test OneDrive upload is blocked when disabled even with valid credentials."""
|
||||
@@ -289,6 +306,7 @@ class TestGetConfiguredServicesFromValidator:
|
||||
"Dropbox": {"configured": True, "enabled": True},
|
||||
"NextCloud": {"configured": False, "enabled": True},
|
||||
"S3 Storage": {"configured": True, "enabled": True},
|
||||
"Evernote": {"configured": True, "enabled": True},
|
||||
}
|
||||
|
||||
result = get_configured_services_from_validator()
|
||||
@@ -296,6 +314,7 @@ class TestGetConfiguredServicesFromValidator:
|
||||
assert result["dropbox"] is True
|
||||
assert result["nextcloud"] is False
|
||||
assert result["s3"] is True
|
||||
assert result["evernote"] is True
|
||||
|
||||
@patch("app.tasks.send_to_all.get_provider_status")
|
||||
def test_handles_missing_providers(self, mock_get_status):
|
||||
|
||||
@@ -8,6 +8,7 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
from app.tasks.upload_to_evernote import upload_to_evernote
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
@@ -447,6 +448,7 @@ def test_all_upload_tasks_have_consistent_signature(sample_text_file):
|
||||
(upload_to_webdav, "app.tasks.upload_to_webdav"),
|
||||
(upload_to_google_drive, "app.tasks.upload_to_google_drive"),
|
||||
(upload_to_email, "app.tasks.upload_to_email"),
|
||||
(upload_to_evernote, "app.tasks.upload_to_evernote"),
|
||||
]
|
||||
|
||||
import inspect
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import hashlib
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.upload_to_evernote import _build_enml, upload_to_evernote
|
||||
|
||||
|
||||
class _FakeTypes:
|
||||
class Data:
|
||||
pass
|
||||
|
||||
class Resource:
|
||||
pass
|
||||
|
||||
class ResourceAttributes:
|
||||
def __init__(self, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
class Note:
|
||||
pass
|
||||
|
||||
|
||||
class _FakeNoteStore:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def createNote(self, auth_token, note):
|
||||
self.calls.append((auth_token, note))
|
||||
note.guid = "note-guid-123"
|
||||
return note
|
||||
|
||||
|
||||
_fake_note_store = _FakeNoteStore()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_fake_store():
|
||||
global _fake_note_store
|
||||
_fake_note_store = _FakeNoteStore()
|
||||
|
||||
|
||||
def test_build_enml_escapes_metadata():
|
||||
enml = _build_enml({"sender": "A&B <Corp>"}, "abc123", "application/pdf", include_metadata=True)
|
||||
|
||||
assert "A&B <Corp>" in enml
|
||||
assert '<en-media type="application/pdf" hash="abc123"/>' in enml
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_upload_to_evernote_creates_note_with_metadata_and_pdf(tmp_path):
|
||||
pdf_path = tmp_path / "invoice.pdf"
|
||||
pdf_bytes = b"%PDF-1.4 test content"
|
||||
pdf_path.write_bytes(pdf_bytes)
|
||||
pdf_path.with_suffix(".json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"title": "Invoice May",
|
||||
"absender": "Example GmbH",
|
||||
"tags": ["invoice", "finance"],
|
||||
"empty": "Unknown",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.tasks.upload_to_evernote._get_note_store", return_value=(_fake_note_store, _FakeTypes)),
|
||||
patch("app.tasks.upload_to_evernote.log_task_progress"),
|
||||
patch("app.tasks.upload_to_evernote.settings") as mock_settings,
|
||||
):
|
||||
mock_settings.evernote_auth_token = "auth-token"
|
||||
mock_settings.evernote_sandbox = True
|
||||
mock_settings.evernote_notebook_guid = "notebook-guid"
|
||||
mock_settings.evernote_default_tags = "docuelevate,archive"
|
||||
mock_settings.evernote_include_metadata = True
|
||||
|
||||
result = upload_to_evernote.apply(args=[str(pdf_path)], kwargs={"file_id": 7}).get()
|
||||
|
||||
created_note = _fake_note_store.calls[0][1]
|
||||
resource = created_note.resources[0]
|
||||
|
||||
assert result["status"] == "Completed"
|
||||
assert result["evernote_note_guid"] == "note-guid-123"
|
||||
assert created_note.title == "Invoice May"
|
||||
assert created_note.notebookGuid == "notebook-guid"
|
||||
assert created_note.tagNames == ["docuelevate", "archive", "invoice", "finance"]
|
||||
assert "Example GmbH" in created_note.content
|
||||
assert "empty" not in created_note.content
|
||||
assert f'hash="{hashlib.md5(pdf_bytes).hexdigest()}"' in created_note.content # noqa: S324
|
||||
assert resource.mime == "application/pdf"
|
||||
assert resource.attributes.fileName == "invoice.pdf"
|
||||
assert resource.data.body == pdf_bytes
|
||||
assert resource.data.bodyHash == hashlib.md5(pdf_bytes).digest() # noqa: S324
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_upload_to_evernote_requires_token(tmp_path):
|
||||
pdf_path = tmp_path / "document.pdf"
|
||||
pdf_path.write_bytes(b"%PDF-1.4")
|
||||
|
||||
with (
|
||||
patch("app.tasks.upload_to_evernote.log_task_progress"),
|
||||
patch("app.tasks.upload_to_evernote.settings", SimpleNamespace(evernote_auth_token=None)),
|
||||
):
|
||||
result = upload_to_evernote.apply(args=[str(pdf_path)])
|
||||
|
||||
assert result.failed()
|
||||
assert isinstance(result.result, ValueError)
|
||||
assert "EVERNOTE_AUTH_TOKEN" in str(result.result)
|
||||
@@ -465,6 +465,19 @@ class TestURLUploadEndpoint:
|
||||
data = response.json()
|
||||
assert "Failed to download file" in data["detail"]
|
||||
|
||||
@patch("app.api.url_upload.httpx.AsyncClient.stream")
|
||||
def test_process_url_unsafe_redirect_returns_400(self, mock_stream, client):
|
||||
"""Test unsafe redirects are reported as a client error instead of HTTP 500."""
|
||||
from app.api.url_upload import UnsafeRedirectError
|
||||
|
||||
mock_stream.side_effect = UnsafeRedirectError("Redirect to unsafe URL blocked: Unsafe URL")
|
||||
|
||||
response = client.post("/api/process-url", json={"url": "https://example.com/file.pdf"})
|
||||
|
||||
assert response.status_code == 400
|
||||
data = response.json()
|
||||
assert "Redirect to unsafe URL blocked" in data["detail"]
|
||||
|
||||
@patch("app.api.url_upload.httpx.AsyncClient.stream")
|
||||
def test_process_url_oserror_during_save(self, mock_stream, client, tmp_path, monkeypatch):
|
||||
"""Test handling of OSError when saving file"""
|
||||
|
||||
@@ -235,7 +235,10 @@ class TestSendWebhookNotification:
|
||||
mock_response.status_code = 200
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch("app.utils.user_notification.httpx.post", return_value=mock_response) as mock_post:
|
||||
with (
|
||||
patch("app.utils.user_notification.is_private_ip", return_value=False),
|
||||
patch("app.utils.user_notification.httpx.post", return_value=mock_response) as mock_post,
|
||||
):
|
||||
result = _send_webhook_notification(
|
||||
{"url": "https://hook.example.com/test", "secret": "mysecret"},
|
||||
"document.processed",
|
||||
@@ -256,7 +259,10 @@ class TestSendWebhookNotification:
|
||||
mock_response.status_code = 200
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch("app.utils.user_notification.httpx.post", return_value=mock_response) as mock_post:
|
||||
with (
|
||||
patch("app.utils.user_notification.is_private_ip", return_value=False),
|
||||
patch("app.utils.user_notification.httpx.post", return_value=mock_response) as mock_post,
|
||||
):
|
||||
result = _send_webhook_notification(
|
||||
{"url": "https://hook.example.com/test"},
|
||||
"document.failed",
|
||||
@@ -272,9 +278,12 @@ class TestSendWebhookNotification:
|
||||
"""_send_webhook_notification returns False when httpx raises."""
|
||||
from app.utils.user_notification import _send_webhook_notification
|
||||
|
||||
with patch(
|
||||
"app.utils.user_notification.httpx.post",
|
||||
side_effect=Exception("connection error"),
|
||||
with (
|
||||
patch("app.utils.user_notification.is_private_ip", return_value=False),
|
||||
patch(
|
||||
"app.utils.user_notification.httpx.post",
|
||||
side_effect=Exception("connection error"),
|
||||
),
|
||||
):
|
||||
result = _send_webhook_notification(
|
||||
{"url": "https://hook.example.com/test"},
|
||||
@@ -300,7 +309,10 @@ class TestSendWebhookNotification:
|
||||
)
|
||||
)
|
||||
|
||||
with patch("app.utils.user_notification.httpx.post", return_value=mock_response):
|
||||
with (
|
||||
patch("app.utils.user_notification.is_private_ip", return_value=False),
|
||||
patch("app.utils.user_notification.httpx.post", return_value=mock_response),
|
||||
):
|
||||
result = _send_webhook_notification(
|
||||
{"url": "https://hook.example.com/test"},
|
||||
"document.processed",
|
||||
@@ -310,6 +322,69 @@ class TestSendWebhookNotification:
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_blocks_private_webhook_target(self):
|
||||
"""Webhook delivery is skipped for private network targets."""
|
||||
from app.utils.user_notification import _send_webhook_notification
|
||||
|
||||
with (
|
||||
patch("app.utils.user_notification.is_private_ip", return_value=True),
|
||||
patch("app.utils.user_notification.httpx.post") as mock_post,
|
||||
):
|
||||
result = _send_webhook_notification(
|
||||
{"url": "https://10.0.0.5/test"},
|
||||
"document.processed",
|
||||
"T",
|
||||
"M",
|
||||
)
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_blocks_metadata_webhook_target(self):
|
||||
"""Webhook delivery is skipped for cloud metadata endpoints."""
|
||||
from app.utils.user_notification import _send_webhook_notification
|
||||
|
||||
with patch("app.utils.user_notification.httpx.post") as mock_post:
|
||||
result = _send_webhook_notification(
|
||||
{"url": "http://169.254.169.254/latest/meta-data"},
|
||||
"document.processed",
|
||||
"T",
|
||||
"M",
|
||||
)
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_blocks_invalid_webhook_scheme(self):
|
||||
"""Webhook delivery is skipped for unsupported URL schemes."""
|
||||
from app.utils.user_notification import _send_webhook_notification
|
||||
|
||||
with patch("app.utils.user_notification.httpx.post") as mock_post:
|
||||
result = _send_webhook_notification(
|
||||
{"url": "file:///etc/passwd"},
|
||||
"document.processed",
|
||||
"T",
|
||||
"M",
|
||||
)
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_blocks_webhook_without_hostname(self):
|
||||
"""Webhook delivery is skipped when the URL has no hostname."""
|
||||
from app.utils.user_notification import _send_webhook_notification
|
||||
|
||||
with patch("app.utils.user_notification.httpx.post") as mock_post:
|
||||
result = _send_webhook_notification(
|
||||
{"url": "https:///missing-host"},
|
||||
"document.processed",
|
||||
"T",
|
||||
"M",
|
||||
)
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# dispatch_user_notification – preference loop
|
||||
|
||||
@@ -81,6 +81,7 @@ class TestDeliverWebhook:
|
||||
|
||||
def test_success_returns_true(self, mocker):
|
||||
"""A 200 response returns True."""
|
||||
mocker.patch("app.utils.webhook.is_private_ip", return_value=False)
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
mock_post.return_value = MagicMock(ok=True, status_code=200)
|
||||
|
||||
@@ -90,6 +91,7 @@ class TestDeliverWebhook:
|
||||
|
||||
def test_non_2xx_returns_false(self, mocker):
|
||||
"""A non-2xx response returns False."""
|
||||
mocker.patch("app.utils.webhook.is_private_ip", return_value=False)
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
mock_post.return_value = MagicMock(ok=False, status_code=500)
|
||||
|
||||
@@ -100,6 +102,7 @@ class TestDeliverWebhook:
|
||||
"""A network error returns False."""
|
||||
import requests
|
||||
|
||||
mocker.patch("app.utils.webhook.is_private_ip", return_value=False)
|
||||
mocker.patch("app.utils.webhook.requests.post", side_effect=requests.ConnectionError("fail"))
|
||||
|
||||
result = deliver_webhook("https://example.com/hook", {"event": "test"})
|
||||
@@ -107,6 +110,7 @@ class TestDeliverWebhook:
|
||||
|
||||
def test_signature_header_included_when_secret(self, mocker):
|
||||
"""X-Webhook-Signature header is present when a secret is supplied."""
|
||||
mocker.patch("app.utils.webhook.is_private_ip", return_value=False)
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
mock_post.return_value = MagicMock(ok=True, status_code=200)
|
||||
|
||||
@@ -118,6 +122,7 @@ class TestDeliverWebhook:
|
||||
|
||||
def test_no_signature_header_without_secret(self, mocker):
|
||||
"""X-Webhook-Signature header is absent when no secret is supplied."""
|
||||
mocker.patch("app.utils.webhook.is_private_ip", return_value=False)
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
mock_post.return_value = MagicMock(ok=True, status_code=200)
|
||||
|
||||
@@ -126,6 +131,43 @@ class TestDeliverWebhook:
|
||||
headers = call_kwargs.kwargs.get("headers") or call_kwargs[1].get("headers")
|
||||
assert "X-Webhook-Signature" not in headers
|
||||
|
||||
def test_private_target_is_blocked(self, mocker):
|
||||
"""Private network webhook targets are not called."""
|
||||
mocker.patch("app.utils.webhook.is_private_ip", return_value=True)
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
|
||||
result = deliver_webhook("https://10.0.0.5/hook", {"event": "test"})
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_metadata_target_is_blocked(self, mocker):
|
||||
"""Cloud metadata webhook targets are not called."""
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
|
||||
result = deliver_webhook("http://169.254.169.254/latest/meta-data", {"event": "test"})
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_invalid_scheme_is_blocked(self, mocker):
|
||||
"""Unsupported webhook URL schemes are not called."""
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
|
||||
result = deliver_webhook("file:///etc/passwd", {"event": "test"})
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
def test_missing_hostname_is_blocked(self, mocker):
|
||||
"""Webhook URLs without a hostname are not called."""
|
||||
mock_post = mocker.patch("app.utils.webhook.requests.post")
|
||||
|
||||
result = deliver_webhook("https:///missing-host", {"event": "test"})
|
||||
|
||||
assert result is False
|
||||
mock_post.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests – get_active_webhooks_for_event (DB)
|
||||
|
||||
Reference in New Issue
Block a user