gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore all code deleted/truncated by d2217531 Jules SSRF commit

Browse Source 
Commit d2217531 (google-labs-jules SSRF fix) catastrophically deleted
11,500+ lines across 100+ files while fixing an unrelated IMAP issue.

Restored from d2217531^ (pre-bad-commit state):

Deleted files (fully restored):
- app/api/{automation,classification_rules,comments,sharing}.py
- app/middleware/upload_rate_limit.py
- app/tasks/{automation_tasks,classify_document}.py
- app/utils/{automation_hooks,classification_rules}.py
- docs/AppleAppStoreCompliance.md
- frontend/input.css, package.json, package-lock.json, tailwind.config.js
- frontend/static/js/{annotations,claim,comments,sharing}.js
- frontend/templates/{admin_connections,file_annotations,file_summary}.html
- tests/{test_api_files_comprehensive,test_auth_extended,test_sharing,
         test_comments,test_connections,test_imap_profiles,test_api_sessions,
         test_automation,test_classification_rules,test_api_advanced_filters,
         test_api_classification_rules,test_upload_rate_limit,test_api_dropbox,
         test_classify_document,test_comments_ui,test_upload_to_icloud,
         test_api_onedrive_comprehensive,test_frontend_build,test_sentry,
         test_diagnostic,test_database,test_views_dropbox,test_local_auth}.py

Truncated files (content restored):
- app/{auth,config,main,models,celery_worker,database}.py
- app/api/{__init__,api_tokens,diagnostic,dropbox,files,google_drive,
           integrations,local_auth,mobile,onedrive,pipelines,qr_auth,
           settings,url_upload}.py
- app/middleware/upload_rate_limit.py
- app/tasks/upload_to_nextcloud.py
- app/utils/{allowed_types,settings_service,settings_sync,user_scope,webhook}.py
- app/views/{base,dropbox,files,google_drive,onedrive,settings}.py
- docs/{API,AuthenticationSetup,ConfigurationGuide,DatabaseConfiguration,
        DeploymentGuide,DropboxSetup,GoogleDriveSetup,KubernetesDeployment,
        MobileApp,OneDriveSetup,ProductionReadiness,SentrySetup,
        SocialLoginSetup,UserGuide}.md
- frontend/static/{js/upload.js,styles.css}
- frontend/templates/{api_tokens,base,devices,dropbox,dropbox_callback,
                      file_view,files,google_drive,onedrive,onedrive_callback,
                      signup}.html
- frontend/translations/en.json
- migrations/env.py
- tests/{conftest,test_api_integrations,test_api_mobile,test_api_settings,
         test_api_tokens,test_audit_logs,test_duplicates,test_imap_tasks,
         test_setup_wizard,test_views_files_comprehensive}.py

Security fixes kept from post-d2217531 commits:
- app/utils/network.py: DNS SSRF fail-secure fix (06b0fced)
- app/utils/file_operations.py: path traversal fix (1018ea17)
- tests/test_imap_tasks.py: re-applied 4 is_private_ip mock patches

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/51133dd8-9bec-41ab-aa10-3de753634187
This commit is contained in:
copilot-swe-agent[bot]
2026-03-23 23:52:39 +00:00
parent 11a49eb7fd
commit c7d3ec57c3
115 changed files with 23532 additions and 1043 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_duplicates.py
+62 -26
View File
@@ -3,11 +3,12 @@
Covers:
- ``GET /api/duplicates`` — list all exact-duplicate groups
- ``GET /api/files/{id}/duplicates`` — per-file exact + near-duplicate info
- ``POST /api/ui-upload`` — exact-duplicate warning in upload response
- ``POST /api/ui-upload`` — exact-duplicate rejection at upload time
- ``GET /duplicates`` — duplicate management UI page
"""
import json
import os
from unittest.mock import patch
import pytest
@@ -283,17 +284,25 @@ class TestGetFileDuplicates:
# ---------------------------------------------------------------------------
# POST /api/ui-upload — exact-duplicate warning
# POST /api/ui-upload — exact-duplicate rejection
# ---------------------------------------------------------------------------
class TestUploadDuplicateWarning:
"""Tests for duplicate warning injected into the upload response."""
class TestUploadDuplicateRejection:
"""Tests for duplicate rejection at upload time.
When ``ENABLE_DEDUPLICATION`` is ``True`` (the default) and the uploaded
file's SHA-256 hash matches an already-processed document, the upload
endpoint must:
- return ``status: "duplicate"`` instead of ``"queued"``
- **not** enqueue a Celery task
- clean up the temporary file from disk
"""
@pytest.mark.integration
@patch("app.tasks.process_document.process_document.delay")
def test_no_warning_for_unique_file(self, mock_delay, client: TestClient, tmp_path):
"""Uploading a unique file should not produce a duplicate_warning."""
"""Uploading a unique file should not produce a duplicate response."""
mock_delay.return_value.id = "task-unique"
pdf = tmp_path / "unique.pdf"
pdf.write_bytes(b"%PDF-1.4\n%%EOF")
@@ -306,14 +315,12 @@ class TestUploadDuplicateWarning:
assert response.status_code == 200
data = response.json()
assert "duplicate_warning" not in data or data.get("duplicate_warning") is None
assert data["status"] == "queued"
assert "duplicate_of" not in data
@pytest.mark.integration
@patch("app.tasks.process_document.process_document.delay")
def test_warning_for_exact_duplicate(self, mock_delay, client: TestClient, db_session, tmp_path):
"""Uploading a file with the same hash as an existing record returns a warning."""
mock_delay.return_value.id = "task-dup"
def test_exact_duplicate_rejected(self, client: TestClient, db_session, tmp_path):
"""Uploading a file with the same hash as an existing record is rejected."""
# Create a real PDF with known content
pdf_bytes = b"%PDF-1.4\nsome unique content for test\n%%EOF"
pdf = tmp_path / "existing.pdf"
@@ -335,16 +342,14 @@ class TestUploadDuplicateWarning:
assert response.status_code == 200
data = response.json()
assert "duplicate_warning" in data
assert data["duplicate_warning"]["duplicate_type"] == "exact"
assert data["duplicate_warning"]["original_file_id"] == existing.id
assert data["status"] == "duplicate"
assert "duplicate_of" in data
assert data["duplicate_of"]["duplicate_type"] == "exact"
assert data["duplicate_of"]["original_file_id"] == existing.id
@pytest.mark.integration
@patch("app.tasks.process_document.process_document.delay")
def test_upload_still_queued_despite_warning(self, mock_delay, client: TestClient, db_session, tmp_path):
"""Even when a duplicate is detected, the file should still be queued."""
mock_delay.return_value.id = "task-still-queued"
def test_duplicate_not_enqueued(self, client: TestClient, db_session, tmp_path):
"""When a duplicate is detected, no Celery task should be created."""
pdf_bytes = b"%PDF-1.4\nqueue test content\n%%EOF"
pdf = tmp_path / "queue_test.pdf"
pdf.write_bytes(pdf_bytes)
@@ -354,16 +359,47 @@ class TestUploadDuplicateWarning:
filehash = hash_file(str(pdf))
_make_file(db_session, filehash=filehash, filename="queue_orig.pdf")
with open(pdf, "rb") as f:
response = client.post(
"/api/ui-upload",
files={"file": ("queue_test.pdf", f, "application/pdf")},
)
with patch("app.tasks.process_document.process_document.delay") as mock_delay:
with open(pdf, "rb") as f:
response = client.post(
"/api/ui-upload",
files={"file": ("queue_test.pdf", f, "application/pdf")},
)
assert response.status_code == 200
data = response.json()
assert "task_id" in data
assert data["status"] == "queued"
assert data["status"] == "duplicate"
assert "task_id" not in data
mock_delay.assert_not_called()
@pytest.mark.integration
def test_duplicate_temp_file_cleaned_up(self, client: TestClient, db_session, tmp_path):
"""The temporary file saved to disk should be removed for a duplicate."""
pdf_bytes = b"%PDF-1.4\ncleanup test content\n%%EOF"
pdf = tmp_path / "cleanup_test.pdf"
pdf.write_bytes(pdf_bytes)
from app.utils.file_operations import hash_file
filehash = hash_file(str(pdf))
_make_file(db_session, filehash=filehash, filename="cleanup_orig.pdf")
with patch("app.tasks.process_document.process_document.delay"):
with open(pdf, "rb") as f:
response = client.post(
"/api/ui-upload",
files={"file": ("cleanup_test.pdf", f, "application/pdf")},
)
assert response.status_code == 200
data = response.json()
# The stored_filename is returned so we can verify cleanup
stored = data.get("stored_filename")
assert stored is not None
from app.config import settings
assert not os.path.exists(os.path.join(settings.workdir, stored))
# ---------------------------------------------------------------------------