fix: restore all code deleted/truncated by d2217531 Jules SSRF commit

Commit d2217531 (google-labs-jules SSRF fix) catastrophically deleted
11,500+ lines across 100+ files while fixing an unrelated IMAP issue.

Restored from d2217531^ (pre-bad-commit state):

Deleted files (fully restored):
- app/api/{automation,classification_rules,comments,sharing}.py
- app/middleware/upload_rate_limit.py
- app/tasks/{automation_tasks,classify_document}.py
- app/utils/{automation_hooks,classification_rules}.py
- docs/AppleAppStoreCompliance.md
- frontend/input.css, package.json, package-lock.json, tailwind.config.js
- frontend/static/js/{annotations,claim,comments,sharing}.js
- frontend/templates/{admin_connections,file_annotations,file_summary}.html
- tests/{test_api_files_comprehensive,test_auth_extended,test_sharing,
         test_comments,test_connections,test_imap_profiles,test_api_sessions,
         test_automation,test_classification_rules,test_api_advanced_filters,
         test_api_classification_rules,test_upload_rate_limit,test_api_dropbox,
         test_classify_document,test_comments_ui,test_upload_to_icloud,
         test_api_onedrive_comprehensive,test_frontend_build,test_sentry,
         test_diagnostic,test_database,test_views_dropbox,test_local_auth}.py

Truncated files (content restored):
- app/{auth,config,main,models,celery_worker,database}.py
- app/api/{__init__,api_tokens,diagnostic,dropbox,files,google_drive,
           integrations,local_auth,mobile,onedrive,pipelines,qr_auth,
           settings,url_upload}.py
- app/middleware/upload_rate_limit.py
- app/tasks/upload_to_nextcloud.py
- app/utils/{allowed_types,settings_service,settings_sync,user_scope,webhook}.py
- app/views/{base,dropbox,files,google_drive,onedrive,settings}.py
- docs/{API,AuthenticationSetup,ConfigurationGuide,DatabaseConfiguration,
        DeploymentGuide,DropboxSetup,GoogleDriveSetup,KubernetesDeployment,
        MobileApp,OneDriveSetup,ProductionReadiness,SentrySetup,
        SocialLoginSetup,UserGuide}.md
- frontend/static/{js/upload.js,styles.css}
- frontend/templates/{api_tokens,base,devices,dropbox,dropbox_callback,
                      file_view,files,google_drive,onedrive,onedrive_callback,
                      signup}.html
- frontend/translations/en.json
- migrations/env.py
- tests/{conftest,test_api_integrations,test_api_mobile,test_api_settings,
         test_api_tokens,test_audit_logs,test_duplicates,test_imap_tasks,
         test_setup_wizard,test_views_files_comprehensive}.py

Security fixes kept from post-d2217531 commits:
- app/utils/network.py: DNS SSRF fail-secure fix (06b0fced)
- app/utils/file_operations.py: path traversal fix (1018ea17)
- tests/test_imap_tasks.py: re-applied 4 is_private_ip mock patches

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/51133dd8-9bec-41ab-aa10-3de753634187
This commit is contained in:
copilot-swe-agent[bot]
2026-03-23 23:52:39 +00:00
parent 11a49eb7fd
commit c7d3ec57c3
115 changed files with 23532 additions and 1043 deletions
+44
View File
@@ -0,0 +1,44 @@
"""Celery task for asynchronous automation hook delivery with retry and backoff.
Uses :class:`~app.tasks.retry_config.BaseTaskWithRetry` so failed deliveries
are automatically retried with exponential backoff (default: 60 s, 300 s,
900 s) and ±20 % jitter.
"""
import logging
from typing import Any
from app.celery_app import celery
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.webhook import deliver_webhook
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True, name="automation.deliver_hook")
def deliver_automation_hook_task(self, url: str, payload: dict[str, Any], secret: str | None = None) -> dict[str, Any]:
"""Deliver an automation hook payload to *url* with automatic retries.
Args:
url: Target webhook URL (provided by Zapier / Make.com).
payload: The flat Zapier-compatible payload.
secret: Optional shared secret for HMAC-SHA256 signing.
Returns:
A dict with ``status`` and ``url`` on success.
Raises:
RuntimeError: Re-raised to trigger Celery retry on delivery failure.
"""
logger.info(
"Delivering automation hook to %s (attempt %d/%d)",
url,
self.request.retries + 1,
self.max_retries + 1,
)
success = deliver_webhook(url, payload, secret)
if success:
return {"status": "delivered", "url": url}
raise RuntimeError(f"Automation hook delivery to {url} failed")
+174
View File
@@ -0,0 +1,174 @@
"""Celery task for rule-based document classification.
This task is executed as a pipeline step (``step_type="classify"``). It
applies built-in and user-defined classification rules against the document's
filename, OCR text, and existing AI metadata to assign a ``document_type``
category.
The result is stored in the ``ai_metadata`` JSON blob on the
:class:`~app.models.FileRecord` (field ``classification``).
"""
from __future__ import annotations
import json
import logging
from typing import Any
from app.celery_app import celery
from app.database import SessionLocal
from app.models import ClassificationRuleModel, FileRecord
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
from app.utils.classification_rules import (
ClassificationResult,
classify_document,
db_rule_to_engine_rule,
)
logger = logging.getLogger(__name__)
STEP_NAME = "classify_document"
def _load_custom_rules(owner_id: str | None) -> list[Any]:
"""Load enabled custom classification rules from the database.
Returns engine-level :class:`ClassificationRule` dataclass instances.
Rules are loaded in priority-descending order. System rules
(``owner_id IS NULL``) and the user's own rules are both included.
"""
with SessionLocal() as db:
query = db.query(ClassificationRuleModel).filter(ClassificationRuleModel.enabled.is_(True))
if owner_id:
query = query.filter(
(ClassificationRuleModel.owner_id.is_(None)) | (ClassificationRuleModel.owner_id == owner_id)
)
else:
query = query.filter(ClassificationRuleModel.owner_id.is_(None))
rules = query.order_by(ClassificationRuleModel.priority.desc()).all()
return [db_rule_to_engine_rule(r) for r in rules]
@celery.task(base=BaseTaskWithRetry, bind=True)
def classify_document_task(
self: Any,
file_id: int,
owner_id: str | None = None,
) -> dict[str, Any]:
"""Classify a document using rule-based matching.
This task:
1. Loads the :class:`FileRecord` from the database.
2. Gathers filename, OCR text, and existing AI metadata.
3. Loads built-in + user-defined classification rules.
4. Runs the classification engine.
5. Persists the result into ``ai_metadata.classification``.
Args:
file_id: Primary key of the :class:`FileRecord` to classify.
owner_id: Owner identifier for loading user-specific rules.
Returns:
Dict with ``category``, ``confidence``, and ``matched_rules``.
"""
task_id = self.request.id
log_task_progress(
task_id,
STEP_NAME,
"in_progress",
f"Starting classification for file {file_id}",
file_id=file_id,
)
try:
with SessionLocal() as db:
file_record: FileRecord | None = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if file_record is None:
log_task_progress(
task_id,
STEP_NAME,
"failure",
f"FileRecord {file_id} not found",
file_id=file_id,
)
return {"status": "error", "detail": "File not found"}
# Gather inputs
filename = file_record.original_filename or ""
text = file_record.ocr_text or ""
existing_metadata: dict[str, Any] = {}
if file_record.ai_metadata:
try:
existing_metadata = json.loads(file_record.ai_metadata)
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to parse ai_metadata for file %s, starting fresh", file_id)
existing_metadata = {}
# Load custom rules
effective_owner = owner_id or file_record.owner_id
custom_rules = _load_custom_rules(effective_owner)
# Run classification engine
result: ClassificationResult = classify_document(
filename=filename,
text=text,
metadata=existing_metadata,
custom_rules=custom_rules,
)
# Persist result into ai_metadata
classification_data = {
"category": result.category,
"confidence": result.confidence,
"matched_rules": [
{
"rule_name": m.rule_name,
"rule_type": m.rule_type,
"category": m.category,
"confidence": m.confidence,
}
for m in result.matched_rules
],
}
existing_metadata["classification"] = classification_data
# If no document_type was set yet, populate it from the classification
if not existing_metadata.get("document_type"):
from app.utils.classification_rules import BUILTIN_CATEGORIES
existing_metadata["document_type"] = BUILTIN_CATEGORIES.get(
result.category, result.category.replace("_", " ").title()
)
file_record.ai_metadata = json.dumps(existing_metadata, ensure_ascii=False)
db.commit()
log_task_progress(
task_id,
STEP_NAME,
"success",
f"Classified as '{result.category}' with confidence {result.confidence}",
file_id=file_id,
detail=f"Matched {len(result.matched_rules)} rule(s)",
)
return {
"status": "success",
"category": result.category,
"confidence": result.confidence,
"matched_rules": len(result.matched_rules),
}
except Exception as e:
logger.exception("Classification failed for file %s: %s", file_id, e)
log_task_progress(
task_id,
STEP_NAME,
"failure",
f"Classification failed: {e}",
file_id=file_id,
)
raise
+157 -141
View File
@@ -1,141 +1,157 @@
#!/usr/bin/env python3
import logging
import os
import requests
from requests.auth import HTTPBasicAuth
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import UploadTaskWithRetry
from app.utils import log_task_progress
from app.utils.filename_utils import extract_remote_path, get_unique_filename
from app.utils.network import join_url
logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_nextcloud(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Upload a file to Nextcloud WebDAV.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Nextcloud upload: {file_path}")
log_task_progress(
task_id,
"upload_to_nextcloud",
"in_progress",
f"Uploading to Nextcloud: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
# This is what's shown in your env view
if not (
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
):
logger.info(f"[{task_id}] Nextcloud upload skipped: Missing configuration")
log_task_progress(task_id, "upload_to_nextcloud", "success", "Skipped: Not configured", file_id=file_id)
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
filename = os.path.basename(file_path)
try:
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
webdav_url = settings.nextcloud_upload_url
if not webdav_url.endswith("/"):
webdav_url += "/"
# Calculate remote path based on local file structure
remote_base = (
folder_override if folder_override is not None else (getattr(settings, "nextcloud_folder", "") or "")
)
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
full_url = join_url(webdav_url, remote_path)
# Function to check if file exists in Nextcloud
def check_exists_in_nextcloud(path):
check_url = join_url(webdav_url, os.path.dirname(path))
try:
response = requests.request(
"PROPFIND",
check_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={"Depth": "1"},
timeout=10,
)
return path in response.text
except Exception:
# If we can't check, assume it doesn't exist
return False
# Check for potential file collision and get a unique name if needed
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
full_url = join_url(webdav_url, remote_path)
# Create necessary parent folders
parent_dirs = os.path.dirname(remote_path)
if parent_dirs:
current_path = ""
for folder in parent_dirs.split("/"):
if not folder:
continue
current_path += f"{folder}/"
mkdir_url = join_url(webdav_url, current_path)
requests.request(
"MKCOL",
mkdir_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
timeout=10,
)
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to Nextcloud at {full_url}")
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {remote_path}", file_id=file_id)
with open(file_path, "rb") as file_data:
response = requests.put(
full_url,
data=file_data,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={"Content-Type": "application/octet-stream"},
timeout=settings.http_request_timeout, # Use configured timeout for large files
)
if response.status_code in (201, 204): # Created or No Content
logger.info(f"[{task_id}] Successfully uploaded {filename} to Nextcloud at {remote_path}")
log_task_progress(
task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"nextcloud_path": remote_path,
"response_code": response.status_code,
}
else:
error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Failed to upload {filename} to Nextcloud: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)
#!/usr/bin/env python3
import logging
import os
import requests
from requests.auth import HTTPBasicAuth
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import UploadTaskWithRetry
from app.utils import log_task_progress
from app.utils.filename_utils import extract_remote_path, get_unique_filename
logger = logging.getLogger(__name__)
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_nextcloud(self, file_path: str, file_id: int = None, folder_override: str = None):
"""
Upload a file to Nextcloud WebDAV.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Nextcloud upload: {file_path}")
log_task_progress(
task_id,
"upload_to_nextcloud",
"in_progress",
f"Uploading to Nextcloud: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
# This is what's shown in your env view
if not (
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
):
logger.info(f"[{task_id}] Nextcloud upload skipped: Missing configuration")
log_task_progress(task_id, "upload_to_nextcloud", "success", "Skipped: Not configured", file_id=file_id)
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
filename = os.path.basename(file_path)
try:
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
webdav_url = settings.nextcloud_upload_url
if not webdav_url.endswith("/"):
webdav_url += "/"
# Calculate remote path based on local file structure
remote_base = (
folder_override if folder_override is not None else (getattr(settings, "nextcloud_folder", "") or "")
)
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
full_url = f"{webdav_url}/{remote_path}"
# Remove any double slashes (except in http://)
full_url = full_url.replace("://", "$PLACEHOLDER$")
while "//" in full_url:
full_url = full_url.replace("//", "/")
full_url = full_url.replace("$PLACEHOLDER$", "://")
# Function to check if file exists in Nextcloud
def check_exists_in_nextcloud(path):
check_url = f"{webdav_url}{os.path.dirname(path)}"
try:
response = requests.request(
"PROPFIND",
check_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={"Depth": "1"},
timeout=10,
)
return path in response.text
except Exception:
# If we can't check, assume it doesn't exist
return False
# Check for potential file collision and get a unique name if needed
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
full_url = f"{webdav_url}/{remote_path}"
# Fix double slashes again
full_url = full_url.replace("://", "$PLACEHOLDER$")
while "//" in full_url:
full_url = full_url.replace("//", "/")
full_url = full_url.replace("$PLACEHOLDER$", "://")
# Create necessary parent folders
parent_dirs = os.path.dirname(remote_path)
if parent_dirs:
current_path = ""
for folder in parent_dirs.split("/"):
if not folder:
continue
current_path += f"{folder}/"
mkdir_url = f"{webdav_url}/{current_path}"
# Fix double slashes
mkdir_url = mkdir_url.replace("://", "$PLACEHOLDER$")
while "//" in mkdir_url:
mkdir_url = mkdir_url.replace("//", "/")
mkdir_url = mkdir_url.replace("$PLACEHOLDER$", "://")
requests.request(
"MKCOL",
mkdir_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
timeout=10,
)
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to Nextcloud at {full_url}")
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {remote_path}", file_id=file_id)
with open(file_path, "rb") as file_data:
response = requests.put(
full_url,
data=file_data,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={"Content-Type": "application/octet-stream"},
timeout=settings.http_request_timeout, # Use configured timeout for large files
)
if response.status_code in (201, 204): # Created or No Content
logger.info(f"[{task_id}] Successfully uploaded {filename} to Nextcloud at {remote_path}")
log_task_progress(
task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"nextcloud_path": remote_path,
"response_code": response.status_code,
}
else:
error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Failed to upload {filename} to Nextcloud: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)