gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore all code deleted/truncated by d2217531 Jules SSRF commit

Browse Source 
Commit d2217531 (google-labs-jules SSRF fix) catastrophically deleted
11,500+ lines across 100+ files while fixing an unrelated IMAP issue.

Restored from d2217531^ (pre-bad-commit state):

Deleted files (fully restored):
- app/api/{automation,classification_rules,comments,sharing}.py
- app/middleware/upload_rate_limit.py
- app/tasks/{automation_tasks,classify_document}.py
- app/utils/{automation_hooks,classification_rules}.py
- docs/AppleAppStoreCompliance.md
- frontend/input.css, package.json, package-lock.json, tailwind.config.js
- frontend/static/js/{annotations,claim,comments,sharing}.js
- frontend/templates/{admin_connections,file_annotations,file_summary}.html
- tests/{test_api_files_comprehensive,test_auth_extended,test_sharing,
         test_comments,test_connections,test_imap_profiles,test_api_sessions,
         test_automation,test_classification_rules,test_api_advanced_filters,
         test_api_classification_rules,test_upload_rate_limit,test_api_dropbox,
         test_classify_document,test_comments_ui,test_upload_to_icloud,
         test_api_onedrive_comprehensive,test_frontend_build,test_sentry,
         test_diagnostic,test_database,test_views_dropbox,test_local_auth}.py

Truncated files (content restored):
- app/{auth,config,main,models,celery_worker,database}.py
- app/api/{__init__,api_tokens,diagnostic,dropbox,files,google_drive,
           integrations,local_auth,mobile,onedrive,pipelines,qr_auth,
           settings,url_upload}.py
- app/middleware/upload_rate_limit.py
- app/tasks/upload_to_nextcloud.py
- app/utils/{allowed_types,settings_service,settings_sync,user_scope,webhook}.py
- app/views/{base,dropbox,files,google_drive,onedrive,settings}.py
- docs/{API,AuthenticationSetup,ConfigurationGuide,DatabaseConfiguration,
        DeploymentGuide,DropboxSetup,GoogleDriveSetup,KubernetesDeployment,
        MobileApp,OneDriveSetup,ProductionReadiness,SentrySetup,
        SocialLoginSetup,UserGuide}.md
- frontend/static/{js/upload.js,styles.css}
- frontend/templates/{api_tokens,base,devices,dropbox,dropbox_callback,
                      file_view,files,google_drive,onedrive,onedrive_callback,
                      signup}.html
- frontend/translations/en.json
- migrations/env.py
- tests/{conftest,test_api_integrations,test_api_mobile,test_api_settings,
         test_api_tokens,test_audit_logs,test_duplicates,test_imap_tasks,
         test_setup_wizard,test_views_files_comprehensive}.py

Security fixes kept from post-d2217531 commits:
- app/utils/network.py: DNS SSRF fail-secure fix (06b0fced)
- app/utils/file_operations.py: path traversal fix (1018ea17)
- tests/test_imap_tasks.py: re-applied 4 is_private_ip mock patches

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/51133dd8-9bec-41ab-aa10-3de753634187
This commit is contained in:
copilot-swe-agent[bot]
2026-03-23 23:52:39 +00:00
parent 11a49eb7fd
commit c7d3ec57c3
115 changed files with 23532 additions and 1043 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+64 -14
View File
@@ -20,6 +20,7 @@ from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.middleware.upload_rate_limit import require_upload_rate_limit
from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.tasks.convert_to_pdf import convert_to_pdf
from app.tasks.process_document import process_document
@@ -29,7 +30,7 @@ from app.utils.file_queries import apply_status_filter
from app.utils.file_status import get_files_processing_status
from app.utils.filename_utils import sanitize_filename
from app.utils.input_validation import validate_search_query, validate_sort_field, validate_sort_order
from app.utils.user_scope import apply_owner_filter, get_current_owner_id
from app.utils.user_scope import apply_owner_filter, get_current_owner_id, get_file_role
# Set up logging
logger = logging.getLogger(__name__)
@@ -299,6 +300,7 @@ def delete_file_record(request: Request, file_id: int, db: DbSession):
"""
Delete a file record from the database.
This only removes the database entry, not the actual file.
Only the file owner (or an admin) may delete a document.
"""
# Check if file deletion is allowed
if not settings.allow_file_delete:
@@ -313,6 +315,18 @@ def delete_file_record(request: Request, file_id: int, db: DbSession):
if not file_record:
raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
# Enforce owner-only deletion in multi-user mode
user = request.session.get("user")
is_admin = isinstance(user, dict) and bool(user.get("is_admin"))
if not is_admin:
owner_id = get_current_owner_id(request)
role = get_file_role(file_record, owner_id, db)
if role != "owner":
raise HTTPException(
status_code=403,
detail="Only the file owner can delete this document",
)
# Log the deletion
logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")
@@ -339,6 +353,7 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
"""
Delete multiple file records from the database.
This only removes the database entries, not the actual files.
Only the file owner (or an admin) may delete each document.
"""
# Check if file deletion is allowed
if not settings.allow_file_delete:
@@ -353,6 +368,18 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
if not file_records:
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
# Enforce owner-only deletion in multi-user mode
user = request.session.get("user")
is_admin = isinstance(user, dict) and bool(user.get("is_admin"))
if not is_admin:
owner_id = get_current_owner_id(request)
non_owner_ids = [f.id for f in file_records if get_file_role(f, owner_id, db) != "owner"]
if non_owner_ids:
raise HTTPException(
status_code=403,
detail=f"You can only delete files you own. Not owner of file IDs: {non_owner_ids}",
)
deleted_count = len(file_records)
deleted_ids = [f.id for f in file_records]
@@ -1267,7 +1294,12 @@ async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size:
def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
"""Check for an exact duplicate of the uploaded file and return a warning if found."""
"""Check for an exact duplicate of the uploaded file.
Returns a dict with duplicate info when the file's SHA-256 hash matches an
already-processed document, or ``None`` when no duplicate is found (or
deduplication is disabled).
"""
if not settings.enable_deduplication:
return None
@@ -1286,8 +1318,8 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
"This file is an exact duplicate of an already-processed document. "
"It has not been queued for processing again."
),
}
except Exception as e:
@@ -1298,7 +1330,12 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s
@router.post("/ui-upload")
@require_login
async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
async def ui_upload(
request: Request,
db: DbSession,
file: UploadFile = File(...),
_rate_ok: None = Depends(require_upload_rate_limit),
):
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
workdir = settings.workdir
@@ -1384,6 +1421,25 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
file_size = written_size
# ── Early duplicate rejection ──────────────────────────────────────────
# Check for exact duplicates (same SHA-256 hash) BEFORE enqueuing a
# processing task. When deduplication is enabled and the file already
# exists, we skip processing entirely, clean up the temp file, and
# return the existing file's information to the caller.
exact_duplicate = _check_for_exact_duplicate(db, target_path, safe_filename)
if exact_duplicate:
# Remove the just-saved temp file — it's a duplicate.
try:
os.remove(target_path)
except OSError:
pass
return {
"status": "duplicate",
"original_filename": safe_filename,
"stored_filename": target_filename,
"duplicate_of": exact_duplicate,
}
# Determine if the file is a PDF or needs conversion
mime_type, _ = mimetypes.guess_type(target_path)
file_ext = os.path.splitext(target_path)[1].lower()
@@ -1447,6 +1503,8 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
".tif",
".webp",
".svg",
".heic",
".heif",
}:
# If it's an image, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id)
@@ -1460,20 +1518,12 @@ async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id)
# Check for exact duplicates (same SHA-256 hash) before returning.
# This gives the caller an immediate warning without waiting for the pipeline.
# Only performed when deduplication is enabled in settings.
exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)
response: dict = {
return {
"task_id": task.id,
"status": "queued",
"original_filename": safe_filename,
"stored_filename": target_filename,
}
if exact_duplicate_warning:
response["duplicate_warning"] = exact_duplicate_warning
return response
# ---------------------------------------------------------------------------