fix: restore all code deleted/truncated by d2217531 Jules SSRF commit
Commitd2217531(google-labs-jules SSRF fix) catastrophically deleted 11,500+ lines across 100+ files while fixing an unrelated IMAP issue. Restored from d2217531^ (pre-bad-commit state): Deleted files (fully restored): - app/api/{automation,classification_rules,comments,sharing}.py - app/middleware/upload_rate_limit.py - app/tasks/{automation_tasks,classify_document}.py - app/utils/{automation_hooks,classification_rules}.py - docs/AppleAppStoreCompliance.md - frontend/input.css, package.json, package-lock.json, tailwind.config.js - frontend/static/js/{annotations,claim,comments,sharing}.js - frontend/templates/{admin_connections,file_annotations,file_summary}.html - tests/{test_api_files_comprehensive,test_auth_extended,test_sharing, test_comments,test_connections,test_imap_profiles,test_api_sessions, test_automation,test_classification_rules,test_api_advanced_filters, test_api_classification_rules,test_upload_rate_limit,test_api_dropbox, test_classify_document,test_comments_ui,test_upload_to_icloud, test_api_onedrive_comprehensive,test_frontend_build,test_sentry, test_diagnostic,test_database,test_views_dropbox,test_local_auth}.py Truncated files (content restored): - app/{auth,config,main,models,celery_worker,database}.py - app/api/{__init__,api_tokens,diagnostic,dropbox,files,google_drive, integrations,local_auth,mobile,onedrive,pipelines,qr_auth, settings,url_upload}.py - app/middleware/upload_rate_limit.py - app/tasks/upload_to_nextcloud.py - app/utils/{allowed_types,settings_service,settings_sync,user_scope,webhook}.py - app/views/{base,dropbox,files,google_drive,onedrive,settings}.py - docs/{API,AuthenticationSetup,ConfigurationGuide,DatabaseConfiguration, DeploymentGuide,DropboxSetup,GoogleDriveSetup,KubernetesDeployment, MobileApp,OneDriveSetup,ProductionReadiness,SentrySetup, SocialLoginSetup,UserGuide}.md - frontend/static/{js/upload.js,styles.css} - frontend/templates/{api_tokens,base,devices,dropbox,dropbox_callback, file_view,files,google_drive,onedrive,onedrive_callback, signup}.html - frontend/translations/en.json - migrations/env.py - tests/{conftest,test_api_integrations,test_api_mobile,test_api_settings, test_api_tokens,test_audit_logs,test_duplicates,test_imap_tasks, test_setup_wizard,test_views_files_comprehensive}.py Security fixes kept from post-d2217531 commits: - app/utils/network.py: DNS SSRF fail-secure fix (06b0fced) - app/utils/file_operations.py: path traversal fix (1018ea17) - tests/test_imap_tasks.py: re-applied 4 is_private_ip mock patches Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/51133dd8-9bec-41ab-aa10-3de753634187
This commit is contained in:
copilot-swe-agent[bot]
+157
-141
@@ -1,141 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import requests
|
||||
from requests.auth import HTTPBasicAuth
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import UploadTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.filename_utils import extract_remote_path, get_unique_filename
|
||||
from app.utils.network import join_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=UploadTaskWithRetry, bind=True)
|
||||
def upload_to_nextcloud(self, file_path: str, file_id: int = None, folder_override: str = None):
|
||||
"""
|
||||
Upload a file to Nextcloud WebDAV.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting Nextcloud upload: {file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_to_nextcloud",
|
||||
"in_progress",
|
||||
f"Uploading to Nextcloud: {os.path.basename(file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
|
||||
# This is what's shown in your env view
|
||||
if not (
|
||||
getattr(settings, "nextcloud_upload_url", None)
|
||||
and getattr(settings, "nextcloud_username", None)
|
||||
and getattr(settings, "nextcloud_password", None)
|
||||
):
|
||||
logger.info(f"[{task_id}] Nextcloud upload skipped: Missing configuration")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "success", "Skipped: Not configured", file_id=file_id)
|
||||
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
try:
|
||||
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
|
||||
webdav_url = settings.nextcloud_upload_url
|
||||
if not webdav_url.endswith("/"):
|
||||
webdav_url += "/"
|
||||
|
||||
# Calculate remote path based on local file structure
|
||||
remote_base = (
|
||||
folder_override if folder_override is not None else (getattr(settings, "nextcloud_folder", "") or "")
|
||||
)
|
||||
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
|
||||
full_url = join_url(webdav_url, remote_path)
|
||||
|
||||
# Function to check if file exists in Nextcloud
|
||||
def check_exists_in_nextcloud(path):
|
||||
check_url = join_url(webdav_url, os.path.dirname(path))
|
||||
try:
|
||||
response = requests.request(
|
||||
"PROPFIND",
|
||||
check_url,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
headers={"Depth": "1"},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
return path in response.text
|
||||
except Exception:
|
||||
# If we can't check, assume it doesn't exist
|
||||
return False
|
||||
|
||||
# Check for potential file collision and get a unique name if needed
|
||||
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
|
||||
full_url = join_url(webdav_url, remote_path)
|
||||
|
||||
# Create necessary parent folders
|
||||
parent_dirs = os.path.dirname(remote_path)
|
||||
if parent_dirs:
|
||||
current_path = ""
|
||||
for folder in parent_dirs.split("/"):
|
||||
if not folder:
|
||||
continue
|
||||
current_path += f"{folder}/"
|
||||
mkdir_url = join_url(webdav_url, current_path)
|
||||
|
||||
requests.request(
|
||||
"MKCOL",
|
||||
mkdir_url,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
# Upload the file
|
||||
logger.info(f"[{task_id}] Uploading {filename} to Nextcloud at {full_url}")
|
||||
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {remote_path}", file_id=file_id)
|
||||
with open(file_path, "rb") as file_data:
|
||||
response = requests.put(
|
||||
full_url,
|
||||
data=file_data,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
headers={"Content-Type": "application/octet-stream"},
|
||||
timeout=settings.http_request_timeout, # Use configured timeout for large files
|
||||
)
|
||||
|
||||
if response.status_code in (201, 204): # Created or No Content
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to Nextcloud at {remote_path}")
|
||||
log_task_progress(
|
||||
task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id
|
||||
)
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"nextcloud_path": remote_path,
|
||||
"response_code": response.status_code,
|
||||
}
|
||||
else:
|
||||
error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to upload {filename} to Nextcloud: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import requests
|
||||
from requests.auth import HTTPBasicAuth
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import UploadTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.filename_utils import extract_remote_path, get_unique_filename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=UploadTaskWithRetry, bind=True)
|
||||
def upload_to_nextcloud(self, file_path: str, file_id: int = None, folder_override: str = None):
|
||||
"""
|
||||
Upload a file to Nextcloud WebDAV.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting Nextcloud upload: {file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_to_nextcloud",
|
||||
"in_progress",
|
||||
f"Uploading to Nextcloud: {os.path.basename(file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
|
||||
# This is what's shown in your env view
|
||||
if not (
|
||||
getattr(settings, "nextcloud_upload_url", None)
|
||||
and getattr(settings, "nextcloud_username", None)
|
||||
and getattr(settings, "nextcloud_password", None)
|
||||
):
|
||||
logger.info(f"[{task_id}] Nextcloud upload skipped: Missing configuration")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "success", "Skipped: Not configured", file_id=file_id)
|
||||
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
try:
|
||||
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
|
||||
webdav_url = settings.nextcloud_upload_url
|
||||
if not webdav_url.endswith("/"):
|
||||
webdav_url += "/"
|
||||
|
||||
# Calculate remote path based on local file structure
|
||||
remote_base = (
|
||||
folder_override if folder_override is not None else (getattr(settings, "nextcloud_folder", "") or "")
|
||||
)
|
||||
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
|
||||
full_url = f"{webdav_url}/{remote_path}"
|
||||
|
||||
# Remove any double slashes (except in http://)
|
||||
full_url = full_url.replace("://", "$PLACEHOLDER$")
|
||||
while "//" in full_url:
|
||||
full_url = full_url.replace("//", "/")
|
||||
full_url = full_url.replace("$PLACEHOLDER$", "://")
|
||||
|
||||
# Function to check if file exists in Nextcloud
|
||||
def check_exists_in_nextcloud(path):
|
||||
check_url = f"{webdav_url}{os.path.dirname(path)}"
|
||||
try:
|
||||
response = requests.request(
|
||||
"PROPFIND",
|
||||
check_url,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
headers={"Depth": "1"},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
return path in response.text
|
||||
except Exception:
|
||||
# If we can't check, assume it doesn't exist
|
||||
return False
|
||||
|
||||
# Check for potential file collision and get a unique name if needed
|
||||
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
|
||||
full_url = f"{webdav_url}/{remote_path}"
|
||||
|
||||
# Fix double slashes again
|
||||
full_url = full_url.replace("://", "$PLACEHOLDER$")
|
||||
while "//" in full_url:
|
||||
full_url = full_url.replace("//", "/")
|
||||
full_url = full_url.replace("$PLACEHOLDER$", "://")
|
||||
|
||||
# Create necessary parent folders
|
||||
parent_dirs = os.path.dirname(remote_path)
|
||||
if parent_dirs:
|
||||
current_path = ""
|
||||
for folder in parent_dirs.split("/"):
|
||||
if not folder:
|
||||
continue
|
||||
current_path += f"{folder}/"
|
||||
mkdir_url = f"{webdav_url}/{current_path}"
|
||||
# Fix double slashes
|
||||
mkdir_url = mkdir_url.replace("://", "$PLACEHOLDER$")
|
||||
while "//" in mkdir_url:
|
||||
mkdir_url = mkdir_url.replace("//", "/")
|
||||
mkdir_url = mkdir_url.replace("$PLACEHOLDER$", "://")
|
||||
|
||||
requests.request(
|
||||
"MKCOL",
|
||||
mkdir_url,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
# Upload the file
|
||||
logger.info(f"[{task_id}] Uploading {filename} to Nextcloud at {full_url}")
|
||||
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {remote_path}", file_id=file_id)
|
||||
with open(file_path, "rb") as file_data:
|
||||
response = requests.put(
|
||||
full_url,
|
||||
data=file_data,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
headers={"Content-Type": "application/octet-stream"},
|
||||
timeout=settings.http_request_timeout, # Use configured timeout for large files
|
||||
)
|
||||
|
||||
if response.status_code in (201, 204): # Created or No Content
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to Nextcloud at {remote_path}")
|
||||
log_task_progress(
|
||||
task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id
|
||||
)
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"nextcloud_path": remote_path,
|
||||
"response_code": response.status_code,
|
||||
}
|
||||
else:
|
||||
error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to upload {filename} to Nextcloud: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
|
||||
Reference in New Issue
Block a user