gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: fix all flake8 linter errors across app/ directory

Browse Source 
- Run Black formatter and isort on all app/ files
- Remove unused imports (F401) across multiple files
- Add # noqa: F401 for intentional re-exports in celery_worker.py,
  tasks/__init__.py, utils.py, frontend.py, views/base.py
- Fix f-strings without placeholders (F541) in azure.py, notification.py,
  check_credentials.py, upload_to_onedrive.py, settings.py
- Fix bare except (E722) in upload_to_sftp.py
- Fix block comment format (E265) in models.py
- Move imports to top of file to fix E402 in celery_app.py, celery_worker.py
- Fix line-too-long (E501) by wrapping strings in multiple files
- Remove unused variable (F841) in upload_to_nextcloud.py

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-08 17:42:33 +00:00
parent 7827b97e06
commit d08040ac4a
73 changed files with 2200 additions and 2185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/upload_to_google_drive.py
+85 -77
View File
@@ -2,24 +2,25 @@
app/tasks/upload_to_google_drive.py
"""
import os
import json
import logging
import os
from google.auth.exceptions import RefreshError
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials as OAuthCredentials
from google.oauth2.service_account import Credentials
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from google.oauth2.service_account import Credentials
from google.oauth2.credentials import Credentials as OAuthCredentials
from google_auth_oauthlib.flow import Flow
from google.auth.transport.requests import Request
from google.auth.exceptions import RefreshError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_drive_service_oauth():
"""
Get Google Drive service using OAuth credentials.
@@ -27,12 +28,14 @@ def get_drive_service_oauth():
"""
try:
# Check for required OAuth settings
if not (settings.google_drive_client_id and
settings.google_drive_client_secret and
settings.google_drive_refresh_token):
if not (
settings.google_drive_client_id
and settings.google_drive_client_secret
and settings.google_drive_refresh_token
):
logger.error("Google Drive OAuth credentials not fully configured")
return None
# Create credentials object from refresh token
credentials = OAuthCredentials(
None, # No access token initially, will be refreshed
@@ -41,16 +44,16 @@ def get_drive_service_oauth():
client_id=settings.google_drive_client_id,
client_secret=settings.google_drive_client_secret,
# Use only drive.file scope
scopes=['https://www.googleapis.com/auth/drive.file']
scopes=["https://www.googleapis.com/auth/drive.file"],
)
# Refresh the access token
credentials.refresh(Request())
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
service = build("drive", "v3", credentials=credentials)
return service
except RefreshError as e:
logger.error(f"Failed to refresh Google Drive token: {str(e)}")
raise
@@ -58,6 +61,7 @@ def get_drive_service_oauth():
logger.error(f"Failed to authenticate with Google Drive OAuth: {str(e)}")
return None
def get_google_drive_service():
"""
Authenticate with Google Drive API using service account credentials
@@ -65,52 +69,53 @@ def get_google_drive_service():
"""
try:
# Check if we should use OAuth instead of service account
if getattr(settings, 'google_drive_use_oauth', False):
if getattr(settings, "google_drive_use_oauth", False):
return get_drive_service_oauth()
# Load service account credentials from settings
if not settings.google_drive_credentials_json:
logger.error("Google Drive credentials not configured")
return None
credentials_dict = json.loads(settings.google_drive_credentials_json)
credentials = Credentials.from_service_account_info(
credentials_dict,
scopes=['https://www.googleapis.com/auth/drive']
credentials_dict, scopes=["https://www.googleapis.com/auth/drive"]
)
# Delegate to user if specified
if settings.google_drive_delegate_to:
credentials = credentials.with_subject(settings.google_drive_delegate_to)
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
service = build("drive", "v3", credentials=credentials)
return service
except Exception as e:
logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
return None
def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
1. Check for a .json metadata file with the same name
Returns a dictionary of metadata or empty dict if not found
"""
# Check for separate metadata JSON file
metadata_path = os.path.splitext(file_path)[0] + '.json'
metadata_path = os.path.splitext(file_path)[0] + ".json"
if os.path.exists(metadata_path):
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
except Exception as e:
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
return {}
def truncate_property_value(key, value, max_bytes=100):
"""
Truncate a property value to ensure the key+value stays under the byte limit.
@@ -119,35 +124,36 @@ def truncate_property_value(key, value, max_bytes=100):
"""
# Convert to string if not already
str_value = str(value)
# Calculate current size of key and value in bytes
key_bytes = len(key.encode('utf-8'))
value_bytes = len(str_value.encode('utf-8'))
key_bytes = len(key.encode("utf-8"))
value_bytes = len(str_value.encode("utf-8"))
total_bytes = key_bytes + value_bytes
# If under limit, return original value
if total_bytes <= max_bytes:
return str_value
# Calculate how many bytes we need to trim from value
# Leave a small buffer to be safe
bytes_to_trim = total_bytes - max_bytes + 4
# Iteratively truncate the string until it's under the byte limit
while len(str_value.encode('utf-8')) > value_bytes - bytes_to_trim:
while len(str_value.encode("utf-8")) > value_bytes - bytes_to_trim:
str_value = str_value[:-1]
# Add ellipsis to indicate truncation
if str_value != str(value):
str_value = str_value[:-3] + "..."
return str_value
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id: int = None):
"""
Uploads a file to Google Drive in the configured folder with optional metadata.
Args:
file_path: Path to the file to upload
include_metadata: Whether to include metadata in the upload
@@ -156,7 +162,11 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
task_id = self.request.id
logger.info(f"[{task_id}] Starting Google Drive upload: {file_path}")
log_task_progress(
task_id, "upload_to_google_drive", "in_progress", f"Uploading to Google Drive: {os.path.basename(file_path)}", file_id=file_id
task_id,
"upload_to_google_drive",
"in_progress",
f"Uploading to Google Drive: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
@@ -184,18 +194,18 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
# Prepare the file metadata
file_metadata = {
'name': filename,
"name": filename,
}
# If folder ID is specified, set parent folder
if settings.google_drive_folder_id:
file_metadata['parents'] = [settings.google_drive_folder_id]
file_metadata["parents"] = [settings.google_drive_folder_id]
# Add custom properties if metadata exists
if metadata:
# Google Drive properties must be strings and can't be nested objects
file_metadata['properties'] = {}
file_metadata["properties"] = {}
# Only add a few important top-level metadata fields as properties
# Skip nested objects and long values to avoid the 124-byte limit
safe_properties = {}
@@ -203,61 +213,59 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
# Skip nested structures completely - they'll be in the description
if isinstance(value, (dict, list)):
continue
# Try to add simple values with truncation if needed
try:
truncated_value = truncate_property_value(key, value)
safe_properties[key] = truncated_value
except Exception as e:
logger.warning(f"Skipping metadata property {key}: {str(e)}")
# Only use the safe properties
file_metadata['properties'] = safe_properties
file_metadata["properties"] = safe_properties
# Add minimal appProperties
file_metadata['appProperties'] = {
'docuelevate': 'true'
}
file_metadata["appProperties"] = {"docuelevate": "true"}
# Add metadata to file description for better visibility in Google Drive UI
# Description has much higher size limits than properties
formatted_json = json.dumps(metadata, indent=2)
file_metadata['description'] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
file_metadata["description"] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
logger.debug(f"Adding metadata to Google Drive file: {json.dumps(file_metadata['properties'])}")
# Upload file with metadata
media = MediaFileUpload(
file_path,
mimetype='application/pdf',
resumable=True
media = MediaFileUpload(file_path, mimetype="application/pdf", resumable=True)
file = (
service.files()
.create(
body=file_metadata, media_body=media, fields="id,name,webViewLink,properties,appProperties,description"
)
.execute()
)
file = service.files().create(
body=file_metadata,
media_body=media,
fields='id,name,webViewLink,properties,appProperties,description'
).execute()
# Log success details
google_drive_file_id = file.get('id')
web_view_link = file.get('webViewLink')
google_drive_file_id = file.get("id")
web_view_link = file.get("webViewLink")
logger.info(f"[{task_id}] Successfully uploaded {filename} to Google Drive with ID: {google_drive_file_id}")
logger.info(f"[{task_id}] File accessible at: {web_view_link}")
log_task_progress(task_id, "upload_to_google_drive", "success", f"Uploaded to Google Drive: {filename}", file_id=file_id)
log_task_progress(
task_id, "upload_to_google_drive", "success", f"Uploaded to Google Drive: {filename}", file_id=file_id
)
result = {
"status": "Completed",
"status": "Completed",
"file_path": file_path,
"google_drive_file_id": google_drive_file_id,
"google_drive_web_link": web_view_link
"google_drive_web_link": web_view_link,
}
# Add metadata info to result if included
if metadata:
result["metadata_included"] = True
return result
except Exception as e: