Code review feedback: improve readability of S3 upload call
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -10,8 +10,6 @@ from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
from app.utils import log_task_progress
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -229,7 +227,13 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting OneDrive upload: {file_path}")
|
||||
log_task_progress(task_id, "upload_to_onedrive", "in_progress", f"Uploading to OneDrive: {os.path.basename(file_path)}", file_id=file_id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_to_onedrive",
|
||||
"in_progress",
|
||||
f"Uploading to OneDrive: {os.path.basename(file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
@@ -261,7 +265,9 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
|
||||
web_url = result.get("webUrl", "Not available")
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
|
||||
logger.info(f"[{task_id}] File accessible at: {web_url}")
|
||||
log_task_progress(task_id, "upload_to_onedrive", "success", f"Uploaded to OneDrive: {filename}", file_id=file_id)
|
||||
log_task_progress(
|
||||
task_id, "upload_to_onedrive", "success", f"Uploaded to OneDrive: {filename}", file_id=file_id
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "Completed",
|
||||
|
||||
+26
-25
@@ -8,24 +8,25 @@ from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
from app.utils import log_task_progress
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def upload_to_s3(self, file_path: str, file_id: int = None):
|
||||
"""
|
||||
Uploads a file to Amazon S3 in the configured bucket and folder.
|
||||
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting S3 upload: {file_path}")
|
||||
log_task_progress(task_id, "upload_to_s3", "in_progress", f"Uploading to S3: {os.path.basename(file_path)}", file_id=file_id)
|
||||
|
||||
log_task_progress(
|
||||
task_id, "upload_to_s3", "in_progress", f"Uploading to S3: {os.path.basename(file_path)}", file_id=file_id
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
@@ -51,59 +52,59 @@ def upload_to_s3(self, file_path: str, file_id: int = None):
|
||||
try:
|
||||
# Create S3 client
|
||||
s3_client = boto3.client(
|
||||
's3',
|
||||
"s3",
|
||||
region_name=settings.aws_region,
|
||||
aws_access_key_id=settings.aws_access_key_id,
|
||||
aws_secret_access_key=settings.aws_secret_access_key
|
||||
aws_secret_access_key=settings.aws_secret_access_key,
|
||||
)
|
||||
|
||||
|
||||
# Construct the S3 key (path within the bucket)
|
||||
if settings.s3_folder_prefix:
|
||||
# Ensure folder prefix ends with a slash
|
||||
folder_prefix = settings.s3_folder_prefix
|
||||
if not folder_prefix.endswith('/'):
|
||||
folder_prefix += '/'
|
||||
if not folder_prefix.endswith("/"):
|
||||
folder_prefix += "/"
|
||||
s3_key = f"{folder_prefix}{filename}"
|
||||
else:
|
||||
s3_key = filename
|
||||
|
||||
|
||||
# Prepare extra arguments
|
||||
extra_args = {
|
||||
'StorageClass': settings.s3_storage_class
|
||||
}
|
||||
|
||||
extra_args = {"StorageClass": settings.s3_storage_class}
|
||||
|
||||
# Add ACL if configured
|
||||
if settings.s3_acl:
|
||||
extra_args['ACL'] = settings.s3_acl
|
||||
|
||||
extra_args["ACL"] = settings.s3_acl
|
||||
|
||||
# Upload file
|
||||
s3_client.upload_file(
|
||||
file_path,
|
||||
settings.s3_bucket_name,
|
||||
file_path,
|
||||
settings.s3_bucket_name,
|
||||
s3_key,
|
||||
ExtraArgs=extra_args
|
||||
)
|
||||
|
||||
|
||||
# Generate URL to the file (useful for public files)
|
||||
# For private files, this is just a reference and won't be accessible directly
|
||||
s3_url = f"https://{settings.s3_bucket_name}.s3.{settings.aws_region}.amazonaws.com/{s3_key}"
|
||||
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to S3 bucket {settings.s3_bucket_name} at path {s3_key}")
|
||||
|
||||
logger.info(
|
||||
f"[{task_id}] Successfully uploaded {filename} to S3 bucket {settings.s3_bucket_name} at path {s3_key}"
|
||||
)
|
||||
log_task_progress(task_id, "upload_to_s3", "success", f"Uploaded to S3: {filename}", file_id=file_id)
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"s3_bucket": settings.s3_bucket_name,
|
||||
"s3_key": s3_key,
|
||||
"s3_url": s3_url
|
||||
"s3_url": s3_url,
|
||||
}
|
||||
|
||||
|
||||
except ClientError as e:
|
||||
error_msg = f"Failed to upload {filename} to S3: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_s3", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error uploading {filename} to S3: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
|
||||
Reference in New Issue
Block a user