feat(storage): add SharePoint integration for document storage

Add Microsoft SharePoint Online as a storage provider using the
Microsoft Graph API. Includes OAuth2 authentication via MSAL,
site/drive resolution, chunked upload sessions, and metadata sync.

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-10 00:17:42 +00:00
parent 76353349e7
commit b85fc1d277
12 changed files with 1143 additions and 3 deletions
+9
View File
@@ -351,6 +351,15 @@ ONEDRIVE_TENANT_ID=common
ONEDRIVE_REFRESH_TOKEN=your-refresh-token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
# SharePoint
SHAREPOINT_CLIENT_ID=your-client-id
SHAREPOINT_CLIENT_SECRET=your-client-secret
SHAREPOINT_TENANT_ID=common
SHAREPOINT_REFRESH_TOKEN=your-refresh-token
SHAREPOINT_SITE_URL=https://tenant.sharepoint.com/sites/sitename
SHAREPOINT_DOCUMENT_LIBRARY=Documents
SHAREPOINT_FOLDER_PATH=Uploads
# WebDAV
WEBDAV_URL=https://webdav.example.com/path
WEBDAV_USERNAME=webdav_user
+1
View File
@@ -50,6 +50,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401
from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
from app.tasks.upload_to_sharepoint import upload_to_sharepoint # noqa: F401
from app.tasks.upload_to_user_integration import upload_to_user_integration # noqa: F401
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401
+9
View File
@@ -457,6 +457,15 @@ class Settings(BaseSettings):
onedrive_refresh_token: Optional[str] = None # Required for personal accounts
onedrive_folder_path: Optional[str] = None
# SharePoint settings
sharepoint_client_id: Optional[str] = None
sharepoint_client_secret: Optional[str] = None
sharepoint_tenant_id: Optional[str] = "common"
sharepoint_refresh_token: Optional[str] = None
sharepoint_site_url: Optional[str] = None # e.g. https://tenant.sharepoint.com/sites/sitename
sharepoint_document_library: Optional[str] = "Documents" # Document library name
sharepoint_folder_path: Optional[str] = None # Subfolder inside the library
# AWS S3 settings
aws_access_key_id: Optional[str] = None
aws_secret_access_key: Optional[str] = None
+2
View File
@@ -505,6 +505,7 @@ class IntegrationType:
EMAIL = "EMAIL"
PAPERLESS = "PAPERLESS"
RCLONE = "RCLONE"
SHAREPOINT = "SHAREPOINT"
ALL = {
IMAP,
@@ -521,6 +522,7 @@ class IntegrationType:
EMAIL,
PAPERLESS,
RCLONE,
SHAREPOINT,
}
+19
View File
@@ -17,6 +17,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
from app.tasks.upload_to_webdav import upload_to_webdav
from app.utils.config_validator import get_provider_status
from app.utils.logging import log_task_progress
@@ -79,6 +80,18 @@ def _should_upload_to_s3():
return bool(settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key)
def _should_upload_to_sharepoint():
return bool(
settings.sharepoint_client_id
and settings.sharepoint_client_secret
and settings.sharepoint_site_url
and (
settings.sharepoint_refresh_token
or (settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common")
)
)
def get_configured_services_from_validator():
"""
Use the config validator to determine which services are configured properly.
@@ -98,6 +111,7 @@ def get_configured_services_from_validator():
"Email": "email",
"OneDrive": "onedrive",
"S3 Storage": "s3",
"SharePoint": "sharepoint",
}
result = {}
@@ -206,6 +220,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
"should_upload": _should_upload_to_s3,
"upload_func": upload_to_s3,
},
{
"name": "sharepoint",
"should_upload": _should_upload_to_sharepoint,
"upload_func": upload_to_sharepoint,
},
]
# Optionally get configuration status from validator
+338
View File
@@ -0,0 +1,338 @@
#!/usr/bin/env python3
"""Upload documents to Microsoft SharePoint via the Microsoft Graph API.
This module authenticates using MSAL (same OAuth2 flow as OneDrive) and
uploads files to a configurable SharePoint Online document library using
the chunked upload session approach for reliability with large files.
Key differences from the OneDrive provider:
- Uses ``/sites/{siteId}/drives/{driveId}`` instead of ``/me/drive``
- Requires a SharePoint site URL to resolve the site and drive IDs
- Targets a named document library (default: ``Documents``)
"""
import logging
import os
import time
import urllib.parse
import msal
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import UploadTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_sharepoint_token() -> str:
"""Acquire a Microsoft Graph API access token for SharePoint.
Uses MSAL ``ConfidentialClientApplication`` with the refresh-token flow
(delegated permissions) or the client-credentials flow (application
permissions) depending on configuration.
Returns:
A valid access token string.
Raises:
ValueError: When required settings are missing or token acquisition fails.
"""
if not settings.sharepoint_client_id or not settings.sharepoint_client_secret:
raise ValueError("SharePoint client ID and client secret must be configured")
tenant = settings.sharepoint_tenant_id or "common"
logger.info("Using SharePoint tenant: %s", tenant)
scopes = ["https://graph.microsoft.com/.default"]
if settings.sharepoint_refresh_token:
app = msal.ConfidentialClientApplication(
client_id=settings.sharepoint_client_id,
client_credential=settings.sharepoint_client_secret,
authority=f"https://login.microsoftonline.com/{tenant}",
)
logger.info("Attempting to acquire SharePoint token using refresh token")
token_response = app.acquire_token_by_refresh_token(
refresh_token=settings.sharepoint_refresh_token, scopes=scopes
)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
logger.error("Failed to get SharePoint access token: %s - %s", error, error_desc)
raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}")
if "refresh_token" in token_response:
settings.sharepoint_refresh_token = token_response["refresh_token"]
logger.info("Updated SharePoint refresh token in memory")
return token_response["access_token"]
elif settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common":
authority = f"https://login.microsoftonline.com/{settings.sharepoint_tenant_id}"
app = msal.ConfidentialClientApplication(
client_id=settings.sharepoint_client_id,
client_credential=settings.sharepoint_client_secret,
authority=authority,
)
token_response = app.acquire_token_for_client(scopes=scopes)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}")
return token_response["access_token"]
else:
raise ValueError("For SharePoint, either a refresh token or a non-'common' tenant ID is required")
def resolve_sharepoint_drive(access_token: str, site_url: str, library_name: str) -> tuple[str, str]:
"""Resolve the Graph API site ID and drive ID for a SharePoint site.
Args:
access_token: Valid Microsoft Graph API token.
site_url: Full SharePoint site URL, e.g.
``https://tenant.sharepoint.com/sites/sitename``.
library_name: Display name of the document library (e.g. ``Documents``).
Returns:
A ``(site_id, drive_id)`` tuple.
Raises:
ValueError: When the site URL cannot be parsed.
RuntimeError: When the Graph API call fails.
"""
parsed = urllib.parse.urlparse(site_url)
hostname = parsed.hostname
site_path = parsed.path.rstrip("/")
if not hostname or not site_path:
raise ValueError(
f"Invalid SharePoint site URL '{site_url}'. Expected format: https://tenant.sharepoint.com/sites/sitename"
)
headers = {"Authorization": f"Bearer {access_token}"}
# Resolve site ID
site_api_url = f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}"
logger.info("Resolving SharePoint site: %s", site_api_url)
resp = requests.get(site_api_url, headers=headers, timeout=settings.http_request_timeout)
if resp.status_code != 200:
raise RuntimeError(f"Failed to resolve SharePoint site: {resp.status_code} - {resp.text}")
site_id = resp.json()["id"]
logger.info("Resolved SharePoint site ID: %s", site_id)
# Resolve drive ID from the document library name
drives_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives"
resp = requests.get(drives_url, headers=headers, timeout=settings.http_request_timeout)
if resp.status_code != 200:
raise RuntimeError(f"Failed to list SharePoint drives: {resp.status_code} - {resp.text}")
drives = resp.json().get("value", [])
drive_id = None
for drive in drives:
if drive.get("name", "").lower() == library_name.lower():
drive_id = drive["id"]
break
if not drive_id:
available = [d.get("name") for d in drives]
raise RuntimeError(f"Document library '{library_name}' not found on site. Available libraries: {available}")
logger.info("Resolved SharePoint drive ID: %s (library: %s)", drive_id, library_name)
return site_id, drive_id
def create_sharepoint_upload_session(
filename: str, folder_path: str | None, drive_id: str, site_id: str, access_token: str
) -> str:
"""Create a resumable upload session on a SharePoint document library.
Args:
filename: Name of the file to upload.
folder_path: Optional subfolder path inside the library.
drive_id: Graph API drive ID of the document library.
site_id: Graph API site ID.
access_token: Valid access token.
Returns:
The upload session URL for chunked PUT requests.
Raises:
RuntimeError: When session creation fails.
"""
base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}"
if folder_path:
folder_path = folder_path.strip("/")
path_components = folder_path.split("/")
encoded_path = "/".join(urllib.parse.quote(component) for component in path_components)
encoded_filename = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_path}/{encoded_filename}:/createUploadSession"
else:
encoded_filename = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_filename}:/createUploadSession"
url = f"{base_url}{item_path}"
request_body = {"item": {"@microsoft.graph.conflictBehavior": "replace"}}
headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
logger.info("Creating SharePoint upload session for %s at path %s", filename, folder_path)
response = requests.post(url, headers=headers, json=request_body, timeout=settings.http_request_timeout)
if response.status_code == 200:
upload_url = response.json().get("uploadUrl")
logger.info("SharePoint upload session created for %s", filename)
return upload_url
else:
raise RuntimeError(f"Failed to create SharePoint upload session: {response.status_code} - {response.text}")
def upload_large_file_sharepoint(file_path: str, upload_url: str) -> dict:
"""Upload a file to SharePoint using a chunked upload session.
Args:
file_path: Local path to the file.
upload_url: The upload session URL from ``create_sharepoint_upload_session``.
Returns:
The Graph API response dict containing file metadata.
Raises:
RuntimeError: When a chunk upload fails after retries.
"""
file_size = os.path.getsize(file_path)
chunk_size = 10 * 1024 * 1024 # 10 MB
response = None
with open(file_path, "rb") as f:
chunk_number = 0
while True:
chunk = f.read(chunk_size)
if not chunk:
break
chunk_start = chunk_number * chunk_size
chunk_end = chunk_start + len(chunk) - 1
content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
headers = {"Content-Length": str(len(chunk)), "Content-Range": content_range}
max_retries = 3
retry_delay = 2
for attempt in range(max_retries):
try:
response = requests.put(
upload_url, headers=headers, data=chunk, timeout=settings.http_request_timeout
)
if response.status_code in (201, 202):
break
else:
logger.warning(
"SharePoint chunk upload failed (attempt %d): %d", attempt + 1, response.status_code
)
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
except Exception as e:
logger.warning("SharePoint chunk upload error (attempt %d): %s", attempt + 1, str(e))
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
if response is None or response.status_code not in (201, 202):
status = response.status_code if response else "no response"
text = response.text if response else ""
raise RuntimeError(f"Failed to upload chunk after {max_retries} attempts: {status} - {text}")
chunk_number += 1
return response.json() if response else {}
@celery.task(base=UploadTaskWithRetry, bind=True)
def upload_to_sharepoint(self, file_path: str, file_id: int = None, folder_override: str = None):
"""Upload a file to SharePoint Online.
Args:
file_path: Path to the file to upload.
file_id: Optional file ID to associate with logs.
folder_override: Optional folder path override.
Returns:
A dict with upload status and file details.
Raises:
FileNotFoundError: When the file does not exist.
ValueError: When SharePoint is not configured.
RuntimeError: When the upload fails.
"""
task_id = self.request.id
logger.info("[%s] Starting SharePoint upload: %s", task_id, file_path)
log_task_progress(
task_id,
"upload_to_sharepoint",
"in_progress",
f"Uploading to SharePoint: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
filename = os.path.basename(file_path)
if not settings.sharepoint_client_id:
error_msg = "SharePoint client ID is not configured"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise ValueError(error_msg)
if not settings.sharepoint_site_url:
error_msg = "SharePoint site URL is not configured"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise ValueError(error_msg)
try:
access_token = get_sharepoint_token()
library_name = settings.sharepoint_document_library or "Documents"
site_id, drive_id = resolve_sharepoint_drive(access_token, settings.sharepoint_site_url, library_name)
folder_path = folder_override if folder_override is not None else settings.sharepoint_folder_path
upload_url = create_sharepoint_upload_session(filename, folder_path, drive_id, site_id, access_token)
result = upload_large_file_sharepoint(file_path, upload_url)
web_url = result.get("webUrl", "Not available")
logger.info("[%s] Successfully uploaded %s to SharePoint", task_id, filename)
logger.info("[%s] File accessible at: %s", task_id, web_url)
log_task_progress(
task_id, "upload_to_sharepoint", "success", f"Uploaded to SharePoint: {filename}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"sharepoint_path": f"{folder_path or ''}/{filename}",
"web_url": web_url,
}
except Exception as e:
error_msg = f"Failed to upload {filename} to SharePoint: {str(e)}"
logger.error("[%s] %s", task_id, error_msg)
log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)
+108
View File
@@ -571,6 +571,113 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t
return {"status": "Completed", "rclone_dest": dest}
def _upload_sharepoint(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]:
"""Upload *file_path* to SharePoint using per-user MSAL credentials."""
import urllib.parse
import msal
import requests as _requests
client_id = creds.get("client_id") or ""
client_secret = creds.get("client_secret") or ""
refresh_token = creds.get("refresh_token") or ""
tenant = cfg.get("tenant_id") or "common"
site_url = cfg.get("site_url") or ""
library_name = cfg.get("document_library") or "Documents"
folder_path = cfg.get("folder_path") or ""
if not (client_id and client_secret):
raise ValueError("SharePoint integration is missing client_id or client_secret in credentials")
if not site_url:
raise ValueError("SharePoint integration is missing site_url in config")
scopes = ["https://graph.microsoft.com/.default"]
msal_app = msal.ConfidentialClientApplication(
client_id=client_id,
client_credential=client_secret,
authority=f"https://login.microsoftonline.com/{tenant}",
)
if refresh_token:
token_resp = msal_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=scopes)
else:
token_resp = msal_app.acquire_token_for_client(scopes=scopes)
if "access_token" not in token_resp:
raise ValueError(f"SharePoint token acquisition failed: {token_resp.get('error_description', 'unknown')}")
access_token = token_resp["access_token"]
headers = {"Authorization": f"Bearer {access_token}"}
# Resolve site ID
parsed = urllib.parse.urlparse(site_url)
hostname = parsed.hostname
site_path = parsed.path.rstrip("/")
if not hostname or not site_path:
raise ValueError(f"Invalid SharePoint site URL: {site_url}")
resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}", headers=headers, timeout=30)
resp.raise_for_status()
site_id = resp.json()["id"]
# Resolve drive ID
resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives", headers=headers, timeout=30)
resp.raise_for_status()
drive_id = None
for drive in resp.json().get("value", []):
if drive.get("name", "").lower() == library_name.lower():
drive_id = drive["id"]
break
if not drive_id:
raise RuntimeError(f"Document library '{library_name}' not found on SharePoint site")
filename = os.path.basename(file_path)
# Build upload-session URL
base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}"
if folder_path:
folder_path = folder_path.strip("/")
encoded_path = "/".join(urllib.parse.quote(p) for p in folder_path.split("/"))
encoded_file = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_path}/{encoded_file}:/createUploadSession"
else:
encoded_file = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_file}:/createUploadSession"
session_url = f"{base_url}{item_path}"
session_headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
resp = _requests.post(
session_url,
headers=session_headers,
json={"item": {"@microsoft.graph.conflictBehavior": "replace"}},
timeout=30,
)
resp.raise_for_status()
upload_url = resp.json()["uploadUrl"]
file_size = os.path.getsize(file_path)
chunk_size = 10 * 1024 * 1024
with open(file_path, "rb") as fh:
chunk_num = 0
while True:
chunk = fh.read(chunk_size)
if not chunk:
break
start = chunk_num * chunk_size
end = start + len(chunk) - 1
upload_headers = {
"Content-Length": str(len(chunk)),
"Content-Range": f"bytes {start}-{end}/{file_size}",
}
upload_resp = _requests.put(upload_url, headers=upload_headers, data=chunk, timeout=120)
if upload_resp.status_code not in (201, 202):
raise RuntimeError(f"SharePoint chunk upload failed: {upload_resp.status_code}")
chunk_num += 1
logger.info("[%s] SharePoint upload complete: %s/%s", task_id, folder_path, filename)
return {"status": "Completed", "sharepoint_folder": folder_path, "filename": filename}
# Map IntegrationType → upload helper
_UPLOAD_HANDLERS = {
IntegrationType.DROPBOX: _upload_dropbox,
@@ -584,6 +691,7 @@ _UPLOAD_HANDLERS = {
IntegrationType.PAPERLESS: _upload_paperless,
IntegrationType.EMAIL: _upload_email,
IntegrationType.RCLONE: _upload_rclone,
IntegrationType.SHAREPOINT: _upload_sharepoint,
}
+22
View File
@@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]:
},
}
# Check SharePoint configuration
providers["SharePoint"] = {
"name": "SharePoint",
"icon": "fa-brands fa-microsoft",
"configured": bool(
getattr(settings, "sharepoint_client_id", None)
and getattr(settings, "sharepoint_client_secret", None)
and getattr(settings, "sharepoint_site_url", None)
),
"enabled": True,
"description": "Store documents in Microsoft SharePoint Online",
"details": {
"client_id": getattr(settings, "sharepoint_client_id", "Not set"),
"client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)),
"tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"),
"refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)),
"site_url": getattr(settings, "sharepoint_site_url", "Not set"),
"document_library": getattr(settings, "sharepoint_document_library", "Not set"),
"folder_path": getattr(settings, "sharepoint_folder_path", "Not set"),
},
}
# Check S3 configuration
providers["S3 Storage"] = {
"name": "S3 Storage",
+71 -2
View File
@@ -675,6 +675,63 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - SharePoint
"sharepoint_client_id": {
"category": "Storage Providers",
"description": "SharePoint Azure AD application (client) ID",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_client_secret": {
"category": "Storage Providers",
"description": "SharePoint Azure AD client secret",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"sharepoint_tenant_id": {
"category": "Storage Providers",
"description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_refresh_token": {
"category": "Storage Providers",
"description": "SharePoint OAuth refresh token",
"type": "string",
"sensitive": True,
"required": False,
"restart_required": False,
},
"sharepoint_site_url": {
"category": "Storage Providers",
"description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_document_library": {
"category": "Storage Providers",
"description": "SharePoint document library name (default: 'Documents')",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
"sharepoint_folder_path": {
"category": "Storage Providers",
"description": "Subfolder path inside the SharePoint document library",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_url": {
"category": "Storage Providers",
@@ -1618,14 +1675,26 @@ SETTING_METADATA = {
"category": "Backup",
"description": (
"Storage provider for remote backup copies. "
"Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. "
"Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. "
"Leave empty to keep backups local only."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
"options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"],
"options": [
"",
"s3",
"dropbox",
"google_drive",
"onedrive",
"sharepoint",
"nextcloud",
"webdav",
"ftp",
"sftp",
"email",
],
},
"backup_remote_folder": {
"category": "Backup",
+1
View File
@@ -525,6 +525,7 @@
<option value="webdav" {% if storage_provider == "webdav" %}selected{% endif %}>WebDAV</option>
<option value="ftp" {% if storage_provider == "ftp" %}selected{% endif %}>FTP</option>
<option value="sftp" {% if storage_provider == "sftp" %}selected{% endif %}>SFTP</option>
<option value="sharepoint" {% if storage_provider == "sharepoint" %}selected{% endif %}>SharePoint</option>
</select>
</div>
+9 -1
View File
@@ -33,7 +33,7 @@
"name": "Which cloud storage providers does DocuElevate support?",
"acceptedAnswer": {
"@type": "Answer",
"text": "DocuElevate integrates with Dropbox, Google Drive, OneDrive, Amazon S3, Nextcloud, WebDAV, SFTP, FTP, and Paperless-ngx."
"text": "DocuElevate integrates with Dropbox, Google Drive, OneDrive, SharePoint, Amazon S3, Nextcloud, WebDAV, SFTP, FTP, and Paperless-ngx."
}
},
{
@@ -196,6 +196,14 @@
</div>
</div>
<div class="flex items-start bg-white shadow rounded-lg p-5">
<i class="fab fa-microsoft text-purple-600 text-xl mr-3 mt-0.5" aria-hidden="true"></i>
<div>
<h3 class="font-semibold text-gray-800 text-sm">SharePoint</h3>
<p class="text-gray-500 text-xs">Upload to SharePoint Online document libraries via Graph API.</p>
</div>
</div>
<div class="flex items-start bg-white shadow rounded-lg p-5">
<i class="fab fa-aws text-yellow-600 text-xl mr-3 mt-0.5" aria-hidden="true"></i>
<div>
+554
View File
@@ -0,0 +1,554 @@
"""
Tests for app/tasks/upload_to_sharepoint.py module.
Covers get_sharepoint_token, resolve_sharepoint_drive,
create_sharepoint_upload_session, upload_large_file_sharepoint,
and upload_to_sharepoint Celery task.
"""
from unittest.mock import Mock, patch
import pytest
@pytest.mark.unit
class TestGetSharepointToken:
"""Tests for get_sharepoint_token function."""
@patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_refresh_token_flow(self, mock_settings, mock_msal):
"""Test token acquisition using refresh token."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "client-secret"
mock_settings.sharepoint_refresh_token = "refresh-token"
mock_settings.sharepoint_tenant_id = "common"
mock_app = Mock()
mock_app.acquire_token_by_refresh_token.return_value = {
"access_token": "new-access-token",
}
mock_msal.return_value = mock_app
token = get_sharepoint_token()
assert token == "new-access-token"
@patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_refresh_token_updates_new_token(self, mock_settings, mock_msal):
"""Test that a new refresh token updates settings."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "client-secret"
mock_settings.sharepoint_refresh_token = "old-refresh-token"
mock_settings.sharepoint_tenant_id = "common"
mock_app = Mock()
mock_app.acquire_token_by_refresh_token.return_value = {
"access_token": "access-token",
"refresh_token": "new-refresh-token",
}
mock_msal.return_value = mock_app
get_sharepoint_token()
assert mock_settings.sharepoint_refresh_token == "new-refresh-token"
@patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_refresh_token_failure(self, mock_settings, mock_msal):
"""Test error handling when refresh token fails."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "client-secret"
mock_settings.sharepoint_refresh_token = "expired-token"
mock_settings.sharepoint_tenant_id = "common"
mock_app = Mock()
mock_app.acquire_token_by_refresh_token.return_value = {
"error": "invalid_grant",
"error_description": "Token expired",
}
mock_msal.return_value = mock_app
with pytest.raises(ValueError, match="Failed to get SharePoint access token"):
get_sharepoint_token()
@patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_client_credentials_flow(self, mock_settings, mock_msal):
"""Test token acquisition using client credentials (org accounts)."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "client-secret"
mock_settings.sharepoint_refresh_token = ""
mock_settings.sharepoint_tenant_id = "org-tenant-id"
mock_app = Mock()
mock_app.acquire_token_for_client.return_value = {
"access_token": "client-cred-token",
}
mock_msal.return_value = mock_app
token = get_sharepoint_token()
assert token == "client-cred-token"
@patch("app.tasks.upload_to_sharepoint.msal.ConfidentialClientApplication")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_client_credentials_failure(self, mock_settings, mock_msal):
"""Test error handling when client credentials flow fails."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "client-secret"
mock_settings.sharepoint_refresh_token = ""
mock_settings.sharepoint_tenant_id = "org-tenant-id"
mock_app = Mock()
mock_app.acquire_token_for_client.return_value = {
"error": "unauthorized_client",
"error_description": "Not authorized",
}
mock_msal.return_value = mock_app
with pytest.raises(ValueError, match="Failed to get SharePoint access token"):
get_sharepoint_token()
@patch("app.tasks.upload_to_sharepoint.settings")
def test_missing_client_id(self, mock_settings):
"""Test error when client ID is missing."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = ""
mock_settings.sharepoint_client_secret = "secret"
with pytest.raises(ValueError, match="client ID and client secret"):
get_sharepoint_token()
@patch("app.tasks.upload_to_sharepoint.settings")
def test_no_refresh_token_common_tenant(self, mock_settings):
"""Test error for common tenant without refresh token."""
from app.tasks.upload_to_sharepoint import get_sharepoint_token
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "secret"
mock_settings.sharepoint_refresh_token = ""
mock_settings.sharepoint_tenant_id = "common"
with pytest.raises(ValueError, match="either a refresh token or a non-'common' tenant ID"):
get_sharepoint_token()
@pytest.mark.unit
class TestResolveSharepointDrive:
"""Tests for resolve_sharepoint_drive function."""
@patch("app.tasks.upload_to_sharepoint.requests.get")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_successful_resolution(self, mock_settings, mock_get):
"""Test successful site and drive resolution."""
from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive
mock_settings.http_request_timeout = 30
site_resp = Mock()
site_resp.status_code = 200
site_resp.json.return_value = {"id": "site-id-123"}
drives_resp = Mock()
drives_resp.status_code = 200
drives_resp.json.return_value = {
"value": [
{"id": "drive-1", "name": "Documents"},
{"id": "drive-2", "name": "Site Assets"},
]
}
mock_get.side_effect = [site_resp, drives_resp]
site_id, drive_id = resolve_sharepoint_drive(
"access-token", "https://tenant.sharepoint.com/sites/mysite", "Documents"
)
assert site_id == "site-id-123"
assert drive_id == "drive-1"
@patch("app.tasks.upload_to_sharepoint.requests.get")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_library_not_found(self, mock_settings, mock_get):
"""Test error when document library is not found."""
from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive
mock_settings.http_request_timeout = 30
site_resp = Mock()
site_resp.status_code = 200
site_resp.json.return_value = {"id": "site-id-123"}
drives_resp = Mock()
drives_resp.status_code = 200
drives_resp.json.return_value = {
"value": [
{"id": "drive-1", "name": "Documents"},
]
}
mock_get.side_effect = [site_resp, drives_resp]
with pytest.raises(RuntimeError, match="not found on site"):
resolve_sharepoint_drive("access-token", "https://tenant.sharepoint.com/sites/mysite", "NonExistentLibrary")
@patch("app.tasks.upload_to_sharepoint.requests.get")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_site_resolution_failure(self, mock_settings, mock_get):
"""Test error when site resolution fails."""
from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive
mock_settings.http_request_timeout = 30
site_resp = Mock()
site_resp.status_code = 404
site_resp.text = "Site not found"
mock_get.return_value = site_resp
with pytest.raises(RuntimeError, match="Failed to resolve SharePoint site"):
resolve_sharepoint_drive("access-token", "https://tenant.sharepoint.com/sites/nonexistent", "Documents")
def test_invalid_site_url(self):
"""Test error with invalid site URL."""
from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive
with pytest.raises(ValueError, match="Invalid SharePoint site URL"):
resolve_sharepoint_drive("access-token", "not-a-url", "Documents")
@patch("app.tasks.upload_to_sharepoint.requests.get")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_case_insensitive_library_match(self, mock_settings, mock_get):
"""Test that library name matching is case-insensitive."""
from app.tasks.upload_to_sharepoint import resolve_sharepoint_drive
mock_settings.http_request_timeout = 30
site_resp = Mock()
site_resp.status_code = 200
site_resp.json.return_value = {"id": "site-id"}
drives_resp = Mock()
drives_resp.status_code = 200
drives_resp.json.return_value = {
"value": [
{"id": "drive-1", "name": "Shared Documents"},
]
}
mock_get.side_effect = [site_resp, drives_resp]
site_id, drive_id = resolve_sharepoint_drive(
"access-token", "https://tenant.sharepoint.com/sites/mysite", "shared documents"
)
assert drive_id == "drive-1"
@pytest.mark.unit
class TestCreateSharepointUploadSession:
"""Tests for create_sharepoint_upload_session function."""
@patch("app.tasks.upload_to_sharepoint.requests.post")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_successful_session_creation(self, mock_settings, mock_post):
"""Test successful upload session creation."""
from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session
mock_settings.http_request_timeout = 30
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {"uploadUrl": "https://upload.url/session123"}
mock_post.return_value = mock_response
url = create_sharepoint_upload_session("test.pdf", "Uploads", "drive-id", "site-id", "access-token")
assert url == "https://upload.url/session123"
@patch("app.tasks.upload_to_sharepoint.requests.post")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_session_without_folder(self, mock_settings, mock_post):
"""Test upload session creation without folder path."""
from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session
mock_settings.http_request_timeout = 30
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {"uploadUrl": "https://upload.url/session456"}
mock_post.return_value = mock_response
url = create_sharepoint_upload_session("test.pdf", None, "drive-id", "site-id", "access-token")
assert url == "https://upload.url/session456"
@patch("app.tasks.upload_to_sharepoint.requests.post")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_session_creation_failure(self, mock_settings, mock_post):
"""Test error handling when session creation fails."""
from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session
mock_settings.http_request_timeout = 30
mock_response = Mock()
mock_response.status_code = 403
mock_response.text = "Access denied"
mock_post.return_value = mock_response
with pytest.raises(RuntimeError, match="Failed to create SharePoint upload session"):
create_sharepoint_upload_session("test.pdf", "Uploads", "drive-id", "site-id", "access-token")
@patch("app.tasks.upload_to_sharepoint.requests.post")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_url_encoding_special_characters(self, mock_settings, mock_post):
"""Test that special characters in folder path are URL-encoded."""
from app.tasks.upload_to_sharepoint import create_sharepoint_upload_session
mock_settings.http_request_timeout = 30
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {"uploadUrl": "https://upload.url/session"}
mock_post.return_value = mock_response
create_sharepoint_upload_session("file with spaces.pdf", "My Documents/Uploads", "drive-id", "site-id", "token")
call_url = mock_post.call_args[0][0]
assert "My%20Documents" in call_url
assert "file%20with%20spaces.pdf" in call_url
@pytest.mark.unit
class TestUploadLargeFileSharepoint:
"""Tests for upload_large_file_sharepoint function."""
@patch("app.tasks.upload_to_sharepoint.requests.put")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_small_single_chunk_upload(self, mock_settings, mock_put, tmp_path):
"""Test uploading a file that fits in a single chunk."""
from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint
mock_settings.http_request_timeout = 30
test_file = tmp_path / "small.pdf"
test_file.write_bytes(b"small content")
mock_response = Mock()
mock_response.status_code = 201
mock_response.json.return_value = {"id": "file123", "name": "small.pdf"}
mock_put.return_value = mock_response
result = upload_large_file_sharepoint(str(test_file), "https://upload.url/session")
assert result["id"] == "file123"
@patch("app.tasks.upload_to_sharepoint.time.sleep")
@patch("app.tasks.upload_to_sharepoint.requests.put")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_chunk_upload_retry_on_failure(self, mock_settings, mock_put, mock_sleep, tmp_path):
"""Test retry logic when a chunk upload fails."""
from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint
mock_settings.http_request_timeout = 30
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
mock_fail = Mock()
mock_fail.status_code = 500
mock_success = Mock()
mock_success.status_code = 201
mock_success.json.return_value = {"id": "file123"}
mock_put.side_effect = [mock_fail, mock_success]
result = upload_large_file_sharepoint(str(test_file), "https://upload.url/session")
assert result["id"] == "file123"
@patch("app.tasks.upload_to_sharepoint.time.sleep")
@patch("app.tasks.upload_to_sharepoint.requests.put")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_chunk_upload_retry_on_exception(self, mock_settings, mock_put, mock_sleep, tmp_path):
"""Test retry logic when an exception occurs during upload."""
from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint
mock_settings.http_request_timeout = 30
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
mock_success = Mock()
mock_success.status_code = 201
mock_success.json.return_value = {"id": "file123"}
mock_put.side_effect = [Exception("Network error"), mock_success]
result = upload_large_file_sharepoint(str(test_file), "https://upload.url/session")
assert result["id"] == "file123"
@patch("app.tasks.upload_to_sharepoint.time.sleep")
@patch("app.tasks.upload_to_sharepoint.requests.put")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_all_retries_exhausted(self, mock_settings, mock_put, mock_sleep, tmp_path):
"""Test that exhausting all retries raises an exception."""
from app.tasks.upload_to_sharepoint import upload_large_file_sharepoint
mock_settings.http_request_timeout = 30
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
mock_fail = Mock()
mock_fail.status_code = 500
mock_fail.text = "Server Error"
mock_put.return_value = mock_fail
with pytest.raises(RuntimeError, match="Failed to upload chunk"):
upload_large_file_sharepoint(str(test_file), "https://upload.url/session")
@pytest.mark.unit
class TestUploadToSharepoint:
"""Tests for upload_to_sharepoint Celery task."""
@patch("app.tasks.upload_to_sharepoint.log_task_progress")
def test_file_not_found(self, mock_log):
"""Test that missing file raises FileNotFoundError."""
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
with pytest.raises(FileNotFoundError):
upload_to_sharepoint.__wrapped__("/nonexistent/file.pdf", file_id=1)
@patch("app.tasks.upload_to_sharepoint.log_task_progress")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_missing_client_id(self, mock_settings, mock_log, tmp_path):
"""Test error when SharePoint client ID is not configured."""
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
mock_settings.sharepoint_client_id = ""
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
with pytest.raises(ValueError, match="client ID is not configured"):
upload_to_sharepoint.__wrapped__(str(test_file), file_id=1)
@patch("app.tasks.upload_to_sharepoint.log_task_progress")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_missing_site_url(self, mock_settings, mock_log, tmp_path):
"""Test error when SharePoint site URL is not configured."""
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_site_url = ""
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
with pytest.raises(ValueError, match="site URL is not configured"):
upload_to_sharepoint.__wrapped__(str(test_file), file_id=1)
@patch("app.tasks.upload_to_sharepoint.upload_large_file_sharepoint")
@patch("app.tasks.upload_to_sharepoint.create_sharepoint_upload_session")
@patch("app.tasks.upload_to_sharepoint.resolve_sharepoint_drive")
@patch("app.tasks.upload_to_sharepoint.get_sharepoint_token")
@patch("app.tasks.upload_to_sharepoint.log_task_progress")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_successful_upload(
self, mock_settings, mock_log, mock_token, mock_resolve, mock_session, mock_upload, tmp_path
):
"""Test successful SharePoint upload."""
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "secret"
mock_settings.sharepoint_refresh_token = "token"
mock_settings.sharepoint_site_url = "https://tenant.sharepoint.com/sites/mysite"
mock_settings.sharepoint_document_library = "Documents"
mock_settings.sharepoint_folder_path = "Uploads"
mock_settings.sharepoint_tenant_id = "common"
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
mock_token.return_value = "access-token"
mock_resolve.return_value = ("site-id", "drive-id")
mock_session.return_value = "https://upload.url/session"
mock_upload.return_value = {"webUrl": "https://tenant.sharepoint.com/sites/mysite/test.pdf"}
result = upload_to_sharepoint.apply(args=[str(test_file)], kwargs={"file_id": 1}).get()
assert result["status"] == "Completed"
assert "Uploads" in result["sharepoint_path"]
assert result["web_url"] == "https://tenant.sharepoint.com/sites/mysite/test.pdf"
@patch("app.tasks.upload_to_sharepoint.get_sharepoint_token")
@patch("app.tasks.upload_to_sharepoint.log_task_progress")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_upload_exception_handling(self, mock_settings, mock_log, mock_token, tmp_path):
"""Test that upload errors are properly handled."""
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "secret"
mock_settings.sharepoint_site_url = "https://tenant.sharepoint.com/sites/mysite"
mock_settings.sharepoint_folder_path = "Uploads"
mock_settings.sharepoint_document_library = "Documents"
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
mock_token.side_effect = ValueError("Token error")
with pytest.raises(Exception, match="Failed to upload"):
upload_to_sharepoint.apply(args=[str(test_file)], kwargs={"file_id": 1}).get()
@patch("app.tasks.upload_to_sharepoint.upload_large_file_sharepoint")
@patch("app.tasks.upload_to_sharepoint.create_sharepoint_upload_session")
@patch("app.tasks.upload_to_sharepoint.resolve_sharepoint_drive")
@patch("app.tasks.upload_to_sharepoint.get_sharepoint_token")
@patch("app.tasks.upload_to_sharepoint.log_task_progress")
@patch("app.tasks.upload_to_sharepoint.settings")
def test_folder_override(
self, mock_settings, mock_log, mock_token, mock_resolve, mock_session, mock_upload, tmp_path
):
"""Test that folder_override is used instead of settings."""
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
mock_settings.sharepoint_client_id = "client-id"
mock_settings.sharepoint_client_secret = "secret"
mock_settings.sharepoint_refresh_token = "token"
mock_settings.sharepoint_site_url = "https://tenant.sharepoint.com/sites/mysite"
mock_settings.sharepoint_document_library = "Documents"
mock_settings.sharepoint_folder_path = "DefaultFolder"
mock_settings.sharepoint_tenant_id = "common"
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"test content")
mock_token.return_value = "access-token"
mock_resolve.return_value = ("site-id", "drive-id")
mock_session.return_value = "https://upload.url/session"
mock_upload.return_value = {"webUrl": "https://example.com/test.pdf"}
result = upload_to_sharepoint.apply(
args=[str(test_file)], kwargs={"file_id": 1, "folder_override": "CustomFolder"}
).get()
# Verify the session was created with the override folder
mock_session.assert_called_once_with("test.pdf", "CustomFolder", "drive-id", "site-id", "access-token")
assert result["status"] == "Completed"