diff --git a/.env.demo b/.env.demo index 1d04bab7..f92b6d8e 100644 --- a/.env.demo +++ b/.env.demo @@ -447,6 +447,15 @@ ONEDRIVE_TENANT_ID=common ONEDRIVE_REFRESH_TOKEN=your-refresh-token ONEDRIVE_FOLDER_PATH=Documents/Uploads +# SharePoint +SHAREPOINT_CLIENT_ID=your-client-id +SHAREPOINT_CLIENT_SECRET=your-client-secret +SHAREPOINT_TENANT_ID=common +SHAREPOINT_REFRESH_TOKEN=your-refresh-token +SHAREPOINT_SITE_URL=https://tenant.sharepoint.com/sites/sitename +SHAREPOINT_DOCUMENT_LIBRARY=Documents +SHAREPOINT_FOLDER_PATH=Uploads + # WebDAV # WEBDAV_ENABLED=true # Set to false to disable WebDAV uploads without removing credentials WEBDAV_URL=https://webdav.example.com/path diff --git a/app/celery_worker.py b/app/celery_worker.py index 17110a66..cca7083b 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -52,6 +52,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401 from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401 from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401 from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401 +from app.tasks.upload_to_sharepoint import upload_to_sharepoint # noqa: F401 from app.tasks.upload_to_user_integration import upload_to_user_integration # noqa: F401 from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401 from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401 diff --git a/app/config.py b/app/config.py index 1f5a25be..cff42826 100644 --- a/app/config.py +++ b/app/config.py @@ -609,6 +609,15 @@ class Settings(BaseSettings): onedrive_refresh_token: Optional[str] = None # Required for personal accounts onedrive_folder_path: Optional[str] = None + # SharePoint settings + sharepoint_client_id: Optional[str] = None + sharepoint_client_secret: Optional[str] = None + sharepoint_tenant_id: Optional[str] = "common" + sharepoint_refresh_token: Optional[str] = None + sharepoint_site_url: Optional[str] = None # e.g. https://tenant.sharepoint.com/sites/sitename + sharepoint_document_library: Optional[str] = "Documents" # Document library name + sharepoint_folder_path: Optional[str] = None # Subfolder inside the library + # AWS S3 settings s3_enabled: bool = Field( default=True, diff --git a/app/models.py b/app/models.py index fe5b9c41..5f406953 100644 --- a/app/models.py +++ b/app/models.py @@ -603,6 +603,7 @@ class IntegrationType: EMAIL = "EMAIL" PAPERLESS = "PAPERLESS" RCLONE = "RCLONE" + SHAREPOINT = "SHAREPOINT" ICLOUD = "ICLOUD" ALL = { @@ -620,6 +621,7 @@ class IntegrationType: EMAIL, PAPERLESS, RCLONE, + SHAREPOINT, ICLOUD, } diff --git a/app/tasks/send_to_all.py b/app/tasks/send_to_all.py index 214e0225..17f60c84 100644 --- a/app/tasks/send_to_all.py +++ b/app/tasks/send_to_all.py @@ -18,6 +18,7 @@ from app.tasks.upload_to_onedrive import upload_to_onedrive from app.tasks.upload_to_paperless import upload_to_paperless from app.tasks.upload_to_s3 import upload_to_s3 from app.tasks.upload_to_sftp import upload_to_sftp +from app.tasks.upload_to_sharepoint import upload_to_sharepoint from app.tasks.upload_to_webdav import upload_to_webdav from app.utils.config_validator import get_provider_status from app.utils.logging import log_task_progress @@ -121,6 +122,18 @@ def _should_upload_to_icloud(): return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password) +def _should_upload_to_sharepoint(): + return bool( + settings.sharepoint_client_id + and settings.sharepoint_client_secret + and settings.sharepoint_site_url + and ( + settings.sharepoint_refresh_token + or (settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common") + ) + ) + + def get_configured_services_from_validator(): """ Use the config validator to determine which services are configured and enabled. @@ -140,6 +153,7 @@ def get_configured_services_from_validator(): "Email": "email", "OneDrive": "onedrive", "S3 Storage": "s3", + "SharePoint": "sharepoint", "iCloud Drive": "icloud", } @@ -250,6 +264,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: "should_upload": _should_upload_to_s3, "upload_func": upload_to_s3, }, + { + "name": "sharepoint", + "should_upload": _should_upload_to_sharepoint, + "upload_func": upload_to_sharepoint, + }, { "name": "icloud", "should_upload": _should_upload_to_icloud, diff --git a/app/tasks/upload_to_sharepoint.py b/app/tasks/upload_to_sharepoint.py new file mode 100644 index 00000000..0e2593c1 --- /dev/null +++ b/app/tasks/upload_to_sharepoint.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Upload documents to Microsoft SharePoint via the Microsoft Graph API. + +This module authenticates using MSAL (same OAuth2 flow as OneDrive) and +uploads files to a configurable SharePoint Online document library using +the chunked upload session approach for reliability with large files. + +Key differences from the OneDrive provider: +- Uses ``/sites/{siteId}/drives/{driveId}`` instead of ``/me/drive`` +- Requires a SharePoint site URL to resolve the site and drive IDs +- Targets a named document library (default: ``Documents``) +""" + +import logging +import os +import time +import urllib.parse + +import msal +import requests + +from app.celery_app import celery +from app.config import settings +from app.tasks.retry_config import UploadTaskWithRetry +from app.utils import log_task_progress + +logger = logging.getLogger(__name__) + + +def get_sharepoint_token() -> str: + """Acquire a Microsoft Graph API access token for SharePoint. + + Uses MSAL ``ConfidentialClientApplication`` with the refresh-token flow + (delegated permissions) or the client-credentials flow (application + permissions) depending on configuration. + + Returns: + A valid access token string. + + Raises: + ValueError: When required settings are missing or token acquisition fails. + """ + if not settings.sharepoint_client_id or not settings.sharepoint_client_secret: + raise ValueError("SharePoint client ID and client secret must be configured") + + tenant = settings.sharepoint_tenant_id or "common" + logger.info("Using SharePoint tenant: %s", tenant) + + scopes = ["https://graph.microsoft.com/.default"] + + if settings.sharepoint_refresh_token: + app = msal.ConfidentialClientApplication( + client_id=settings.sharepoint_client_id, + client_credential=settings.sharepoint_client_secret, + authority=f"https://login.microsoftonline.com/{tenant}", + ) + + logger.info("Attempting to acquire SharePoint token using refresh token") + token_response = app.acquire_token_by_refresh_token( + refresh_token=settings.sharepoint_refresh_token, scopes=scopes + ) + + if "access_token" not in token_response: + error = token_response.get("error", "") + error_desc = token_response.get("error_description", "Unknown error") + logger.error("Failed to get SharePoint access token: %s - %s", error, error_desc) + raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}") + + if "refresh_token" in token_response: + settings.sharepoint_refresh_token = token_response["refresh_token"] + logger.info("Updated SharePoint refresh token in memory") + + return token_response["access_token"] + + elif settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common": + authority = f"https://login.microsoftonline.com/{settings.sharepoint_tenant_id}" + app = msal.ConfidentialClientApplication( + client_id=settings.sharepoint_client_id, + client_credential=settings.sharepoint_client_secret, + authority=authority, + ) + + token_response = app.acquire_token_for_client(scopes=scopes) + + if "access_token" not in token_response: + error = token_response.get("error", "") + error_desc = token_response.get("error_description", "Unknown error") + raise ValueError(f"Failed to get SharePoint access token: {error} - {error_desc}") + + return token_response["access_token"] + + else: + raise ValueError("For SharePoint, either a refresh token or a non-'common' tenant ID is required") + + +def resolve_sharepoint_drive(access_token: str, site_url: str, library_name: str) -> tuple[str, str]: + """Resolve the Graph API site ID and drive ID for a SharePoint site. + + Args: + access_token: Valid Microsoft Graph API token. + site_url: Full SharePoint site URL, e.g. + ``https://tenant.sharepoint.com/sites/sitename``. + library_name: Display name of the document library (e.g. ``Documents``). + + Returns: + A ``(site_id, drive_id)`` tuple. + + Raises: + ValueError: When the site URL cannot be parsed. + RuntimeError: When the Graph API call fails. + """ + parsed = urllib.parse.urlparse(site_url) + hostname = parsed.hostname + site_path = parsed.path.rstrip("/") + + if not hostname or not site_path: + raise ValueError( + f"Invalid SharePoint site URL '{site_url}'. Expected format: https://tenant.sharepoint.com/sites/sitename" + ) + + headers = {"Authorization": f"Bearer {access_token}"} + + # Resolve site ID + site_api_url = f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}" + logger.info("Resolving SharePoint site: %s", site_api_url) + resp = requests.get(site_api_url, headers=headers, timeout=settings.http_request_timeout) + + if resp.status_code != 200: + raise RuntimeError(f"Failed to resolve SharePoint site: {resp.status_code} - {resp.text}") + + site_id = resp.json()["id"] + logger.info("Resolved SharePoint site ID: %s", site_id) + + # Resolve drive ID from the document library name + drives_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives" + resp = requests.get(drives_url, headers=headers, timeout=settings.http_request_timeout) + + if resp.status_code != 200: + raise RuntimeError(f"Failed to list SharePoint drives: {resp.status_code} - {resp.text}") + + drives = resp.json().get("value", []) + drive_id = None + for drive in drives: + if drive.get("name", "").lower() == library_name.lower(): + drive_id = drive["id"] + break + + if not drive_id: + available = [d.get("name") for d in drives] + raise RuntimeError(f"Document library '{library_name}' not found on site. Available libraries: {available}") + + logger.info("Resolved SharePoint drive ID: %s (library: %s)", drive_id, library_name) + return site_id, drive_id + + +def create_sharepoint_upload_session( + filename: str, folder_path: str | None, drive_id: str, site_id: str, access_token: str +) -> str: + """Create a resumable upload session on a SharePoint document library. + + Args: + filename: Name of the file to upload. + folder_path: Optional subfolder path inside the library. + drive_id: Graph API drive ID of the document library. + site_id: Graph API site ID. + access_token: Valid access token. + + Returns: + The upload session URL for chunked PUT requests. + + Raises: + RuntimeError: When session creation fails. + """ + base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}" + + if folder_path: + folder_path = folder_path.strip("/") + path_components = folder_path.split("/") + encoded_path = "/".join(urllib.parse.quote(component) for component in path_components) + encoded_filename = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_path}/{encoded_filename}:/createUploadSession" + else: + encoded_filename = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_filename}:/createUploadSession" + + url = f"{base_url}{item_path}" + request_body = {"item": {"@microsoft.graph.conflictBehavior": "replace"}} + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + + logger.info("Creating SharePoint upload session for %s at path %s", filename, folder_path) + response = requests.post(url, headers=headers, json=request_body, timeout=settings.http_request_timeout) + + if response.status_code == 200: + upload_url = response.json().get("uploadUrl") + logger.info("SharePoint upload session created for %s", filename) + return upload_url + else: + raise RuntimeError(f"Failed to create SharePoint upload session: {response.status_code} - {response.text}") + + +def upload_large_file_sharepoint(file_path: str, upload_url: str) -> dict: + """Upload a file to SharePoint using a chunked upload session. + + Args: + file_path: Local path to the file. + upload_url: The upload session URL from ``create_sharepoint_upload_session``. + + Returns: + The Graph API response dict containing file metadata. + + Raises: + RuntimeError: When a chunk upload fails after retries. + """ + file_size = os.path.getsize(file_path) + chunk_size = 10 * 1024 * 1024 # 10 MB + + response = None + with open(file_path, "rb") as f: + chunk_number = 0 + while True: + chunk = f.read(chunk_size) + if not chunk: + break + + chunk_start = chunk_number * chunk_size + chunk_end = chunk_start + len(chunk) - 1 + content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}" + + headers = {"Content-Length": str(len(chunk)), "Content-Range": content_range} + + max_retries = 3 + retry_delay = 2 + + for attempt in range(max_retries): + try: + response = requests.put( + upload_url, headers=headers, data=chunk, timeout=settings.http_request_timeout + ) + if response.status_code in (201, 202): + break + else: + logger.warning( + "SharePoint chunk upload failed (attempt %d): %d", attempt + 1, response.status_code + ) + if attempt < max_retries - 1: + time.sleep(retry_delay * (attempt + 1)) + except Exception as e: + logger.warning("SharePoint chunk upload error (attempt %d): %s", attempt + 1, str(e)) + if attempt < max_retries - 1: + time.sleep(retry_delay * (attempt + 1)) + + if response is None or response.status_code not in (201, 202): + status = response.status_code if response else "no response" + text = response.text if response else "" + raise RuntimeError(f"Failed to upload chunk after {max_retries} attempts: {status} - {text}") + + chunk_number += 1 + + return response.json() if response else {} + + +@celery.task(base=UploadTaskWithRetry, bind=True) +def upload_to_sharepoint(self, file_path: str, file_id: int = None, folder_override: str = None): + """Upload a file to SharePoint Online. + + Args: + file_path: Path to the file to upload. + file_id: Optional file ID to associate with logs. + folder_override: Optional folder path override. + + Returns: + A dict with upload status and file details. + + Raises: + FileNotFoundError: When the file does not exist. + ValueError: When SharePoint is not configured. + RuntimeError: When the upload fails. + """ + task_id = self.request.id + logger.info("[%s] Starting SharePoint upload: %s", task_id, file_path) + log_task_progress( + task_id, + "upload_to_sharepoint", + "in_progress", + f"Uploading to SharePoint: {os.path.basename(file_path)}", + file_id=file_id, + ) + + if not os.path.exists(file_path): + error_msg = f"File not found: {file_path}" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise FileNotFoundError(error_msg) + + filename = os.path.basename(file_path) + + if not settings.sharepoint_client_id: + error_msg = "SharePoint client ID is not configured" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise ValueError(error_msg) + + if not settings.sharepoint_site_url: + error_msg = "SharePoint site URL is not configured" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise ValueError(error_msg) + + try: + access_token = get_sharepoint_token() + + library_name = settings.sharepoint_document_library or "Documents" + site_id, drive_id = resolve_sharepoint_drive(access_token, settings.sharepoint_site_url, library_name) + + folder_path = folder_override if folder_override is not None else settings.sharepoint_folder_path + + upload_url = create_sharepoint_upload_session(filename, folder_path, drive_id, site_id, access_token) + result = upload_large_file_sharepoint(file_path, upload_url) + + web_url = result.get("webUrl", "Not available") + logger.info("[%s] Successfully uploaded %s to SharePoint", task_id, filename) + logger.info("[%s] File accessible at: %s", task_id, web_url) + log_task_progress( + task_id, "upload_to_sharepoint", "success", f"Uploaded to SharePoint: {filename}", file_id=file_id + ) + + return { + "status": "Completed", + "file_path": file_path, + "sharepoint_path": f"{folder_path or ''}/{filename}", + "web_url": web_url, + } + + except Exception as e: + error_msg = f"Failed to upload {filename} to SharePoint: {str(e)}" + logger.error("[%s] %s", task_id, error_msg) + log_task_progress(task_id, "upload_to_sharepoint", "failure", error_msg, file_id=file_id) + raise RuntimeError(error_msg) from e diff --git a/app/tasks/upload_to_user_integration.py b/app/tasks/upload_to_user_integration.py index db21701d..1b23d321 100644 --- a/app/tasks/upload_to_user_integration.py +++ b/app/tasks/upload_to_user_integration.py @@ -571,6 +571,113 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t return {"status": "Completed", "rclone_dest": dest} +def _upload_sharepoint(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]: + """Upload *file_path* to SharePoint using per-user MSAL credentials.""" + import urllib.parse + + import msal + import requests as _requests + + client_id = creds.get("client_id") or "" + client_secret = creds.get("client_secret") or "" + refresh_token = creds.get("refresh_token") or "" + tenant = cfg.get("tenant_id") or "common" + site_url = cfg.get("site_url") or "" + library_name = cfg.get("document_library") or "Documents" + folder_path = cfg.get("folder_path") or "" + + if not (client_id and client_secret): + raise ValueError("SharePoint integration is missing client_id or client_secret in credentials") + if not site_url: + raise ValueError("SharePoint integration is missing site_url in config") + + scopes = ["https://graph.microsoft.com/.default"] + msal_app = msal.ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"https://login.microsoftonline.com/{tenant}", + ) + + if refresh_token: + token_resp = msal_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=scopes) + else: + token_resp = msal_app.acquire_token_for_client(scopes=scopes) + + if "access_token" not in token_resp: + raise ValueError(f"SharePoint token acquisition failed: {token_resp.get('error_description', 'unknown')}") + + access_token = token_resp["access_token"] + headers = {"Authorization": f"Bearer {access_token}"} + + # Resolve site ID + parsed = urllib.parse.urlparse(site_url) + hostname = parsed.hostname + site_path = parsed.path.rstrip("/") + if not hostname or not site_path: + raise ValueError(f"Invalid SharePoint site URL: {site_url}") + + resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{hostname}:{site_path}", headers=headers, timeout=30) + resp.raise_for_status() + site_id = resp.json()["id"] + + # Resolve drive ID + resp = _requests.get(f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives", headers=headers, timeout=30) + resp.raise_for_status() + drive_id = None + for drive in resp.json().get("value", []): + if drive.get("name", "").lower() == library_name.lower(): + drive_id = drive["id"] + break + if not drive_id: + raise RuntimeError(f"Document library '{library_name}' not found on SharePoint site") + + filename = os.path.basename(file_path) + + # Build upload-session URL + base_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}" + if folder_path: + folder_path = folder_path.strip("/") + encoded_path = "/".join(urllib.parse.quote(p) for p in folder_path.split("/")) + encoded_file = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_path}/{encoded_file}:/createUploadSession" + else: + encoded_file = urllib.parse.quote(filename) + item_path = f"/root:/{encoded_file}:/createUploadSession" + + session_url = f"{base_url}{item_path}" + session_headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + resp = _requests.post( + session_url, + headers=session_headers, + json={"item": {"@microsoft.graph.conflictBehavior": "replace"}}, + timeout=30, + ) + resp.raise_for_status() + upload_url = resp.json()["uploadUrl"] + + file_size = os.path.getsize(file_path) + chunk_size = 10 * 1024 * 1024 + with open(file_path, "rb") as fh: + chunk_num = 0 + while True: + chunk = fh.read(chunk_size) + if not chunk: + break + start = chunk_num * chunk_size + end = start + len(chunk) - 1 + upload_headers = { + "Content-Length": str(len(chunk)), + "Content-Range": f"bytes {start}-{end}/{file_size}", + } + upload_resp = _requests.put(upload_url, headers=upload_headers, data=chunk, timeout=120) + if upload_resp.status_code not in (201, 202): + raise RuntimeError(f"SharePoint chunk upload failed: {upload_resp.status_code}") + chunk_num += 1 + + logger.info("[%s] SharePoint upload complete: %s/%s", task_id, folder_path, filename) + return {"status": "Completed", "sharepoint_folder": folder_path, "filename": filename} + + def _upload_icloud(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]: """Upload *file_path* to iCloud Drive using per-user credentials. @@ -615,6 +722,7 @@ _UPLOAD_HANDLERS = { IntegrationType.PAPERLESS: _upload_paperless, IntegrationType.EMAIL: _upload_email, IntegrationType.RCLONE: _upload_rclone, + IntegrationType.SHAREPOINT: _upload_sharepoint, IntegrationType.ICLOUD: _upload_icloud, } diff --git a/app/utils/config_validator/providers.py b/app/utils/config_validator/providers.py index 9a1e7069..91383438 100644 --- a/app/utils/config_validator/providers.py +++ b/app/utils/config_validator/providers.py @@ -296,6 +296,28 @@ def get_provider_status() -> dict[str, dict[str, object]]: }, } + # Check SharePoint configuration + providers["SharePoint"] = { + "name": "SharePoint", + "icon": "fa-brands fa-microsoft", + "configured": bool( + getattr(settings, "sharepoint_client_id", None) + and getattr(settings, "sharepoint_client_secret", None) + and getattr(settings, "sharepoint_site_url", None) + ), + "enabled": True, + "description": "Store documents in Microsoft SharePoint Online", + "details": { + "client_id": getattr(settings, "sharepoint_client_id", "Not set"), + "client_secret": mask_sensitive_value(getattr(settings, "sharepoint_client_secret", None)), + "tenant_id": getattr(settings, "sharepoint_tenant_id", "Not set"), + "refresh_token": mask_sensitive_value(getattr(settings, "sharepoint_refresh_token", None)), + "site_url": getattr(settings, "sharepoint_site_url", "Not set"), + "document_library": getattr(settings, "sharepoint_document_library", "Not set"), + "folder_path": getattr(settings, "sharepoint_folder_path", "Not set"), + }, + } + # Check S3 configuration providers["S3 Storage"] = { "name": "S3 Storage", diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 577fed6c..41e870ec 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -899,6 +899,63 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + # Storage Providers - SharePoint + "sharepoint_client_id": { + "category": "Storage Providers", + "description": "SharePoint Azure AD application (client) ID", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_client_secret": { + "category": "Storage Providers", + "description": "SharePoint Azure AD client secret", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": False, + }, + "sharepoint_tenant_id": { + "category": "Storage Providers", + "description": "SharePoint Azure AD tenant ID (use 'common' for multi-tenant apps)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_refresh_token": { + "category": "Storage Providers", + "description": "SharePoint OAuth refresh token", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": False, + }, + "sharepoint_site_url": { + "category": "Storage Providers", + "description": "SharePoint site URL (e.g. https://tenant.sharepoint.com/sites/sitename)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_document_library": { + "category": "Storage Providers", + "description": "SharePoint document library name (default: 'Documents')", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "sharepoint_folder_path": { + "category": "Storage Providers", + "description": "Subfolder path inside the SharePoint document library", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Storage Providers - WebDAV "webdav_enabled": { "category": "Storage Providers", @@ -1969,14 +2026,26 @@ SETTING_METADATA = { "category": "Backup", "description": ( "Storage provider for remote backup copies. " - "Accepted values: s3, dropbox, google_drive, onedrive, nextcloud, webdav, ftp, sftp, email. " + "Accepted values: s3, dropbox, google_drive, onedrive, sharepoint, nextcloud, webdav, ftp, sftp, email. " "Leave empty to keep backups local only." ), "type": "string", "sensitive": False, "required": False, "restart_required": False, - "options": ["", "s3", "dropbox", "google_drive", "onedrive", "nextcloud", "webdav", "ftp", "sftp", "email"], + "options": [ + "", + "s3", + "dropbox", + "google_drive", + "onedrive", + "sharepoint", + "nextcloud", + "webdav", + "ftp", + "sftp", + "email", + ], }, "backup_remote_folder": { "category": "Backup", diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 1874abfb..4e68e91a 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -1289,6 +1289,20 @@ For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveS For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md). +### SharePoint Online + +| **Variable** | **Description** | +|---------------------------------|-------------------------------------------------------| +| `SHAREPOINT_CLIENT_ID` | Azure AD application client ID | +| `SHAREPOINT_CLIENT_SECRET` | Azure AD application client secret | +| `SHAREPOINT_TENANT_ID` | Azure AD tenant ID (use "common" for multi-tenant apps) | +| `SHAREPOINT_REFRESH_TOKEN` | OAuth 2.0 refresh token | +| `SHAREPOINT_SITE_URL` | SharePoint site URL (e.g. `https://tenant.sharepoint.com/sites/sitename`) | +| `SHAREPOINT_DOCUMENT_LIBRARY` | Document library name (default: `Documents`) | +| `SHAREPOINT_FOLDER_PATH` | Subfolder path inside the document library | + +SharePoint uses the same Microsoft Graph API as OneDrive. See the [OneDrive Setup Guide](OneDriveSetup.md) for Azure AD app registration instructions — the same app registration can be reused for SharePoint with the `Sites.ReadWrite.All` permission. + ### Amazon S3 | **Variable** | **Description** | @@ -1628,6 +1642,7 @@ For example: | S3 | `docs/uploads/` | `docs/uploads/pdfa/` | | Nextcloud | `/Files` | `/Files/pdfa` | | OneDrive | `Documents/Uploads` | `Documents/Uploads/pdfa` | +| SharePoint | `Uploads` | `Uploads/pdfa` | | Google Drive | *(folder ID)* | `GOOGLE_DRIVE_PDFA_FOLDER_ID` | Set `PDFA_UPLOAD_FOLDER` to an empty string to upload PDF/A files into the @@ -1834,6 +1849,15 @@ ONEDRIVE_TENANT_ID=common ONEDRIVE_REFRESH_TOKEN=your_refresh_token ONEDRIVE_FOLDER_PATH=Documents/Uploads +# SharePoint Online +SHAREPOINT_CLIENT_ID=12345678-1234-1234-1234-123456789012 +SHAREPOINT_CLIENT_SECRET=your_client_secret +SHAREPOINT_TENANT_ID=your-tenant-id +SHAREPOINT_REFRESH_TOKEN=your_refresh_token +SHAREPOINT_SITE_URL=https://tenant.sharepoint.com/sites/sitename +SHAREPOINT_DOCUMENT_LIBRARY=Documents +SHAREPOINT_FOLDER_PATH=Uploads + # Amazon S3 AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY diff --git a/docs/CredentialRotationGuide.md b/docs/CredentialRotationGuide.md index bb911752..3dbe3c20 100644 --- a/docs/CredentialRotationGuide.md +++ b/docs/CredentialRotationGuide.md @@ -11,7 +11,7 @@ Credentials fall into two categories: | Category | Examples | |---|---| | **API keys** | OpenAI API key, Azure AI key, Paperless-ngx API token, AWS access keys | -| **OAuth tokens / secrets** | Dropbox, Google Drive, OneDrive, Authentik client secrets and refresh tokens | +| **OAuth tokens / secrets** | Dropbox, Google Drive, OneDrive, SharePoint, Authentik client secrets and refresh tokens | | **Passwords** | Admin password, Nextcloud, Email (SMTP), IMAP, FTP, SFTP, WebDAV | | **Private keys** | SFTP private key and passphrase | @@ -119,6 +119,15 @@ For service-account credentials (`google_drive_credentials_json`): 4. Re-authorize via the OAuth flow to get a fresh `onedrive_refresh_token`. 5. Delete the old client secret in Azure. +### SharePoint (Microsoft OAuth) + +1. SharePoint uses the same Azure AD app registration as OneDrive. +2. In **Azure App Registrations**, navigate to **Certificates & secrets** for your app. +3. Add a new client secret. +4. Update `sharepoint_client_secret` in DocuElevate. +5. Re-authorize via the OAuth flow to get a fresh `sharepoint_refresh_token`. +6. Delete the old client secret in Azure. + ### Authentik (OIDC) 1. In your Authentik admin panel, navigate to the DocuElevate application and regenerate the client secret. diff --git a/docs/DeploymentGuide.md b/docs/DeploymentGuide.md index cb66bfa9..94dc634f 100644 --- a/docs/DeploymentGuide.md +++ b/docs/DeploymentGuide.md @@ -19,7 +19,7 @@ This guide covers all supported deployment methods for DocuElevate. - Access to required external services (if configured): - AI provider API key (OpenAI, Anthropic, Gemini, or other configured provider) - Azure Document Intelligence - - Dropbox, Google Drive, OneDrive, S3, or other storage APIs + - Dropbox, Google Drive, OneDrive, SharePoint, S3, or other storage APIs - SMTP / IMAP server (for email processing) - Notification services (Discord, Telegram, etc.) diff --git a/docs/README.md b/docs/README.md index a3a6a96c..bc210343 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ Welcome to the DocuElevate documentation. This directory contains comprehensive - [Google Drive Setup](GoogleDriveSetup.md) - How to set up Google Drive integration - [Dropbox Setup](DropboxSetup.md) - How to set up Dropbox integration - [OneDrive Setup](OneDriveSetup.md) - How to set up Microsoft OneDrive/Graph integration + - [SharePoint Setup](SharePointSetup.md) - How to set up Microsoft SharePoint Online integration - [Amazon S3 Setup](AmazonS3Setup.md) - How to set up Amazon S3 integration - [Authentication Setup](AuthenticationSetup.md) - How to set up user authentication - [Notifications Setup](NotificationsSetup.md) - How to set up system notifications diff --git a/docs/SettingsManagement.md b/docs/SettingsManagement.md index b64c1ef4..288c261e 100644 --- a/docs/SettingsManagement.md +++ b/docs/SettingsManagement.md @@ -28,7 +28,7 @@ Settings are organized into logical categories for easy navigation: - **Authentication**: Login settings, session secrets, OAuth configuration, admin group - **AI Services**: AI provider selection, model configuration, embeddings, and credentials (OpenAI, Azure, Anthropic, Gemini, Ollama, OpenRouter, Portkey, LiteLLM) - **OCR Engines**: OCR provider selection and configuration (Tesseract, EasyOCR, Mistral, Google DocAI, AWS Textract) -- **Storage Providers**: Dropbox, Google Drive, OneDrive, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless +- **Storage Providers**: Dropbox, Google Drive, OneDrive, SharePoint, S3, FTP, SFTP, WebDAV, Nextcloud, Paperless - **Email**: SMTP configuration for sending emails - **IMAP**: Email ingestion configuration (supports two mailbox accounts) - **Monitoring**: Uptime Kuma integration diff --git a/docs/SharePointSetup.md b/docs/SharePointSetup.md new file mode 100644 index 00000000..560f1671 --- /dev/null +++ b/docs/SharePointSetup.md @@ -0,0 +1,185 @@ +# Setting up SharePoint Integration + +This guide explains how to set up the Microsoft SharePoint Online integration for DocuElevate. + +## Required Configuration Parameters + +| **Variable** | **Description** | +|---------------------------------|-------------------------------------------------------| +| `SHAREPOINT_CLIENT_ID` | Azure AD application client ID | +| `SHAREPOINT_CLIENT_SECRET` | Azure AD application client secret | +| `SHAREPOINT_TENANT_ID` | Azure AD tenant ID (use "common" for multi-tenant apps) | +| `SHAREPOINT_REFRESH_TOKEN` | OAuth 2.0 refresh token | +| `SHAREPOINT_SITE_URL` | SharePoint site URL (e.g. `https://tenant.sharepoint.com/sites/sitename`) | +| `SHAREPOINT_DOCUMENT_LIBRARY` | Document library name (default: `Documents`) | +| `SHAREPOINT_FOLDER_PATH` | Subfolder path inside the document library | + +For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md). + +## Overview + +SharePoint Online integration uses the same Microsoft Graph API as OneDrive. The key difference is that SharePoint targets a **site-specific document library** rather than a personal OneDrive. Documents are uploaded via chunked upload sessions for reliability with large files. + +> **Tip:** If you already have an Azure AD app registration for OneDrive, you can reuse it for SharePoint — just add the `Sites.ReadWrite.All` permission. + +## Setup Steps + +### 1. Register an application in Azure Active Directory + +If you don't already have an app registration (e.g. from OneDrive setup): + +1. Go to the [Azure Portal](https://portal.azure.com/) +2. Navigate to **Azure Active Directory** > **App registrations** +3. Click **New registration** +4. Enter a name for your application (e.g., "DocuElevate") +5. For **Supported account types**, select: + - **Single tenant**: "Accounts in this organizational directory only" + - **Multi-tenant**: "Accounts in any organizational directory" +6. For **Redirect URI**, select "Web" and enter your callback URL (e.g., `https://your-domain.com/onedrive-callback`) +7. Click **Register** + +### 2. Get Application (client) ID + +1. After registration, note the **Application (client) ID** from the overview page +2. Set this value as `SHAREPOINT_CLIENT_ID` + +### 3. Create a client secret + +1. In your application page, go to **Certificates & secrets** +2. Under **Client secrets**, click **New client secret** +3. Add a description and select an expiration period +4. Click **Add** and immediately copy the secret value (it will only be shown once) +5. Set this value as `SHAREPOINT_CLIENT_SECRET` + +### 4. Configure API permissions + +1. In your application page, go to **API permissions** +2. Click **Add a permission** +3. Select **Microsoft Graph** +4. For **delegated permissions** (user-context access), add: + - `Sites.ReadWrite.All` — Read and write items in all site collections + - `offline_access` — Required for refresh tokens +5. For **application permissions** (app-only access without a user), add: + - `Sites.ReadWrite.All` — Read and write items in all site collections +6. Click **Add permissions** +7. Click **Grant admin consent** (requires admin privileges) + +> **Important:** SharePoint access requires `Sites.ReadWrite.All` rather than the `Files.ReadWrite` permission used by OneDrive. + +### 5. Get your Tenant ID + +1. In the Azure Portal, find your **Tenant ID** (also called "Directory ID") +2. It is on the **Azure Active Directory** overview page +3. Set this value as `SHAREPOINT_TENANT_ID` + +### 6. Generate a Refresh Token + +#### Using the OneDrive Auth Wizard + +The SharePoint integration reuses the same MSAL token flow as OneDrive: + +1. Navigate to `/onedrive-setup` +2. Enter your SharePoint Client ID and Tenant ID +3. Click **Start Authentication Flow** and follow the prompts +4. Copy the generated refresh token and set it as `SHAREPOINT_REFRESH_TOKEN` + +#### Manual Method + +1. Open the following URL in your browser (replace placeholders): + ``` + https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize?client_id=YOUR_CLIENT_ID&response_type=code&redirect_uri=YOUR_REDIRECT_URI&response_mode=query&scope=https://graph.microsoft.com/.default offline_access&prompt=consent + ``` +2. Sign in with your Microsoft work account +3. After authentication, copy the `code` parameter from the redirect URL +4. Exchange the code for tokens: + ```bash + curl -X POST https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "client_id=YOUR_CLIENT_ID&scope=https://graph.microsoft.com/.default offline_access&code=YOUR_AUTH_CODE&redirect_uri=YOUR_REDIRECT_URI&grant_type=authorization_code&client_secret=YOUR_CLIENT_SECRET" + ``` +5. From the response JSON, copy the `refresh_token` value +6. Set this as `SHAREPOINT_REFRESH_TOKEN` + +### 7. Find your SharePoint Site URL + +Your SharePoint site URL follows the pattern: +``` +https://YOUR-TENANT.sharepoint.com/sites/SITE-NAME +``` + +For example: +- `https://contoso.sharepoint.com/sites/documents` +- `https://contoso.sharepoint.com/sites/engineering-team` + +Set this as `SHAREPOINT_SITE_URL`. + +### 8. Choose your Document Library + +Each SharePoint site has one or more document libraries. The default library is usually called `Documents` (or `Shared Documents`). You can find your library names by navigating to your SharePoint site in a browser and looking at the left sidebar. + +Set the library name as `SHAREPOINT_DOCUMENT_LIBRARY` (default: `Documents`). + +### 9. Set the Upload Folder (Optional) + +If you want documents to be uploaded into a subfolder inside the library, set `SHAREPOINT_FOLDER_PATH`. For example, `Uploads` or `DocuElevate/Processed`. + +## App-Only Access (No User Token) + +For fully automated scenarios without user interaction: + +1. Add **Application permissions** (not Delegated) for `Sites.ReadWrite.All` +2. Grant admin consent +3. Set `SHAREPOINT_TENANT_ID` to your organization's tenant ID +4. Leave `SHAREPOINT_REFRESH_TOKEN` empty — the app will use the client credentials flow + +> **Note:** Client credentials flow requires a specific tenant ID (not "common"). + +## Configuration Examples + +**With Refresh Token (Delegated Permissions):** +```dotenv +SHAREPOINT_CLIENT_ID=12345678-1234-1234-1234-123456789012 +SHAREPOINT_CLIENT_SECRET=your_client_secret +SHAREPOINT_TENANT_ID=87654321-4321-4321-4321-210987654321 +SHAREPOINT_REFRESH_TOKEN=your_refresh_token +SHAREPOINT_SITE_URL=https://contoso.sharepoint.com/sites/documents +SHAREPOINT_DOCUMENT_LIBRARY=Documents +SHAREPOINT_FOLDER_PATH=Uploads +``` + +**App-Only Access (Application Permissions):** +```dotenv +SHAREPOINT_CLIENT_ID=12345678-1234-1234-1234-123456789012 +SHAREPOINT_CLIENT_SECRET=your_client_secret +SHAREPOINT_TENANT_ID=87654321-4321-4321-4321-210987654321 +# No refresh token needed for app-only access +SHAREPOINT_SITE_URL=https://contoso.sharepoint.com/sites/documents +SHAREPOINT_DOCUMENT_LIBRARY=Shared Documents +SHAREPOINT_FOLDER_PATH=DocuElevate/Processed +``` + +## Troubleshooting + +### "Failed to resolve SharePoint site" + +- Verify `SHAREPOINT_SITE_URL` is correct and accessible +- Ensure your app has `Sites.ReadWrite.All` permission with admin consent +- Check that the site exists and your account has access to it + +### "Document library not found" + +- Verify the library name in `SHAREPOINT_DOCUMENT_LIBRARY` matches exactly (case-insensitive) +- Navigate to your SharePoint site in a browser to confirm the library name +- Common names: `Documents`, `Shared Documents` + +### Token errors + +- If using a refresh token, try re-authorizing via the OAuth flow +- Ensure `offline_access` scope is included in your permissions +- For app-only access, verify the tenant ID is not set to "common" + +### Permission errors + +- Ensure an admin has granted consent for `Sites.ReadWrite.All` +- Verify the app registration has the correct permissions +- Check that the site's sharing settings allow API access diff --git a/docs/StorageArchitecture.md b/docs/StorageArchitecture.md index af055406..399b89a8 100644 --- a/docs/StorageArchitecture.md +++ b/docs/StorageArchitecture.md @@ -341,6 +341,7 @@ in task messages or logs. | `S3` | boto3 `upload_file`, per-user access key | | `GOOGLE_DRIVE` | Google Drive API v3, OAuth or service account | | `ONEDRIVE` | Microsoft Graph API, MSAL confidential-client | +| `SHAREPOINT` | Microsoft Graph API, site/drive resolution + chunked upload | | `WEBDAV` | HTTP PUT request, Basic Auth | | `NEXTCLOUD` | WebDAV (same as WEBDAV, Nextcloud-compatible path) | | `FTP` | ftplib FTPS (TLS preferred, plaintext configurable) | diff --git a/docs/UserGuide.md b/docs/UserGuide.md index 8d6b0a82..cd8afdd1 100644 --- a/docs/UserGuide.md +++ b/docs/UserGuide.md @@ -170,7 +170,7 @@ The **Integrations** page (`/integrations`) provides a unified view of all your - **S3** — bucket, region, access key, secret key - **WebDAV / Nextcloud** — URL, folder, username, password - **FTP / SFTP** — host, port, remote path, username, password - - **Dropbox / Google Drive / OneDrive** — folder path, with a link to the OAuth setup page + - **Dropbox / Google Drive / OneDrive / SharePoint** — folder path, with a link to the OAuth setup page - **Email Forward** — recipient email address - **Watch Folder** — source type (Local, S3, Dropbox, Google Drive, OneDrive, Nextcloud, WebDAV), per-type config fields, delete after processing toggle - **Paperless NGX** — URL and API token diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 51d5d48a..1d142a03 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -524,6 +524,7 @@ + diff --git a/frontend/templates/help.html b/frontend/templates/help.html index 277c654a..aa696092 100644 --- a/frontend/templates/help.html +++ b/frontend/templates/help.html @@ -33,7 +33,7 @@ "name": "Which cloud storage providers does DocuElevate support?", "acceptedAnswer": { "@type": "Answer", - "text": "DocuElevate integrates with Dropbox, Google Drive, OneDrive, Amazon S3, Nextcloud, WebDAV, SFTP, FTP, and Paperless-ngx." + "text": "DocuElevate integrates with Dropbox, Google Drive, OneDrive, SharePoint, Amazon S3, Nextcloud, WebDAV, SFTP, FTP, and Paperless-ngx." } }, { @@ -188,6 +188,14 @@ +
Upload to SharePoint Online document libraries via Graph API.
+