feat(api): add advanced filtering and saved searches

- Add date range (date_from/date_to), storage provider, and tags filters to GET /api/files - Add SavedSearch model and migration (005_add_saved_searches) - Add CRUD API endpoints for saved searches at /api/saved-searches - Update files.html template with new filter controls and saved searches UI - Update files view to pass new filter parameters to template - Add comprehensive tests for all new functionality (26 tests) Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
2026-03-01 11:22:07 +00:00
parent f45e9bfd70
commit e5a4c6c64a
9 changed files with 912 additions and 9 deletions
@@ -16,6 +16,7 @@ from app.api.onedrive import router as onedrive_router
 from app.api.openai import router as openai_router
 from app.api.process import router as process_router
 from app.api.queue import router as queue_router
+from app.api.saved_searches import router as saved_searches_router
 from app.api.search import router as search_router
 from app.api.settings import router as settings_router
 from app.api.url_upload import router as url_upload_router
@@ -44,3 +45,4 @@ router.include_router(settings_router)
 router.include_router(url_upload_router)
 router.include_router(search_router)
 router.include_router(queue_router)
+router.include_router(saved_searches_router)
@@ -6,6 +6,7 @@ import logging
 import mimetypes
 import os
 import uuid
+from datetime import datetime, timezone
 from typing import Annotated, List, Optional

 from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
@@ -15,7 +16,7 @@ from sqlalchemy.orm import Session
 from app.auth import require_login
 from app.config import settings
 from app.database import get_db
-from app.models import FileRecord, ProcessingLog
+from app.models import FileProcessingStep, FileRecord, ProcessingLog
 from app.tasks.convert_to_pdf import convert_to_pdf
 from app.tasks.process_document import process_document
 from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
@@ -54,6 +55,10 @@ def list_files_api(
    search: Optional[str] = Query(None, description="Search in filename"),
    mime_type: Optional[str] = Query(None, description="Filter by MIME type"),
    status: Optional[str] = Query(None, description="Filter by processing status"),
+    date_from: Optional[str] = Query(None, description="Filter files created on or after this date (ISO 8601)"),
+    date_to: Optional[str] = Query(None, description="Filter files created on or before this date (ISO 8601)"),
+    storage_provider: Optional[str] = Query(None, description="Filter by storage provider (e.g. dropbox, s3)"),
+    tags: Optional[str] = Query(None, description="Filter by tag (comma-separated for multiple, AND logic)"),
 ):
    """
    Returns a paginated JSON list of FileRecord entries with processing status.
@@ -67,6 +72,10 @@ def list_files_api(
    - search: Search in filename
    - mime_type: Filter by MIME type
    - status: Filter by processing status (pending, processing, completed, failed)
+    - date_from: Filter files created on or after this date (ISO 8601, e.g. 2026-01-01)
+    - date_to: Filter files created on or before this date (ISO 8601, e.g. 2026-12-31)
+    - storage_provider: Filter by storage provider (e.g. dropbox, s3, google_drive)
+    - tags: Filter by tags (comma-separated, AND logic)

    Example response:
    {
@@ -95,6 +104,47 @@ def list_files_api(
    if mime_type:
        query = query.filter(FileRecord.mime_type == mime_type)

+    # Apply date range filters
+    if date_from:
+        try:
+            dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc)
+            query = query.filter(FileRecord.created_at >= dt_from)
+        except ValueError:
+            raise HTTPException(
+                status_code=422,
+                detail="Invalid date_from format. Use ISO 8601 (e.g. 2026-01-01)",
+            )
+
+    if date_to:
+        try:
+            dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc)
+            query = query.filter(FileRecord.created_at <= dt_to)
+        except ValueError:
+            raise HTTPException(
+                status_code=422,
+                detail="Invalid date_to format. Use ISO 8601 (e.g. 2026-12-31)",
+            )
+
+    # Apply storage provider filter (files that have a successful upload_to_{provider} step)
+    if storage_provider:
+        step_name = f"upload_to_{storage_provider}"
+        uploaded_file_ids = (
+            db.query(FileProcessingStep.file_id)
+            .filter(
+                FileProcessingStep.step_name == step_name,
+                FileProcessingStep.status == "success",
+            )
+            .distinct()
+            .subquery()
+        )
+        query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id)))
+
+    # Apply tags filter (AND logic: all specified tags must be present in ai_metadata)
+    if tags:
+        tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()]
+        for tag in tag_list:
+            query = query.filter(FileRecord.ai_metadata.ilike(f"%{tag}%"))
+
    # Apply status filter (before pagination for correct counts)
    query = apply_status_filter(query, db, status)

@@ -0,0 +1,294 @@
+"""
+Saved searches API endpoints.
+
+Provides CRUD operations for user-defined saved search filters.
+Each user can save, list, update, and delete named filter combinations
+for quick access on the files page.
+"""
+
+import json
+import logging
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
+from sqlalchemy.orm import Session
+
+from app.auth import get_current_user, require_login
+from app.database import get_db
+from app.models import SavedSearch
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/saved-searches", tags=["saved-searches"])
+
+DbSession = Annotated[Session, Depends(get_db)]
+
+# Allowed filter keys that can be saved
+ALLOWED_FILTER_KEYS = frozenset(
+    {
+        "search",
+        "mime_type",
+        "status",
+        "date_from",
+        "date_to",
+        "storage_provider",
+        "tags",
+        "sort_by",
+        "sort_order",
+    }
+)
+
+# Maximum number of saved searches per user
+MAX_SAVED_SEARCHES_PER_USER = 50
+
+# Maximum length for saved search name
+MAX_NAME_LENGTH = 100
+
+
+def _get_user_id(request: Request) -> str:
+    """Extract user identifier from the session.
+
+    Returns the preferred_username, email, or 'anonymous' if auth is disabled.
+
+    Args:
+        request: The incoming HTTP request.
+
+    Returns:
+        A string identifying the current user.
+    """
+    user = get_current_user(request)
+    if user:
+        return user.get("preferred_username") or user.get("email") or user.get("name", "anonymous")
+    return "anonymous"
+
+
+def _validate_filters(filters: Any) -> dict:
+    """Validate and sanitize filter parameters.
+
+    Ensures only allowed filter keys are present and values are strings.
+
+    Args:
+        filters: The raw filter value from the client.
+
+    Returns:
+        A sanitized filter dictionary with only allowed keys.
+
+    Raises:
+        HTTPException: If filters is not a dict or contains invalid values.
+    """
+    if not isinstance(filters, dict):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="filters must be a JSON object",
+        )
+    sanitized = {}
+    for key, value in filters.items():
+        if key in ALLOWED_FILTER_KEYS and isinstance(value, str) and value.strip():
+            sanitized[key] = value.strip()
+    return sanitized
+
+
+def _serialize_saved_search(s: SavedSearch) -> dict:
+    """Serialize a SavedSearch model instance to a JSON-compatible dict.
+
+    Args:
+        s: The SavedSearch model instance.
+
+    Returns:
+        A dictionary with id, name, filters, created_at, and updated_at.
+    """
+    return {
+        "id": s.id,
+        "name": s.name,
+        "filters": json.loads(s.filters),
+        "created_at": s.created_at.isoformat() if s.created_at else None,
+        "updated_at": s.updated_at.isoformat() if s.updated_at else None,
+    }
+
+
+@router.get("")
+@require_login
+def list_saved_searches(request: Request, db: DbSession):
+    """List all saved searches for the current user.
+
+    Returns:
+        A list of saved search objects with id, name, filters, and timestamps.
+    """
+    user_id = _get_user_id(request)
+    searches = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).order_by(SavedSearch.name).all()
+    return [_serialize_saved_search(s) for s in searches]
+
+
+@router.post("", status_code=status.HTTP_201_CREATED)
+@require_login
+def create_saved_search(
+    request: Request,
+    db: DbSession,
+    name: str = Body(..., embed=True),
+    filters: dict = Body(..., embed=True),
+):
+    """Create a new saved search for the current user.
+
+    Request body (JSON):
+        name: Display name for the saved search (required, max 100 chars)
+        filters: Dictionary of filter parameters (required)
+
+    Returns:
+        The created saved search object.
+
+    Raises:
+        HTTPException 422: If name or filters are invalid.
+        HTTPException 409: If a saved search with the same name already exists.
+    """
+    user_id = _get_user_id(request)
+
+    name = name.strip() if isinstance(name, str) else ""
+    if not name or len(name) > MAX_NAME_LENGTH:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"name is required and must be at most {MAX_NAME_LENGTH} characters",
+        )
+
+    sanitized_filters = _validate_filters(filters)
+    if not sanitized_filters:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="At least one filter parameter is required",
+        )
+
+    # Check user limit
+    count = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).count()
+    if count >= MAX_SAVED_SEARCHES_PER_USER:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Maximum of {MAX_SAVED_SEARCHES_PER_USER} saved searches reached",
+        )
+
+    # Check for duplicate name
+    existing = db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == name).first()
+    if existing:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"A saved search named '{name}' already exists",
+        )
+
+    saved_search = SavedSearch(
+        user_id=user_id,
+        name=name,
+        filters=json.dumps(sanitized_filters),
+    )
+
+    try:
+        db.add(saved_search)
+        db.commit()
+        db.refresh(saved_search)
+    except Exception:
+        db.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to save search",
+        )
+
+    logger.info(f"Saved search created: user={user_id}, name={name!r}")
+    return _serialize_saved_search(saved_search)
+
+
+@router.put("/{search_id}")
+@require_login
+def update_saved_search(
+    search_id: int,
+    request: Request,
+    db: DbSession,
+    name: str | None = Body(None, embed=True),
+    filters: dict | None = Body(None, embed=True),
+):
+    """Update an existing saved search.
+
+    Path Parameters:
+        search_id: The ID of the saved search to update.
+
+    Request body (JSON):
+        name: New display name (optional)
+        filters: New filter parameters (optional)
+
+    Returns:
+        The updated saved search object.
+
+    Raises:
+        HTTPException 404: If the saved search is not found.
+        HTTPException 409: If the new name conflicts with an existing saved search.
+    """
+    user_id = _get_user_id(request)
+    saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
+    if not saved_search:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
+
+    if name is not None:
+        new_name = name.strip() if isinstance(name, str) else ""
+        if not new_name or len(new_name) > MAX_NAME_LENGTH:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"name must be non-empty and at most {MAX_NAME_LENGTH} characters",
+            )
+        # Check for name conflict
+        if new_name != saved_search.name:
+            existing = (
+                db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == new_name).first()
+            )
+            if existing:
+                raise HTTPException(
+                    status_code=status.HTTP_409_CONFLICT,
+                    detail=f"A saved search named '{new_name}' already exists",
+                )
+        saved_search.name = new_name
+
+    if filters is not None:
+        sanitized_filters = _validate_filters(filters)
+        if not sanitized_filters:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail="At least one filter parameter is required",
+            )
+        saved_search.filters = json.dumps(sanitized_filters)
+
+    try:
+        db.commit()
+        db.refresh(saved_search)
+    except Exception:
+        db.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to update saved search",
+        )
+
+    logger.info(f"Saved search updated: id={search_id}, user={user_id}")
+    return _serialize_saved_search(saved_search)
+
+
+@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
+@require_login
+def delete_saved_search(search_id: int, request: Request, db: DbSession):
+    """Delete a saved search.
+
+    Path Parameters:
+        search_id: The ID of the saved search to delete.
+
+    Raises:
+        HTTPException 404: If the saved search is not found.
+    """
+    user_id = _get_user_id(request)
+    saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
+    if not saved_search:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
+
+    try:
+        db.delete(saved_search)
+        db.commit()
+    except Exception:
+        db.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to delete saved search",
+        )
+
+    logger.info(f"Saved search deleted: id={search_id}, user={user_id}")
@@ -125,3 +125,18 @@ class SettingsAuditLog(Base):
    changed_by = Column(String, nullable=False)  # Username of the admin who made the change
    changed_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
    action = Column(String, nullable=False)  # "update" or "delete"
+
+
+class SavedSearch(Base):
+    """User-defined saved search filters for quick access to frequently used filter combinations."""
+
+    __tablename__ = "saved_searches"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String, nullable=False, index=True)  # Username or user identifier from session
+    name = Column(String, nullable=False)  # Human-readable name for the saved search
+    filters = Column(Text, nullable=False)  # JSON-encoded filter parameters
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+    __table_args__ = (UniqueConstraint("user_id", "name", name="unique_user_search_name"),)
@@ -2,9 +2,10 @@
 File management views for displaying and managing files.
 """

+from datetime import datetime, timezone
 from typing import Optional

-from fastapi import Depends, Query, Request
+from fastapi import Depends, HTTPException, Query, Request
 from sqlalchemy.orm import Session

 from app.utils.file_queries import apply_status_filter
@@ -29,6 +30,10 @@ def files_page(
    search: Optional[str] = Query(None),
    mime_type: Optional[str] = Query(None),
    status: Optional[str] = Query(None),
+    date_from: Optional[str] = Query(None),
+    date_to: Optional[str] = Query(None),
+    storage_provider: Optional[str] = Query(None),
+    tags: Optional[str] = Query(None),
 ):
    """
    Return the 'files.html' template with server-side pagination, sorting, and filtering
@@ -39,7 +44,7 @@ def files_page(
        # Import the model here to avoid circular imports
        from sqlalchemy import asc, desc

-        from app.models import FileRecord
+        from app.models import FileProcessingStep, FileRecord

        # Start with base query
        query = db.query(FileRecord)
@@ -52,6 +57,41 @@ def files_page(
        if mime_type:
            query = query.filter(FileRecord.mime_type == mime_type)

+        # Apply date range filters
+        if date_from:
+            try:
+                dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc)
+                query = query.filter(FileRecord.created_at >= dt_from)
+            except ValueError:
+                pass  # Silently ignore invalid dates in view
+
+        if date_to:
+            try:
+                dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc)
+                query = query.filter(FileRecord.created_at <= dt_to)
+            except ValueError:
+                pass
+
+        # Apply storage provider filter
+        if storage_provider:
+            step_name = f"upload_to_{storage_provider}"
+            uploaded_file_ids = (
+                db.query(FileProcessingStep.file_id)
+                .filter(
+                    FileProcessingStep.step_name == step_name,
+                    FileProcessingStep.status == "success",
+                )
+                .distinct()
+                .subquery()
+            )
+            query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id)))
+
+        # Apply tags filter (AND logic)
+        if tags:
+            tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()]
+            for tag in tag_list:
+                query = query.filter(FileRecord.ai_metadata.ilike(f"%{tag}%"))
+
        # Apply status filter (before pagination for correct counts)
        query = apply_status_filter(query, db, status)

@@ -112,6 +152,10 @@ def files_page(
                "search": search or "",
                "mime_type": mime_type or "",
                "status": status or "",
+                "date_from": date_from or "",
+                "date_to": date_to or "",
+                "storage_provider": storage_provider or "",
+                "tags": tags or "",
                "mime_types": mime_types,
                "upload_concurrency": settings.upload_concurrency,
                "upload_queue_delay_ms": settings.upload_queue_delay_ms,
@@ -524,7 +568,7 @@ def preview_original_file(request: Request, file_id: int, db: Session = Depends(
    """
    import os

-    from fastapi import HTTPException, status
+    from fastapi import status
    from fastapi.responses import FileResponse

    from app.models import FileRecord
@@ -551,7 +595,7 @@ def preview_processed_file(request: Request, file_id: int, db: Session = Depends
    """
    import os

-    from fastapi import HTTPException, status
+    from fastapi import status
    from fastapi.responses import FileResponse

    from app.models import FileRecord
@@ -578,7 +622,7 @@ def get_original_text(request: Request, file_id: int, db: Session = Depends(get_
    """
    import os

-    from fastapi import HTTPException, status
+    from fastapi import status
    from fastapi.responses import JSONResponse

    from app.models import FileRecord
@@ -618,7 +662,7 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
    """
    import os

-    from fastapi import HTTPException, status
+    from fastapi import status
    from fastapi.responses import JSONResponse

    from app.models import FileRecord