feat(api): add advanced filtering and saved searches
- Add date range (date_from/date_to), storage provider, and tags filters to GET /api/files - Add SavedSearch model and migration (005_add_saved_searches) - Add CRUD API endpoints for saved searches at /api/saved-searches - Update files.html template with new filter controls and saved searches UI - Update files view to pass new filter parameters to template - Add comprehensive tests for all new functionality (26 tests) Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from app.api.onedrive import router as onedrive_router
|
||||
from app.api.openai import router as openai_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.queue import router as queue_router
|
||||
from app.api.saved_searches import router as saved_searches_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
@@ -44,3 +45,4 @@ router.include_router(settings_router)
|
||||
router.include_router(url_upload_router)
|
||||
router.include_router(search_router)
|
||||
router.include_router(queue_router)
|
||||
router.include_router(saved_searches_router)
|
||||
|
||||
+51
-1
@@ -6,6 +6,7 @@ import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
|
||||
@@ -15,7 +16,7 @@ from sqlalchemy.orm import Session
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord, ProcessingLog
|
||||
from app.models import FileProcessingStep, FileRecord, ProcessingLog
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
|
||||
@@ -54,6 +55,10 @@ def list_files_api(
|
||||
search: Optional[str] = Query(None, description="Search in filename"),
|
||||
mime_type: Optional[str] = Query(None, description="Filter by MIME type"),
|
||||
status: Optional[str] = Query(None, description="Filter by processing status"),
|
||||
date_from: Optional[str] = Query(None, description="Filter files created on or after this date (ISO 8601)"),
|
||||
date_to: Optional[str] = Query(None, description="Filter files created on or before this date (ISO 8601)"),
|
||||
storage_provider: Optional[str] = Query(None, description="Filter by storage provider (e.g. dropbox, s3)"),
|
||||
tags: Optional[str] = Query(None, description="Filter by tag (comma-separated for multiple, AND logic)"),
|
||||
):
|
||||
"""
|
||||
Returns a paginated JSON list of FileRecord entries with processing status.
|
||||
@@ -67,6 +72,10 @@ def list_files_api(
|
||||
- search: Search in filename
|
||||
- mime_type: Filter by MIME type
|
||||
- status: Filter by processing status (pending, processing, completed, failed)
|
||||
- date_from: Filter files created on or after this date (ISO 8601, e.g. 2026-01-01)
|
||||
- date_to: Filter files created on or before this date (ISO 8601, e.g. 2026-12-31)
|
||||
- storage_provider: Filter by storage provider (e.g. dropbox, s3, google_drive)
|
||||
- tags: Filter by tags (comma-separated, AND logic)
|
||||
|
||||
Example response:
|
||||
{
|
||||
@@ -95,6 +104,47 @@ def list_files_api(
|
||||
if mime_type:
|
||||
query = query.filter(FileRecord.mime_type == mime_type)
|
||||
|
||||
# Apply date range filters
|
||||
if date_from:
|
||||
try:
|
||||
dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc)
|
||||
query = query.filter(FileRecord.created_at >= dt_from)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="Invalid date_from format. Use ISO 8601 (e.g. 2026-01-01)",
|
||||
)
|
||||
|
||||
if date_to:
|
||||
try:
|
||||
dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc)
|
||||
query = query.filter(FileRecord.created_at <= dt_to)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="Invalid date_to format. Use ISO 8601 (e.g. 2026-12-31)",
|
||||
)
|
||||
|
||||
# Apply storage provider filter (files that have a successful upload_to_{provider} step)
|
||||
if storage_provider:
|
||||
step_name = f"upload_to_{storage_provider}"
|
||||
uploaded_file_ids = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(
|
||||
FileProcessingStep.step_name == step_name,
|
||||
FileProcessingStep.status == "success",
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id)))
|
||||
|
||||
# Apply tags filter (AND logic: all specified tags must be present in ai_metadata)
|
||||
if tags:
|
||||
tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()]
|
||||
for tag in tag_list:
|
||||
query = query.filter(FileRecord.ai_metadata.ilike(f"%{tag}%"))
|
||||
|
||||
# Apply status filter (before pagination for correct counts)
|
||||
query = apply_status_filter(query, db, status)
|
||||
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
Saved searches API endpoints.
|
||||
|
||||
Provides CRUD operations for user-defined saved search filters.
|
||||
Each user can save, list, update, and delete named filter combinations
|
||||
for quick access on the files page.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import get_current_user, require_login
|
||||
from app.database import get_db
|
||||
from app.models import SavedSearch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/saved-searches", tags=["saved-searches"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# Allowed filter keys that can be saved
|
||||
ALLOWED_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"search",
|
||||
"mime_type",
|
||||
"status",
|
||||
"date_from",
|
||||
"date_to",
|
||||
"storage_provider",
|
||||
"tags",
|
||||
"sort_by",
|
||||
"sort_order",
|
||||
}
|
||||
)
|
||||
|
||||
# Maximum number of saved searches per user
|
||||
MAX_SAVED_SEARCHES_PER_USER = 50
|
||||
|
||||
# Maximum length for saved search name
|
||||
MAX_NAME_LENGTH = 100
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
"""Extract user identifier from the session.
|
||||
|
||||
Returns the preferred_username, email, or 'anonymous' if auth is disabled.
|
||||
|
||||
Args:
|
||||
request: The incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
A string identifying the current user.
|
||||
"""
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return user.get("preferred_username") or user.get("email") or user.get("name", "anonymous")
|
||||
return "anonymous"
|
||||
|
||||
|
||||
def _validate_filters(filters: Any) -> dict:
|
||||
"""Validate and sanitize filter parameters.
|
||||
|
||||
Ensures only allowed filter keys are present and values are strings.
|
||||
|
||||
Args:
|
||||
filters: The raw filter value from the client.
|
||||
|
||||
Returns:
|
||||
A sanitized filter dictionary with only allowed keys.
|
||||
|
||||
Raises:
|
||||
HTTPException: If filters is not a dict or contains invalid values.
|
||||
"""
|
||||
if not isinstance(filters, dict):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="filters must be a JSON object",
|
||||
)
|
||||
sanitized = {}
|
||||
for key, value in filters.items():
|
||||
if key in ALLOWED_FILTER_KEYS and isinstance(value, str) and value.strip():
|
||||
sanitized[key] = value.strip()
|
||||
return sanitized
|
||||
|
||||
|
||||
def _serialize_saved_search(s: SavedSearch) -> dict:
|
||||
"""Serialize a SavedSearch model instance to a JSON-compatible dict.
|
||||
|
||||
Args:
|
||||
s: The SavedSearch model instance.
|
||||
|
||||
Returns:
|
||||
A dictionary with id, name, filters, created_at, and updated_at.
|
||||
"""
|
||||
return {
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"filters": json.loads(s.filters),
|
||||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||||
"updated_at": s.updated_at.isoformat() if s.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
@require_login
|
||||
def list_saved_searches(request: Request, db: DbSession):
|
||||
"""List all saved searches for the current user.
|
||||
|
||||
Returns:
|
||||
A list of saved search objects with id, name, filters, and timestamps.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
searches = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).order_by(SavedSearch.name).all()
|
||||
return [_serialize_saved_search(s) for s in searches]
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
@require_login
|
||||
def create_saved_search(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
name: str = Body(..., embed=True),
|
||||
filters: dict = Body(..., embed=True),
|
||||
):
|
||||
"""Create a new saved search for the current user.
|
||||
|
||||
Request body (JSON):
|
||||
name: Display name for the saved search (required, max 100 chars)
|
||||
filters: Dictionary of filter parameters (required)
|
||||
|
||||
Returns:
|
||||
The created saved search object.
|
||||
|
||||
Raises:
|
||||
HTTPException 422: If name or filters are invalid.
|
||||
HTTPException 409: If a saved search with the same name already exists.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
name = name.strip() if isinstance(name, str) else ""
|
||||
if not name or len(name) > MAX_NAME_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"name is required and must be at most {MAX_NAME_LENGTH} characters",
|
||||
)
|
||||
|
||||
sanitized_filters = _validate_filters(filters)
|
||||
if not sanitized_filters:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="At least one filter parameter is required",
|
||||
)
|
||||
|
||||
# Check user limit
|
||||
count = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).count()
|
||||
if count >= MAX_SAVED_SEARCHES_PER_USER:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Maximum of {MAX_SAVED_SEARCHES_PER_USER} saved searches reached",
|
||||
)
|
||||
|
||||
# Check for duplicate name
|
||||
existing = db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == name).first()
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A saved search named '{name}' already exists",
|
||||
)
|
||||
|
||||
saved_search = SavedSearch(
|
||||
user_id=user_id,
|
||||
name=name,
|
||||
filters=json.dumps(sanitized_filters),
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(saved_search)
|
||||
db.commit()
|
||||
db.refresh(saved_search)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to save search",
|
||||
)
|
||||
|
||||
logger.info(f"Saved search created: user={user_id}, name={name!r}")
|
||||
return _serialize_saved_search(saved_search)
|
||||
|
||||
|
||||
@router.put("/{search_id}")
|
||||
@require_login
|
||||
def update_saved_search(
|
||||
search_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
name: str | None = Body(None, embed=True),
|
||||
filters: dict | None = Body(None, embed=True),
|
||||
):
|
||||
"""Update an existing saved search.
|
||||
|
||||
Path Parameters:
|
||||
search_id: The ID of the saved search to update.
|
||||
|
||||
Request body (JSON):
|
||||
name: New display name (optional)
|
||||
filters: New filter parameters (optional)
|
||||
|
||||
Returns:
|
||||
The updated saved search object.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the saved search is not found.
|
||||
HTTPException 409: If the new name conflicts with an existing saved search.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
|
||||
if not saved_search:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
|
||||
|
||||
if name is not None:
|
||||
new_name = name.strip() if isinstance(name, str) else ""
|
||||
if not new_name or len(new_name) > MAX_NAME_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"name must be non-empty and at most {MAX_NAME_LENGTH} characters",
|
||||
)
|
||||
# Check for name conflict
|
||||
if new_name != saved_search.name:
|
||||
existing = (
|
||||
db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == new_name).first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A saved search named '{new_name}' already exists",
|
||||
)
|
||||
saved_search.name = new_name
|
||||
|
||||
if filters is not None:
|
||||
sanitized_filters = _validate_filters(filters)
|
||||
if not sanitized_filters:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="At least one filter parameter is required",
|
||||
)
|
||||
saved_search.filters = json.dumps(sanitized_filters)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(saved_search)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update saved search",
|
||||
)
|
||||
|
||||
logger.info(f"Saved search updated: id={search_id}, user={user_id}")
|
||||
return _serialize_saved_search(saved_search)
|
||||
|
||||
|
||||
@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
def delete_saved_search(search_id: int, request: Request, db: DbSession):
|
||||
"""Delete a saved search.
|
||||
|
||||
Path Parameters:
|
||||
search_id: The ID of the saved search to delete.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the saved search is not found.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
|
||||
if not saved_search:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
|
||||
|
||||
try:
|
||||
db.delete(saved_search)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete saved search",
|
||||
)
|
||||
|
||||
logger.info(f"Saved search deleted: id={search_id}, user={user_id}")
|
||||
@@ -125,3 +125,18 @@ class SettingsAuditLog(Base):
|
||||
changed_by = Column(String, nullable=False) # Username of the admin who made the change
|
||||
changed_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
action = Column(String, nullable=False) # "update" or "delete"
|
||||
|
||||
|
||||
class SavedSearch(Base):
|
||||
"""User-defined saved search filters for quick access to frequently used filter combinations."""
|
||||
|
||||
__tablename__ = "saved_searches"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(String, nullable=False, index=True) # Username or user identifier from session
|
||||
name = Column(String, nullable=False) # Human-readable name for the saved search
|
||||
filters = Column(Text, nullable=False) # JSON-encoded filter parameters
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
__table_args__ = (UniqueConstraint("user_id", "name", name="unique_user_search_name"),)
|
||||
|
||||
+50
-6
@@ -2,9 +2,10 @@
|
||||
File management views for displaying and managing files.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, Query, Request
|
||||
from fastapi import Depends, HTTPException, Query, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.utils.file_queries import apply_status_filter
|
||||
@@ -29,6 +30,10 @@ def files_page(
|
||||
search: Optional[str] = Query(None),
|
||||
mime_type: Optional[str] = Query(None),
|
||||
status: Optional[str] = Query(None),
|
||||
date_from: Optional[str] = Query(None),
|
||||
date_to: Optional[str] = Query(None),
|
||||
storage_provider: Optional[str] = Query(None),
|
||||
tags: Optional[str] = Query(None),
|
||||
):
|
||||
"""
|
||||
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
||||
@@ -39,7 +44,7 @@ def files_page(
|
||||
# Import the model here to avoid circular imports
|
||||
from sqlalchemy import asc, desc
|
||||
|
||||
from app.models import FileRecord
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
# Start with base query
|
||||
query = db.query(FileRecord)
|
||||
@@ -52,6 +57,41 @@ def files_page(
|
||||
if mime_type:
|
||||
query = query.filter(FileRecord.mime_type == mime_type)
|
||||
|
||||
# Apply date range filters
|
||||
if date_from:
|
||||
try:
|
||||
dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc)
|
||||
query = query.filter(FileRecord.created_at >= dt_from)
|
||||
except ValueError:
|
||||
pass # Silently ignore invalid dates in view
|
||||
|
||||
if date_to:
|
||||
try:
|
||||
dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc)
|
||||
query = query.filter(FileRecord.created_at <= dt_to)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Apply storage provider filter
|
||||
if storage_provider:
|
||||
step_name = f"upload_to_{storage_provider}"
|
||||
uploaded_file_ids = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(
|
||||
FileProcessingStep.step_name == step_name,
|
||||
FileProcessingStep.status == "success",
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id)))
|
||||
|
||||
# Apply tags filter (AND logic)
|
||||
if tags:
|
||||
tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()]
|
||||
for tag in tag_list:
|
||||
query = query.filter(FileRecord.ai_metadata.ilike(f"%{tag}%"))
|
||||
|
||||
# Apply status filter (before pagination for correct counts)
|
||||
query = apply_status_filter(query, db, status)
|
||||
|
||||
@@ -112,6 +152,10 @@ def files_page(
|
||||
"search": search or "",
|
||||
"mime_type": mime_type or "",
|
||||
"status": status or "",
|
||||
"date_from": date_from or "",
|
||||
"date_to": date_to or "",
|
||||
"storage_provider": storage_provider or "",
|
||||
"tags": tags or "",
|
||||
"mime_types": mime_types,
|
||||
"upload_concurrency": settings.upload_concurrency,
|
||||
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
|
||||
@@ -524,7 +568,7 @@ def preview_original_file(request: Request, file_id: int, db: Session = Depends(
|
||||
"""
|
||||
import os
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from fastapi import status
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.models import FileRecord
|
||||
@@ -551,7 +595,7 @@ def preview_processed_file(request: Request, file_id: int, db: Session = Depends
|
||||
"""
|
||||
import os
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from fastapi import status
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.models import FileRecord
|
||||
@@ -578,7 +622,7 @@ def get_original_text(request: Request, file_id: int, db: Session = Depends(get_
|
||||
"""
|
||||
import os
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from fastapi import status
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.models import FileRecord
|
||||
@@ -618,7 +662,7 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
|
||||
"""
|
||||
import os
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from fastapi import status
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.models import FileRecord
|
||||
|
||||
@@ -475,7 +475,7 @@
|
||||
|
||||
<!-- Filters Section -->
|
||||
<div class="filters-section">
|
||||
<form method="get" action="/files" class="filter-group">
|
||||
<form method="get" action="/files" class="filter-group" role="search" aria-label="Filter files">
|
||||
<div class="filter-item">
|
||||
<label for="search">Search Filename</label>
|
||||
<input type="text" id="search" name="search" value="{{ search }}" placeholder="Enter filename...">
|
||||
@@ -503,6 +503,36 @@
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label for="date_from">Date From</label>
|
||||
<input type="date" id="date_from" name="date_from" value="{{ date_from }}" aria-label="Filter from date">
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label for="date_to">Date To</label>
|
||||
<input type="date" id="date_to" name="date_to" value="{{ date_to }}" aria-label="Filter to date">
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label for="storage_provider">Storage Provider</label>
|
||||
<select id="storage_provider" name="storage_provider">
|
||||
<option value="">All Providers</option>
|
||||
<option value="dropbox" {% if storage_provider == "dropbox" %}selected{% endif %}>Dropbox</option>
|
||||
<option value="google_drive" {% if storage_provider == "google_drive" %}selected{% endif %}>Google Drive</option>
|
||||
<option value="onedrive" {% if storage_provider == "onedrive" %}selected{% endif %}>OneDrive</option>
|
||||
<option value="s3" {% if storage_provider == "s3" %}selected{% endif %}>S3</option>
|
||||
<option value="nextcloud" {% if storage_provider == "nextcloud" %}selected{% endif %}>Nextcloud</option>
|
||||
<option value="webdav" {% if storage_provider == "webdav" %}selected{% endif %}>WebDAV</option>
|
||||
<option value="ftp" {% if storage_provider == "ftp" %}selected{% endif %}>FTP</option>
|
||||
<option value="sftp" {% if storage_provider == "sftp" %}selected{% endif %}>SFTP</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label for="tags">Tags</label>
|
||||
<input type="text" id="tags" name="tags" value="{{ tags }}" placeholder="e.g. invoice,amazon" aria-label="Filter by tags (comma-separated)">
|
||||
</div>
|
||||
|
||||
<div class="filter-item">
|
||||
<label> </label>
|
||||
<button type="submit">Apply Filters</button>
|
||||
@@ -520,6 +550,21 @@
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<!-- Saved Searches Section -->
|
||||
<div class="filters-section" style="margin-top: 0.5rem;">
|
||||
<div style="display: flex; align-items: center; gap: 0.5rem; flex-wrap: wrap; width: 100%;">
|
||||
<label style="font-weight: 600; font-size: 0.85rem; white-space: nowrap;">
|
||||
<i class="fas fa-bookmark" aria-hidden="true"></i> Saved Searches:
|
||||
</label>
|
||||
<div id="saved-searches-list" style="display: flex; gap: 0.25rem; flex-wrap: wrap;" aria-live="polite">
|
||||
<span style="color: var(--text-muted); font-size: 0.85rem;">Loading...</span>
|
||||
</div>
|
||||
<button type="button" onclick="saveCurrentFilters()" class="btn-save-search" style="margin-left: auto; font-size: 0.8rem; padding: 0.25rem 0.5rem; cursor: pointer;" aria-label="Save current filters as a saved search">
|
||||
<i class="fas fa-plus" aria-hidden="true"></i> Save Current
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Full-Text Search Section -->
|
||||
<div class="filters-section" style="margin-top: 0.75rem;">
|
||||
<div style="width: 100%;">
|
||||
@@ -1055,6 +1100,73 @@
|
||||
window.location.href = '/files';
|
||||
}
|
||||
|
||||
// Saved searches functionality
|
||||
function loadSavedSearches() {
|
||||
fetch('/api/saved-searches')
|
||||
.then(response => response.json())
|
||||
.then(searches => {
|
||||
const container = document.getElementById('saved-searches-list');
|
||||
if (!container) return;
|
||||
if (!searches || searches.length === 0) {
|
||||
container.innerHTML = '<span style="color: var(--text-muted); font-size: 0.85rem;">No saved searches yet</span>';
|
||||
return;
|
||||
}
|
||||
container.innerHTML = searches.map(s => {
|
||||
const params = new URLSearchParams(s.filters);
|
||||
return `<span class="saved-search-tag" style="display: inline-flex; align-items: center; gap: 0.25rem; background: var(--bg-tertiary, #e5e7eb); padding: 0.2rem 0.5rem; border-radius: 0.25rem; font-size: 0.8rem;">
|
||||
<a href="/files?${params.toString()}" style="text-decoration: none; color: inherit;">${s.name}</a>
|
||||
<button type="button" onclick="deleteSavedSearch(${s.id})" style="border: none; background: none; cursor: pointer; color: var(--text-muted); padding: 0; line-height: 1;" aria-label="Delete saved search ${s.name}">
|
||||
<i class="fas fa-times" aria-hidden="true"></i>
|
||||
</button>
|
||||
</span>`;
|
||||
}).join('');
|
||||
})
|
||||
.catch(() => {
|
||||
const container = document.getElementById('saved-searches-list');
|
||||
if (container) container.innerHTML = '<span style="color: var(--text-muted); font-size: 0.85rem;">Could not load saved searches</span>';
|
||||
});
|
||||
}
|
||||
|
||||
function saveCurrentFilters() {
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
const filters = {};
|
||||
const filterKeys = ['search', 'mime_type', 'status', 'date_from', 'date_to', 'storage_provider', 'tags', 'sort_by', 'sort_order'];
|
||||
filterKeys.forEach(key => {
|
||||
const val = urlParams.get(key);
|
||||
if (val) filters[key] = val;
|
||||
});
|
||||
if (Object.keys(filters).length === 0) {
|
||||
alert('No filters to save. Apply some filters first.');
|
||||
return;
|
||||
}
|
||||
const name = prompt('Enter a name for this saved search:');
|
||||
if (!name || !name.trim()) return;
|
||||
fetch('/api/saved-searches', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name: name.trim(), filters: filters })
|
||||
})
|
||||
.then(response => {
|
||||
if (!response.ok) return response.json().then(d => { throw new Error(d.detail || 'Failed to save'); });
|
||||
return response.json();
|
||||
})
|
||||
.then(() => loadSavedSearches())
|
||||
.catch(error => alert(error.message));
|
||||
}
|
||||
|
||||
function deleteSavedSearch(id) {
|
||||
if (!confirm('Delete this saved search?')) return;
|
||||
fetch(`/api/saved-searches/${id}`, { method: 'DELETE' })
|
||||
.then(response => {
|
||||
if (!response.ok) throw new Error('Failed to delete');
|
||||
loadSavedSearches();
|
||||
})
|
||||
.catch(error => alert(error.message));
|
||||
}
|
||||
|
||||
// Load saved searches on page load
|
||||
document.addEventListener('DOMContentLoaded', loadSavedSearches);
|
||||
|
||||
// Bulk selection functionality
|
||||
function toggleSelectAll() {
|
||||
const selectAll = document.getElementById('selectAll');
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Add saved_searches table for user-defined filter combinations
|
||||
|
||||
Revision ID: 005_add_saved_searches
|
||||
Revises: 004_add_search_fields
|
||||
Create Date: 2026-03-01
|
||||
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "005_add_saved_searches"
|
||||
down_revision: Union[str, None] = "004_add_search_fields"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create saved_searches table."""
|
||||
op.create_table(
|
||||
"saved_searches",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("user_id", sa.String(), nullable=False, index=True),
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("filters", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.UniqueConstraint("user_id", "name", name="unique_user_search_name"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop saved_searches table."""
|
||||
op.drop_table("saved_searches")
|
||||
+1
-1
@@ -59,7 +59,7 @@ from app.database import Base # noqa: E402
|
||||
from app.main import app as fastapi_app # noqa: E402
|
||||
|
||||
# Import models to register them with SQLAlchemy Base
|
||||
from app.models import DocumentMetadata, FileRecord, ProcessingLog # noqa: F401, E402
|
||||
from app.models import DocumentMetadata, FileRecord, ProcessingLog, SavedSearch # noqa: F401, E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""Tests for advanced filtering on GET /api/files and saved searches CRUD API.
|
||||
|
||||
Tests cover:
|
||||
- Date range filtering (date_from, date_to)
|
||||
- Storage provider filtering
|
||||
- Tags filtering (AND logic)
|
||||
- Combined filters
|
||||
- Saved searches CRUD (create, list, update, delete)
|
||||
- Saved searches validation and error handling
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Advanced filtering tests for GET /api/files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestFilesAdvancedFiltering:
|
||||
"""Tests for advanced filtering on the files list endpoint."""
|
||||
|
||||
def _create_file(self, db_session, **kwargs):
|
||||
"""Helper to create a FileRecord in the test database."""
|
||||
defaults = {
|
||||
"filehash": "abc123",
|
||||
"original_filename": "test.pdf",
|
||||
"local_filename": "/tmp/test.pdf",
|
||||
"file_size": 1024,
|
||||
"mime_type": "application/pdf",
|
||||
}
|
||||
defaults.update(kwargs)
|
||||
record = FileRecord(**defaults)
|
||||
db_session.add(record)
|
||||
db_session.commit()
|
||||
db_session.refresh(record)
|
||||
return record
|
||||
|
||||
def test_filter_by_date_from(self, client: TestClient, db_session):
|
||||
"""Test filtering files created after a specific date."""
|
||||
self._create_file(db_session, original_filename="old.pdf")
|
||||
response = client.get("/api/files?date_from=2020-01-01")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "files" in data
|
||||
|
||||
def test_filter_by_date_to(self, client: TestClient, db_session):
|
||||
"""Test filtering files created before a specific date."""
|
||||
self._create_file(db_session, original_filename="recent.pdf")
|
||||
response = client.get("/api/files?date_to=2099-12-31")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
|
||||
def test_filter_by_date_range(self, client: TestClient, db_session):
|
||||
"""Test filtering files within a date range."""
|
||||
self._create_file(db_session, original_filename="in_range.pdf")
|
||||
response = client.get("/api/files?date_from=2020-01-01&date_to=2099-12-31")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
|
||||
def test_filter_invalid_date_from(self, client: TestClient):
|
||||
"""Test that invalid date_from returns 422."""
|
||||
response = client.get("/api/files?date_from=not-a-date")
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_filter_invalid_date_to(self, client: TestClient):
|
||||
"""Test that invalid date_to returns 422."""
|
||||
response = client.get("/api/files?date_to=not-a-date")
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_filter_by_storage_provider(self, client: TestClient, db_session):
|
||||
"""Test filtering files by storage provider."""
|
||||
file1 = self._create_file(db_session, original_filename="dropbox_file.pdf")
|
||||
file2 = self._create_file(db_session, original_filename="other_file.pdf", filehash="def456")
|
||||
|
||||
# Add successful upload step for file1
|
||||
step = FileProcessingStep(
|
||||
file_id=file1.id,
|
||||
step_name="upload_to_dropbox",
|
||||
status="success",
|
||||
)
|
||||
db_session.add(step)
|
||||
db_session.commit()
|
||||
|
||||
response = client.get("/api/files?storage_provider=dropbox")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
assert data["files"][0]["original_filename"] == "dropbox_file.pdf"
|
||||
|
||||
def test_filter_by_storage_provider_no_results(self, client: TestClient, db_session):
|
||||
"""Test storage provider filter with no matching files."""
|
||||
self._create_file(db_session)
|
||||
response = client.get("/api/files?storage_provider=s3")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 0
|
||||
|
||||
def test_filter_by_tags(self, client: TestClient, db_session):
|
||||
"""Test filtering files by tags in AI metadata."""
|
||||
self._create_file(
|
||||
db_session,
|
||||
original_filename="invoice.pdf",
|
||||
ai_metadata=json.dumps({"tags": ["invoice", "amazon"]}),
|
||||
)
|
||||
self._create_file(
|
||||
db_session,
|
||||
original_filename="receipt.pdf",
|
||||
filehash="def456",
|
||||
ai_metadata=json.dumps({"tags": ["receipt", "walmart"]}),
|
||||
)
|
||||
|
||||
response = client.get("/api/files?tags=invoice")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
assert data["files"][0]["original_filename"] == "invoice.pdf"
|
||||
|
||||
def test_filter_by_multiple_tags_and_logic(self, client: TestClient, db_session):
|
||||
"""Test filtering with multiple tags using AND logic."""
|
||||
self._create_file(
|
||||
db_session,
|
||||
original_filename="both_tags.pdf",
|
||||
ai_metadata=json.dumps({"tags": ["invoice", "amazon"]}),
|
||||
)
|
||||
self._create_file(
|
||||
db_session,
|
||||
original_filename="one_tag.pdf",
|
||||
filehash="def456",
|
||||
ai_metadata=json.dumps({"tags": ["invoice", "walmart"]}),
|
||||
)
|
||||
|
||||
response = client.get("/api/files?tags=invoice,amazon")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
assert data["files"][0]["original_filename"] == "both_tags.pdf"
|
||||
|
||||
def test_combined_filters(self, client: TestClient, db_session):
|
||||
"""Test combining multiple filters together (AND logic)."""
|
||||
self._create_file(
|
||||
db_session,
|
||||
original_filename="target.pdf",
|
||||
mime_type="application/pdf",
|
||||
ai_metadata=json.dumps({"tags": ["invoice"]}),
|
||||
)
|
||||
self._create_file(
|
||||
db_session,
|
||||
original_filename="other.jpg",
|
||||
filehash="def456",
|
||||
mime_type="image/jpeg",
|
||||
ai_metadata=json.dumps({"tags": ["photo"]}),
|
||||
)
|
||||
|
||||
response = client.get("/api/files?mime_type=application/pdf&tags=invoice")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
assert data["files"][0]["original_filename"] == "target.pdf"
|
||||
|
||||
def test_filter_with_date_and_search(self, client: TestClient, db_session):
|
||||
"""Test combining date filter with filename search."""
|
||||
self._create_file(db_session, original_filename="invoice_2026.pdf")
|
||||
|
||||
response = client.get("/api/files?search=invoice&date_from=2020-01-01")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
|
||||
def test_empty_tags_ignored(self, client: TestClient, db_session):
|
||||
"""Test that empty tags parameter is ignored."""
|
||||
self._create_file(db_session)
|
||||
response = client.get("/api/files?tags=")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["files"]) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Saved searches CRUD tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSavedSearchesCRUD:
|
||||
"""Tests for saved searches CRUD API endpoints."""
|
||||
|
||||
def test_list_saved_searches_empty(self, client: TestClient):
|
||||
"""GET /api/saved-searches returns empty list when no searches exist."""
|
||||
response = client.get("/api/saved-searches")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == []
|
||||
|
||||
def test_create_saved_search(self, client: TestClient):
|
||||
"""POST /api/saved-searches creates a new saved search."""
|
||||
payload = {
|
||||
"name": "My Invoices",
|
||||
"filters": {"tags": "invoice", "status": "completed"},
|
||||
}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 201
|
||||
data = response.json()
|
||||
assert data["name"] == "My Invoices"
|
||||
assert data["filters"]["tags"] == "invoice"
|
||||
assert data["filters"]["status"] == "completed"
|
||||
assert "id" in data
|
||||
|
||||
def test_create_and_list_saved_search(self, client: TestClient):
|
||||
"""Creating a saved search makes it appear in the list."""
|
||||
payload = {
|
||||
"name": "PDF Files",
|
||||
"filters": {"mime_type": "application/pdf"},
|
||||
}
|
||||
client.post("/api/saved-searches", json=payload)
|
||||
|
||||
response = client.get("/api/saved-searches")
|
||||
assert response.status_code == 200
|
||||
searches = response.json()
|
||||
assert len(searches) == 1
|
||||
assert searches[0]["name"] == "PDF Files"
|
||||
|
||||
def test_create_saved_search_missing_name(self, client: TestClient):
|
||||
"""POST /api/saved-searches without name returns 422."""
|
||||
payload = {"filters": {"status": "completed"}}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_create_saved_search_empty_filters(self, client: TestClient):
|
||||
"""POST /api/saved-searches with empty filters returns 422."""
|
||||
payload = {"name": "Empty", "filters": {}}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_create_saved_search_invalid_filter_keys(self, client: TestClient):
|
||||
"""POST /api/saved-searches ignores unknown filter keys."""
|
||||
payload = {
|
||||
"name": "With unknown keys",
|
||||
"filters": {"invalid_key": "value", "status": "completed"},
|
||||
}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 201
|
||||
data = response.json()
|
||||
# Only valid filter key should remain
|
||||
assert "invalid_key" not in data["filters"]
|
||||
assert data["filters"]["status"] == "completed"
|
||||
|
||||
def test_create_saved_search_only_invalid_keys(self, client: TestClient):
|
||||
"""POST with only invalid filter keys returns 422."""
|
||||
payload = {
|
||||
"name": "All invalid",
|
||||
"filters": {"bad_key": "value"},
|
||||
}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_create_duplicate_name(self, client: TestClient):
|
||||
"""POST /api/saved-searches with duplicate name returns 409."""
|
||||
payload = {"name": "My Search", "filters": {"status": "completed"}}
|
||||
response1 = client.post("/api/saved-searches", json=payload)
|
||||
assert response1.status_code == 201
|
||||
|
||||
response2 = client.post("/api/saved-searches", json=payload)
|
||||
assert response2.status_code == 409
|
||||
|
||||
def test_update_saved_search(self, client: TestClient):
|
||||
"""PUT /api/saved-searches/{id} updates the saved search."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/saved-searches",
|
||||
json={"name": "Original", "filters": {"status": "pending"}},
|
||||
)
|
||||
search_id = create_resp.json()["id"]
|
||||
|
||||
# Update
|
||||
update_resp = client.put(
|
||||
f"/api/saved-searches/{search_id}",
|
||||
json={"name": "Updated", "filters": {"status": "completed"}},
|
||||
)
|
||||
assert update_resp.status_code == 200
|
||||
data = update_resp.json()
|
||||
assert data["name"] == "Updated"
|
||||
assert data["filters"]["status"] == "completed"
|
||||
|
||||
def test_update_saved_search_not_found(self, client: TestClient):
|
||||
"""PUT /api/saved-searches/999 returns 404."""
|
||||
response = client.put(
|
||||
"/api/saved-searches/999",
|
||||
json={"name": "Nope", "filters": {"status": "completed"}},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_delete_saved_search(self, client: TestClient):
|
||||
"""DELETE /api/saved-searches/{id} removes the saved search."""
|
||||
# Create
|
||||
create_resp = client.post(
|
||||
"/api/saved-searches",
|
||||
json={"name": "To Delete", "filters": {"status": "failed"}},
|
||||
)
|
||||
search_id = create_resp.json()["id"]
|
||||
|
||||
# Delete
|
||||
del_resp = client.delete(f"/api/saved-searches/{search_id}")
|
||||
assert del_resp.status_code == 204
|
||||
|
||||
# Verify it's gone
|
||||
list_resp = client.get("/api/saved-searches")
|
||||
assert len(list_resp.json()) == 0
|
||||
|
||||
def test_delete_saved_search_not_found(self, client: TestClient):
|
||||
"""DELETE /api/saved-searches/999 returns 404."""
|
||||
response = client.delete("/api/saved-searches/999")
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_create_name_too_long(self, client: TestClient):
|
||||
"""POST /api/saved-searches with name > 100 chars returns 422."""
|
||||
payload = {
|
||||
"name": "x" * 101,
|
||||
"filters": {"status": "completed"},
|
||||
}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 422
|
||||
|
||||
def test_saved_search_filters_sanitized(self, client: TestClient):
|
||||
"""Saved search filters are sanitized to allowed keys only."""
|
||||
payload = {
|
||||
"name": "Sanitized",
|
||||
"filters": {
|
||||
"search": "invoice",
|
||||
"mime_type": "application/pdf",
|
||||
"date_from": "2026-01-01",
|
||||
"date_to": "2026-12-31",
|
||||
"storage_provider": "dropbox",
|
||||
"tags": "invoice,amazon",
|
||||
"sort_by": "created_at",
|
||||
"sort_order": "desc",
|
||||
},
|
||||
}
|
||||
response = client.post("/api/saved-searches", json=payload)
|
||||
assert response.status_code == 201
|
||||
data = response.json()
|
||||
assert len(data["filters"]) == 8
|
||||
assert data["filters"]["search"] == "invoice"
|
||||
assert data["filters"]["tags"] == "invoice,amazon"
|
||||
Reference in New Issue
Block a user