feat(api): add advanced filtering and saved searches

- Add date range (date_from/date_to), storage provider, and tags filters to GET /api/files
- Add SavedSearch model and migration (005_add_saved_searches)
- Add CRUD API endpoints for saved searches at /api/saved-searches
- Update files.html template with new filter controls and saved searches UI
- Update files view to pass new filter parameters to template
- Add comprehensive tests for all new functionality (26 tests)

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-01 11:22:07 +00:00
parent f45e9bfd70
commit e5a4c6c64a
9 changed files with 912 additions and 9 deletions
+2
View File
@@ -16,6 +16,7 @@ from app.api.onedrive import router as onedrive_router
from app.api.openai import router as openai_router
from app.api.process import router as process_router
from app.api.queue import router as queue_router
from app.api.saved_searches import router as saved_searches_router
from app.api.search import router as search_router
from app.api.settings import router as settings_router
from app.api.url_upload import router as url_upload_router
@@ -44,3 +45,4 @@ router.include_router(settings_router)
router.include_router(url_upload_router)
router.include_router(search_router)
router.include_router(queue_router)
router.include_router(saved_searches_router)
+51 -1
View File
@@ -6,6 +6,7 @@ import logging
import mimetypes
import os
import uuid
from datetime import datetime, timezone
from typing import Annotated, List, Optional
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
@@ -15,7 +16,7 @@ from sqlalchemy.orm import Session
from app.auth import require_login
from app.config import settings
from app.database import get_db
from app.models import FileRecord, ProcessingLog
from app.models import FileProcessingStep, FileRecord, ProcessingLog
from app.tasks.convert_to_pdf import convert_to_pdf
from app.tasks.process_document import process_document
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
@@ -54,6 +55,10 @@ def list_files_api(
search: Optional[str] = Query(None, description="Search in filename"),
mime_type: Optional[str] = Query(None, description="Filter by MIME type"),
status: Optional[str] = Query(None, description="Filter by processing status"),
date_from: Optional[str] = Query(None, description="Filter files created on or after this date (ISO 8601)"),
date_to: Optional[str] = Query(None, description="Filter files created on or before this date (ISO 8601)"),
storage_provider: Optional[str] = Query(None, description="Filter by storage provider (e.g. dropbox, s3)"),
tags: Optional[str] = Query(None, description="Filter by tag (comma-separated for multiple, AND logic)"),
):
"""
Returns a paginated JSON list of FileRecord entries with processing status.
@@ -67,6 +72,10 @@ def list_files_api(
- search: Search in filename
- mime_type: Filter by MIME type
- status: Filter by processing status (pending, processing, completed, failed)
- date_from: Filter files created on or after this date (ISO 8601, e.g. 2026-01-01)
- date_to: Filter files created on or before this date (ISO 8601, e.g. 2026-12-31)
- storage_provider: Filter by storage provider (e.g. dropbox, s3, google_drive)
- tags: Filter by tags (comma-separated, AND logic)
Example response:
{
@@ -95,6 +104,47 @@ def list_files_api(
if mime_type:
query = query.filter(FileRecord.mime_type == mime_type)
# Apply date range filters
if date_from:
try:
dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc)
query = query.filter(FileRecord.created_at >= dt_from)
except ValueError:
raise HTTPException(
status_code=422,
detail="Invalid date_from format. Use ISO 8601 (e.g. 2026-01-01)",
)
if date_to:
try:
dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc)
query = query.filter(FileRecord.created_at <= dt_to)
except ValueError:
raise HTTPException(
status_code=422,
detail="Invalid date_to format. Use ISO 8601 (e.g. 2026-12-31)",
)
# Apply storage provider filter (files that have a successful upload_to_{provider} step)
if storage_provider:
step_name = f"upload_to_{storage_provider}"
uploaded_file_ids = (
db.query(FileProcessingStep.file_id)
.filter(
FileProcessingStep.step_name == step_name,
FileProcessingStep.status == "success",
)
.distinct()
.subquery()
)
query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id)))
# Apply tags filter (AND logic: all specified tags must be present in ai_metadata)
if tags:
tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()]
for tag in tag_list:
query = query.filter(FileRecord.ai_metadata.ilike(f"%{tag}%"))
# Apply status filter (before pagination for correct counts)
query = apply_status_filter(query, db, status)
+294
View File
@@ -0,0 +1,294 @@
"""
Saved searches API endpoints.
Provides CRUD operations for user-defined saved search filters.
Each user can save, list, update, and delete named filter combinations
for quick access on the files page.
"""
import json
import logging
from typing import Annotated, Any
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
from sqlalchemy.orm import Session
from app.auth import get_current_user, require_login
from app.database import get_db
from app.models import SavedSearch
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/saved-searches", tags=["saved-searches"])
DbSession = Annotated[Session, Depends(get_db)]
# Allowed filter keys that can be saved
ALLOWED_FILTER_KEYS = frozenset(
{
"search",
"mime_type",
"status",
"date_from",
"date_to",
"storage_provider",
"tags",
"sort_by",
"sort_order",
}
)
# Maximum number of saved searches per user
MAX_SAVED_SEARCHES_PER_USER = 50
# Maximum length for saved search name
MAX_NAME_LENGTH = 100
def _get_user_id(request: Request) -> str:
"""Extract user identifier from the session.
Returns the preferred_username, email, or 'anonymous' if auth is disabled.
Args:
request: The incoming HTTP request.
Returns:
A string identifying the current user.
"""
user = get_current_user(request)
if user:
return user.get("preferred_username") or user.get("email") or user.get("name", "anonymous")
return "anonymous"
def _validate_filters(filters: Any) -> dict:
"""Validate and sanitize filter parameters.
Ensures only allowed filter keys are present and values are strings.
Args:
filters: The raw filter value from the client.
Returns:
A sanitized filter dictionary with only allowed keys.
Raises:
HTTPException: If filters is not a dict or contains invalid values.
"""
if not isinstance(filters, dict):
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="filters must be a JSON object",
)
sanitized = {}
for key, value in filters.items():
if key in ALLOWED_FILTER_KEYS and isinstance(value, str) and value.strip():
sanitized[key] = value.strip()
return sanitized
def _serialize_saved_search(s: SavedSearch) -> dict:
"""Serialize a SavedSearch model instance to a JSON-compatible dict.
Args:
s: The SavedSearch model instance.
Returns:
A dictionary with id, name, filters, created_at, and updated_at.
"""
return {
"id": s.id,
"name": s.name,
"filters": json.loads(s.filters),
"created_at": s.created_at.isoformat() if s.created_at else None,
"updated_at": s.updated_at.isoformat() if s.updated_at else None,
}
@router.get("")
@require_login
def list_saved_searches(request: Request, db: DbSession):
"""List all saved searches for the current user.
Returns:
A list of saved search objects with id, name, filters, and timestamps.
"""
user_id = _get_user_id(request)
searches = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).order_by(SavedSearch.name).all()
return [_serialize_saved_search(s) for s in searches]
@router.post("", status_code=status.HTTP_201_CREATED)
@require_login
def create_saved_search(
request: Request,
db: DbSession,
name: str = Body(..., embed=True),
filters: dict = Body(..., embed=True),
):
"""Create a new saved search for the current user.
Request body (JSON):
name: Display name for the saved search (required, max 100 chars)
filters: Dictionary of filter parameters (required)
Returns:
The created saved search object.
Raises:
HTTPException 422: If name or filters are invalid.
HTTPException 409: If a saved search with the same name already exists.
"""
user_id = _get_user_id(request)
name = name.strip() if isinstance(name, str) else ""
if not name or len(name) > MAX_NAME_LENGTH:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"name is required and must be at most {MAX_NAME_LENGTH} characters",
)
sanitized_filters = _validate_filters(filters)
if not sanitized_filters:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="At least one filter parameter is required",
)
# Check user limit
count = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).count()
if count >= MAX_SAVED_SEARCHES_PER_USER:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Maximum of {MAX_SAVED_SEARCHES_PER_USER} saved searches reached",
)
# Check for duplicate name
existing = db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == name).first()
if existing:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"A saved search named '{name}' already exists",
)
saved_search = SavedSearch(
user_id=user_id,
name=name,
filters=json.dumps(sanitized_filters),
)
try:
db.add(saved_search)
db.commit()
db.refresh(saved_search)
except Exception:
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to save search",
)
logger.info(f"Saved search created: user={user_id}, name={name!r}")
return _serialize_saved_search(saved_search)
@router.put("/{search_id}")
@require_login
def update_saved_search(
search_id: int,
request: Request,
db: DbSession,
name: str | None = Body(None, embed=True),
filters: dict | None = Body(None, embed=True),
):
"""Update an existing saved search.
Path Parameters:
search_id: The ID of the saved search to update.
Request body (JSON):
name: New display name (optional)
filters: New filter parameters (optional)
Returns:
The updated saved search object.
Raises:
HTTPException 404: If the saved search is not found.
HTTPException 409: If the new name conflicts with an existing saved search.
"""
user_id = _get_user_id(request)
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
if not saved_search:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
if name is not None:
new_name = name.strip() if isinstance(name, str) else ""
if not new_name or len(new_name) > MAX_NAME_LENGTH:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"name must be non-empty and at most {MAX_NAME_LENGTH} characters",
)
# Check for name conflict
if new_name != saved_search.name:
existing = (
db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == new_name).first()
)
if existing:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"A saved search named '{new_name}' already exists",
)
saved_search.name = new_name
if filters is not None:
sanitized_filters = _validate_filters(filters)
if not sanitized_filters:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="At least one filter parameter is required",
)
saved_search.filters = json.dumps(sanitized_filters)
try:
db.commit()
db.refresh(saved_search)
except Exception:
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to update saved search",
)
logger.info(f"Saved search updated: id={search_id}, user={user_id}")
return _serialize_saved_search(saved_search)
@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
@require_login
def delete_saved_search(search_id: int, request: Request, db: DbSession):
"""Delete a saved search.
Path Parameters:
search_id: The ID of the saved search to delete.
Raises:
HTTPException 404: If the saved search is not found.
"""
user_id = _get_user_id(request)
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
if not saved_search:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
try:
db.delete(saved_search)
db.commit()
except Exception:
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to delete saved search",
)
logger.info(f"Saved search deleted: id={search_id}, user={user_id}")
+15
View File
@@ -125,3 +125,18 @@ class SettingsAuditLog(Base):
changed_by = Column(String, nullable=False) # Username of the admin who made the change
changed_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
action = Column(String, nullable=False) # "update" or "delete"
class SavedSearch(Base):
"""User-defined saved search filters for quick access to frequently used filter combinations."""
__tablename__ = "saved_searches"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(String, nullable=False, index=True) # Username or user identifier from session
name = Column(String, nullable=False) # Human-readable name for the saved search
filters = Column(Text, nullable=False) # JSON-encoded filter parameters
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
__table_args__ = (UniqueConstraint("user_id", "name", name="unique_user_search_name"),)
+50 -6
View File
@@ -2,9 +2,10 @@
File management views for displaying and managing files.
"""
from datetime import datetime, timezone
from typing import Optional
from fastapi import Depends, Query, Request
from fastapi import Depends, HTTPException, Query, Request
from sqlalchemy.orm import Session
from app.utils.file_queries import apply_status_filter
@@ -29,6 +30,10 @@ def files_page(
search: Optional[str] = Query(None),
mime_type: Optional[str] = Query(None),
status: Optional[str] = Query(None),
date_from: Optional[str] = Query(None),
date_to: Optional[str] = Query(None),
storage_provider: Optional[str] = Query(None),
tags: Optional[str] = Query(None),
):
"""
Return the 'files.html' template with server-side pagination, sorting, and filtering
@@ -39,7 +44,7 @@ def files_page(
# Import the model here to avoid circular imports
from sqlalchemy import asc, desc
from app.models import FileRecord
from app.models import FileProcessingStep, FileRecord
# Start with base query
query = db.query(FileRecord)
@@ -52,6 +57,41 @@ def files_page(
if mime_type:
query = query.filter(FileRecord.mime_type == mime_type)
# Apply date range filters
if date_from:
try:
dt_from = datetime.fromisoformat(date_from).replace(tzinfo=timezone.utc)
query = query.filter(FileRecord.created_at >= dt_from)
except ValueError:
pass # Silently ignore invalid dates in view
if date_to:
try:
dt_to = datetime.fromisoformat(date_to).replace(tzinfo=timezone.utc)
query = query.filter(FileRecord.created_at <= dt_to)
except ValueError:
pass
# Apply storage provider filter
if storage_provider:
step_name = f"upload_to_{storage_provider}"
uploaded_file_ids = (
db.query(FileProcessingStep.file_id)
.filter(
FileProcessingStep.step_name == step_name,
FileProcessingStep.status == "success",
)
.distinct()
.subquery()
)
query = query.filter(FileRecord.id.in_(db.query(uploaded_file_ids.c.file_id)))
# Apply tags filter (AND logic)
if tags:
tag_list = [t.strip().lower() for t in tags.split(",") if t.strip()]
for tag in tag_list:
query = query.filter(FileRecord.ai_metadata.ilike(f"%{tag}%"))
# Apply status filter (before pagination for correct counts)
query = apply_status_filter(query, db, status)
@@ -112,6 +152,10 @@ def files_page(
"search": search or "",
"mime_type": mime_type or "",
"status": status or "",
"date_from": date_from or "",
"date_to": date_to or "",
"storage_provider": storage_provider or "",
"tags": tags or "",
"mime_types": mime_types,
"upload_concurrency": settings.upload_concurrency,
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
@@ -524,7 +568,7 @@ def preview_original_file(request: Request, file_id: int, db: Session = Depends(
"""
import os
from fastapi import HTTPException, status
from fastapi import status
from fastapi.responses import FileResponse
from app.models import FileRecord
@@ -551,7 +595,7 @@ def preview_processed_file(request: Request, file_id: int, db: Session = Depends
"""
import os
from fastapi import HTTPException, status
from fastapi import status
from fastapi.responses import FileResponse
from app.models import FileRecord
@@ -578,7 +622,7 @@ def get_original_text(request: Request, file_id: int, db: Session = Depends(get_
"""
import os
from fastapi import HTTPException, status
from fastapi import status
from fastapi.responses import JSONResponse
from app.models import FileRecord
@@ -618,7 +662,7 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
"""
import os
from fastapi import HTTPException, status
from fastapi import status
from fastapi.responses import JSONResponse
from app.models import FileRecord
+113 -1
View File
@@ -475,7 +475,7 @@
<!-- Filters Section -->
<div class="filters-section">
<form method="get" action="/files" class="filter-group">
<form method="get" action="/files" class="filter-group" role="search" aria-label="Filter files">
<div class="filter-item">
<label for="search">Search Filename</label>
<input type="text" id="search" name="search" value="{{ search }}" placeholder="Enter filename...">
@@ -503,6 +503,36 @@
</select>
</div>
<div class="filter-item">
<label for="date_from">Date From</label>
<input type="date" id="date_from" name="date_from" value="{{ date_from }}" aria-label="Filter from date">
</div>
<div class="filter-item">
<label for="date_to">Date To</label>
<input type="date" id="date_to" name="date_to" value="{{ date_to }}" aria-label="Filter to date">
</div>
<div class="filter-item">
<label for="storage_provider">Storage Provider</label>
<select id="storage_provider" name="storage_provider">
<option value="">All Providers</option>
<option value="dropbox" {% if storage_provider == "dropbox" %}selected{% endif %}>Dropbox</option>
<option value="google_drive" {% if storage_provider == "google_drive" %}selected{% endif %}>Google Drive</option>
<option value="onedrive" {% if storage_provider == "onedrive" %}selected{% endif %}>OneDrive</option>
<option value="s3" {% if storage_provider == "s3" %}selected{% endif %}>S3</option>
<option value="nextcloud" {% if storage_provider == "nextcloud" %}selected{% endif %}>Nextcloud</option>
<option value="webdav" {% if storage_provider == "webdav" %}selected{% endif %}>WebDAV</option>
<option value="ftp" {% if storage_provider == "ftp" %}selected{% endif %}>FTP</option>
<option value="sftp" {% if storage_provider == "sftp" %}selected{% endif %}>SFTP</option>
</select>
</div>
<div class="filter-item">
<label for="tags">Tags</label>
<input type="text" id="tags" name="tags" value="{{ tags }}" placeholder="e.g. invoice,amazon" aria-label="Filter by tags (comma-separated)">
</div>
<div class="filter-item">
<label>&nbsp;</label>
<button type="submit">Apply Filters</button>
@@ -520,6 +550,21 @@
</form>
</div>
<!-- Saved Searches Section -->
<div class="filters-section" style="margin-top: 0.5rem;">
<div style="display: flex; align-items: center; gap: 0.5rem; flex-wrap: wrap; width: 100%;">
<label style="font-weight: 600; font-size: 0.85rem; white-space: nowrap;">
<i class="fas fa-bookmark" aria-hidden="true"></i> Saved Searches:
</label>
<div id="saved-searches-list" style="display: flex; gap: 0.25rem; flex-wrap: wrap;" aria-live="polite">
<span style="color: var(--text-muted); font-size: 0.85rem;">Loading...</span>
</div>
<button type="button" onclick="saveCurrentFilters()" class="btn-save-search" style="margin-left: auto; font-size: 0.8rem; padding: 0.25rem 0.5rem; cursor: pointer;" aria-label="Save current filters as a saved search">
<i class="fas fa-plus" aria-hidden="true"></i> Save Current
</button>
</div>
</div>
<!-- Full-Text Search Section -->
<div class="filters-section" style="margin-top: 0.75rem;">
<div style="width: 100%;">
@@ -1055,6 +1100,73 @@
window.location.href = '/files';
}
// Saved searches functionality
function loadSavedSearches() {
fetch('/api/saved-searches')
.then(response => response.json())
.then(searches => {
const container = document.getElementById('saved-searches-list');
if (!container) return;
if (!searches || searches.length === 0) {
container.innerHTML = '<span style="color: var(--text-muted); font-size: 0.85rem;">No saved searches yet</span>';
return;
}
container.innerHTML = searches.map(s => {
const params = new URLSearchParams(s.filters);
return `<span class="saved-search-tag" style="display: inline-flex; align-items: center; gap: 0.25rem; background: var(--bg-tertiary, #e5e7eb); padding: 0.2rem 0.5rem; border-radius: 0.25rem; font-size: 0.8rem;">
<a href="/files?${params.toString()}" style="text-decoration: none; color: inherit;">${s.name}</a>
<button type="button" onclick="deleteSavedSearch(${s.id})" style="border: none; background: none; cursor: pointer; color: var(--text-muted); padding: 0; line-height: 1;" aria-label="Delete saved search ${s.name}">
<i class="fas fa-times" aria-hidden="true"></i>
</button>
</span>`;
}).join('');
})
.catch(() => {
const container = document.getElementById('saved-searches-list');
if (container) container.innerHTML = '<span style="color: var(--text-muted); font-size: 0.85rem;">Could not load saved searches</span>';
});
}
function saveCurrentFilters() {
const urlParams = new URLSearchParams(window.location.search);
const filters = {};
const filterKeys = ['search', 'mime_type', 'status', 'date_from', 'date_to', 'storage_provider', 'tags', 'sort_by', 'sort_order'];
filterKeys.forEach(key => {
const val = urlParams.get(key);
if (val) filters[key] = val;
});
if (Object.keys(filters).length === 0) {
alert('No filters to save. Apply some filters first.');
return;
}
const name = prompt('Enter a name for this saved search:');
if (!name || !name.trim()) return;
fetch('/api/saved-searches', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: name.trim(), filters: filters })
})
.then(response => {
if (!response.ok) return response.json().then(d => { throw new Error(d.detail || 'Failed to save'); });
return response.json();
})
.then(() => loadSavedSearches())
.catch(error => alert(error.message));
}
function deleteSavedSearch(id) {
if (!confirm('Delete this saved search?')) return;
fetch(`/api/saved-searches/${id}`, { method: 'DELETE' })
.then(response => {
if (!response.ok) throw new Error('Failed to delete');
loadSavedSearches();
})
.catch(error => alert(error.message));
}
// Load saved searches on page load
document.addEventListener('DOMContentLoaded', loadSavedSearches);
// Bulk selection functionality
function toggleSelectAll() {
const selectAll = document.getElementById('selectAll');
@@ -0,0 +1,36 @@
"""Add saved_searches table for user-defined filter combinations
Revision ID: 005_add_saved_searches
Revises: 004_add_search_fields
Create Date: 2026-03-01
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "005_add_saved_searches"
down_revision: Union[str, None] = "004_add_search_fields"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Create saved_searches table."""
op.create_table(
"saved_searches",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("user_id", sa.String(), nullable=False, index=True),
sa.Column("name", sa.String(), nullable=False),
sa.Column("filters", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.UniqueConstraint("user_id", "name", name="unique_user_search_name"),
)
def downgrade() -> None:
"""Drop saved_searches table."""
op.drop_table("saved_searches")
+1 -1
View File
@@ -59,7 +59,7 @@ from app.database import Base # noqa: E402
from app.main import app as fastapi_app # noqa: E402
# Import models to register them with SQLAlchemy Base
from app.models import DocumentMetadata, FileRecord, ProcessingLog # noqa: F401, E402
from app.models import DocumentMetadata, FileRecord, ProcessingLog, SavedSearch # noqa: F401, E402
@pytest.fixture(scope="session")
+350
View File
@@ -0,0 +1,350 @@
"""Tests for advanced filtering on GET /api/files and saved searches CRUD API.
Tests cover:
- Date range filtering (date_from, date_to)
- Storage provider filtering
- Tags filtering (AND logic)
- Combined filters
- Saved searches CRUD (create, list, update, delete)
- Saved searches validation and error handling
"""
import json
import pytest
from fastapi.testclient import TestClient
from app.models import FileProcessingStep, FileRecord
# ---------------------------------------------------------------------------
# Advanced filtering tests for GET /api/files
# ---------------------------------------------------------------------------
@pytest.mark.unit
class TestFilesAdvancedFiltering:
"""Tests for advanced filtering on the files list endpoint."""
def _create_file(self, db_session, **kwargs):
"""Helper to create a FileRecord in the test database."""
defaults = {
"filehash": "abc123",
"original_filename": "test.pdf",
"local_filename": "/tmp/test.pdf",
"file_size": 1024,
"mime_type": "application/pdf",
}
defaults.update(kwargs)
record = FileRecord(**defaults)
db_session.add(record)
db_session.commit()
db_session.refresh(record)
return record
def test_filter_by_date_from(self, client: TestClient, db_session):
"""Test filtering files created after a specific date."""
self._create_file(db_session, original_filename="old.pdf")
response = client.get("/api/files?date_from=2020-01-01")
assert response.status_code == 200
data = response.json()
assert "files" in data
def test_filter_by_date_to(self, client: TestClient, db_session):
"""Test filtering files created before a specific date."""
self._create_file(db_session, original_filename="recent.pdf")
response = client.get("/api/files?date_to=2099-12-31")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
def test_filter_by_date_range(self, client: TestClient, db_session):
"""Test filtering files within a date range."""
self._create_file(db_session, original_filename="in_range.pdf")
response = client.get("/api/files?date_from=2020-01-01&date_to=2099-12-31")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
def test_filter_invalid_date_from(self, client: TestClient):
"""Test that invalid date_from returns 422."""
response = client.get("/api/files?date_from=not-a-date")
assert response.status_code == 422
def test_filter_invalid_date_to(self, client: TestClient):
"""Test that invalid date_to returns 422."""
response = client.get("/api/files?date_to=not-a-date")
assert response.status_code == 422
def test_filter_by_storage_provider(self, client: TestClient, db_session):
"""Test filtering files by storage provider."""
file1 = self._create_file(db_session, original_filename="dropbox_file.pdf")
file2 = self._create_file(db_session, original_filename="other_file.pdf", filehash="def456")
# Add successful upload step for file1
step = FileProcessingStep(
file_id=file1.id,
step_name="upload_to_dropbox",
status="success",
)
db_session.add(step)
db_session.commit()
response = client.get("/api/files?storage_provider=dropbox")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
assert data["files"][0]["original_filename"] == "dropbox_file.pdf"
def test_filter_by_storage_provider_no_results(self, client: TestClient, db_session):
"""Test storage provider filter with no matching files."""
self._create_file(db_session)
response = client.get("/api/files?storage_provider=s3")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 0
def test_filter_by_tags(self, client: TestClient, db_session):
"""Test filtering files by tags in AI metadata."""
self._create_file(
db_session,
original_filename="invoice.pdf",
ai_metadata=json.dumps({"tags": ["invoice", "amazon"]}),
)
self._create_file(
db_session,
original_filename="receipt.pdf",
filehash="def456",
ai_metadata=json.dumps({"tags": ["receipt", "walmart"]}),
)
response = client.get("/api/files?tags=invoice")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
assert data["files"][0]["original_filename"] == "invoice.pdf"
def test_filter_by_multiple_tags_and_logic(self, client: TestClient, db_session):
"""Test filtering with multiple tags using AND logic."""
self._create_file(
db_session,
original_filename="both_tags.pdf",
ai_metadata=json.dumps({"tags": ["invoice", "amazon"]}),
)
self._create_file(
db_session,
original_filename="one_tag.pdf",
filehash="def456",
ai_metadata=json.dumps({"tags": ["invoice", "walmart"]}),
)
response = client.get("/api/files?tags=invoice,amazon")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
assert data["files"][0]["original_filename"] == "both_tags.pdf"
def test_combined_filters(self, client: TestClient, db_session):
"""Test combining multiple filters together (AND logic)."""
self._create_file(
db_session,
original_filename="target.pdf",
mime_type="application/pdf",
ai_metadata=json.dumps({"tags": ["invoice"]}),
)
self._create_file(
db_session,
original_filename="other.jpg",
filehash="def456",
mime_type="image/jpeg",
ai_metadata=json.dumps({"tags": ["photo"]}),
)
response = client.get("/api/files?mime_type=application/pdf&tags=invoice")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
assert data["files"][0]["original_filename"] == "target.pdf"
def test_filter_with_date_and_search(self, client: TestClient, db_session):
"""Test combining date filter with filename search."""
self._create_file(db_session, original_filename="invoice_2026.pdf")
response = client.get("/api/files?search=invoice&date_from=2020-01-01")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
def test_empty_tags_ignored(self, client: TestClient, db_session):
"""Test that empty tags parameter is ignored."""
self._create_file(db_session)
response = client.get("/api/files?tags=")
assert response.status_code == 200
data = response.json()
assert len(data["files"]) == 1
# ---------------------------------------------------------------------------
# Saved searches CRUD tests
# ---------------------------------------------------------------------------
@pytest.mark.unit
class TestSavedSearchesCRUD:
"""Tests for saved searches CRUD API endpoints."""
def test_list_saved_searches_empty(self, client: TestClient):
"""GET /api/saved-searches returns empty list when no searches exist."""
response = client.get("/api/saved-searches")
assert response.status_code == 200
assert response.json() == []
def test_create_saved_search(self, client: TestClient):
"""POST /api/saved-searches creates a new saved search."""
payload = {
"name": "My Invoices",
"filters": {"tags": "invoice", "status": "completed"},
}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 201
data = response.json()
assert data["name"] == "My Invoices"
assert data["filters"]["tags"] == "invoice"
assert data["filters"]["status"] == "completed"
assert "id" in data
def test_create_and_list_saved_search(self, client: TestClient):
"""Creating a saved search makes it appear in the list."""
payload = {
"name": "PDF Files",
"filters": {"mime_type": "application/pdf"},
}
client.post("/api/saved-searches", json=payload)
response = client.get("/api/saved-searches")
assert response.status_code == 200
searches = response.json()
assert len(searches) == 1
assert searches[0]["name"] == "PDF Files"
def test_create_saved_search_missing_name(self, client: TestClient):
"""POST /api/saved-searches without name returns 422."""
payload = {"filters": {"status": "completed"}}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 422
def test_create_saved_search_empty_filters(self, client: TestClient):
"""POST /api/saved-searches with empty filters returns 422."""
payload = {"name": "Empty", "filters": {}}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 422
def test_create_saved_search_invalid_filter_keys(self, client: TestClient):
"""POST /api/saved-searches ignores unknown filter keys."""
payload = {
"name": "With unknown keys",
"filters": {"invalid_key": "value", "status": "completed"},
}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 201
data = response.json()
# Only valid filter key should remain
assert "invalid_key" not in data["filters"]
assert data["filters"]["status"] == "completed"
def test_create_saved_search_only_invalid_keys(self, client: TestClient):
"""POST with only invalid filter keys returns 422."""
payload = {
"name": "All invalid",
"filters": {"bad_key": "value"},
}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 422
def test_create_duplicate_name(self, client: TestClient):
"""POST /api/saved-searches with duplicate name returns 409."""
payload = {"name": "My Search", "filters": {"status": "completed"}}
response1 = client.post("/api/saved-searches", json=payload)
assert response1.status_code == 201
response2 = client.post("/api/saved-searches", json=payload)
assert response2.status_code == 409
def test_update_saved_search(self, client: TestClient):
"""PUT /api/saved-searches/{id} updates the saved search."""
# Create
create_resp = client.post(
"/api/saved-searches",
json={"name": "Original", "filters": {"status": "pending"}},
)
search_id = create_resp.json()["id"]
# Update
update_resp = client.put(
f"/api/saved-searches/{search_id}",
json={"name": "Updated", "filters": {"status": "completed"}},
)
assert update_resp.status_code == 200
data = update_resp.json()
assert data["name"] == "Updated"
assert data["filters"]["status"] == "completed"
def test_update_saved_search_not_found(self, client: TestClient):
"""PUT /api/saved-searches/999 returns 404."""
response = client.put(
"/api/saved-searches/999",
json={"name": "Nope", "filters": {"status": "completed"}},
)
assert response.status_code == 404
def test_delete_saved_search(self, client: TestClient):
"""DELETE /api/saved-searches/{id} removes the saved search."""
# Create
create_resp = client.post(
"/api/saved-searches",
json={"name": "To Delete", "filters": {"status": "failed"}},
)
search_id = create_resp.json()["id"]
# Delete
del_resp = client.delete(f"/api/saved-searches/{search_id}")
assert del_resp.status_code == 204
# Verify it's gone
list_resp = client.get("/api/saved-searches")
assert len(list_resp.json()) == 0
def test_delete_saved_search_not_found(self, client: TestClient):
"""DELETE /api/saved-searches/999 returns 404."""
response = client.delete("/api/saved-searches/999")
assert response.status_code == 404
def test_create_name_too_long(self, client: TestClient):
"""POST /api/saved-searches with name > 100 chars returns 422."""
payload = {
"name": "x" * 101,
"filters": {"status": "completed"},
}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 422
def test_saved_search_filters_sanitized(self, client: TestClient):
"""Saved search filters are sanitized to allowed keys only."""
payload = {
"name": "Sanitized",
"filters": {
"search": "invoice",
"mime_type": "application/pdf",
"date_from": "2026-01-01",
"date_to": "2026-12-31",
"storage_provider": "dropbox",
"tags": "invoice,amazon",
"sort_by": "created_at",
"sort_order": "desc",
},
}
response = client.post("/api/saved-searches", json=payload)
assert response.status_code == 201
data = response.json()
assert len(data["filters"]) == 8
assert data["filters"]["search"] == "invoice"
assert data["filters"]["tags"] == "invoice,amazon"