From 5bf0a4c0b990c3c4915696ae01448bf1ca12f87b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 13:46:15 +0000 Subject: [PATCH] feat(search): add content-finding filters, saved searches, and text quality to Search view - Add tags, sender, text_quality filters to search API and Meilisearch client - Add sender and ocr_text_length to Meilisearch filterable attributes - Expand saved search allowed filter keys to include q, document_type, language, sender, text_quality - Add filters panel and saved searches UI to the Search view template - Add tests for new search filters, saved search keys, and search view elements Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/saved_searches.py | 10 +- app/api/search.py | 14 +- app/utils/meilisearch_client.py | 25 +++ frontend/templates/search.html | 300 ++++++++++++++++++++++++++--- tests/test_api_advanced_filters.py | 33 ++++ tests/test_api_search.py | 90 +++++++++ tests/test_views_search.py | 30 +++ 7 files changed, 468 insertions(+), 34 deletions(-) diff --git a/app/api/saved_searches.py b/app/api/saved_searches.py index 92caba31..20c1f004 100644 --- a/app/api/saved_searches.py +++ b/app/api/saved_searches.py @@ -23,10 +23,14 @@ router = APIRouter(prefix="/saved-searches", tags=["saved-searches"]) DbSession = Annotated[Session, Depends(get_db)] -# Allowed filter keys that can be saved +# Allowed filter keys that can be saved. +# Files-view keys: search, mime_type, status, storage_provider, sort_by, sort_order +# Search-view keys: q, document_type, language, sender, text_quality +# Shared keys: tags, date_from, date_to ALLOWED_FILTER_KEYS = frozenset( { "search", + "q", "mime_type", "status", "date_from", @@ -35,6 +39,10 @@ ALLOWED_FILTER_KEYS = frozenset( "tags", "sort_by", "sort_order", + "document_type", + "language", + "sender", + "text_quality", } ) diff --git a/app/api/search.py b/app/api/search.py index 2b6a5117..0cec03de 100644 --- a/app/api/search.py +++ b/app/api/search.py @@ -29,6 +29,12 @@ def search_api( mime_type: Optional[str] = Query(None, description="Filter by MIME type (e.g. application/pdf)"), document_type: Optional[str] = Query(None, description="Filter by document type (e.g. Invoice)"), language: Optional[str] = Query(None, description="Filter by language code (e.g. de, en)"), + tags: Optional[str] = Query(None, description="Filter by tag (exact match)"), + sender: Optional[str] = Query(None, description="Filter by sender/absender (exact match)"), + text_quality: Optional[str] = Query( + None, + description="Filter by OCR text quality: no_text, low, medium, high", + ), date_from: Optional[int] = Query(None, description="Filter results created after this Unix timestamp"), date_to: Optional[int] = Query(None, description="Filter results created before this Unix timestamp"), page: int = Query(1, ge=1, description="Page number (1-based)"), @@ -50,6 +56,9 @@ def search_api( - mime_type: Filter by MIME type - document_type: Filter by document type - language: Filter by language code + - tags: Filter by tag (exact match on a single tag) + - sender: Filter by sender/absender (exact match) + - text_quality: Filter by OCR text quality (no_text, low, medium, high) - date_from: Unix timestamp lower bound - date_to: Unix timestamp upper bound - page: Page number (default: 1) @@ -57,7 +66,7 @@ def search_api( Example: ``` - GET /api/search?q=invoice&document_type=Invoice&date_from=1704067200&page=1&per_page=20 + GET /api/search?q=invoice&document_type=Invoice&tags=amazon&date_from=1704067200&page=1&per_page=20 ``` Response: @@ -90,6 +99,9 @@ def search_api( mime_type=mime_type, document_type=document_type, language=language, + tags=tags, + sender=sender, + text_quality=text_quality, date_from=date_from, date_to=date_to, page=page, diff --git a/app/utils/meilisearch_client.py b/app/utils/meilisearch_client.py index c99beb87..eccd2993 100644 --- a/app/utils/meilisearch_client.py +++ b/app/utils/meilisearch_client.py @@ -33,8 +33,10 @@ _INDEX_SETTINGS = { "document_type", "language", "tags", + "sender", "created_at_ts", "file_id", + "ocr_text_length", ], "sortableAttributes": [ "created_at_ts", @@ -55,6 +57,7 @@ _INDEX_SETTINGS = { "file_size", "created_at_ts", "ocr_text", + "ocr_text_length", ], "rankingRules": [ "words", @@ -142,6 +145,7 @@ def _build_document(file_record: "FileRecord", text: str, metadata: dict) -> dic "file_size": file_record.file_size or 0, "created_at_ts": created_at_ts, "ocr_text": text or "", + "ocr_text_length": len(text) if text else 0, } @@ -202,6 +206,9 @@ def search_documents( mime_type: Optional[str] = None, document_type: Optional[str] = None, language: Optional[str] = None, + tags: Optional[str] = None, + sender: Optional[str] = None, + text_quality: Optional[str] = None, date_from: Optional[int] = None, date_to: Optional[int] = None, page: int = 1, @@ -214,6 +221,9 @@ def search_documents( mime_type: Optional MIME-type filter. document_type: Optional document type filter. language: Optional language filter (ISO 639-1, e.g. "de"). + tags: Optional tag filter (exact match on a single tag). + sender: Optional sender/absender filter (exact match). + text_quality: Optional text quality filter: no_text, low, medium, high. date_from: Optional lower bound Unix timestamp for created_at. date_to: Optional upper bound Unix timestamp for created_at. page: 1-based page number. @@ -240,6 +250,21 @@ def search_documents( filters.append(f'document_type = "{document_type}"') if language: filters.append(f'language = "{language}"') + if tags: + filters.append(f'tags = "{tags}"') + if sender: + filters.append(f'sender = "{sender}"') + if text_quality: + # Translate text_quality labels into ocr_text_length ranges + _tq_filters = { + "no_text": "ocr_text_length = 0", + "low": "ocr_text_length > 0 AND ocr_text_length < 500", + "medium": "ocr_text_length >= 500 AND ocr_text_length < 2000", + "high": "ocr_text_length >= 2000", + } + tq_expr = _tq_filters.get(text_quality) + if tq_expr: + filters.append(tq_expr) if date_from is not None: filters.append(f"created_at_ts >= {date_from}") if date_to is not None: diff --git a/frontend/templates/search.html b/frontend/templates/search.html index e78b28e8..1735deda 100644 --- a/frontend/templates/search.html +++ b/frontend/templates/search.html @@ -7,7 +7,7 @@ .search-container { max-width: 800px; margin: 0 auto; } .search-box { display: flex; gap: 0.5rem; align-items: center; - margin-bottom: 1.5rem; + margin-bottom: 0.75rem; } .search-box input { flex: 1; padding: 0.75rem 1rem; @@ -24,11 +24,48 @@ white-space: nowrap; } .search-box button:hover { background-color: #2563eb; } + .search-filters { + display: flex; flex-wrap: wrap; gap: 0.5rem; align-items: flex-end; + margin-bottom: 0.75rem; padding: 0.75rem; + border: 1px solid #e5e7eb; border-radius: 0.5rem; + background: #f9fafb; + } + .search-filter-item { display: flex; flex-direction: column; gap: 0.2rem; } + .search-filter-item label { + font-size: 0.75rem; font-weight: 600; color: #4b5563; + } + .search-filter-item select, + .search-filter-item input { + padding: 0.4rem 0.5rem; border: 1px solid #d1d5db; + border-radius: 0.375rem; font-size: 0.8rem; min-width: 120px; + } + .search-saved-bar { + display: flex; align-items: center; gap: 0.5rem; flex-wrap: wrap; + margin-bottom: 1rem; font-size: 0.85rem; + } + .search-saved-bar .saved-label { + font-weight: 600; white-space: nowrap; + } + .saved-search-tag { + display: inline-flex; align-items: center; gap: 0.25rem; + background: #e5e7eb; padding: 0.2rem 0.5rem; border-radius: 0.25rem; + font-size: 0.8rem; + } + .saved-search-tag a { text-decoration: none; color: inherit; } + .saved-search-tag button { + border: none; background: none; cursor: pointer; color: #6b7280; + padding: 0; line-height: 1; + } + .btn-save-search { + margin-left: auto; font-size: 0.8rem; padding: 0.25rem 0.5rem; + cursor: pointer; border: 1px solid #d1d5db; border-radius: 0.25rem; + background: white; color: #374151; white-space: nowrap; + } + .btn-save-search:hover { background: #f3f4f6; } .search-summary { font-size: 0.875rem; color: #6b7280; margin-bottom: 1rem; } - /* Google-style result cards */ .search-result { margin-bottom: 1.5rem; } @@ -101,6 +138,60 @@ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+ Saved Searches: + + Loading... + + +
+ @@ -113,18 +204,58 @@ {% endblock %} diff --git a/tests/test_api_advanced_filters.py b/tests/test_api_advanced_filters.py index 514d99cd..03bd7b24 100644 --- a/tests/test_api_advanced_filters.py +++ b/tests/test_api_advanced_filters.py @@ -348,3 +348,36 @@ class TestSavedSearchesCRUD: assert len(data["filters"]) == 8 assert data["filters"]["search"] == "invoice" assert data["filters"]["tags"] == "invoice,amazon" + + def test_saved_search_with_fulltext_query(self, client: TestClient): + """Saved search can include full-text query (q) for the search view.""" + payload = { + "name": "Invoice Search", + "filters": {"q": "invoice total amount", "document_type": "Invoice"}, + } + response = client.post("/api/saved-searches", json=payload) + assert response.status_code == 201 + data = response.json() + assert data["filters"]["q"] == "invoice total amount" + assert data["filters"]["document_type"] == "Invoice" + + def test_saved_search_content_finding_filters(self, client: TestClient): + """Saved search accepts content-finding filter keys (language, sender, text_quality).""" + payload = { + "name": "German Invoices", + "filters": { + "q": "rechnung", + "language": "de", + "sender": "ACME GmbH", + "text_quality": "high", + "tags": "invoice", + }, + } + response = client.post("/api/saved-searches", json=payload) + assert response.status_code == 201 + data = response.json() + assert data["filters"]["q"] == "rechnung" + assert data["filters"]["language"] == "de" + assert data["filters"]["sender"] == "ACME GmbH" + assert data["filters"]["text_quality"] == "high" + assert data["filters"]["tags"] == "invoice" diff --git a/tests/test_api_search.py b/tests/test_api_search.py index c44a4db6..43322cb8 100644 --- a/tests/test_api_search.py +++ b/tests/test_api_search.py @@ -97,6 +97,7 @@ class TestMeilisearchIndexDocument: assert doc["file_id"] == 1 assert doc["document_title"] == "Invoice January 2026" assert "invoice" in doc["tags"] + assert doc["ocr_text_length"] == len("This is an invoice for services rendered") def test_index_document_meilisearch_error(self): """index_document returns False on Meilisearch exception.""" @@ -179,6 +180,62 @@ class TestMeilisearchSearchDocuments: assert 'mime_type = "application/pdf"' in search_params["filter"] assert 'language = "de"' in search_params["filter"] + def test_search_with_tags_filter(self): + """search_documents passes tags filter to Meilisearch.""" + mock_client, mock_index = self._make_mock_client(hits=[], total=0) + + with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client): + from app.utils.meilisearch_client import search_documents + + search_documents("test", tags="invoice", page=1, per_page=10) + + call_kwargs = mock_index.search.call_args + search_params = call_kwargs[0][1] + assert "filter" in search_params + assert 'tags = "invoice"' in search_params["filter"] + + def test_search_with_sender_filter(self): + """search_documents passes sender filter to Meilisearch.""" + mock_client, mock_index = self._make_mock_client(hits=[], total=0) + + with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client): + from app.utils.meilisearch_client import search_documents + + search_documents("test", sender="ACME Corp", page=1, per_page=10) + + call_kwargs = mock_index.search.call_args + search_params = call_kwargs[0][1] + assert "filter" in search_params + assert 'sender = "ACME Corp"' in search_params["filter"] + + def test_search_with_text_quality_high(self): + """search_documents translates text_quality=high to ocr_text_length filter.""" + mock_client, mock_index = self._make_mock_client(hits=[], total=0) + + with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client): + from app.utils.meilisearch_client import search_documents + + search_documents("test", text_quality="high", page=1, per_page=10) + + call_kwargs = mock_index.search.call_args + search_params = call_kwargs[0][1] + assert "filter" in search_params + assert "ocr_text_length >= 2000" in search_params["filter"] + + def test_search_with_text_quality_no_text(self): + """search_documents translates text_quality=no_text to ocr_text_length filter.""" + mock_client, mock_index = self._make_mock_client(hits=[], total=0) + + with patch("app.utils.meilisearch_client.get_meilisearch_client", return_value=mock_client): + from app.utils.meilisearch_client import search_documents + + search_documents("test", text_quality="no_text", page=1, per_page=10) + + call_kwargs = mock_index.search.call_args + search_params = call_kwargs[0][1] + assert "filter" in search_params + assert "ocr_text_length = 0" in search_params["filter"] + def test_search_pagination(self): """search_documents applies correct offset for page 2.""" mock_client, mock_index = self._make_mock_client(hits=[], total=50) @@ -271,6 +328,9 @@ class TestSearchAPIEndpoint: mime_type="application/pdf", document_type=None, language="en", + tags=None, + sender=None, + text_quality=None, date_from=None, date_to=None, page=2, @@ -294,6 +354,9 @@ class TestSearchAPIEndpoint: mime_type=None, document_type=None, language=None, + tags=None, + sender=None, + text_quality=None, date_from=None, date_to=None, page=1, @@ -310,3 +373,30 @@ class TestSearchAPIEndpoint: call_kwargs = mock_search.call_args assert call_kwargs[1]["date_from"] == 1704067200 assert call_kwargs[1]["date_to"] == 1735689600 + + def test_search_endpoint_tags_filter(self, client): + """GET /api/search?q=...&tags=invoice passes tags to search_documents.""" + mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "test"} + with patch("app.api.search.search_documents", return_value=mock_result) as mock_search: + response = client.get("/api/search?q=test&tags=invoice") + + assert response.status_code == 200 + assert mock_search.call_args[1]["tags"] == "invoice" + + def test_search_endpoint_sender_filter(self, client): + """GET /api/search?q=...&sender=ACME passes sender to search_documents.""" + mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "test"} + with patch("app.api.search.search_documents", return_value=mock_result) as mock_search: + response = client.get("/api/search?q=test&sender=ACME") + + assert response.status_code == 200 + assert mock_search.call_args[1]["sender"] == "ACME" + + def test_search_endpoint_text_quality_filter(self, client): + """GET /api/search?q=...&text_quality=high passes text_quality to search_documents.""" + mock_result = {"results": [], "total": 0, "page": 1, "pages": 0, "query": "test"} + with patch("app.api.search.search_documents", return_value=mock_result) as mock_search: + response = client.get("/api/search?q=test&text_quality=high") + + assert response.status_code == 200 + assert mock_search.call_args[1]["text_quality"] == "high" diff --git a/tests/test_views_search.py b/tests/test_views_search.py index b3d6bbe3..788ec4c8 100644 --- a/tests/test_views_search.py +++ b/tests/test_views_search.py @@ -39,3 +39,33 @@ class TestSearchPage: long_query = "a" * 600 response = client.get(f"/search?q={long_query}") assert response.status_code == 422 + + def test_search_page_contains_filter_elements(self, client): + """GET /search contains content-finding filter UI elements.""" + response = client.get("/search") + assert response.status_code == 200 + assert 'id="filter-document-type"' in response.text + assert 'id="filter-tags"' in response.text + assert 'id="filter-sender"' in response.text + assert 'id="filter-language"' in response.text + assert 'id="filter-text-quality"' in response.text + assert 'id="filter-date-from"' in response.text + assert 'id="filter-date-to"' in response.text + + def test_search_page_contains_saved_searches(self, client): + """GET /search contains saved searches UI elements.""" + response = client.get("/search") + assert response.status_code == 200 + assert 'id="saved-searches-list"' in response.text + assert 'id="save-search-btn"' in response.text + assert "/api/saved-searches" in response.text + + def test_search_page_text_quality_options(self, client): + """GET /search contains text quality filter with expected options.""" + response = client.get("/search") + assert response.status_code == 200 + text = response.text + assert 'value="high"' in text + assert 'value="medium"' in text + assert 'value="low"' in text + assert 'value="no_text"' in text