feat(queue): add queue monitoring dashboard and pending banner on files page
- Add /api/queue/stats endpoint with Redis queue lengths, Celery worker inspection, and DB processing summaries - Add /api/queue/pending-count lightweight endpoint for the files page banner - Add /admin/queue admin-only view with auto-refreshing queue dashboard - Add queue pending banner on /files page showing queued/processing count - Add Queue Monitor link to admin dropdown in navigation (desktop + mobile) - Add comprehensive tests for all new endpoints and views Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -17,6 +17,7 @@ from app.api.openai import router as openai_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.queue import router as queue_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
|
||||
# Import all the individual routers
|
||||
@@ -42,3 +43,4 @@ router.include_router(logs_router)
|
||||
router.include_router(settings_router)
|
||||
router.include_router(url_upload_router)
|
||||
router.include_router(search_router)
|
||||
router.include_router(queue_router)
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
"""
|
||||
Queue monitoring API endpoints.
|
||||
|
||||
Provides endpoints to query Celery/Redis queue statistics and
|
||||
database-level processing status for document pipeline visibility.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import redis
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/queue", tags=["queue"])
|
||||
|
||||
|
||||
def _get_redis_queue_length(redis_client: redis.Redis, queue_name: str) -> int:
|
||||
"""Get the number of messages in a Redis-backed Celery queue.
|
||||
|
||||
Args:
|
||||
redis_client: Connected Redis client instance.
|
||||
queue_name: Name of the Celery queue to inspect.
|
||||
|
||||
Returns:
|
||||
Number of messages (tasks) waiting in the queue.
|
||||
"""
|
||||
try:
|
||||
return redis_client.llen(queue_name)
|
||||
except Exception:
|
||||
logger.debug(f"Could not read queue length for '{queue_name}'")
|
||||
return 0
|
||||
|
||||
|
||||
def _get_celery_inspect_stats() -> dict[str, Any]:
|
||||
"""Query the Celery inspect API for active, reserved, and scheduled tasks.
|
||||
|
||||
Returns:
|
||||
Dictionary with active, reserved, and scheduled task summaries.
|
||||
"""
|
||||
from app.celery_app import celery
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"active": [],
|
||||
"reserved": [],
|
||||
"scheduled": [],
|
||||
"workers_online": 0,
|
||||
}
|
||||
|
||||
try:
|
||||
inspector = celery.control.inspect(timeout=2.0)
|
||||
|
||||
active = inspector.active() or {}
|
||||
reserved = inspector.reserved() or {}
|
||||
scheduled = inspector.scheduled() or {}
|
||||
|
||||
result["workers_online"] = len(active)
|
||||
|
||||
for _worker, tasks in active.items():
|
||||
for task in tasks:
|
||||
result["active"].append(
|
||||
{
|
||||
"id": task.get("id", ""),
|
||||
"name": task.get("name", "unknown"),
|
||||
"args": str(task.get("args", []))[:200],
|
||||
"started": task.get("time_start"),
|
||||
}
|
||||
)
|
||||
|
||||
for _worker, tasks in reserved.items():
|
||||
for task in tasks:
|
||||
result["reserved"].append(
|
||||
{
|
||||
"id": task.get("id", ""),
|
||||
"name": task.get("name", "unknown"),
|
||||
"args": str(task.get("args", []))[:200],
|
||||
}
|
||||
)
|
||||
|
||||
for _worker, tasks in scheduled.items():
|
||||
for task in tasks:
|
||||
req = task.get("request", {})
|
||||
result["scheduled"].append(
|
||||
{
|
||||
"id": req.get("id", ""),
|
||||
"name": req.get("name", "unknown"),
|
||||
"eta": task.get("eta"),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Celery inspect failed (workers may be offline): {exc}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _get_db_processing_summary(db: Session) -> dict[str, Any]:
|
||||
"""Query the database for a summary of file processing states.
|
||||
|
||||
Args:
|
||||
db: SQLAlchemy database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with counts of files by processing state.
|
||||
"""
|
||||
try:
|
||||
total_files = db.query(func.count(FileRecord.id)).scalar() or 0
|
||||
|
||||
# Count files with at least one in_progress step
|
||||
processing_count = (
|
||||
db.query(func.count(func.distinct(FileProcessingStep.file_id)))
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Count files with at least one failure and no in_progress
|
||||
failed_subq = (
|
||||
db.query(FileProcessingStep.file_id).filter(FileProcessingStep.status == "failure").distinct().subquery()
|
||||
)
|
||||
in_progress_subq = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
failed_count = (
|
||||
db.query(func.count(func.distinct(failed_subq.c.file_id)))
|
||||
.filter(~failed_subq.c.file_id.in_(db.query(in_progress_subq.c.file_id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Count files that have steps and all steps are success/skipped
|
||||
all_step_files = db.query(FileProcessingStep.file_id).distinct().subquery()
|
||||
# Files with any non-terminal step
|
||||
non_terminal = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status.in_(["in_progress", "pending", "failure"]))
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
completed_count = (
|
||||
db.query(func.count(func.distinct(all_step_files.c.file_id)))
|
||||
.filter(~all_step_files.c.file_id.in_(db.query(non_terminal.c.file_id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Files with no processing steps at all
|
||||
files_with_steps = db.query(FileProcessingStep.file_id).distinct().subquery()
|
||||
pending_count = (
|
||||
db.query(func.count(FileRecord.id))
|
||||
.filter(~FileRecord.id.in_(db.query(files_with_steps.c.file_id)))
|
||||
.filter(FileRecord.is_duplicate.is_(False))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Recent files being processed (last 20 in_progress or pending)
|
||||
recent_processing = (
|
||||
db.query(FileRecord.id, FileRecord.original_filename, FileProcessingStep.step_name)
|
||||
.join(FileProcessingStep, FileRecord.id == FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.order_by(FileProcessingStep.updated_at.desc())
|
||||
.limit(20)
|
||||
.all()
|
||||
)
|
||||
|
||||
recent_list = [
|
||||
{"file_id": r[0], "filename": r[1] or f"File #{r[0]}", "current_step": r[2]} for r in recent_processing
|
||||
]
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"processing": processing_count,
|
||||
"failed": failed_count,
|
||||
"completed": completed_count,
|
||||
"pending": pending_count,
|
||||
"recent_processing": recent_list,
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error querying DB processing summary: {exc}")
|
||||
return {
|
||||
"total_files": 0,
|
||||
"processing": 0,
|
||||
"failed": 0,
|
||||
"completed": 0,
|
||||
"pending": 0,
|
||||
"recent_processing": [],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_queue_stats(db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
"""Get comprehensive queue and processing statistics.
|
||||
|
||||
Returns queue lengths from Redis, Celery worker inspection data,
|
||||
and database-level processing summaries for the document pipeline.
|
||||
|
||||
Returns:
|
||||
Dictionary containing redis queue info, celery worker info,
|
||||
and database processing summary.
|
||||
"""
|
||||
# 1. Redis queue lengths
|
||||
queue_lengths: dict[str, int] = {}
|
||||
try:
|
||||
redis_client = redis.Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
for queue_name in ["document_processor", "default", "celery"]:
|
||||
queue_lengths[queue_name] = _get_redis_queue_length(redis_client, queue_name)
|
||||
redis_client.close()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not connect to Redis: {exc}")
|
||||
|
||||
total_queued = sum(queue_lengths.values())
|
||||
|
||||
# 2. Celery inspect
|
||||
celery_stats = _get_celery_inspect_stats()
|
||||
|
||||
# 3. DB summary
|
||||
db_summary = _get_db_processing_summary(db)
|
||||
|
||||
return {
|
||||
"queues": queue_lengths,
|
||||
"total_queued": total_queued,
|
||||
"celery": celery_stats,
|
||||
"db_summary": db_summary,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pending-count")
|
||||
def get_pending_count(db: Session = Depends(get_db)) -> dict[str, int]:
|
||||
"""Get a lightweight count of queued + in-progress items for the files page banner.
|
||||
|
||||
Returns:
|
||||
Dictionary with total_pending count (queued in Redis + processing in DB).
|
||||
"""
|
||||
total_pending = 0
|
||||
|
||||
# Redis queue lengths
|
||||
try:
|
||||
redis_client = redis.Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
for queue_name in ["document_processor", "default", "celery"]:
|
||||
total_pending += _get_redis_queue_length(redis_client, queue_name)
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
logger.debug("Could not connect to Redis for pending count")
|
||||
|
||||
# DB in-progress count
|
||||
try:
|
||||
processing_count = (
|
||||
db.query(func.count(func.distinct(FileProcessingStep.file_id)))
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
total_pending += processing_count
|
||||
except Exception:
|
||||
logger.debug("Could not query DB for processing count")
|
||||
|
||||
return {"total_pending": total_pending}
|
||||
@@ -12,6 +12,7 @@ from app.views.general import router as general_router
|
||||
from app.views.google_drive import router as google_drive_router
|
||||
from app.views.license_routes import router as license_router # Add the license router
|
||||
from app.views.onedrive import router as onedrive_router
|
||||
from app.views.queue import router as queue_router
|
||||
from app.views.search import router as search_router
|
||||
from app.views.settings import router as settings_router
|
||||
from app.views.status import router as status_router
|
||||
@@ -29,3 +30,4 @@ router.include_router(license_router) # Include the license router
|
||||
router.include_router(settings_router)
|
||||
router.include_router(filemanager_router)
|
||||
router.include_router(search_router)
|
||||
router.include_router(queue_router)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""
|
||||
Queue monitoring view for the admin dashboard.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.views.base import APIRouter, get_db, require_login, templates
|
||||
from app.views.settings import require_admin_access
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/admin/queue")
|
||||
@require_login
|
||||
@require_admin_access
|
||||
async def queue_dashboard(request: Request, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Queue monitoring dashboard — admin only.
|
||||
|
||||
Displays Celery/Redis queue statistics and database processing summaries
|
||||
so administrators can monitor the document processing pipeline.
|
||||
"""
|
||||
return templates.TemplateResponse(
|
||||
"queue_dashboard.html",
|
||||
{
|
||||
"request": request,
|
||||
},
|
||||
)
|
||||
@@ -95,6 +95,9 @@
|
||||
<a href="/admin/files" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-folder-open w-4 mr-2 text-gray-500"></i> File Manager
|
||||
</a>
|
||||
<a href="/admin/queue" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-stream w-4 mr-2 text-blue-500"></i> Queue Monitor
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -168,6 +171,9 @@
|
||||
<a href="/admin/files" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
|
||||
<i class="fas fa-folder-open mr-2 text-gray-400"></i> File Manager
|
||||
</a>
|
||||
<a href="/admin/queue" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
|
||||
<i class="fas fa-stream mr-2 text-blue-400"></i> Queue Monitor
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -370,6 +370,23 @@
|
||||
<div class="container mx-auto px-4 py-8">
|
||||
<h2 class="text-3xl font-bold mb-6">File Records</h2>
|
||||
|
||||
<!-- Queue pending banner (populated via JS) -->
|
||||
<div id="queueBanner" class="hidden bg-blue-50 border-l-4 border-blue-400 text-blue-800 p-4 mb-6 rounded-r-lg" role="status">
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="flex items-center">
|
||||
<i class="fas fa-spinner fa-spin mr-3 text-blue-500"></i>
|
||||
<span>
|
||||
<strong id="queueBannerCount">0</strong> item(s) are currently queued or being processed.
|
||||
Files will appear here once processing completes.
|
||||
</span>
|
||||
</div>
|
||||
<a href="/admin/queue" class="text-blue-600 hover:text-blue-800 text-sm font-medium whitespace-nowrap ml-4"
|
||||
id="queueBannerLink" style="display:none;">
|
||||
<i class="fas fa-external-link-alt mr-1"></i>View Queue
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if error %}
|
||||
<div class="error-message">
|
||||
<p><strong>Error:</strong> {{ error }}</p>
|
||||
@@ -1050,5 +1067,35 @@
|
||||
_searchCurrentPage = 1;
|
||||
}
|
||||
</script>
|
||||
|
||||
<!-- Queue banner updater -->
|
||||
<script>
|
||||
(function () {
|
||||
function updateQueueBanner() {
|
||||
fetch('/api/queue/pending-count')
|
||||
.then(function (r) { return r.ok ? r.json() : null; })
|
||||
.then(function (data) {
|
||||
if (!data) return;
|
||||
var banner = document.getElementById('queueBanner');
|
||||
var count = data.total_pending || 0;
|
||||
if (count > 0) {
|
||||
document.getElementById('queueBannerCount').textContent = count;
|
||||
banner.classList.remove('hidden');
|
||||
// Show admin link if user is admin (adminMenuContainer visible)
|
||||
var adminMenu = document.getElementById('adminMenuContainer');
|
||||
if (adminMenu && !adminMenu.classList.contains('hidden')) {
|
||||
var link = document.getElementById('queueBannerLink');
|
||||
if (link) link.style.display = '';
|
||||
}
|
||||
} else {
|
||||
banner.classList.add('hidden');
|
||||
}
|
||||
})
|
||||
.catch(function () { /* silently ignore */ });
|
||||
}
|
||||
updateQueueBanner();
|
||||
setInterval(updateQueueBanner, 15000);
|
||||
})();
|
||||
</script>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Queue Monitor — DocuElevate{% endblock %}
|
||||
|
||||
{% block head_extra %}
|
||||
<script src="/static/js/common.js"></script>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container mx-auto px-4 py-8">
|
||||
|
||||
<!-- Header -->
|
||||
<div class="mb-6">
|
||||
<h1 class="text-3xl font-bold mb-2">
|
||||
<i class="fas fa-stream mr-2 text-blue-500"></i>Queue Monitor
|
||||
</h1>
|
||||
<p class="text-gray-600">Real-time view of the document processing pipeline and Celery task queues.</p>
|
||||
</div>
|
||||
|
||||
<!-- Loading state -->
|
||||
<div id="queueLoading" class="text-center py-12">
|
||||
<i class="fas fa-spinner fa-spin fa-2x text-blue-400"></i>
|
||||
<p class="mt-2 text-gray-500">Loading queue statistics…</p>
|
||||
</div>
|
||||
|
||||
<!-- Error state -->
|
||||
<div id="queueError" class="hidden bg-red-100 border-l-4 border-red-500 text-red-700 p-4 mb-6" role="alert">
|
||||
<p><strong>Error:</strong> <span id="queueErrorMsg"></span></p>
|
||||
</div>
|
||||
|
||||
<!-- Dashboard content (hidden until loaded) -->
|
||||
<div id="queueContent" class="hidden">
|
||||
|
||||
<!-- Summary Cards -->
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4 mb-8">
|
||||
<!-- Queued -->
|
||||
<div class="bg-white rounded-lg shadow p-5">
|
||||
<div class="flex items-center">
|
||||
<div class="flex-shrink-0 bg-yellow-100 rounded-full p-3">
|
||||
<i class="fas fa-inbox text-yellow-600 text-xl"></i>
|
||||
</div>
|
||||
<div class="ml-4">
|
||||
<p class="text-sm font-medium text-gray-500">Queued Tasks</p>
|
||||
<p id="totalQueued" class="text-2xl font-bold text-gray-900">0</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Active / Running -->
|
||||
<div class="bg-white rounded-lg shadow p-5">
|
||||
<div class="flex items-center">
|
||||
<div class="flex-shrink-0 bg-blue-100 rounded-full p-3">
|
||||
<i class="fas fa-cogs text-blue-600 text-xl"></i>
|
||||
</div>
|
||||
<div class="ml-4">
|
||||
<p class="text-sm font-medium text-gray-500">Active Tasks</p>
|
||||
<p id="activeTasks" class="text-2xl font-bold text-gray-900">0</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Processing Files -->
|
||||
<div class="bg-white rounded-lg shadow p-5">
|
||||
<div class="flex items-center">
|
||||
<div class="flex-shrink-0 bg-indigo-100 rounded-full p-3">
|
||||
<i class="fas fa-file-alt text-indigo-600 text-xl"></i>
|
||||
</div>
|
||||
<div class="ml-4">
|
||||
<p class="text-sm font-medium text-gray-500">Files Processing</p>
|
||||
<p id="filesProcessing" class="text-2xl font-bold text-gray-900">0</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Workers Online -->
|
||||
<div class="bg-white rounded-lg shadow p-5">
|
||||
<div class="flex items-center">
|
||||
<div class="flex-shrink-0 bg-green-100 rounded-full p-3">
|
||||
<i class="fas fa-server text-green-600 text-xl"></i>
|
||||
</div>
|
||||
<div class="ml-4">
|
||||
<p class="text-sm font-medium text-gray-500">Workers Online</p>
|
||||
<p id="workersOnline" class="text-2xl font-bold text-gray-900">0</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Pipeline Summary -->
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-8">
|
||||
|
||||
<!-- Queue Breakdown -->
|
||||
<div class="bg-white rounded-lg shadow">
|
||||
<div class="px-5 py-4 border-b border-gray-200">
|
||||
<h2 class="text-lg font-semibold text-gray-900">
|
||||
<i class="fas fa-list-ol mr-2 text-gray-500"></i>Redis Queues
|
||||
</h2>
|
||||
</div>
|
||||
<div class="p-5">
|
||||
<table class="w-full text-sm">
|
||||
<thead>
|
||||
<tr class="text-left text-gray-500 border-b">
|
||||
<th class="pb-2">Queue</th>
|
||||
<th class="pb-2 text-right">Pending</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="queueRows">
|
||||
<tr><td colspan="2" class="py-3 text-gray-400 text-center">No data</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- DB Processing Summary -->
|
||||
<div class="bg-white rounded-lg shadow">
|
||||
<div class="px-5 py-4 border-b border-gray-200">
|
||||
<h2 class="text-lg font-semibold text-gray-900">
|
||||
<i class="fas fa-database mr-2 text-gray-500"></i>Processing Pipeline
|
||||
</h2>
|
||||
</div>
|
||||
<div class="p-5">
|
||||
<table class="w-full text-sm">
|
||||
<thead>
|
||||
<tr class="text-left text-gray-500 border-b">
|
||||
<th class="pb-2">State</th>
|
||||
<th class="pb-2 text-right">Files</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="dbSummaryRows">
|
||||
<tr><td colspan="2" class="py-3 text-gray-400 text-center">No data</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Active Tasks -->
|
||||
<div class="bg-white rounded-lg shadow mb-8">
|
||||
<div class="px-5 py-4 border-b border-gray-200">
|
||||
<h2 class="text-lg font-semibold text-gray-900">
|
||||
<i class="fas fa-play-circle mr-2 text-blue-500"></i>Active Tasks
|
||||
<span id="activeCount" class="ml-1 text-sm font-normal text-gray-500">(0)</span>
|
||||
</h2>
|
||||
</div>
|
||||
<div class="p-5 overflow-x-auto">
|
||||
<table class="w-full text-sm" id="activeTable">
|
||||
<thead>
|
||||
<tr class="text-left text-gray-500 border-b">
|
||||
<th class="pb-2">Task</th>
|
||||
<th class="pb-2">Arguments</th>
|
||||
<th class="pb-2">Task ID</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="activeRows">
|
||||
<tr><td colspan="3" class="py-3 text-gray-400 text-center">No active tasks</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Recently Processing Files -->
|
||||
<div class="bg-white rounded-lg shadow mb-8">
|
||||
<div class="px-5 py-4 border-b border-gray-200">
|
||||
<h2 class="text-lg font-semibold text-gray-900">
|
||||
<i class="fas fa-file-medical-alt mr-2 text-indigo-500"></i>Recently Processing Files
|
||||
</h2>
|
||||
</div>
|
||||
<div class="p-5 overflow-x-auto">
|
||||
<table class="w-full text-sm">
|
||||
<thead>
|
||||
<tr class="text-left text-gray-500 border-b">
|
||||
<th class="pb-2">File</th>
|
||||
<th class="pb-2">Current Step</th>
|
||||
<th class="pb-2 text-right">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="recentRows">
|
||||
<tr><td colspan="3" class="py-3 text-gray-400 text-center">No files currently processing</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Auto-refresh info -->
|
||||
<div class="text-center text-gray-400 text-xs mb-4">
|
||||
<i class="fas fa-sync-alt mr-1"></i>
|
||||
Auto-refreshes every <span id="refreshInterval">10</span> seconds —
|
||||
Last updated: <span id="lastUpdated">—</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const REFRESH_SECONDS = 10;
|
||||
let timer = null;
|
||||
|
||||
function escapeHtml(str) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = str;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function shortName(taskName) {
|
||||
// Show only the last part of the dotted task name
|
||||
const parts = (taskName || 'unknown').split('.');
|
||||
return parts[parts.length - 1];
|
||||
}
|
||||
|
||||
function renderQueues(queues) {
|
||||
const tbody = document.getElementById('queueRows');
|
||||
if (!queues || Object.keys(queues).length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="2" class="py-3 text-gray-400 text-center">No queues found</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = Object.entries(queues).map(([name, count]) => `
|
||||
<tr class="border-b border-gray-100">
|
||||
<td class="py-2 font-mono text-gray-700">${escapeHtml(name)}</td>
|
||||
<td class="py-2 text-right">
|
||||
<span class="inline-block px-2 py-0.5 rounded-full text-xs font-semibold ${count > 0 ? 'bg-yellow-100 text-yellow-800' : 'bg-gray-100 text-gray-500'}">${count}</span>
|
||||
</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
function renderDbSummary(summary) {
|
||||
const tbody = document.getElementById('dbSummaryRows');
|
||||
if (!summary) {
|
||||
tbody.innerHTML = '<tr><td colspan="2" class="py-3 text-gray-400 text-center">No data</td></tr>';
|
||||
return;
|
||||
}
|
||||
const rows = [
|
||||
{ label: 'Completed', value: summary.completed, color: 'bg-green-100 text-green-800', icon: 'fa-check-circle text-green-500' },
|
||||
{ label: 'Processing', value: summary.processing, color: 'bg-blue-100 text-blue-800', icon: 'fa-spinner text-blue-500' },
|
||||
{ label: 'Pending', value: summary.pending, color: 'bg-yellow-100 text-yellow-800', icon: 'fa-clock text-yellow-500' },
|
||||
{ label: 'Failed', value: summary.failed, color: 'bg-red-100 text-red-800', icon: 'fa-times-circle text-red-500' },
|
||||
{ label: 'Total Files', value: summary.total_files, color: 'bg-gray-100 text-gray-700', icon: 'fa-folder text-gray-500' },
|
||||
];
|
||||
tbody.innerHTML = rows.map(r => `
|
||||
<tr class="border-b border-gray-100">
|
||||
<td class="py-2"><i class="fas ${r.icon} mr-2"></i>${r.label}</td>
|
||||
<td class="py-2 text-right">
|
||||
<span class="inline-block px-2 py-0.5 rounded-full text-xs font-semibold ${r.color}">${r.value}</span>
|
||||
</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
function renderActiveTasks(tasks) {
|
||||
const tbody = document.getElementById('activeRows');
|
||||
document.getElementById('activeCount').textContent = `(${tasks.length})`;
|
||||
if (tasks.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="3" class="py-3 text-gray-400 text-center">No active tasks</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = tasks.map(t => `
|
||||
<tr class="border-b border-gray-100">
|
||||
<td class="py-2 font-medium text-gray-800">${escapeHtml(shortName(t.name))}</td>
|
||||
<td class="py-2 text-gray-500 truncate max-w-xs" title="${escapeHtml(t.args)}">${escapeHtml(t.args)}</td>
|
||||
<td class="py-2 font-mono text-xs text-gray-400">${escapeHtml((t.id || '').substring(0, 8))}</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
function renderRecentFiles(files) {
|
||||
const tbody = document.getElementById('recentRows');
|
||||
if (!files || files.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="3" class="py-3 text-gray-400 text-center">No files currently processing</td></tr>';
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = files.map(f => `
|
||||
<tr class="border-b border-gray-100">
|
||||
<td class="py-2 text-gray-800">${escapeHtml(f.filename)}</td>
|
||||
<td class="py-2"><span class="inline-block px-2 py-0.5 rounded bg-blue-50 text-blue-700 text-xs font-mono">${escapeHtml(f.current_step)}</span></td>
|
||||
<td class="py-2 text-right">
|
||||
<a href="/files/${f.file_id}" class="text-blue-500 hover:text-blue-700 text-xs"><i class="fas fa-external-link-alt mr-1"></i>View</a>
|
||||
</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
async function fetchStats() {
|
||||
try {
|
||||
const resp = await fetch('/api/queue/stats');
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
const data = await resp.json();
|
||||
|
||||
// Hide loading, show content
|
||||
document.getElementById('queueLoading').classList.add('hidden');
|
||||
document.getElementById('queueError').classList.add('hidden');
|
||||
document.getElementById('queueContent').classList.remove('hidden');
|
||||
|
||||
// Summary cards
|
||||
document.getElementById('totalQueued').textContent = data.total_queued || 0;
|
||||
document.getElementById('activeTasks').textContent = (data.celery && data.celery.active) ? data.celery.active.length : 0;
|
||||
document.getElementById('filesProcessing').textContent = (data.db_summary && data.db_summary.processing) || 0;
|
||||
document.getElementById('workersOnline').textContent = (data.celery && data.celery.workers_online) || 0;
|
||||
|
||||
// Sections
|
||||
renderQueues(data.queues);
|
||||
renderDbSummary(data.db_summary);
|
||||
renderActiveTasks(data.celery ? data.celery.active || [] : []);
|
||||
renderRecentFiles(data.db_summary ? data.db_summary.recent_processing || [] : []);
|
||||
|
||||
document.getElementById('lastUpdated').textContent = new Date().toLocaleTimeString();
|
||||
} catch (err) {
|
||||
document.getElementById('queueLoading').classList.add('hidden');
|
||||
document.getElementById('queueError').classList.remove('hidden');
|
||||
document.getElementById('queueErrorMsg').textContent = err.message || 'Failed to load queue data';
|
||||
}
|
||||
}
|
||||
|
||||
// Initial fetch and auto-refresh
|
||||
fetchStats();
|
||||
timer = setInterval(fetchStats, REFRESH_SECONDS * 1000);
|
||||
|
||||
// Cleanup on page unload
|
||||
window.addEventListener('beforeunload', function () {
|
||||
if (timer) clearInterval(timer);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Tests for app/api/queue.py and app/views/queue.py modules."""
|
||||
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGetRedisQueueLength:
|
||||
"""Tests for the _get_redis_queue_length helper."""
|
||||
|
||||
def test_returns_queue_length(self):
|
||||
"""Test returns queue length from Redis."""
|
||||
from app.api.queue import _get_redis_queue_length
|
||||
|
||||
mock_redis = MagicMock()
|
||||
mock_redis.llen.return_value = 42
|
||||
assert _get_redis_queue_length(mock_redis, "document_processor") == 42
|
||||
mock_redis.llen.assert_called_once_with("document_processor")
|
||||
|
||||
def test_returns_zero_on_error(self):
|
||||
"""Test returns 0 when Redis call fails."""
|
||||
from app.api.queue import _get_redis_queue_length
|
||||
|
||||
mock_redis = MagicMock()
|
||||
mock_redis.llen.side_effect = Exception("Connection refused")
|
||||
assert _get_redis_queue_length(mock_redis, "default") == 0
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGetCeleryInspectStats:
|
||||
"""Tests for the _get_celery_inspect_stats helper."""
|
||||
|
||||
@patch("app.celery_app.celery")
|
||||
def test_returns_worker_stats(self, mock_celery_mod):
|
||||
"""Test returns active, reserved, scheduled tasks."""
|
||||
from app.api.queue import _get_celery_inspect_stats
|
||||
|
||||
mock_inspector = MagicMock()
|
||||
mock_inspector.active.return_value = {
|
||||
"worker1": [
|
||||
{"id": "task-1", "name": "app.tasks.process_document.process_document", "args": [1], "time_start": 123}
|
||||
]
|
||||
}
|
||||
mock_inspector.reserved.return_value = {
|
||||
"worker1": [{"id": "task-2", "name": "app.tasks.upload_to_s3.upload_to_s3", "args": [2]}]
|
||||
}
|
||||
mock_inspector.scheduled.return_value = {
|
||||
"worker1": [{"request": {"id": "task-3", "name": "app.tasks.check_credentials.check_credentials"}, "eta": "2026-01-01"}]
|
||||
}
|
||||
mock_celery_mod.control.inspect.return_value = mock_inspector
|
||||
|
||||
result = _get_celery_inspect_stats()
|
||||
|
||||
assert result["workers_online"] == 1
|
||||
assert len(result["active"]) == 1
|
||||
assert result["active"][0]["id"] == "task-1"
|
||||
assert len(result["reserved"]) == 1
|
||||
assert len(result["scheduled"]) == 1
|
||||
|
||||
@patch("app.celery_app.celery")
|
||||
def test_handles_no_workers(self, mock_celery_mod):
|
||||
"""Test handles case where no workers are online."""
|
||||
from app.api.queue import _get_celery_inspect_stats
|
||||
|
||||
mock_inspector = MagicMock()
|
||||
mock_inspector.active.return_value = None
|
||||
mock_inspector.reserved.return_value = None
|
||||
mock_inspector.scheduled.return_value = None
|
||||
mock_celery_mod.control.inspect.return_value = mock_inspector
|
||||
|
||||
result = _get_celery_inspect_stats()
|
||||
|
||||
assert result["workers_online"] == 0
|
||||
assert result["active"] == []
|
||||
assert result["reserved"] == []
|
||||
assert result["scheduled"] == []
|
||||
|
||||
@patch("app.celery_app.celery")
|
||||
def test_handles_inspect_exception(self, mock_celery_mod):
|
||||
"""Test handles exception during inspect."""
|
||||
from app.api.queue import _get_celery_inspect_stats
|
||||
|
||||
mock_celery_mod.control.inspect.side_effect = Exception("Broker unreachable")
|
||||
|
||||
result = _get_celery_inspect_stats()
|
||||
|
||||
assert result["workers_online"] == 0
|
||||
assert result["active"] == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGetDbProcessingSummary:
|
||||
"""Tests for the _get_db_processing_summary helper."""
|
||||
|
||||
def test_returns_summary(self, db_session):
|
||||
"""Test returns processing summary from DB."""
|
||||
from app.api.queue import _get_db_processing_summary
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
# Create some test files
|
||||
file1 = FileRecord(filehash="abc1", local_filename="f1.pdf", file_size=100, is_duplicate=False)
|
||||
file2 = FileRecord(filehash="abc2", local_filename="f2.pdf", file_size=200, is_duplicate=False)
|
||||
db_session.add_all([file1, file2])
|
||||
db_session.commit()
|
||||
|
||||
# Add steps: file1 completed, file2 in_progress
|
||||
step1 = FileProcessingStep(file_id=file1.id, step_name="extract_text", status="success")
|
||||
step2 = FileProcessingStep(file_id=file2.id, step_name="extract_text", status="in_progress")
|
||||
db_session.add_all([step1, step2])
|
||||
db_session.commit()
|
||||
|
||||
result = _get_db_processing_summary(db_session)
|
||||
|
||||
assert result["total_files"] == 2
|
||||
assert result["processing"] == 1
|
||||
assert isinstance(result["recent_processing"], list)
|
||||
|
||||
def test_returns_empty_on_error(self):
|
||||
"""Test returns empty summary on DB error."""
|
||||
from app.api.queue import _get_db_processing_summary
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.query.side_effect = Exception("DB error")
|
||||
|
||||
result = _get_db_processing_summary(mock_db)
|
||||
|
||||
assert result["total_files"] == 0
|
||||
assert result["processing"] == 0
|
||||
assert result["recent_processing"] == []
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestQueueStatsEndpoint:
|
||||
"""Tests for the GET /api/queue/stats endpoint."""
|
||||
|
||||
@patch("app.api.queue.redis.Redis")
|
||||
@patch("app.celery_app.celery")
|
||||
def test_queue_stats_returns_200(self, mock_celery_mod, mock_redis_cls, client):
|
||||
"""Test queue stats endpoint returns 200 with data."""
|
||||
# Mock Redis
|
||||
mock_redis_instance = MagicMock()
|
||||
mock_redis_instance.llen.return_value = 5
|
||||
mock_redis_cls.from_url.return_value = mock_redis_instance
|
||||
|
||||
# Mock Celery inspector
|
||||
mock_inspector = MagicMock()
|
||||
mock_inspector.active.return_value = {}
|
||||
mock_inspector.reserved.return_value = {}
|
||||
mock_inspector.scheduled.return_value = {}
|
||||
mock_celery_mod.control.inspect.return_value = mock_inspector
|
||||
|
||||
response = client.get("/api/queue/stats")
|
||||
assert response.status_code == 200
|
||||
|
||||
data = response.json()
|
||||
assert "queues" in data
|
||||
assert "total_queued" in data
|
||||
assert "celery" in data
|
||||
assert "db_summary" in data
|
||||
|
||||
@patch("app.api.queue.redis.Redis")
|
||||
def test_queue_stats_handles_redis_error(self, mock_redis_cls, client):
|
||||
"""Test queue stats handles Redis connection failure."""
|
||||
mock_redis_cls.from_url.side_effect = Exception("Redis unavailable")
|
||||
|
||||
response = client.get("/api/queue/stats")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_queued"] == 0
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestPendingCountEndpoint:
|
||||
"""Tests for the GET /api/queue/pending-count endpoint."""
|
||||
|
||||
@patch("app.api.queue.redis.Redis")
|
||||
def test_pending_count_returns_200(self, mock_redis_cls, client):
|
||||
"""Test pending count endpoint returns 200."""
|
||||
mock_redis_instance = MagicMock()
|
||||
mock_redis_instance.llen.return_value = 3
|
||||
mock_redis_cls.from_url.return_value = mock_redis_instance
|
||||
|
||||
response = client.get("/api/queue/pending-count")
|
||||
assert response.status_code == 200
|
||||
|
||||
data = response.json()
|
||||
assert "total_pending" in data
|
||||
assert data["total_pending"] >= 0
|
||||
|
||||
@patch("app.api.queue.redis.Redis")
|
||||
def test_pending_count_redis_failure_still_works(self, mock_redis_cls, client):
|
||||
"""Test pending count still works when Redis is down."""
|
||||
mock_redis_cls.from_url.side_effect = Exception("Connection refused")
|
||||
|
||||
response = client.get("/api/queue/pending-count")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "total_pending" in data
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestQueueDashboardView:
|
||||
"""Tests for the GET /admin/queue view."""
|
||||
|
||||
def test_queue_dashboard_requires_login(self, client):
|
||||
"""Test queue dashboard requires authentication."""
|
||||
response = client.get("/admin/queue", follow_redirects=False)
|
||||
assert response.status_code in [200, 302, 401]
|
||||
|
||||
def test_queue_dashboard_non_admin_redirect(self, client):
|
||||
"""Test queue dashboard redirects non-admin users."""
|
||||
# Set non-admin session
|
||||
with client:
|
||||
client.cookies.set("session", "test")
|
||||
response = client.get("/admin/queue", follow_redirects=False)
|
||||
# Should redirect or deny since no admin session
|
||||
assert response.status_code in [200, 302, 401]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestQueueDashboardViewFunction:
|
||||
"""Tests for the queue_dashboard view function."""
|
||||
|
||||
@patch("app.views.queue.templates")
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_dashboard_returns_template(self, mock_templates):
|
||||
"""Test queue dashboard returns template response."""
|
||||
from app.views.queue import queue_dashboard
|
||||
|
||||
mock_request = Mock()
|
||||
mock_request.session = {"user": {"is_admin": True}}
|
||||
mock_db = MagicMock()
|
||||
|
||||
result = await queue_dashboard(mock_request, db=mock_db)
|
||||
|
||||
mock_templates.TemplateResponse.assert_called_once()
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
assert call_args[0][0] == "queue_dashboard.html"
|
||||
context = call_args[0][1]
|
||||
assert "request" in context
|
||||
Reference in New Issue
Block a user