feat(pipelines): add custom processing pipeline engine

- Add Pipeline and PipelineStep models with user-specific ownership
- Add pipeline_id FK column to FileRecord
- Migration 017_add_pipelines (batch mode for SQLite FK compat)
- Pipeline CRUD API at /api/pipelines with step management endpoints
- Reorder steps PUT endpoint placed before parameterised {step_id} routes
- POST /api/files/{id}/assign-pipeline for per-file pipeline assignment
- Admin-only POST /api/pipelines/admin/system for system-level pipelines
- Management UI at /pipelines (Jinja2 + Alpine.js + Tailwind)
- Pipelines link added to desktop and mobile navigation
- 41 new tests in tests/test_api_pipelines.py (all passing)

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-07 10:32:00 +00:00
parent 2c12af95fc
commit 89e0c2fb50
11 changed files with 2369 additions and 0 deletions
+70
View File
@@ -6,6 +6,7 @@ from app.database import Base
# Foreign key constants
_FILES_ID_FK = "files.id"
_PIPELINES_ID_FK = "pipelines.id"
class DocumentMetadata(Base):
@@ -79,6 +80,9 @@ class FileRecord(Base):
# Pre-computed text embedding vector stored as JSON array of floats
embedding = Column(Text, nullable=True)
# Processing pipeline assigned to this file (NULL = use system default)
pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=True, index=True)
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
@@ -259,3 +263,69 @@ class SubscriptionPlan(Base):
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class Pipeline(Base):
"""User-defined processing pipeline: an ordered set of steps.
Pipelines are user-specific. A pipeline with ``owner_id = NULL`` is a
*system default* pipeline that only admins may create. Regular users
create pipelines under their own ``owner_id``. When a file has no
explicit pipeline assigned, the active system default is used.
"""
__tablename__ = "pipelines"
id = Column(Integer, primary_key=True, index=True)
# Owner of this pipeline. NULL = system/admin pipeline visible to everyone.
owner_id = Column(String, nullable=True, index=True)
# Human-readable name (unique per owner)
name = Column(String(255), nullable=False)
# Optional description
description = Column(Text, nullable=True)
# When True this pipeline is the default for new files belonging to the owner
# (or the global default when owner_id is NULL). Only one pipeline per
# owner may be active default at a time — enforced at the application level.
is_default = Column(Boolean, nullable=False, default=False)
# Soft-disable without deleting
is_active = Column(Boolean, nullable=False, default=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class PipelineStep(Base):
"""A single step in a processing pipeline.
Steps are executed in ascending ``position`` order. Each step has a
``step_type`` that maps to a built-in processing action and an optional
``config`` JSON blob with step-specific parameters.
"""
__tablename__ = "pipeline_steps"
id = Column(Integer, primary_key=True, index=True)
pipeline_id = Column(Integer, ForeignKey(_PIPELINES_ID_FK), nullable=False, index=True)
# Execution order within the pipeline (lower = earlier)
position = Column(Integer, nullable=False, default=0)
# One of the recognised step types (see PIPELINE_STEP_TYPES in pipelines.py)
step_type = Column(String(100), nullable=False)
# Optional human-readable label override (defaults to step_type label)
label = Column(String(255), nullable=True)
# JSON-encoded step-specific configuration dict
config = Column(Text, nullable=True)
# When False this step is skipped during execution
enabled = Column(Boolean, nullable=False, default=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())