Initial commit

apps/svc_forms/Dockerfile

@@ -0,0 +1,53 @@
+# Multi-stage build for svc_forms
+FROM python:3.12-slim AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy requirements and install dependencies
+COPY libs/requirements-base.txt /tmp/libs-requirements.txt
+COPY apps/svc_forms/requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r /tmp/libs-requirements.txt -r /tmp/requirements.txt
+
+# Production stage
+FROM python:3.12-slim
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && groupadd -r appuser \
+    && useradd -r -g appuser appuser
+
+# Copy virtual environment from builder
+COPY --from=builder /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Set working directory
+WORKDIR /app
+
+# Copy application code
+COPY libs/ ./libs/
+COPY apps/svc_forms/ ./apps/svc_forms/
+
+# Create non-root user and set permissions
+RUN chown -R appuser:appuser /app
+USER appuser
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/healthz || exit 1
+
+# Expose port
+EXPOSE 8000
+
+# Run the application
+CMD ["python", "-m", "uvicorn", "apps.svc_forms.main:app", "--host", "0.0.0.0", "--port", "8000"]

apps/svc_forms/main.py

@@ -0,0 +1,625 @@
+"""PDF form filling with evidence pack generation."""
+
+# FILE: apps/svc-forms/main.py
+# pylint: disable=wrong-import-position,import-error,too-few-public-methods,global-statement
+# pylint: disable=global-variable-not-assigned,raise-missing-from,unused-argument
+# pylint: disable=broad-exception-caught,no-else-return,too-many-arguments,too-many-positional-arguments
+# pylint: disable=too-many-locals,import-outside-toplevel
+# mypy: disable-error-code=union-attr
+
+
+import os
+
+# Import shared libraries
+import sys
+from datetime import datetime
+from io import BytesIO
+from typing import Any
+
+import structlog
+import ulid
+from fastapi import BackgroundTasks, Depends, HTTPException, Request
+from fastapi.responses import JSONResponse, Response
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from libs.app_factory import create_app
+from libs.config import (
+    BaseAppSettings,
+    create_event_bus,
+    create_minio_client,
+    create_neo4j_client,
+)
+from libs.events import EventBus, EventPayload, EventTopics
+from libs.forms import UK_TAX_FORMS, EvidencePackGenerator, PDFFormFiller
+from libs.neo import Neo4jClient
+from libs.observability import get_metrics, get_tracer, setup_observability
+from libs.schemas import ErrorResponse
+from libs.security import get_current_user, get_tenant_id
+from libs.storage import DocumentStorage, StorageClient
+
+logger = structlog.get_logger()
+
+
+class FormsSettings(BaseAppSettings):
+    """Settings for forms service"""
+
+    service_name: str = "svc-forms"
+
+    # Form templates
+    forms_template_dir: str = "forms/templates"
+    output_bucket: str = "filled-forms"
+    evidence_packs_bucket: str = "evidence-packs"
+
+    # Supported forms
+    supported_forms: list[str] = ["SA100", "SA103", "SA105", "SA106"]
+
+    # PDF configuration
+    pdf_quality: str = "high"
+    flatten_forms: bool = True
+
+
+# Create app and settings
+app, settings = create_app(
+    service_name="svc-forms",
+    title="Tax Agent Forms Service",
+    description="PDF form filling and evidence pack generation",
+    settings_class=FormsSettings,
+)
+
+# Global clients
+storage_client: StorageClient | None = None
+document_storage: DocumentStorage | None = None
+neo4j_client: Neo4jClient | None = None
+pdf_form_filler: PDFFormFiller | None = None
+evidence_pack_generator: EvidencePackGenerator | None = None
+event_bus: EventBus | None = None
+tracer = get_tracer("svc-forms")
+metrics = get_metrics()
+
+
+@app.on_event("startup")
+async def startup_event() -> None:
+    """Initialize service dependencies"""
+    global storage_client, document_storage, neo4j_client, pdf_form_filler  # pylint: disable=line-too-long
+    global evidence_pack_generator, event_bus
+
+    logger.info("Starting forms service")
+
+    # Setup observability
+    setup_observability(settings)
+
+    # Initialize MinIO client
+    minio_client = create_minio_client(settings)
+    storage_client = StorageClient(minio_client)
+    document_storage = DocumentStorage(storage_client)
+
+    # Initialize Neo4j client
+    neo4j_driver = create_neo4j_client(settings)
+    neo4j_client = Neo4jClient(neo4j_driver)
+
+    # Initialize PDF form filler
+    pdf_form_filler = PDFFormFiller()
+
+    # Load form templates
+    for form_id in settings.supported_forms:
+        template_path = os.path.join(settings.forms_template_dir, f"{form_id}.pdf")
+        if os.path.exists(template_path):
+            pdf_form_filler.load_template(form_id, template_path)
+        else:
+            logger.warning(
+                "Form template not found", form_id=form_id, path=template_path
+            )
+
+    # Initialize evidence pack generator
+    evidence_pack_generator = EvidencePackGenerator(storage_client)
+
+    # Initialize event bus
+    event_bus = create_event_bus(settings)
+    await event_bus.start() # fmt: skip # pyright: ignore[reportOptionalMemberAccess]
+
+    # Subscribe to calculation completion events
+    await event_bus.subscribe(  # fmt: skip # pyright: ignore[reportOptionalMemberAccess]
+        EventTopics.CALC_SCHEDULE_READY, _handle_calculation_ready
+    )
+
+    # Ensure buckets exist
+    await storage_client.ensure_bucket(settings.output_bucket)
+    await storage_client.ensure_bucket(settings.evidence_packs_bucket)
+
+    logger.info("Forms service started successfully")
+
+
+@app.on_event("shutdown")
+async def shutdown_event() -> None:
+    """Cleanup service dependencies"""
+    global neo4j_client, event_bus
+
+    logger.info("Shutting down forms service")
+
+    if neo4j_client:
+        await neo4j_client.close()
+
+    if event_bus:
+        await event_bus.stop()
+
+    logger.info("Forms service shutdown complete")
+
+
+@app.get("/health")
+async def health_check() -> dict[str, Any]:
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "service": settings.service_name,
+        "version": "1.0.0",
+        "timestamp": datetime.now().isoformat(),
+        "supported_forms": settings.supported_forms,
+    }
+
+
+@app.post("/fill/{form_id}")
+async def fill_form(
+    form_id: str,
+    field_values: dict[str, Any],
+    background_tasks: BackgroundTasks,
+    current_user: dict[str, Any] = Depends(get_current_user),
+    tenant_id: str = Depends(get_tenant_id),
+) -> dict[str, Any]:
+    """Fill PDF form with provided values"""
+
+    with tracer.start_as_current_span("fill_form") as span:
+        span.set_attribute("form_id", form_id)
+        span.set_attribute("tenant_id", tenant_id)
+        span.set_attribute("field_count", len(field_values))
+
+        try:
+            # Validate form ID
+            if form_id not in settings.supported_forms:
+                raise HTTPException(
+                    status_code=400, detail=f"Unsupported form: {form_id}"
+                )
+
+            # Generate filling ID
+            filling_id = str(ulid.new())
+            span.set_attribute("filling_id", filling_id)
+
+            # Start background form filling
+            background_tasks.add_task(
+                _fill_form_async,
+                form_id,
+                field_values,
+                tenant_id,
+                filling_id,
+                current_user.get("sub", "system"),
+            )
+
+            logger.info("Form filling started", form_id=form_id, filling_id=filling_id)
+
+            return {
+                "filling_id": filling_id,
+                "form_id": form_id,
+                "status": "filling",
+                "field_count": len(field_values),
+            }
+
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error("Failed to start form filling", form_id=form_id, error=str(e))
+            raise HTTPException(status_code=500, detail="Failed to start form filling")
+
+
+@app.post("/fill-from-calculation/{calculation_id}")
+async def fill_form_from_calculation(
+    calculation_id: str,
+    background_tasks: BackgroundTasks,
+    current_user: dict[str, Any] = Depends(get_current_user),
+    tenant_id: str = Depends(get_tenant_id),
+) -> dict[str, Any]:
+    """Fill form using calculation results"""
+
+    with tracer.start_as_current_span("fill_form_from_calculation") as span:
+        span.set_attribute("calculation_id", calculation_id)
+        span.set_attribute("tenant_id", tenant_id)
+
+        try:
+            # Get calculation from Neo4j
+            calc_query = """
+            MATCH (c:Calculation {calculation_id: $calculation_id, tenant_id: $tenant_id})
+            WHERE c.retracted_at IS NULL
+            RETURN c
+            """
+
+            calc_results = await neo4j_client.run_query(  # pyright: ignore[reportOptionalMemberAccess]
+                calc_query, {"calculation_id": calculation_id, "tenant_id": tenant_id}
+            )
+
+            if not calc_results:
+                raise HTTPException(status_code=404, detail="Calculation not found")
+
+            calculation = calc_results[0]["c"]
+            form_id = calculation.get("schedule")
+
+            if not form_id:
+                raise HTTPException(
+                    status_code=400, detail="No schedule found in calculation"
+                )
+
+            # Get form boxes
+            boxes_query = """
+            MATCH (c:Calculation {calculation_id: $calculation_id})-[:HAS_BOX]->(b:FormBox)
+            WHERE c.retracted_at IS NULL AND b.retracted_at IS NULL
+            RETURN b
+            """
+
+            box_results = await neo4j_client.run_query(  # pyright: ignore[reportOptionalMemberAccess]
+                boxes_query, {"calculation_id": calculation_id}
+            )
+
+            # Convert form boxes to field values
+            field_values = {}
+            for box_result in box_results:
+                box = box_result["b"]
+                field_values[f"box_{box['box']}"] = box["value"]
+
+            # Generate filling ID
+            filling_id = str(ulid.new())
+            span.set_attribute("filling_id", filling_id)
+            span.set_attribute("form_id", form_id)
+
+            # Start background form filling
+            background_tasks.add_task(
+                _fill_form_async,
+                form_id,
+                field_values,
+                tenant_id,
+                filling_id,
+                current_user.get("sub", "system"),
+                calculation_id,
+            )
+
+            logger.info(
+                "Form filling from calculation started",
+                form_id=form_id,
+                filling_id=filling_id,
+                calculation_id=calculation_id,
+            )
+
+            return {
+                "filling_id": filling_id,
+                "form_id": form_id,
+                "calculation_id": calculation_id,
+                "status": "filling",
+                "field_count": len(field_values),
+            }
+
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(
+                "Failed to fill form from calculation",
+                calculation_id=calculation_id,
+                error=str(e),
+            )
+            raise HTTPException(
+                status_code=500, detail="Failed to fill form from calculation"
+            )
+
+
+@app.get("/download/{filling_id}")
+async def download_filled_form(
+    filling_id: str,
+    current_user: dict[str, Any] = Depends(get_current_user),
+    tenant_id: str = Depends(get_tenant_id),
+) -> Response:
+    """Download filled form"""
+
+    with tracer.start_as_current_span("download_filled_form") as span:
+        span.set_attribute("filling_id", filling_id)
+        span.set_attribute("tenant_id", tenant_id)
+
+        try:
+            # Get filled form from storage
+            object_key = f"tenants/{tenant_id}/filled/{filling_id}.pdf"
+
+            form_content = await storage_client.get_object(  # pyright: ignore[reportOptionalMemberAccess]
+                settings.output_bucket, object_key
+            )
+
+            if not form_content:
+                raise HTTPException(status_code=404, detail="Filled form not found")
+
+            return Response(
+                content=form_content,
+                media_type="application/pdf",
+                headers={
+                    "Content-Disposition": f"attachment; filename={filling_id}.pdf"
+                },
+            )
+
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(
+                "Failed to download filled form", filling_id=filling_id, error=str(e)
+            )
+            raise HTTPException(
+                status_code=500, detail="Failed to download filled form"
+            )
+
+
+@app.post("/evidence-pack")
+async def create_evidence_pack(
+    taxpayer_id: str,
+    tax_year: str,
+    scope: str,
+    evidence_items: list[dict[str, Any]],
+    background_tasks: BackgroundTasks,
+    current_user: dict[str, Any] = Depends(get_current_user),
+    tenant_id: str = Depends(get_tenant_id),
+) -> dict[str, Any]:
+    """Create evidence pack with supporting documents"""
+
+    with tracer.start_as_current_span("create_evidence_pack") as span:
+        span.set_attribute("taxpayer_id", taxpayer_id)
+        span.set_attribute("tax_year", tax_year)
+        span.set_attribute("scope", scope)
+        span.set_attribute("tenant_id", tenant_id)
+        span.set_attribute("evidence_count", len(evidence_items))
+
+        try:
+            # Generate pack ID
+            pack_id = str(ulid.new())
+            span.set_attribute("pack_id", pack_id)
+
+            # Start background pack creation
+            background_tasks.add_task(
+                _create_evidence_pack_async,
+                taxpayer_id,
+                tax_year,
+                scope,
+                evidence_items,
+                tenant_id,
+                pack_id,
+                current_user.get("sub", "system"),
+            )
+
+            logger.info(
+                "Evidence pack creation started",
+                pack_id=pack_id,
+                taxpayer_id=taxpayer_id,
+                scope=scope,
+            )
+
+            return {
+                "pack_id": pack_id,
+                "taxpayer_id": taxpayer_id,
+                "tax_year": tax_year,
+                "scope": scope,
+                "status": "creating",
+                "evidence_count": len(evidence_items),
+            }
+
+        except Exception as e:
+            logger.error("Failed to start evidence pack creation", error=str(e))
+            raise HTTPException(
+                status_code=500, detail="Failed to start evidence pack creation"
+            )
+
+
+@app.get("/forms")
+async def list_supported_forms(
+    current_user: dict[str, Any] = Depends(get_current_user),
+    tenant_id: str = Depends(get_tenant_id),
+) -> dict[str, Any]:
+    """List supported forms with field information"""
+
+    try:
+        forms_info = []
+
+        for form_id in settings.supported_forms:
+            form_config = UK_TAX_FORMS.get(form_id, {})
+
+            # Get form fields if template is loaded
+            fields = []
+            if pdf_form_filler and form_id in pdf_form_filler.form_templates:
+                fields = pdf_form_filler.get_form_fields(form_id)
+
+            forms_info.append(
+                {
+                    "form_id": form_id,
+                    "name": form_config.get("name", form_id),
+                    "template_available": form_id
+                    in (pdf_form_filler.form_templates if pdf_form_filler else {}),
+                    "field_count": len(fields),
+                    "fields": fields[:10],  # Limit to first 10 fields for overview
+                }
+            )
+
+        return {"supported_forms": forms_info, "total_forms": len(forms_info)}
+
+    except Exception as e:
+        logger.error("Failed to list forms", error=str(e))
+        raise HTTPException(status_code=500, detail="Failed to list forms")
+
+
+async def _handle_calculation_ready(topic: str, payload: EventPayload) -> None:
+    """Handle calculation completion events for auto-form filling"""
+    try:
+        data = payload.data
+        calculation_id = data.get("calculation_id")
+        schedule = data.get("schedule")
+        tenant_id = data.get("tenant_id")
+
+        if not calculation_id or not schedule or not tenant_id:
+            logger.warning("Invalid calculation ready event", data=data)
+            return
+
+        logger.info(
+            "Auto-filling form from calculation",
+            calculation_id=calculation_id,
+            schedule=schedule,
+        )
+
+        # Get form boxes from event data
+        form_boxes = data.get("form_boxes", {})
+
+        # Convert to field values
+        field_values = {}
+        for box_id, box_data in form_boxes.items():
+            field_values[f"box_{box_id}"] = box_data.get("value")
+
+        await _fill_form_async(
+            form_id=schedule,
+            field_values=field_values,
+            tenant_id=tenant_id,
+            filling_id=str(ulid.new()),
+            actor=payload.actor,
+            calculation_id=calculation_id,
+        )
+
+    except Exception as e:
+        logger.error("Failed to handle calculation ready event", error=str(e))
+
+
+async def _fill_form_async(
+    form_id: str,
+    field_values: dict[str, Any],
+    tenant_id: str,
+    filling_id: str,
+    actor: str,
+    calculation_id: str | None = None,
+) -> None:
+    """Fill form asynchronously"""
+
+    with tracer.start_as_current_span("fill_form_async") as span:
+        span.set_attribute("form_id", form_id)
+        span.set_attribute("filling_id", filling_id)
+        span.set_attribute("tenant_id", tenant_id)
+
+        try:
+            # Fill the form
+            filled_pdf = pdf_form_filler.fill_form(form_id, field_values) # fmt: skip # pyright: ignore[reportOptionalMemberAccess]
+
+            if not filled_pdf:
+                # pylint: disable-next=broad-exception-raised
+                raise Exception("Form filling failed")
+
+            # Store filled form
+            object_key = f"tenants/{tenant_id}/filled/{filling_id}.pdf"
+
+            success = await storage_client.put_object(  # fmt: skip # pyright: ignore[reportOptionalMemberAccess]
+                bucket_name=settings.output_bucket,
+                object_name=object_key,
+                data=BytesIO(filled_pdf),
+                length=len(filled_pdf),
+                content_type="application/pdf",
+                metadata={
+                    "form_id": form_id,
+                    "filling_id": filling_id,
+                    "tenant_id": tenant_id,
+                    "calculation_id": calculation_id or "",
+                    "filled_at": datetime.utcnow().isoformat(),
+                },
+            )
+
+            if not success:
+                # pylint: disable-next=broad-exception-raised
+                raise Exception("Failed to store filled form")
+
+            # Update metrics
+            metrics.counter("forms_filled_total").labels(
+                tenant_id=tenant_id, form_id=form_id
+            ).inc()
+
+            # Publish completion event
+            event_payload = EventPayload(
+                data={
+                    "filling_id": filling_id,
+                    "form_id": form_id,
+                    "tenant_id": tenant_id,
+                    "calculation_id": calculation_id,
+                    "s3_url": f"s3://{settings.output_bucket}/{object_key}",
+                    "field_count": len(field_values),
+                },
+                actor=actor,
+                tenant_id=tenant_id,
+            )
+
+            await event_bus.publish(EventTopics.FORM_FILLED, event_payload) # fmt: skip # pyright: ignore[reportOptionalMemberAccess]
+
+            logger.info(
+                "Form filling completed", filling_id=filling_id, form_id=form_id
+            )
+
+        except Exception as e:
+            logger.error("Form filling failed", filling_id=filling_id, error=str(e))
+
+            # Update error metrics
+            metrics.counter("form_filling_errors_total").labels(
+                tenant_id=tenant_id, form_id=form_id, error_type=type(e).__name__
+            ).inc()
+
+
+async def _create_evidence_pack_async(
+    taxpayer_id: str,
+    tax_year: str,
+    scope: str,
+    evidence_items: list[dict[str, Any]],
+    tenant_id: str,
+    pack_id: str,
+    actor: str,
+) -> None:
+    """Create evidence pack asynchronously"""
+
+    with tracer.start_as_current_span("create_evidence_pack_async") as span:
+        span.set_attribute("pack_id", pack_id)
+        span.set_attribute("taxpayer_id", taxpayer_id)
+        span.set_attribute("scope", scope)
+
+        try:
+            # Create evidence pack
+            pack_result = await evidence_pack_generator.create_evidence_pack(  # fmt: skip # pyright: ignore[reportOptionalMemberAccess]
+                taxpayer_id=taxpayer_id,
+                tax_year=tax_year,
+                scope=scope,
+                evidence_items=evidence_items,
+            )
+
+            # Update metrics
+            metrics.counter("evidence_packs_created_total").labels(
+                tenant_id=tenant_id, scope=scope
+            ).inc()
+
+            logger.info(
+                "Evidence pack created",
+                pack_id=pack_id,
+                pack_size=pack_result["pack_size"],
+                evidence_count=pack_result["evidence_count"],
+            )
+
+        except Exception as e:
+            logger.error("Evidence pack creation failed", pack_id=pack_id, error=str(e))
+
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request: Request, exc: HTTPException) -> JSONResponse:
+    """Handle HTTP exceptions with RFC7807 format"""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=ErrorResponse(
+            type=f"https://httpstatuses.com/{exc.status_code}",
+            title=exc.detail,
+            status=exc.status_code,
+            detail=exc.detail,
+            instance=str(request.url),
+            trace_id="",
+        ).model_dump(),
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("main:app", host="0.0.0.0", port=8009, reload=True, log_config=None)

apps/svc_forms/requirements.txt

@@ -0,0 +1,37 @@
+# FastAPI and server
+fastapi>=0.104.1
+uvicorn[standard]>=0.24.0
+pydantic>=2.5.0
+
+# Service-specific dependencies
+# PDF form filling
+pdfrw>=0.4
+reportlab>=4.0.0
+
+# PDF processing
+PyPDF2>=3.0.0
+pypdf>=3.17.0
+
+# Image processing for overlays
+Pillow>=10.1.0
+
+# ZIP file creation for evidence packs
+zipfile36>=0.1.3
+
+# Template processing
+jinja2>=3.1.0
+
+# QR code generation
+qrcode>=7.4.0
+
+# Barcode generation
+python-barcode>=0.15.0
+
+# Font handling
+fonttools>=4.44.0
+
+# Additional PDF utilities
+pdfminer.six>=20231228
+
+# Document conversion
+python-docx>=1.1.0