ai-tax-agent/.gitea/workflows/ci.yml

# FILE: .gitea/workflows/ci.yml
# Lint → Test → Build → Scan → Push → Deploy (compose up)

name: CI/CD Pipeline

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
  release:
    types: [published]

env:
  REGISTRY: registry.local
  IMAGE_PREFIX: ai-tax-agent

jobs:
  lint:
    name: Code Quality & Linting
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python 3.12
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: Set up Node.js 20
        uses: actions/setup-node@v4
        with:
          node-version: "20"

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install ruff mypy safety bandit
          find apps -name requirements.txt -exec pip install -r {} \;

      - name: Install Node.js dependencies
        run: |
          find apps -name package.json -execdir npm install \;

      - name: Python linting with ruff
        run: |
          ruff check apps/
          ruff format --check apps/

      - name: Python type checking with mypy
        run: |
          find apps -name "*.py" -path "*/svc-*/*" -exec mypy {} \;

      - name: TypeScript linting
        run: |
          find apps -name "*.ts" -o -name "*.tsx" -execdir npx eslint {} \; || true

      - name: YAML linting
        run: |
          pip install yamllint
          yamllint -d relaxed .

      - name: Docker linting
        run: |
          docker run --rm -i hadolint/hadolint < apps/svc-extract/Dockerfile || true

      - name: Security linting
        run: |
          bandit -r apps/ -f json -o bandit-report.json || true
          safety check --json --output safety-report.json || true

      - name: Upload lint reports
        uses: actions/upload-artifact@v3
        with:
          name: lint-reports
          path: |
            bandit-report.json
            safety-report.json

  policy-validate:
    name: Policy Validation
    runs-on: ubuntu-latest
    needs: lint
    services:
      neo4j:
        image: neo4j:5.15-community
        env:
          NEO4J_AUTH: neo4j/testpass
        ports:
          - 7687:7687
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python 3.12
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install yamllint jsonschema pyyaml
          pip install -r libs/requirements.txt

      - name: YAML lint coverage policy
        run: |
          yamllint config/coverage.yaml

      - name: Validate policy schema
        run: |
          python -c "
          import yaml
          import json
          from jsonschema import validate

          # Load policy
          with open('config/coverage.yaml', 'r') as f:
              policy = yaml.safe_load(f)

          # Load schema
          with open('libs/coverage_schema.json', 'r') as f:
              schema = json.load(f)

          # Validate
          validate(instance=policy, schema=schema)
          print('✅ Policy schema validation passed')
          "

      - name: Validate box references (mock)
        run: |
          python -c "
          import yaml

          # Load policy
          with open('config/coverage.yaml', 'r') as f:
              policy = yaml.safe_load(f)

          # Extract all box references
          boxes = set()
          for schedule in policy.get('schedules', {}).values():
              for evidence in schedule.get('evidence', []):
                  boxes.update(evidence.get('boxes', []))

          print(f'Found {len(boxes)} unique box references')

          # Mock validation - in production this would check against KG
          invalid_boxes = [box for box in boxes if not box.startswith('SA')]
          if invalid_boxes:
              print(f'❌ Invalid box format: {invalid_boxes}')
              exit(1)
          else:
              print('✅ Box format validation passed')
          "

  test:
    name: Test Suite
    runs-on: ubuntu-latest
    needs: lint
    services:
      postgres:
        image: postgres:15-alpine
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: test_db
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

      redis:
        image: redis:7-alpine
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379

      neo4j:
        image: neo4j:5.15-community
        env:
          NEO4J_AUTH: neo4j/testpass
        ports:
          - 7687:7687

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python 3.12
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest pytest-cov pytest-asyncio
          find apps -name requirements.txt -exec pip install -r {} \;

      - name: Run unit tests
        env:
          POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/test_db
          REDIS_URL: redis://localhost:6379
          NEO4J_URI: bolt://localhost:7687
          NEO4J_USER: neo4j
          NEO4J_PASSWORD: testpass
        run: |
          pytest apps/ -v --cov=apps --cov-report=xml --cov-report=html

      - name: Run integration tests
        env:
          POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/test_db
          REDIS_URL: redis://localhost:6379
          NEO4J_URI: bolt://localhost:7687
          NEO4J_USER: neo4j
          NEO4J_PASSWORD: testpass
        run: |
          pytest tests/integration/ -v

      - name: Upload coverage reports
        uses: actions/upload-artifact@v3
        with:
          name: coverage-reports
          path: |
            coverage.xml
            htmlcov/

  build:
    name: Build Docker Images
    runs-on: ubuntu-latest
    needs: [test, policy-validate]
    strategy:
      matrix:
        service:
          - svc-ingestion
          - svc-rpa
          - svc-ocr
          - svc-extract
          - svc-normalize-map
          - svc-kg
          - svc-rag-indexer
          - svc-rag-retriever
          - svc-reason
          - svc-forms
          - svc-hmrc
          - svc-firm-connectors
          - svc-coverage
          - ui-review

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.REGISTRY_USERNAME }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }}
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha,prefix={{branch}}-

      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: apps/${{ matrix.service }}
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  security-scan:
    name: Security Scanning
    runs-on: ubuntu-latest
    needs: build
    strategy:
      matrix:
        service:
          - svc-extract
          - svc-kg
          - svc-rag-retriever
          - svc-coverage
          - ui-review

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }}
          format: "sarif"
          output: "trivy-results-${{ matrix.service }}.sarif"

      - name: Upload Trivy scan results
        uses: actions/upload-artifact@v3
        with:
          name: trivy-results-${{ matrix.service }}
          path: trivy-results-${{ matrix.service }}.sarif

      - name: Run Snyk security scan
        uses: snyk/actions/docker@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }}
          args: --severity-threshold=high
        continue-on-error: true

  sbom:
    name: Generate SBOM
    runs-on: ubuntu-latest
    needs: build
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Syft
        run: |
          curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin

      - name: Generate SBOM for key services
        run: |
          syft ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/svc-extract:${{ github.sha }} -o spdx-json=sbom-svc-extract.json
          syft ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/svc-kg:${{ github.sha }} -o spdx-json=sbom-svc-kg.json
          syft ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/ui-review:${{ github.sha }} -o spdx-json=sbom-ui-review.json

      - name: Upload SBOM artifacts
        uses: actions/upload-artifact@v3
        with:
          name: sbom-reports
          path: sbom-*.json

  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: [security-scan, sbom]
    if: github.ref == 'refs/heads/develop'
    environment: staging

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Docker Compose
        run: |
          sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
          sudo chmod +x /usr/local/bin/docker-compose

      - name: Deploy to staging
        env:
          DOCKER_HOST: ${{ secrets.STAGING_DOCKER_HOST }}
          DOCKER_CERT_PATH: ${{ secrets.STAGING_DOCKER_CERT_PATH }}
          DOCKER_TLS_VERIFY: 1
        run: |
          cd infra/compose
          cp env.example .env
          sed -i 's/local/staging.local/g' .env
          docker-compose -f docker-compose.local.yml pull
          docker-compose -f docker-compose.local.yml up -d

      - name: Run smoke tests
        run: |
          sleep 60  # Wait for services to start
          curl -f https://api.staging.local/health || exit 1
          curl -f https://review.staging.local || exit 1

  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    needs: [security-scan, sbom]
    if: github.event_name == 'release'
    environment: production

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Deploy to production
        env:
          KUBECONFIG: ${{ secrets.KUBECONFIG }}
        run: |
          echo "🚀 Production deployment would happen here"
          echo "📝 TODO: Implement Kubernetes deployment with ArgoCD"
          echo "🏷️ Release tag: ${{ github.event.release.tag_name }}"

  notify:
    name: Notifications
    runs-on: ubuntu-latest
    needs: [deploy-staging, deploy-production]
    if: always()
    steps:
      - name: Notify on success
        if: ${{ needs.deploy-staging.result == 'success' || needs.deploy-production.result == 'success' }}
        run: |
          echo "✅ Deployment successful!"
          # Add Slack/Teams notification here

      - name: Notify on failure
        if: ${{ needs.deploy-staging.result == 'failure' || needs.deploy-production.result == 'failure' }}
        run: |
          echo "❌ Deployment failed!"
          # Add Slack/Teams notification here