Initial commit

.gitea/workflows/ci.yml

@@ -0,0 +1,426 @@
+# FILE: .gitea/workflows/ci.yml
+# Lint → Test → Build → Scan → Push → Deploy (compose up)
+
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
+  release:
+    types: [published]
+
+env:
+  REGISTRY: registry.local
+  IMAGE_PREFIX: ai-tax-agent
+
+jobs:
+  lint:
+    name: Code Quality & Linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Set up Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff mypy safety bandit
+          find apps -name requirements.txt -exec pip install -r {} \;
+
+      - name: Install Node.js dependencies
+        run: |
+          find apps -name package.json -execdir npm install \;
+
+      - name: Python linting with ruff
+        run: |
+          ruff check apps/
+          ruff format --check apps/
+
+      - name: Python type checking with mypy
+        run: |
+          find apps -name "*.py" -path "*/svc-*/*" -exec mypy {} \;
+
+      - name: TypeScript linting
+        run: |
+          find apps -name "*.ts" -o -name "*.tsx" -execdir npx eslint {} \; || true
+
+      - name: YAML linting
+        run: |
+          pip install yamllint
+          yamllint -d relaxed .
+
+      - name: Docker linting
+        run: |
+          docker run --rm -i hadolint/hadolint < apps/svc-extract/Dockerfile || true
+
+      - name: Security linting
+        run: |
+          bandit -r apps/ -f json -o bandit-report.json || true
+          safety check --json --output safety-report.json || true
+
+      - name: Upload lint reports
+        uses: actions/upload-artifact@v3
+        with:
+          name: lint-reports
+          path: |
+            bandit-report.json
+            safety-report.json
+
+  policy-validate:
+    name: Policy Validation
+    runs-on: ubuntu-latest
+    needs: lint
+    services:
+      neo4j:
+        image: neo4j:5.15-community
+        env:
+          NEO4J_AUTH: neo4j/testpass
+        ports:
+          - 7687:7687
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install yamllint jsonschema pyyaml
+          pip install -r libs/requirements.txt
+
+      - name: YAML lint coverage policy
+        run: |
+          yamllint config/coverage.yaml
+
+      - name: Validate policy schema
+        run: |
+          python -c "
+          import yaml
+          import json
+          from jsonschema import validate
+
+          # Load policy
+          with open('config/coverage.yaml', 'r') as f:
+              policy = yaml.safe_load(f)
+
+          # Load schema
+          with open('libs/coverage_schema.json', 'r') as f:
+              schema = json.load(f)
+
+          # Validate
+          validate(instance=policy, schema=schema)
+          print('✅ Policy schema validation passed')
+          "
+
+      - name: Validate box references (mock)
+        run: |
+          python -c "
+          import yaml
+
+          # Load policy
+          with open('config/coverage.yaml', 'r') as f:
+              policy = yaml.safe_load(f)
+
+          # Extract all box references
+          boxes = set()
+          for schedule in policy.get('schedules', {}).values():
+              for evidence in schedule.get('evidence', []):
+                  boxes.update(evidence.get('boxes', []))
+
+          print(f'Found {len(boxes)} unique box references')
+
+          # Mock validation - in production this would check against KG
+          invalid_boxes = [box for box in boxes if not box.startswith('SA')]
+          if invalid_boxes:
+              print(f'❌ Invalid box format: {invalid_boxes}')
+              exit(1)
+          else:
+              print('✅ Box format validation passed')
+          "
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    needs: lint
+    services:
+      postgres:
+        image: postgres:15-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+      neo4j:
+        image: neo4j:5.15-community
+        env:
+          NEO4J_AUTH: neo4j/testpass
+        ports:
+          - 7687:7687
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-cov pytest-asyncio
+          find apps -name requirements.txt -exec pip install -r {} \;
+
+      - name: Run unit tests
+        env:
+          POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/test_db
+          REDIS_URL: redis://localhost:6379
+          NEO4J_URI: bolt://localhost:7687
+          NEO4J_USER: neo4j
+          NEO4J_PASSWORD: testpass
+        run: |
+          pytest apps/ -v --cov=apps --cov-report=xml --cov-report=html
+
+      - name: Run integration tests
+        env:
+          POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/test_db
+          REDIS_URL: redis://localhost:6379
+          NEO4J_URI: bolt://localhost:7687
+          NEO4J_USER: neo4j
+          NEO4J_PASSWORD: testpass
+        run: |
+          pytest tests/integration/ -v
+
+      - name: Upload coverage reports
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-reports
+          path: |
+            coverage.xml
+            htmlcov/
+
+  build:
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    needs: [test, policy-validate]
+    strategy:
+      matrix:
+        service:
+          - svc-ingestion
+          - svc-rpa
+          - svc-ocr
+          - svc-extract
+          - svc-normalize-map
+          - svc-kg
+          - svc-rag-indexer
+          - svc-rag-retriever
+          - svc-reason
+          - svc-forms
+          - svc-hmrc
+          - svc-firm-connectors
+          - svc-coverage
+          - ui-review
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix={{branch}}-
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: apps/${{ matrix.service }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  security-scan:
+    name: Security Scanning
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        service:
+          - svc-extract
+          - svc-kg
+          - svc-rag-retriever
+          - svc-coverage
+          - ui-review
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }}
+          format: "sarif"
+          output: "trivy-results-${{ matrix.service }}.sarif"
+
+      - name: Upload Trivy scan results
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-results-${{ matrix.service }}
+          path: trivy-results-${{ matrix.service }}.sarif
+
+      - name: Run Snyk security scan
+        uses: snyk/actions/docker@master
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }}
+          args: --severity-threshold=high
+        continue-on-error: true
+
+  sbom:
+    name: Generate SBOM
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Syft
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
+
+      - name: Generate SBOM for key services
+        run: |
+          syft ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/svc-extract:${{ github.sha }} -o spdx-json=sbom-svc-extract.json
+          syft ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/svc-kg:${{ github.sha }} -o spdx-json=sbom-svc-kg.json
+          syft ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/ui-review:${{ github.sha }} -o spdx-json=sbom-ui-review.json
+
+      - name: Upload SBOM artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: sbom-reports
+          path: sbom-*.json
+
+  deploy-staging:
+    name: Deploy to Staging
+    runs-on: ubuntu-latest
+    needs: [security-scan, sbom]
+    if: github.ref == 'refs/heads/develop'
+    environment: staging
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Deploy to staging
+        env:
+          DOCKER_HOST: ${{ secrets.STAGING_DOCKER_HOST }}
+          DOCKER_CERT_PATH: ${{ secrets.STAGING_DOCKER_CERT_PATH }}
+          DOCKER_TLS_VERIFY: 1
+        run: |
+          cd infra/compose
+          cp env.example .env
+          sed -i 's/local/staging.local/g' .env
+          docker-compose -f docker-compose.local.yml pull
+          docker-compose -f docker-compose.local.yml up -d
+
+      - name: Run smoke tests
+        run: |
+          sleep 60  # Wait for services to start
+          curl -f https://api.staging.local/health || exit 1
+          curl -f https://review.staging.local || exit 1
+
+  deploy-production:
+    name: Deploy to Production
+    runs-on: ubuntu-latest
+    needs: [security-scan, sbom]
+    if: github.event_name == 'release'
+    environment: production
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Deploy to production
+        env:
+          KUBECONFIG: ${{ secrets.KUBECONFIG }}
+        run: |
+          echo "🚀 Production deployment would happen here"
+          echo "📝 TODO: Implement Kubernetes deployment with ArgoCD"
+          echo "🏷️ Release tag: ${{ github.event.release.tag_name }}"
+
+  notify:
+    name: Notifications
+    runs-on: ubuntu-latest
+    needs: [deploy-staging, deploy-production]
+    if: always()
+    steps:
+      - name: Notify on success
+        if: ${{ needs.deploy-staging.result == 'success' || needs.deploy-production.result == 'success' }}
+        run: |
+          echo "✅ Deployment successful!"
+          # Add Slack/Teams notification here
+
+      - name: Notify on failure
+        if: ${{ needs.deploy-staging.result == 'failure' || needs.deploy-production.result == 'failure' }}
+        run: |
+          echo "❌ Deployment failed!"
+          # Add Slack/Teams notification here