Fix production deployment: Authentik blueprint, env vars, and infrastructure config

.gitignore

@@ -88,7 +88,8 @@ instance/
 
 # Sphinx documentation
 docs/_build/
-
+infra/base/certs/
+.provider.env
 # PyBuilder
 .pybuilder/
 target/

infra/base/authentik/bootstrap-prod.yaml

@@ -50,18 +50,57 @@ entries:
       groups:
         - !Find [authentik_core.group, [name, "Administrators"]]
 
-  # --- Scope mappings (find existing ones and get stable IDs) -----------------
+  # --- Scope mappings --------------------------------------------------------
+  - id: scope_openid
+    model: authentik_providers_oauth2.scopemapping
+    identifiers:
+      scope_name: openid
+
+  - id: scope_profile
+    model: authentik_providers_oauth2.scopemapping
+    identifiers:
+      scope_name: profile
+
+  - id: scope_email
+    model: authentik_providers_oauth2.scopemapping
+    identifiers:
+      scope_name: email
+
   - id: scope_groups
     model: authentik_providers_oauth2.scopemapping
     identifiers:
       scope_name: groups
     attrs:
-      name: "groups"
+      name: "authentik default OAuth Mapping: OpenID 'groups'"
       expression: |
         return {
             "groups": [group.name for group in request.user.ak_groups.all()]
         }
 
+  - id: scope_offline
+    model: authentik_providers_oauth2.scopemapping
+    identifiers:
+      scope_name: offline_access
+
+  # Helper finders/definitions
+  - id: default_signing_key
+    model: authentik_crypto.certificatekeypair
+    state: present
+    identifiers:
+      name: "authentik Self-signed Certificate"
+
+  - id: default_authz_flow
+    model: authentik_flows.flow
+    state: present
+    identifiers:
+      slug: "default-authentication-flow"
+
+  - id: default_inval_flow
+    model: authentik_flows.flow
+    state: present
+    identifiers:
+      slug: "default-invalidation-flow"
+
   # --- AI Tax Agent API (Production) -----------------------------------------
   - model: authentik_providers_oauth2.oauth2provider
     state: present
@@ -75,39 +114,18 @@ entries:
       issuer_mode: "per_provider"
       sub_mode: "hashed_user_id"
       include_claims_in_id_token: true
-      signing_key:
+      signing_key: !KeyOf default_signing_key
-        !Find [
-          authentik_crypto.certificatekeypair,
-          [name, "authentik Self-signed Certificate"],
-        ]
       redirect_uris:
         - matching_mode: strict
           url: "https://api.app.harkon.co.uk/auth/callback"
       scope_mappings:
-        - !Find [
+        - !KeyOf scope_openid
-            authentik_providers_oauth2.scopemapping,
+        - !KeyOf scope_profile
-            [managed, "goauthentik.io/providers/oauth2/scope-openid"],
+        - !KeyOf scope_email
-          ]
+        - !KeyOf scope_groups
-        - !Find [
+        - !KeyOf scope_offline
-            authentik_providers_oauth2.scopemapping,
+      authorization_flow: !KeyOf default_authz_flow
-            [managed, "goauthentik.io/providers/oauth2/scope-profile"],
+      invalidation_flow: !KeyOf default_inval_flow
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-email"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [scope_name, "groups"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-offline_access"],
-          ]
-      authorization_flow:
-        !Find [authentik_flows.flow, [slug, "default-authentication-flow"]]
-      invalidation_flow:
-        !Find [authentik_flows.flow, [slug, "default-invalidation-flow"]]
 
   - model: authentik_core.application
     state: present
@@ -139,39 +157,18 @@ entries:
       issuer_mode: "per_provider"
       sub_mode: "hashed_user_id"
       include_claims_in_id_token: true
-      signing_key:
+      signing_key: !KeyOf default_signing_key
-        !Find [
-          authentik_crypto.certificatekeypair,
-          [name, "authentik Self-signed Certificate"],
-        ]
       redirect_uris:
         - matching_mode: strict
           url: "https://minio.app.harkon.co.uk/oauth_callback"
       scope_mappings:
-        - !Find [
+        - !KeyOf scope_openid
-            authentik_providers_oauth2.scopemapping,
+        - !KeyOf scope_profile
-            [managed, "goauthentik.io/providers/oauth2/scope-openid"],
+        - !KeyOf scope_email
-          ]
+        - !KeyOf scope_groups
-        - !Find [
+        - !KeyOf scope_offline
-            authentik_providers_oauth2.scopemapping,
+      authorization_flow: !KeyOf default_authz_flow
-            [managed, "goauthentik.io/providers/oauth2/scope-profile"],
+      invalidation_flow: !KeyOf default_inval_flow
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-email"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [scope_name, "groups"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-offline_access"],
-          ]
-      authorization_flow:
-        !Find [authentik_flows.flow, [slug, "default-authentication-flow"]]
-      invalidation_flow:
-        !Find [authentik_flows.flow, [slug, "default-invalidation-flow"]]
 
   - model: authentik_core.application
     state: present
@@ -203,41 +200,20 @@ entries:
       issuer_mode: "per_provider"
       sub_mode: "hashed_user_id"
       include_claims_in_id_token: true
-      signing_key:
+      signing_key: !KeyOf default_signing_key
-        !Find [
-          authentik_crypto.certificatekeypair,
-          [name, "authentik Self-signed Certificate"],
-        ]
       redirect_uris:
         - matching_mode: strict
           url: "https://vault.app.harkon.co.uk/ui/vault/auth/oidc/oidc/callback"
         - matching_mode: strict
           url: "https://vault.app.harkon.co.uk/oidc/callback"
       scope_mappings:
-        - !Find [
+        - !KeyOf scope_openid
-            authentik_providers_oauth2.scopemapping,
+        - !KeyOf scope_profile
-            [managed, "goauthentik.io/providers/oauth2/scope-openid"],
+        - !KeyOf scope_email
-          ]
+        - !KeyOf scope_groups
-        - !Find [
+        - !KeyOf scope_offline
-            authentik_providers_oauth2.scopemapping,
+      authorization_flow: !KeyOf default_authz_flow
-            [managed, "goauthentik.io/providers/oauth2/scope-profile"],
+      invalidation_flow: !KeyOf default_inval_flow
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-email"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [scope_name, "groups"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-offline_access"],
-          ]
-      authorization_flow:
-        !Find [authentik_flows.flow, [slug, "default-authentication-flow"]]
-      invalidation_flow:
-        !Find [authentik_flows.flow, [slug, "default-invalidation-flow"]]
 
   - model: authentik_core.application
     state: present
@@ -256,6 +232,32 @@ entries:
       policy_engine_mode: "any"
 
   # --- Grafana (Production) --------------------------------------------------
+
+  # Custom Role Mapping for Grafana
+  - id: scope_grafana_roles
+    model: authentik_providers_oauth2.scopemapping
+    state: present
+    identifiers:
+      name: "Grafana Role Mapping"
+    attrs:
+      name: "Grafana Role Mapping"
+      description: "Maps Authentik groups to Grafana roles"
+      scope_name: "role"
+      expression: |
+        # Map Authentik groups to Grafana roles
+        user_groups = [group.name for group in request.user.ak_groups.all()]
+
+        # Admin role mapping
+        if "authentik Admins" in user_groups or "Administrators" in user_groups:
+            return "Admin"
+
+        # Editor role mapping
+        if "Tax Reviewers" in user_groups or "Accountants" in user_groups:
+            return "Editor"
+
+        # Default to Viewer role
+        return "Viewer"
+
   - model: authentik_providers_oauth2.oauth2provider
     state: present
     identifiers:
@@ -271,36 +273,15 @@ entries:
       sub_mode: "hashed_user_id"
       include_claims_in_id_token: true
       issuer_mode: "per_provider"
-      signing_key:
+      signing_key: !KeyOf default_signing_key
-        !Find [
-          authentik_crypto.certificatekeypair,
-          [name, "authentik Self-signed Certificate"],
-        ]
       property_mappings:
-        - !Find [
+        - !KeyOf scope_openid
-            authentik_providers_oauth2.scopemapping,
+        - !KeyOf scope_email
-            [managed, "goauthentik.io/providers/oauth2/scope-openid"],
+        - !KeyOf scope_profile
-          ]
+        - !KeyOf scope_groups
-        - !Find [
+        - !KeyOf scope_grafana_roles
-            authentik_providers_oauth2.scopemapping,
+      authorization_flow: !KeyOf default_authz_flow
-            [managed, "goauthentik.io/providers/oauth2/scope-email"],
+      invalidation_flow: !KeyOf default_inval_flow
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [managed, "goauthentik.io/providers/oauth2/scope-profile"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [scope_name, "groups"],
-          ]
-        - !Find [
-            authentik_providers_oauth2.scopemapping,
-            [name, "Grafana Role Mapping"],
-          ]
-      authorization_flow:
-        !Find [authentik_flows.flow, [slug, "default-authentication-flow"]]
-      invalidation_flow:
-        !Find [authentik_flows.flow, [slug, "default-invalidation-flow"]]
 
   - model: authentik_core.application
     state: present
@@ -319,7 +300,8 @@ entries:
       policy_engine_mode: "any"
 
   # --- Policies --------------------------------------------------------------
-  - model: authentik_policies_expression.expressionpolicy
+  - id: policy_always_allow
+    model: authentik_policies_expression.expressionpolicy
     state: present
     identifiers:
       name: "Always Allow"
@@ -329,11 +311,7 @@ entries:
   - model: authentik_policies.policybinding
     state: present
     identifiers:
-      policy:
+      policy: !KeyOf policy_always_allow
-        !Find [
-          authentik_policies_expression.expressionpolicy,
-          [name, "Always Allow"],
-        ]
       target:
         !Find [authentik_core.application, [slug, "ai-tax-agent-api-prod"]]
     attrs:
@@ -342,11 +320,7 @@ entries:
   - model: authentik_policies.policybinding
     state: present
     identifiers:
-      policy:
+      policy: !KeyOf policy_always_allow
-        !Find [
-          authentik_policies_expression.expressionpolicy,
-          [name, "Always Allow"],
-        ]
       target: !Find [authentik_core.application, [slug, "minio-prod"]]
     attrs:
       order: 0
@@ -354,11 +328,7 @@ entries:
   - model: authentik_policies.policybinding
     state: present
     identifiers:
-      policy:
+      policy: !KeyOf policy_always_allow
-        !Find [
-          authentik_policies_expression.expressionpolicy,
-          [name, "Always Allow"],
-        ]
       target: !Find [authentik_core.application, [slug, "vault-prod"]]
     attrs:
       order: 0
@@ -366,11 +336,7 @@ entries:
   - model: authentik_policies.policybinding
     state: present
     identifiers:
-      policy:
+      policy: !KeyOf policy_always_allow
-        !Find [
-          authentik_policies_expression.expressionpolicy,
-          [name, "Always Allow"],
-        ]
       target: !Find [authentik_core.application, [slug, "grafana-prod"]]
     attrs:
       order: 0

infra/base/infrastructure.yaml

@@ -105,6 +105,7 @@ services:
       AUTHENTIK_GRAFANA_CLIENT_SECRET: ${AUTHENTIK_GRAFANA_CLIENT_SECRET}
       AUTHENTIK_MINIO_CLIENT_SECRET: ${AUTHENTIK_MINIO_CLIENT_SECRET}
       AUTHENTIK_VAULT_CLIENT_SECRET: ${AUTHENTIK_VAULT_CLIENT_SECRET}
+      AUTHENTIK_BOOTSTRAP_FILE: /blueprints/ai-tax-agent-bootstrap.yaml
     depends_on:
       - apa-authentik-db
       - apa-authentik-redis
@@ -139,6 +140,7 @@ services:
       AUTHENTIK_GRAFANA_CLIENT_SECRET: ${AUTHENTIK_GRAFANA_CLIENT_SECRET}
       AUTHENTIK_MINIO_CLIENT_SECRET: ${AUTHENTIK_MINIO_CLIENT_SECRET}
       AUTHENTIK_VAULT_CLIENT_SECRET: ${AUTHENTIK_VAULT_CLIENT_SECRET}
+      AUTHENTIK_BOOTSTRAP_FILE: /blueprints/ai-tax-agent-bootstrap.yaml
     depends_on:
       - apa-authentik-db
       - apa-authentik-redis

infra/base/services.yaml

@@ -44,6 +44,13 @@ services:
       - "traefik.http.routers.svc-ingestion.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-ingestion.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-ingestion.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-nats
 
   # Data Extraction Service
   apa-svc-extract:
@@ -77,6 +84,13 @@ services:
       - "traefik.http.routers.svc-extract.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-extract.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-extract.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-nats
 
   # Knowledge Graph Service
   apa-svc-kg:
@@ -104,6 +118,10 @@ services:
       - "traefik.http.routers.svc-kg.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-kg.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-kg.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-neo4j
+      - apa-nats
 
   # RAG Retrieval Service
   apa-svc-rag-retriever:
@@ -134,6 +152,11 @@ services:
       - "traefik.http.routers.svc-rag-retriever.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-rag-retriever.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-rag-retriever.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-neo4j
+      - apa-qdrant
+      - apa-nats
 
   # Forms Service
   apa-svc-forms:
@@ -167,6 +190,14 @@ services:
       - "traefik.http.routers.svc-forms.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-forms.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-forms.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # HMRC Integration Service
   apa-svc-hmrc:
@@ -201,6 +232,14 @@ services:
       - "traefik.http.routers.svc-hmrc.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-hmrc.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-hmrc.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # OCR Service
   apa-svc-ocr:
@@ -234,6 +273,14 @@ services:
       - "traefik.http.routers.svc-ocr.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-ocr.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-ocr.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # RAG Indexer Service
   apa-svc-rag-indexer:
@@ -267,6 +314,14 @@ services:
       - "traefik.http.routers.svc-rag-indexer.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-rag-indexer.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-rag-indexer.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # Reasoning Service
   apa-svc-reason:
@@ -300,6 +355,14 @@ services:
       - "traefik.http.routers.svc-reason.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-reason.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-reason.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # RPA Service
   apa-svc-rpa:
@@ -333,6 +396,14 @@ services:
       - "traefik.http.routers.svc-rpa.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-rpa.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-rpa.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # Normalize & Map Service
   apa-svc-normalize-map:
@@ -366,6 +437,14 @@ services:
       - "traefik.http.routers.svc-normalize-map.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-normalize-map.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-normalize-map.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # Coverage Service
   apa-svc-coverage:
@@ -399,6 +478,14 @@ services:
       - "traefik.http.routers.svc-coverage.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-coverage.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-coverage.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # Firm Connectors Service
   apa-svc-firm-connectors:
@@ -432,6 +519,14 @@ services:
       - "traefik.http.routers.svc-firm-connectors.tls.certresolver=godaddy"
       - "traefik.http.routers.svc-firm-connectors.middlewares=authentik-forwardauth@file,rate-limit@file,strip-api-prefixes@file"
       - "traefik.http.services.svc-firm-connectors.loadbalancer.server.port=8000"
+    depends_on:
+      - apa-vault
+      - apa-postgres
+      - apa-neo4j
+      - apa-redis
+      - apa-minio
+      - apa-qdrant
+      - apa-nats
 
   # Review UI
   # apa-ui-review:

infra/compose/compose.build.yaml

@@ -85,4 +85,4 @@ services:
     build:
       context: ../../ui_review
       dockerfile: Dockerfile
-    image: gitea.harkon.co.uk/harkon/ui-review:latest
+    image: gitea.harkon.co.uk/harkon/ui-review:1.0.1

infra/environments/development/compose.yaml

@@ -0,0 +1,10 @@
+# FILE: infra/environments/development/compose.yaml
+# Development Environment Unified Compose File
+# Includes base configurations from infra/base/
+
+include:
+  - ../../base/infrastructure.yaml
+  - ../../base/services.yaml
+  - ../../base/monitoring.yaml
+
+name: ai-tax-agent-development

infra/environments/production/compose.override.yaml

@@ -0,0 +1,10 @@
+# FILE: infra/environments/production/compose.override.yaml
+# Production Overrides
+# Bind Traefik to standard ports 80/443
+
+services:
+  apa-traefik:
+    ports:
+      - "80:80"
+      - "443:443"
+      - "8080:8080" # Dashboard (protected by middleware)

infra/environments/production/compose.yaml

@@ -0,0 +1,10 @@
+# FILE: infra/environments/production/compose.yaml
+# Production Environment Unified Compose File
+# Includes base configurations from infra/base/
+
+include:
+  - ../../base/infrastructure.yaml
+  - ../../base/services.yaml
+  - ../../base/monitoring.yaml
+
+name: ai-tax-agent-production

infra/scripts/deploy.sh

@@ -205,9 +205,14 @@ deploy_all() {
     fi
 
     # Deploy in order
+    local unified_compose="$INFRA_DIR/environments/$ENVIRONMENT/compose.yaml"
+
     if [ "$ENVIRONMENT" = "local" ]; then
         log_info "Deploying unified stack for local environment..."
         compose_cmd "all" up -d "$@"
+    elif [ -f "$unified_compose" ]; then
+        log_info "Deploying unified stack for $ENVIRONMENT environment..."
+        docker compose -f "$unified_compose" --env-file "$ENV_FILE" --project-name "ai-tax-agent-$ENVIRONMENT" up -d "$@"
     else
         deploy_infrastructure "$@"
         sleep 5