ai-tax-agent/Makefile

# FILE: Makefile
# bootstrap, run, test, lint, build, deploy, format, seed

.PHONY: help bootstrap run test lint build deploy format seed clean logs status deploy-external

# Default target
help: ## Show this help message
	@echo "AI Tax Agent System - Development Commands"
	@echo ""
	@echo "Usage: make [target]"
	@echo ""
	@echo "Targets:"
	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST)

# Environment setup
bootstrap: ## Bootstrap the development environment
	@echo "🚀 Bootstrapping AI Tax Agent System..."
	@./scripts/generate-secrets.sh
	@mkdir -p data/{postgres,neo4j,qdrant,minio,vault,redis,prometheus,grafana,loki,authentik}
	@mkdir -p logs/{services,infra}
	@mkdir -p certs
	@echo "📁 Created data and log directories"
	@./scripts/create-networks.sh
	@echo "✅ Bootstrap complete! Run 'make run' to start the system"

# Network management
networks: ## Create external Docker networks
	@./scripts/create-networks.sh

generate-secrets: ## Generate secure secrets for deployment
	@./scripts/generate-secrets.sh
	@ln -sf ../environments/local/.env infra/compose/.env

setup-authentik: ## Configure Authentik SSO after deployment
	@./scripts/setup-authentik.sh

complete-authentik-setup: ## Complete Authentik initial setup and get API token
	@./scripts/complete-authentik-setup.sh



setup-sso: ## Complete end-to-end SSO setup (setup + configuration)
	@echo "🔐 Setting up complete SSO configuration..."
	@echo "Step 1: Completing Authentik initial setup..."
	@./scripts/complete-authentik-setup.sh || true

	@echo "Step 3: Importing blueprint configuration..."
	@./scripts/setup-authentik.sh
	@echo "Step 4: Configuring Vault OIDC..."
	@./scripts/setup-vault.sh
	@echo "🎉 SSO setup complete!"

setup-vault: ## Configure Vault OIDC
	@./scripts/setup-vault.sh

fix-databases: ## Fix common database issues
	@echo "🔧 Fixing database issues..."
	@./scripts/fix-database-issues.sh

deploy-with-fixes: ## Deploy with all discovered fixes applied
	@echo "🚀 Deploying with comprehensive fixes..."
	@./scripts/deploy-with-fixes.sh

networks-clean: ## Remove external Docker networks
	@echo "🧹 Removing external Docker networks..."
	@docker network rm apa-frontend 2>/dev/null || true
	@docker network rm apa-backend 2>/dev/null || true
	@echo "✅ Networks removed"

# Development lifecycle
run: ## Start all services in development mode
	@echo "🏃 Starting AI Tax Agent System..."
	@./infra/scripts/deploy.sh local all

run-simple: ## Start all services without fixes (original behavior)
	@echo "🏃 Starting AI Tax Agent System (simple)..."
	@./scripts/create-networks.sh
	@./scripts/generate-dev-certs.sh
	@cd infra/compose && docker compose up -d
	@echo "⏳ Waiting for services to be ready..."
	@sleep 10
	@make status
	@echo "🔧 Run 'make setup-sso' to configure SSO"

setup: generate-secrets deploy-infra ## Complete setup with secrets and infrastructure
	@echo "🎉 Setup complete! Next steps:"
	@echo "  1. Run 'make setup-sso' to configure SSO"
	@echo "  2. Run 'make deploy-services' to start application services"
	@echo "  3. Access Authentik at https://auth.local.lan"
	@echo ""
	@echo "🎉 System is running!"
	@echo "📊 Grafana: https://grafana.local.lan"
	@echo "🔐 Authentik: https://auth.local.lan"
	@echo "📝 Review UI: https://review.local.lan"
	@echo "🔧 Traefik Dashboard: http://localhost:8080"

stop: ## Stop all services
	@echo "🛑 Stopping AI Tax Agent System..."
	@cd infra/compose && docker compose down

restart: ## Restart all services
	@echo "🔄 Restarting AI Tax Agent System..."
	@make stop
	@make run

# Build and deployment
build: ## Build all Docker images
	@echo "🔨 Building Docker images..."
	@cd infra/compose && docker compose build --parallel
	@echo "✅ Build complete"

build-service: ## Build specific service (usage: make build-service SERVICE=svc-ingestion)
	@echo "🔨 Building $(SERVICE)..."
	@cd infra/compose && docker compose build $(SERVICE)
	@echo "✅ Build complete for $(SERVICE)"

deploy-infra: networks ## Deploy only infrastructure services
	@echo "🏗️  Deploying infrastructure services..."
	@./scripts/generate-dev-certs.sh
	@cd infra/compose && docker compose up -d apa-traefik apa-postgres apa-redis apa-authentik-db apa-authentik-redis
	@echo "⏳ Waiting for databases..."
	@sleep 15
	@make fix-databases
	@cd infra/compose && docker compose up -d apa-authentik-server apa-authentik-worker apa-authentik-outpost apa-vault apa-neo4j apa-qdrant apa-minio apa-prometheus apa-grafana apa-loki
	@echo "✅ Infrastructure deployment complete"
	@echo "⏳ Waiting for services to be ready..."
	@sleep 30
	@echo "🔧 Run 'make setup-sso' to configure SSO"

deploy-services: ## Deploy only application services
	@echo "🚀 Deploying application services..."
	@cd infra/compose && docker compose up -d apa-svc-ingestion apa-svc-extract apa-svc-forms apa-svc-hmrc apa-svc-kg apa-svc-normalize-map apa-svc-ocr apa-svc-rag-indexer apa-svc-rag-retriever apa-svc-reason apa-svc-rpa apa-svc-firm-connectors
	@echo "✅ Services deployment complete"

# Development tools
test: ## Run all tests with coverage
	@echo "🧪 Running all tests..."
	@python -m pytest tests/ -v --cov=libs --cov=apps --cov-report=term-missing --cov-report=html:htmlcov

test-unit: ## Run unit tests only
	@echo "📋 Running unit tests..."
	@python -m pytest tests/unit/ -v --cov=libs --cov=apps --cov-report=term-missing

test-integration: ## Run integration tests only
	@echo "🔗 Running integration tests..."
	@python -m pytest tests/integration/ -v

test-e2e: ## Run end-to-end tests only
	@echo "🌐 Running end-to-end tests..."
	@python -m pytest tests/e2e/ -v

test-no-coverage: ## Run all tests without coverage reporting
	@echo "🧪 Running all tests (no coverage)..."
	@python -m pytest tests/ -v

test-fast: ## Run tests without coverage for faster feedback
	@echo "⚡ Running fast tests..."
	@python -m pytest tests/unit/ -v -x

lint: ## Run linting and code quality checks
	@echo "🔍 Running linting and code quality checks..."
	@ruff check apps/ libs/ tests/ || echo "ruff not installed"
	@mypy apps/ libs/ || echo "mypy not installed"

format: ## Format code
	@echo "✨ Formatting code..."
	@echo "🐍 Python formatting..."
	@ruff format apps/ libs/ tests/ || echo "ruff not installed"
	@echo "📜 TypeScript formatting..."
	@find apps -name "*.ts" -o -name "*.tsx" -exec prettier --write {} \; 2>/dev/null || echo "prettier not installed"



deploy: ## Deploy to production (placeholder)
	@echo "🚀 Deploying to production..."
	@echo "⚠️ Production deployment not implemented yet"
	@echo "📝 TODO: Implement K8s deployment with ArgoCD"

# External services deployment (production)
deploy-external: ## Deploy external services (traefik, authentik, gitea, etc.)
	@echo "🚀 Deploying external services..."
	@./scripts/deploy-external.sh all

deploy-traefik: ## Deploy Traefik reverse proxy
	@./scripts/deploy-external.sh traefik

deploy-authentik: ## Deploy Authentik SSO
	@./scripts/deploy-external.sh authentik

deploy-gitea: ## Deploy Gitea (Git + Registry)
	@./scripts/deploy-external.sh gitea

deploy-nextcloud: ## Deploy Nextcloud
	@./scripts/deploy-external.sh nextcloud

deploy-portainer: ## Deploy Portainer
	@./scripts/deploy-external.sh portainer

# Multi-environment infrastructure deployment
deploy-infra-local: ## Deploy application infrastructure (local)
	@echo "🏗️  Deploying local infrastructure..."
	@./infra/scripts/deploy.sh local infrastructure

deploy-infra-dev: ## Deploy application infrastructure (development)
	@echo "🏗️  Deploying development infrastructure..."
	@./infra/scripts/deploy.sh development infrastructure

deploy-infra-prod: ## Deploy application infrastructure (production)
	@echo "🏗️  Deploying production infrastructure..."
	@./infra/scripts/deploy.sh production infrastructure

deploy-services-local: ## Deploy application services (local)
	@echo "🚀 Deploying local services..."
	@./infra/scripts/deploy.sh local services

deploy-services-dev: ## Deploy application services (development)
	@echo "🚀 Deploying development services..."
	@./infra/scripts/deploy.sh development services

deploy-services-prod: ## Deploy application services (production)
	@echo "🚀 Deploying production services..."
	@./infra/scripts/deploy.sh production services

deploy-monitoring-local: ## Deploy monitoring stack (local)
	@echo "📊 Deploying local monitoring..."
	@./infra/scripts/deploy.sh local monitoring

deploy-monitoring-dev: ## Deploy monitoring stack (development)
	@echo "📊 Deploying development monitoring..."
	@./infra/scripts/deploy.sh development monitoring

deploy-monitoring-prod: ## Deploy monitoring stack (production)
	@echo "📊 Deploying production monitoring..."
	@./infra/scripts/deploy.sh production monitoring

# Production (On-Server) Management
prod-setup: ## Run full production setup (secrets + authentik)
	@echo "🚀 Setting up production environment..."
	@./scripts/setup-prod.sh

prod-deploy: ## Deploy full production stack
	@echo "🚀 Deploying production stack..."
	@./infra/scripts/deploy.sh production infrastructure
	@./infra/scripts/deploy.sh production services

prod-clean-authentik: ## WIPE Authentik data to reset admin password (DANGER)
	@echo "⚠️  WARNING: This will delete all Authentik users and configuration!"
	@read -p "Are you sure? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
	@echo "🛑 Stopping Authentik..."
	@docker stop apa-authentik-server apa-authentik-worker apa-authentik-outpost apa-authentik-db apa-authentik-redis 2>/dev/null || true
	@docker rm apa-authentik-server apa-authentik-worker apa-authentik-outpost apa-authentik-db apa-authentik-redis 2>/dev/null || true
	@echo "🧹 Removing Authentik database volume..."
	@docker volume rm infra_apa-authentik-db-data 2>/dev/null || true
	@echo "🔄 Restarting Authentik..."
	@./infra/scripts/deploy.sh production infrastructure
	@echo "✅ Authentik reset complete. Go to https://auth.app.harkon.co.uk/if/flow/initial-setup/ to set admin password."

# Data management
seed: ## Seed the system with initial data
	@echo "🌱 Seeding system with initial data..."
	@echo "📊 Creating Neo4j constraints and indexes..."
	@docker exec apa-neo4j cypher-shell -u neo4j -p $(NEO4J_PASSWORD) -f /var/lib/neo4j/import/schema.cypher 2>/dev/null || echo "Neo4j not ready"
	@echo "🗂️ Creating Qdrant collections..."
	@curl -X PUT "http://localhost:6333/collections/documents" -H "Content-Type: application/json" -d '{"vectors": {"size": 1536, "distance": "Cosine"}}' 2>/dev/null || echo "Qdrant not ready"
	@echo "✅ Seeding complete"

seed-test-data: ## Load test data for development
	@echo "📋 Loading test data..."
	@echo "ℹ️  Test data loading not implemented yet"

# Monitoring and debugging
logs: ## Show logs from all services
	@cd infra/compose && docker compose logs -f


logs-service: ## Show logs from specific service (usage: make logs-service SERVICE=svc-extract)
	@if [ -z "$(SERVICE)" ]; then \
		echo "❌ Please specify SERVICE (e.g., make logs-service SERVICE=svc-extract)"; \
		exit 1; \
	fi
	@cd infra/compose && docker compose logs -f $(SERVICE)

status: ## Show status of all services
	@echo "📊 Service Status:"
	@cd infra/compose && docker compose ps

health: ## Check health of all services
	@echo "🏥 Health Check:"
	@echo "🔗 Traefik: $$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8080/ping || echo 'DOWN')"
	@echo "🗄️ PostgreSQL: $$(docker exec apa-postgres pg_isready -U postgres 2>/dev/null && echo 'UP' || echo 'DOWN')"
	@echo "📊 Neo4j: $$(curl -s -o /dev/null -w '%{http_code}' http://localhost:7474 || echo 'DOWN')"
	@echo "🔍 Qdrant: $$(curl -s -o /dev/null -w '%{http_code}' http://localhost:6333/health || echo 'DOWN')"
	@echo "📦 MinIO: $$(curl -s -o /dev/null -w '%{http_code}' http://localhost:9000/minio/health/live || echo 'DOWN')"
	@echo "🔐 Vault: $$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8200/v1/sys/health || echo 'DOWN')"
	@echo "🏃 Redis: $$(docker exec apa-redis redis-cli ping 2>/dev/null || echo 'DOWN')"
	@echo "🔐 Authentik: $$(curl -s -k -o /dev/null -w '%{http_code}' https://auth.local.lan || echo 'DOWN')"

verify: ## Run comprehensive infrastructure verification
	@echo "🔍 Running infrastructure verification..."
	@./scripts/verify-infra.sh

troubleshoot: ## Run comprehensive troubleshooting and fixes
	@echo "🔍 Running troubleshooting..."
	@./scripts/troubleshoot.sh

restart-authentik: ## Restart Authentik components in correct order
	@echo "🔄 Restarting Authentik components..."
	@cd infra/compose && docker compose stop apa-authentik-server apa-authentik-worker apa-authentik-outpost
	@make fix-databases
	@cd infra/compose && docker compose up -d apa-authentik-server
	@sleep 15
	@cd infra/compose && docker compose up -d apa-authentik-worker apa-authentik-outpost
	@echo "✅ Authentik restart complete"

restart-unleash: ## Restart Unleash with database fixes
	@echo "🔄 Restarting Unleash..."
	@cd infra/compose && docker compose stop apa-unleash
	@make fix-databases
	@cd infra/compose && docker compose up -d apa-unleash
	@echo "✅ Unleash restart complete"

# Cleanup
clean: ## Clean up containers, volumes, and networks
	@echo "🧹 Cleaning up..."
	@cd infra/compose && docker compose down -v --remove-orphans
	@docker system prune -f
	@echo "✅ Cleanup complete"

clean-data: ## Clean up all data volumes (WARNING: This will delete all data!)
	@echo "⚠️ WARNING: This will delete ALL data!"
	@read -p "Are you sure? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
	@make clean
	@docker volume rm $$(docker volume ls -q | grep ai-tax) 2>/dev/null || true
	@rm -rf data/*
	@echo "🗑️ All data deleted"

# Development utilities
shell: ## Open shell in specific service (usage: make shell SERVICE=svc-extract)
	@if [ -z "$(SERVICE)" ]; then \
		echo "❌ Please specify SERVICE (e.g., make shell SERVICE=svc-extract)"; \
		exit 1; \
	fi
	@docker exec -it $(SERVICE) /bin/bash

db-shell: ## Open PostgreSQL shell
	@docker exec -it apa-postgres psql -U postgres -d tax_system

neo4j-shell: ## Open Neo4j shell
	@docker exec -it apa-neo4j cypher-shell -u neo4j -p $(NEO4J_PASSWORD)

redis-shell: ## Open Redis shell
	@docker exec -it apa-redis redis-cli

# Documentation
docs: ## Generate documentation
	@echo "📚 Generating documentation..."
	@mkdocs build 2>/dev/null || echo "MkDocs not installed"
	@echo "📖 Documentation available at docs/site/index.html"

docs-serve: ## Serve documentation locally
	@echo "📚 Serving documentation..."
	@mkdocs serve 2>/dev/null || echo "MkDocs not installed"

# Security
security-scan: ## Run security scans
	@echo "🔒 Running security scans..."
	@echo "🐳 Container scanning..."
	@trivy image ai-tax-agent/svc-extract:latest 2>/dev/null || echo "Trivy not installed"
	@echo "📋 Dependency scanning..."
	@safety check 2>/dev/null || echo "Safety not installed"
	@echo "🔍 Secret scanning..."
	@gitleaks detect 2>/dev/null || echo "Gitleaks not installed"

# Performance
benchmark: ## Run performance benchmarks
	@echo "⚡ Running performance benchmarks..."
	@echo "ℹ️  Benchmark suite not implemented yet"

load-test: ## Run load tests
	@echo "🏋️ Running load tests..."
	@locust -f tests/load/locustfile.py 2>/dev/null || echo "Locust not installed"

# Backup and restore
backup: ## Create backup of all data
	@echo "💾 Creating backup..."
	@mkdir -p backups/$$(date +%Y%m%d_%H%M%S)
	@docker exec apa-postgres pg_dump -U postgres tax_system > backups/$$(date +%Y%m%d_%H%M%S)/postgres.sql
	@docker exec apa-neo4j neo4j-admin dump --database=neo4j --to=/tmp/neo4j.dump
	@docker cp apa-neo4j:/tmp/neo4j.dump backups/$$(date +%Y%m%d_%H%M%S)/
	@echo "✅ Backup created in backups/ directory"

restore: ## Restore from backup (usage: make restore BACKUP=20240101_120000)
	@if [ -z "$(BACKUP)" ]; then \
		echo "❌ Please specify BACKUP directory (e.g., make restore BACKUP=20240101_120000)"; \
		exit 1; \
	fi
	@echo "📥 Restoring from backup $(BACKUP)..."
	@echo "⚠️ This will overwrite existing data!"
	@read -p "Are you sure? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
	@docker exec -i apa-postgres psql -U postgres -d tax_system < backups/$(BACKUP)/postgres.sql
	@docker cp backups/$(BACKUP)/neo4j.dump apa-neo4j:/tmp/
	@docker exec apa-neo4j neo4j-admin load --database=neo4j --from=/tmp/neo4j.dump --force
	@echo "✅ Restore complete"

# Environment variables
env: ## Show current environment configuration
	@echo "🌍 Environment Configuration:"
	@cd infra/compose && cat .env 2>/dev/null || echo ".env file not found - run 'make bootstrap' first"

# Convenience shortcuts
dev-up: ## Full dev bring-up with automation
	@bash ./scripts/dev-up.sh

dev-down: ## Stop dev environment (pass '-v' to remove volumes)
	@bash ./scripts/dev-down.sh $(FLAG)

hosts: ## Add local domains to /etc/hosts
	@bash ./scripts/hosts-setup.sh

dev-service: ## Run single service locally (usage: make dev-service SERVICE=svc_ingestion)
	@echo "🚀 Starting $(SERVICE) locally..."
	@make deploy-infra
	@echo "📝 Loading environment variables from .env file..."
	@cd apps/$(SERVICE) && \
		export $$(cat ../../.env | grep -v '^#' | xargs) && \
		uvicorn main:app --reload --host 0.0.0.0 --port 8000

test-endpoints: ## Test service endpoints with curl
	@echo "🧪 Testing service endpoints..."
	@curl -s http://localhost:8000/health | jq
	@curl -s -X POST http://localhost:8000/v1/coverage/check \
		-H "Content-Type: application/json" \
		-d '{"tax_year":"2024-25","taxpayer_id":"T-001"}' | jq