#!/bin/bash # Cleanup and align infrastructure structure # This script consolidates configurations and removes duplication set -e # Colors GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' RED='\033[0;31m' NC='\033[0m' log_info() { echo -e "${BLUE}ℹ️ $1${NC}" } log_success() { echo -e "${GREEN}✅ $1${NC}" } log_warning() { echo -e "${YELLOW}⚠️ $1${NC}" } log_error() { echo -e "${RED}❌ $1${NC}" } # Script directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" INFRA_DIR="$PROJECT_ROOT/infra" log_info "Cleaning up infrastructure structure..." echo " Project Root: $PROJECT_ROOT" echo " Infra Dir: $INFRA_DIR" echo "" # Step 1: Backup current structure log_info "Step 1: Creating backup..." BACKUP_DIR="$PROJECT_ROOT/infra-backup-$(date +%Y%m%d_%H%M%S)" mkdir -p "$BACKUP_DIR" cp -r "$INFRA_DIR/configs" "$BACKUP_DIR/" 2>/dev/null || true log_success "Backup created at $BACKUP_DIR" # Step 2: Align Traefik configurations log_info "Step 2: Aligning Traefik configurations..." # The source of truth is infra/compose/traefik/config/ # Remove duplicates from infra/configs/traefik/config/ if [ -d "$INFRA_DIR/configs/traefik/config" ]; then log_warning " Removing duplicate Traefik configs from infra/configs/traefik/config/" rm -rf "$INFRA_DIR/configs/traefik/config" log_success " Removed duplicate Traefik configs" fi # Keep only app-specific Traefik middleware in configs mkdir -p "$INFRA_DIR/configs/traefik" cat > "$INFRA_DIR/configs/traefik/app-middlewares.yml" << 'EOF' # Application-specific Traefik middlewares # These are loaded by the application infrastructure, not the external Traefik http: middlewares: # Large upload middleware for Gitea registry gitea-large-upload: buffering: maxRequestBodyBytes: 5368709120 # 5GB memRequestBodyBytes: 104857600 # 100MB maxResponseBodyBytes: 5368709120 # 5GB memResponseBodyBytes: 104857600 # 100MB retryExpression: "IsNetworkError() && Attempts() < 3" # Rate limiting for public APIs api-ratelimit: rateLimit: average: 100 burst: 50 period: 1s # Security headers security-headers: headers: frameDeny: true sslRedirect: true browserXssFilter: true contentTypeNosniff: true stsIncludeSubdomains: true stsPreload: true stsSeconds: 31536000 EOF log_success " Created app-specific Traefik middlewares" # Step 3: Align Authentik configurations log_info "Step 3: Aligning Authentik configurations..." # infra/compose/authentik/ - Production service configs # infra/configs/authentik/ - Application bootstrap configs (keep separate) if [ -d "$INFRA_DIR/configs/authentik" ]; then log_info " Keeping app-specific Authentik bootstrap in infra/configs/authentik/" log_success " Authentik configs aligned" fi # Step 4: Clean up old directories log_info "Step 4: Cleaning up old directories..." # Remove old standalone config directories that were moved OLD_DIRS=( "$INFRA_DIR/traefik" "$INFRA_DIR/grafana" "$INFRA_DIR/prometheus" "$INFRA_DIR/loki" "$INFRA_DIR/promtail" "$INFRA_DIR/vault" "$INFRA_DIR/neo4j" "$INFRA_DIR/postgres" ) for dir in "${OLD_DIRS[@]}"; do if [ -d "$dir" ] && [ -f "$INFRA_DIR/configs/$(basename $dir)/.moved" ]; then log_warning " Removing old directory: $dir" rm -rf "$dir" log_success " Removed $dir" fi done # Step 5: Update .gitignore log_info "Step 5: Updating .gitignore..." cat > "$INFRA_DIR/.gitignore" << 'EOF' # Environment files (contain secrets) environments/*/.env !environments/*/.env.example compose/*/.env !compose/env.example # Certificates certs/*/ !certs/.gitkeep compose/*/certs/ !compose/*/certs/.gitkeep # Provider credentials compose/traefik/.provider.env configs/traefik/.provider.env # Data directories compose/*/data/ compose/*/media/ compose/authentik/media/ compose/authentik/custom-templates/ compose/portainer/portainer/ # Backup files *.backup *.tmp *-backup-*/ # Docker volumes (if mounted locally) volumes/ # Logs *.log logs/ # Moved markers **/.moved EOF log_success ".gitignore updated" # Step 6: Create README for external services log_info "Step 6: Creating documentation..." cat > "$INFRA_DIR/compose/README.md" << 'EOF' # External Services This directory contains Docker Compose configurations for external services that run on the production server. ## Services ### Traefik - **Location**: `traefik/` - **Purpose**: Reverse proxy and load balancer for all services - **Deploy**: `cd traefik && docker compose up -d` - **Access**: https://traefik.harkon.co.uk ### Authentik - **Location**: `authentik/` - **Purpose**: SSO and authentication provider - **Deploy**: `cd authentik && docker compose up -d` - **Access**: https://authentik.harkon.co.uk ### Gitea - **Location**: `gitea/` - **Purpose**: Git repository hosting and container registry - **Deploy**: `cd gitea && docker compose up -d` - **Access**: https://gitea.harkon.co.uk ### Nextcloud - **Location**: `nextcloud/` - **Purpose**: File storage and collaboration - **Deploy**: `cd nextcloud && docker compose up -d` - **Access**: https://nextcloud.harkon.co.uk ### Portainer - **Location**: `portainer/` - **Purpose**: Docker management UI - **Deploy**: `cd portainer && docker compose up -d` - **Access**: https://portainer.harkon.co.uk ## Deployment ### Production (Remote Server) ```bash # SSH to server ssh deploy@141.136.35.199 # Navigate to service directory cd /opt/ai-tax-agent/infra/compose/ # Deploy service docker compose up -d # Check logs docker compose logs -f # Check status docker compose ps ``` ### Local Development For local development, use the all-in-one compose file: ```bash cd infra/compose docker compose -f docker-compose.local.yml up -d ``` ## Configuration Each service has its own `.env` file for environment-specific configuration: - `traefik/.provider.env` - GoDaddy API credentials - `authentik/.env` - Authentik secrets - `gitea/.env` - Gitea database credentials ## Networks All services use shared Docker networks: - `frontend` - Public-facing services - `backend` - Internal services Create networks before deploying: ```bash docker network create frontend docker network create backend ``` ## Maintenance ### Update Service ```bash cd /opt/ai-tax-agent/infra/compose/ docker compose pull docker compose up -d ``` ### Restart Service ```bash cd /opt/ai-tax-agent/infra/compose/ docker compose restart ``` ### View Logs ```bash cd /opt/ai-tax-agent/infra/compose/ docker compose logs -f ``` ### Backup Data ```bash # Backup volumes docker run --rm -v _data:/data -v $(pwd):/backup alpine tar czf /backup/-backup.tar.gz /data ``` ## Integration with Application These external services are used by the application infrastructure: - **Traefik** - Routes traffic to application services - **Authentik** - Provides SSO for application UIs - **Gitea** - Hosts Docker images for application services The application infrastructure is deployed separately using: ```bash ./infra/scripts/deploy.sh production infrastructure ./infra/scripts/deploy.sh production services ``` EOF log_success "Created external services README" # Step 7: Create deployment helper script log_info "Step 7: Creating deployment helper script..." cat > "$SCRIPT_DIR/deploy-external.sh" << 'EOF' #!/bin/bash # Deploy external services on production server # Usage: ./scripts/deploy-external.sh set -e SERVICE=$1 if [ -z "$SERVICE" ]; then echo "Usage: $0 " echo "" echo "Available services:" echo " traefik" echo " authentik" echo " gitea" echo " nextcloud" echo " portainer" echo " all" exit 1 fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" COMPOSE_DIR="$PROJECT_ROOT/infra/compose" deploy_service() { local svc=$1 echo "🚀 Deploying $svc..." if [ ! -d "$COMPOSE_DIR/$svc" ]; then echo "❌ Service directory not found: $COMPOSE_DIR/$svc" return 1 fi cd "$COMPOSE_DIR/$svc" docker compose up -d echo "✅ $svc deployed" } if [ "$SERVICE" = "all" ]; then deploy_service "traefik" sleep 5 deploy_service "authentik" sleep 5 deploy_service "gitea" deploy_service "nextcloud" deploy_service "portainer" else deploy_service "$SERVICE" fi echo "" echo "🎉 Deployment complete!" EOF chmod +x "$SCRIPT_DIR/deploy-external.sh" log_success "Created deploy-external.sh script" # Step 8: Summary echo "" log_success "Cleanup complete!" echo "" log_info "Summary of changes:" echo " ✅ Removed duplicate Traefik configs" echo " ✅ Created app-specific Traefik middlewares" echo " ✅ Aligned Authentik configurations" echo " ✅ Cleaned up old directories" echo " ✅ Updated .gitignore" echo " ✅ Created external services README" echo " ✅ Created deploy-external.sh script" echo "" log_info "Backup location: $BACKUP_DIR" echo "" log_info "Next steps:" echo " 1. Review changes in infra/ directory" echo " 2. Update Makefile with new targets" echo " 3. Test local deployment: make run" echo " 4. Test external service deployment: ./scripts/deploy-external.sh traefik" echo ""