Initial commit

scripts/cleanup-infra-structure.sh

@@ -0,0 +1,401 @@
+#!/bin/bash
+
+# Cleanup and align infrastructure structure
+# This script consolidates configurations and removes duplication
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+log_info() {
+    echo -e "${BLUE}ℹ️  $1${NC}"
+}
+
+log_success() {
+    echo -e "${GREEN}✅ $1${NC}"
+}
+
+log_warning() {
+    echo -e "${YELLOW}⚠️  $1${NC}"
+}
+
+log_error() {
+    echo -e "${RED}❌ $1${NC}"
+}
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+INFRA_DIR="$PROJECT_ROOT/infra"
+
+log_info "Cleaning up infrastructure structure..."
+echo "  Project Root: $PROJECT_ROOT"
+echo "  Infra Dir: $INFRA_DIR"
+echo ""
+
+# Step 1: Backup current structure
+log_info "Step 1: Creating backup..."
+BACKUP_DIR="$PROJECT_ROOT/infra-backup-$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$BACKUP_DIR"
+cp -r "$INFRA_DIR/configs" "$BACKUP_DIR/" 2>/dev/null || true
+log_success "Backup created at $BACKUP_DIR"
+
+# Step 2: Align Traefik configurations
+log_info "Step 2: Aligning Traefik configurations..."
+
+# The source of truth is infra/compose/traefik/config/
+# Remove duplicates from infra/configs/traefik/config/
+if [ -d "$INFRA_DIR/configs/traefik/config" ]; then
+    log_warning "  Removing duplicate Traefik configs from infra/configs/traefik/config/"
+    rm -rf "$INFRA_DIR/configs/traefik/config"
+    log_success "  Removed duplicate Traefik configs"
+fi
+
+# Keep only app-specific Traefik middleware in configs
+mkdir -p "$INFRA_DIR/configs/traefik"
+cat > "$INFRA_DIR/configs/traefik/app-middlewares.yml" << 'EOF'
+# Application-specific Traefik middlewares
+# These are loaded by the application infrastructure, not the external Traefik
+
+http:
+  middlewares:
+    # Large upload middleware for Gitea registry
+    gitea-large-upload:
+      buffering:
+        maxRequestBodyBytes: 5368709120   # 5GB
+        memRequestBodyBytes: 104857600    # 100MB
+        maxResponseBodyBytes: 5368709120  # 5GB
+        memResponseBodyBytes: 104857600   # 100MB
+        retryExpression: "IsNetworkError() && Attempts() < 3"
+    
+    # Rate limiting for public APIs
+    api-ratelimit:
+      rateLimit:
+        average: 100
+        burst: 50
+        period: 1s
+    
+    # Security headers
+    security-headers:
+      headers:
+        frameDeny: true
+        sslRedirect: true
+        browserXssFilter: true
+        contentTypeNosniff: true
+        stsIncludeSubdomains: true
+        stsPreload: true
+        stsSeconds: 31536000
+EOF
+
+log_success "  Created app-specific Traefik middlewares"
+
+# Step 3: Align Authentik configurations
+log_info "Step 3: Aligning Authentik configurations..."
+
+# infra/compose/authentik/ - Production service configs
+# infra/configs/authentik/ - Application bootstrap configs (keep separate)
+
+if [ -d "$INFRA_DIR/configs/authentik" ]; then
+    log_info "  Keeping app-specific Authentik bootstrap in infra/configs/authentik/"
+    log_success "  Authentik configs aligned"
+fi
+
+# Step 4: Clean up old directories
+log_info "Step 4: Cleaning up old directories..."
+
+# Remove old standalone config directories that were moved
+OLD_DIRS=(
+    "$INFRA_DIR/traefik"
+    "$INFRA_DIR/grafana"
+    "$INFRA_DIR/prometheus"
+    "$INFRA_DIR/loki"
+    "$INFRA_DIR/promtail"
+    "$INFRA_DIR/vault"
+    "$INFRA_DIR/neo4j"
+    "$INFRA_DIR/postgres"
+)
+
+for dir in "${OLD_DIRS[@]}"; do
+    if [ -d "$dir" ] && [ -f "$INFRA_DIR/configs/$(basename $dir)/.moved" ]; then
+        log_warning "  Removing old directory: $dir"
+        rm -rf "$dir"
+        log_success "  Removed $dir"
+    fi
+done
+
+# Step 5: Update .gitignore
+log_info "Step 5: Updating .gitignore..."
+
+cat > "$INFRA_DIR/.gitignore" << 'EOF'
+# Environment files (contain secrets)
+environments/*/.env
+!environments/*/.env.example
+compose/*/.env
+!compose/env.example
+
+# Certificates
+certs/*/
+!certs/.gitkeep
+compose/*/certs/
+!compose/*/certs/.gitkeep
+
+# Provider credentials
+compose/traefik/.provider.env
+configs/traefik/.provider.env
+
+# Data directories
+compose/*/data/
+compose/*/media/
+compose/authentik/media/
+compose/authentik/custom-templates/
+compose/portainer/portainer/
+
+# Backup files
+*.backup
+*.tmp
+*-backup-*/
+
+# Docker volumes (if mounted locally)
+volumes/
+
+# Logs
+*.log
+logs/
+
+# Moved markers
+**/.moved
+EOF
+
+log_success ".gitignore updated"
+
+# Step 6: Create README for external services
+log_info "Step 6: Creating documentation..."
+
+cat > "$INFRA_DIR/compose/README.md" << 'EOF'
+# External Services
+
+This directory contains Docker Compose configurations for external services that run on the production server.
+
+## Services
+
+### Traefik
+- **Location**: `traefik/`
+- **Purpose**: Reverse proxy and load balancer for all services
+- **Deploy**: `cd traefik && docker compose up -d`
+- **Access**: https://traefik.harkon.co.uk
+
+### Authentik
+- **Location**: `authentik/`
+- **Purpose**: SSO and authentication provider
+- **Deploy**: `cd authentik && docker compose up -d`
+- **Access**: https://authentik.harkon.co.uk
+
+### Gitea
+- **Location**: `gitea/`
+- **Purpose**: Git repository hosting and container registry
+- **Deploy**: `cd gitea && docker compose up -d`
+- **Access**: https://gitea.harkon.co.uk
+
+### Nextcloud
+- **Location**: `nextcloud/`
+- **Purpose**: File storage and collaboration
+- **Deploy**: `cd nextcloud && docker compose up -d`
+- **Access**: https://nextcloud.harkon.co.uk
+
+### Portainer
+- **Location**: `portainer/`
+- **Purpose**: Docker management UI
+- **Deploy**: `cd portainer && docker compose up -d`
+- **Access**: https://portainer.harkon.co.uk
+
+## Deployment
+
+### Production (Remote Server)
+
+```bash
+# SSH to server
+ssh deploy@141.136.35.199
+
+# Navigate to service directory
+cd /opt/ai-tax-agent/infra/compose/<service>
+
+# Deploy service
+docker compose up -d
+
+# Check logs
+docker compose logs -f
+
+# Check status
+docker compose ps
+```
+
+### Local Development
+
+For local development, use the all-in-one compose file:
+
+```bash
+cd infra/compose
+docker compose -f docker-compose.local.yml up -d
+```
+
+## Configuration
+
+Each service has its own `.env` file for environment-specific configuration:
+
+- `traefik/.provider.env` - GoDaddy API credentials
+- `authentik/.env` - Authentik secrets
+- `gitea/.env` - Gitea database credentials
+
+## Networks
+
+All services use shared Docker networks:
+
+- `frontend` - Public-facing services
+- `backend` - Internal services
+
+Create networks before deploying:
+
+```bash
+docker network create frontend
+docker network create backend
+```
+
+## Maintenance
+
+### Update Service
+
+```bash
+cd /opt/ai-tax-agent/infra/compose/<service>
+docker compose pull
+docker compose up -d
+```
+
+### Restart Service
+
+```bash
+cd /opt/ai-tax-agent/infra/compose/<service>
+docker compose restart
+```
+
+### View Logs
+
+```bash
+cd /opt/ai-tax-agent/infra/compose/<service>
+docker compose logs -f
+```
+
+### Backup Data
+
+```bash
+# Backup volumes
+docker run --rm -v <service>_data:/data -v $(pwd):/backup alpine tar czf /backup/<service>-backup.tar.gz /data
+```
+
+## Integration with Application
+
+These external services are used by the application infrastructure:
+
+- **Traefik** - Routes traffic to application services
+- **Authentik** - Provides SSO for application UIs
+- **Gitea** - Hosts Docker images for application services
+
+The application infrastructure is deployed separately using:
+
+```bash
+./infra/scripts/deploy.sh production infrastructure
+./infra/scripts/deploy.sh production services
+```
+EOF
+
+log_success "Created external services README"
+
+# Step 7: Create deployment helper script
+log_info "Step 7: Creating deployment helper script..."
+
+cat > "$SCRIPT_DIR/deploy-external.sh" << 'EOF'
+#!/bin/bash
+
+# Deploy external services on production server
+# Usage: ./scripts/deploy-external.sh <service>
+
+set -e
+
+SERVICE=$1
+
+if [ -z "$SERVICE" ]; then
+    echo "Usage: $0 <service>"
+    echo ""
+    echo "Available services:"
+    echo "  traefik"
+    echo "  authentik"
+    echo "  gitea"
+    echo "  nextcloud"
+    echo "  portainer"
+    echo "  all"
+    exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+COMPOSE_DIR="$PROJECT_ROOT/infra/compose"
+
+deploy_service() {
+    local svc=$1
+    echo "🚀 Deploying $svc..."
+    
+    if [ ! -d "$COMPOSE_DIR/$svc" ]; then
+        echo "❌ Service directory not found: $COMPOSE_DIR/$svc"
+        return 1
+    fi
+    
+    cd "$COMPOSE_DIR/$svc"
+    docker compose up -d
+    echo "✅ $svc deployed"
+}
+
+if [ "$SERVICE" = "all" ]; then
+    deploy_service "traefik"
+    sleep 5
+    deploy_service "authentik"
+    sleep 5
+    deploy_service "gitea"
+    deploy_service "nextcloud"
+    deploy_service "portainer"
+else
+    deploy_service "$SERVICE"
+fi
+
+echo ""
+echo "🎉 Deployment complete!"
+EOF
+
+chmod +x "$SCRIPT_DIR/deploy-external.sh"
+log_success "Created deploy-external.sh script"
+
+# Step 8: Summary
+echo ""
+log_success "Cleanup complete!"
+echo ""
+log_info "Summary of changes:"
+echo "  ✅ Removed duplicate Traefik configs"
+echo "  ✅ Created app-specific Traefik middlewares"
+echo "  ✅ Aligned Authentik configurations"
+echo "  ✅ Cleaned up old directories"
+echo "  ✅ Updated .gitignore"
+echo "  ✅ Created external services README"
+echo "  ✅ Created deploy-external.sh script"
+echo ""
+log_info "Backup location: $BACKUP_DIR"
+echo ""
+log_info "Next steps:"
+echo "  1. Review changes in infra/ directory"
+echo "  2. Update Makefile with new targets"
+echo "  3. Test local deployment: make run"
+echo "  4. Test external service deployment: ./scripts/deploy-external.sh traefik"
+echo ""
+