ai-tax-agent/scripts/cleanup-infra-structure.sh

#!/bin/bash

# Cleanup and align infrastructure structure
# This script consolidates configurations and removes duplication

set -e

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
RED='\033[0;31m'
NC='\033[0m'

log_info() {
    echo -e "${BLUE}ℹ️  $1${NC}"
}

log_success() {
    echo -e "${GREEN}✅ $1${NC}"
}

log_warning() {
    echo -e "${YELLOW}⚠️  $1${NC}"
}

log_error() {
    echo -e "${RED}❌ $1${NC}"
}

# Script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
INFRA_DIR="$PROJECT_ROOT/infra"

log_info "Cleaning up infrastructure structure..."
echo "  Project Root: $PROJECT_ROOT"
echo "  Infra Dir: $INFRA_DIR"
echo ""

# Step 1: Backup current structure
log_info "Step 1: Creating backup..."
BACKUP_DIR="$PROJECT_ROOT/infra-backup-$(date +%Y%m%d_%H%M%S)"
mkdir -p "$BACKUP_DIR"
cp -r "$INFRA_DIR/configs" "$BACKUP_DIR/" 2>/dev/null || true
log_success "Backup created at $BACKUP_DIR"

# Step 2: Align Traefik configurations
log_info "Step 2: Aligning Traefik configurations..."

# The source of truth is infra/compose/traefik/config/
# Remove duplicates from infra/configs/traefik/config/
if [ -d "$INFRA_DIR/configs/traefik/config" ]; then
    log_warning "  Removing duplicate Traefik configs from infra/configs/traefik/config/"
    rm -rf "$INFRA_DIR/configs/traefik/config"
    log_success "  Removed duplicate Traefik configs"
fi

# Keep only app-specific Traefik middleware in configs
mkdir -p "$INFRA_DIR/configs/traefik"
cat > "$INFRA_DIR/configs/traefik/app-middlewares.yml" << 'EOF'
# Application-specific Traefik middlewares
# These are loaded by the application infrastructure, not the external Traefik

http:
  middlewares:
    # Large upload middleware for Gitea registry
    gitea-large-upload:
      buffering:
        maxRequestBodyBytes: 5368709120   # 5GB
        memRequestBodyBytes: 104857600    # 100MB
        maxResponseBodyBytes: 5368709120  # 5GB
        memResponseBodyBytes: 104857600   # 100MB
        retryExpression: "IsNetworkError() && Attempts() < 3"
    
    # Rate limiting for public APIs
    api-ratelimit:
      rateLimit:
        average: 100
        burst: 50
        period: 1s
    
    # Security headers
    security-headers:
      headers:
        frameDeny: true
        sslRedirect: true
        browserXssFilter: true
        contentTypeNosniff: true
        stsIncludeSubdomains: true
        stsPreload: true
        stsSeconds: 31536000
EOF

log_success "  Created app-specific Traefik middlewares"

# Step 3: Align Authentik configurations
log_info "Step 3: Aligning Authentik configurations..."

# infra/compose/authentik/ - Production service configs
# infra/configs/authentik/ - Application bootstrap configs (keep separate)

if [ -d "$INFRA_DIR/configs/authentik" ]; then
    log_info "  Keeping app-specific Authentik bootstrap in infra/configs/authentik/"
    log_success "  Authentik configs aligned"
fi

# Step 4: Clean up old directories
log_info "Step 4: Cleaning up old directories..."

# Remove old standalone config directories that were moved
OLD_DIRS=(
    "$INFRA_DIR/traefik"
    "$INFRA_DIR/grafana"
    "$INFRA_DIR/prometheus"
    "$INFRA_DIR/loki"
    "$INFRA_DIR/promtail"
    "$INFRA_DIR/vault"
    "$INFRA_DIR/neo4j"
    "$INFRA_DIR/postgres"
)

for dir in "${OLD_DIRS[@]}"; do
    if [ -d "$dir" ] && [ -f "$INFRA_DIR/configs/$(basename $dir)/.moved" ]; then
        log_warning "  Removing old directory: $dir"
        rm -rf "$dir"
        log_success "  Removed $dir"
    fi
done

# Step 5: Update .gitignore
log_info "Step 5: Updating .gitignore..."

cat > "$INFRA_DIR/.gitignore" << 'EOF'
# Environment files (contain secrets)
environments/*/.env
!environments/*/.env.example
compose/*/.env
!compose/env.example

# Certificates
certs/*/
!certs/.gitkeep
compose/*/certs/
!compose/*/certs/.gitkeep

# Provider credentials
compose/traefik/.provider.env
configs/traefik/.provider.env

# Data directories
compose/*/data/
compose/*/media/
compose/authentik/media/
compose/authentik/custom-templates/
compose/portainer/portainer/

# Backup files
*.backup
*.tmp
*-backup-*/

# Docker volumes (if mounted locally)
volumes/

# Logs
*.log
logs/

# Moved markers
**/.moved
EOF

log_success ".gitignore updated"

# Step 6: Create README for external services
log_info "Step 6: Creating documentation..."

cat > "$INFRA_DIR/compose/README.md" << 'EOF'
# External Services

This directory contains Docker Compose configurations for external services that run on the production server.

## Services

### Traefik
- **Location**: `traefik/`
- **Purpose**: Reverse proxy and load balancer for all services
- **Deploy**: `cd traefik && docker compose up -d`
- **Access**: https://traefik.harkon.co.uk

### Authentik
- **Location**: `authentik/`
- **Purpose**: SSO and authentication provider
- **Deploy**: `cd authentik && docker compose up -d`
- **Access**: https://authentik.harkon.co.uk

### Gitea
- **Location**: `gitea/`
- **Purpose**: Git repository hosting and container registry
- **Deploy**: `cd gitea && docker compose up -d`
- **Access**: https://gitea.harkon.co.uk

### Nextcloud
- **Location**: `nextcloud/`
- **Purpose**: File storage and collaboration
- **Deploy**: `cd nextcloud && docker compose up -d`
- **Access**: https://nextcloud.harkon.co.uk

### Portainer
- **Location**: `portainer/`
- **Purpose**: Docker management UI
- **Deploy**: `cd portainer && docker compose up -d`
- **Access**: https://portainer.harkon.co.uk

## Deployment

### Production (Remote Server)

```bash
# SSH to server
ssh deploy@141.136.35.199

# Navigate to service directory
cd /opt/ai-tax-agent/infra/compose/<service>

# Deploy service
docker compose up -d

# Check logs
docker compose logs -f

# Check status
docker compose ps
```

### Local Development

For local development, use the all-in-one compose file:

```bash
cd infra/compose
docker compose -f docker-compose.local.yml up -d
```

## Configuration

Each service has its own `.env` file for environment-specific configuration:

- `traefik/.provider.env` - GoDaddy API credentials
- `authentik/.env` - Authentik secrets
- `gitea/.env` - Gitea database credentials

## Networks

All services use shared Docker networks:

- `frontend` - Public-facing services
- `backend` - Internal services

Create networks before deploying:

```bash
docker network create frontend
docker network create backend
```

## Maintenance

### Update Service

```bash
cd /opt/ai-tax-agent/infra/compose/<service>
docker compose pull
docker compose up -d
```

### Restart Service

```bash
cd /opt/ai-tax-agent/infra/compose/<service>
docker compose restart
```

### View Logs

```bash
cd /opt/ai-tax-agent/infra/compose/<service>
docker compose logs -f
```

### Backup Data

```bash
# Backup volumes
docker run --rm -v <service>_data:/data -v $(pwd):/backup alpine tar czf /backup/<service>-backup.tar.gz /data
```

## Integration with Application

These external services are used by the application infrastructure:

- **Traefik** - Routes traffic to application services
- **Authentik** - Provides SSO for application UIs
- **Gitea** - Hosts Docker images for application services

The application infrastructure is deployed separately using:

```bash
./infra/scripts/deploy.sh production infrastructure
./infra/scripts/deploy.sh production services
```
EOF

log_success "Created external services README"

# Step 7: Create deployment helper script
log_info "Step 7: Creating deployment helper script..."

cat > "$SCRIPT_DIR/deploy-external.sh" << 'EOF'
#!/bin/bash

# Deploy external services on production server
# Usage: ./scripts/deploy-external.sh <service>

set -e

SERVICE=$1

if [ -z "$SERVICE" ]; then
    echo "Usage: $0 <service>"
    echo ""
    echo "Available services:"
    echo "  traefik"
    echo "  authentik"
    echo "  gitea"
    echo "  nextcloud"
    echo "  portainer"
    echo "  all"
    exit 1
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
COMPOSE_DIR="$PROJECT_ROOT/infra/compose"

deploy_service() {
    local svc=$1
    echo "🚀 Deploying $svc..."
    
    if [ ! -d "$COMPOSE_DIR/$svc" ]; then
        echo "❌ Service directory not found: $COMPOSE_DIR/$svc"
        return 1
    fi
    
    cd "$COMPOSE_DIR/$svc"
    docker compose up -d
    echo "✅ $svc deployed"
}

if [ "$SERVICE" = "all" ]; then
    deploy_service "traefik"
    sleep 5
    deploy_service "authentik"
    sleep 5
    deploy_service "gitea"
    deploy_service "nextcloud"
    deploy_service "portainer"
else
    deploy_service "$SERVICE"
fi

echo ""
echo "🎉 Deployment complete!"
EOF

chmod +x "$SCRIPT_DIR/deploy-external.sh"
log_success "Created deploy-external.sh script"

# Step 8: Summary
echo ""
log_success "Cleanup complete!"
echo ""
log_info "Summary of changes:"
echo "  ✅ Removed duplicate Traefik configs"
echo "  ✅ Created app-specific Traefik middlewares"
echo "  ✅ Aligned Authentik configurations"
echo "  ✅ Cleaned up old directories"
echo "  ✅ Updated .gitignore"
echo "  ✅ Created external services README"
echo "  ✅ Created deploy-external.sh script"
echo ""
log_info "Backup location: $BACKUP_DIR"
echo ""
log_info "Next steps:"
echo "  1. Review changes in infra/ directory"
echo "  2. Update Makefile with new targets"
echo "  3. Test local deployment: make run"
echo "  4. Test external service deployment: ./scripts/deploy-external.sh traefik"
echo ""