ai-tax-agent/scripts/verify-deployment.sh

#!/bin/bash

# Deployment Verification Script
# Checks all services are running and healthy

set -e

# Colors
GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

REMOTE_USER="${REMOTE_USER:-deploy}"
REMOTE_HOST="${REMOTE_HOST:-141.136.35.199}"
DOMAIN="${DOMAIN:-harkon.co.uk}"

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}AI Tax Agent - Deployment Verification${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# Function to check service health
check_service() {
    local service_name=$1
    local url=$2
    
    echo -n "Checking $service_name... "
    
    if curl -s -f -o /dev/null "$url"; then
        echo -e "${GREEN}✓ OK${NC}"
        return 0
    else
        echo -e "${RED}✗ FAILED${NC}"
        return 1
    fi
}

# Function to check Docker container status
check_container() {
    local container_name=$1
    
    echo -n "Checking container $container_name... "
    
    status=$(ssh ${REMOTE_USER}@${REMOTE_HOST} "docker ps --filter name=$container_name --format '{{.Status}}'" 2>/dev/null)
    
    if [[ $status == *"Up"* ]]; then
        echo -e "${GREEN}✓ Running${NC}"
        return 0
    else
        echo -e "${RED}✗ Not running${NC}"
        return 1
    fi
}

echo -e "${YELLOW}1. Checking Infrastructure Services${NC}"
echo "-----------------------------------"

# Check containers on remote server
ssh ${REMOTE_USER}@${REMOTE_HOST} "docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'" | grep -E "(apa-vault|apa-minio|apa-postgres|apa-redis|apa-neo4j|apa-qdrant|apa-nats)" || true

echo ""
echo -e "${YELLOW}2. Checking Infrastructure Endpoints${NC}"
echo "------------------------------------"

check_service "Vault" "https://vault.${DOMAIN}/v1/sys/health" || true
check_service "MinIO Console" "https://minio.${DOMAIN}" || true
check_service "Neo4j Browser" "https://neo4j.${DOMAIN}" || true
check_service "Qdrant" "https://qdrant.${DOMAIN}" || true

echo ""
echo -e "${YELLOW}3. Checking Application Services${NC}"
echo "--------------------------------"

# Check application containers
ssh ${REMOTE_USER}@${REMOTE_HOST} "docker ps --format 'table {{.Names}}\t{{.Status}}'" | grep -E "apa-svc-" || true

echo ""
echo -e "${YELLOW}4. Checking Application Endpoints${NC}"
echo "---------------------------------"

check_service "API Gateway" "https://api.${DOMAIN}/health" || true
check_service "UI" "https://app.${DOMAIN}" || true

# Check individual services
services=(
    "ingestion"
    "extract"
    "kg"
    "rag-retriever"
    "rag-indexer"
    "forms"
    "hmrc"
    "ocr"
    "rpa"
    "normalize-map"
    "reason"
    "firm-connectors"
    "coverage"
)

for service in "${services[@]}"; do
    check_service "svc-$service" "https://api.${DOMAIN}/$service/health" || true
done

echo ""
echo -e "${YELLOW}5. Checking Monitoring Stack${NC}"
echo "----------------------------"

check_service "Prometheus" "https://prometheus.${DOMAIN}/-/healthy" || true
check_service "Grafana" "https://grafana.${DOMAIN}/api/health" || true
check_service "Loki" "https://loki.${DOMAIN}/ready" || true

echo ""
echo -e "${YELLOW}6. Checking Docker Networks${NC}"
echo "--------------------------"

ssh ${REMOTE_USER}@${REMOTE_HOST} "docker network ls | grep -E '(apa-frontend|apa-backend)'" || true

echo ""
echo -e "${YELLOW}7. Checking Disk Usage${NC}"
echo "---------------------"

ssh ${REMOTE_USER}@${REMOTE_HOST} "df -h | grep -E '(Filesystem|/opt|/var/lib/docker)'" || true

echo ""
echo -e "${YELLOW}8. Checking Memory Usage${NC}"
echo "-----------------------"

ssh ${REMOTE_USER}@${REMOTE_HOST} "free -h" || true

echo ""
echo -e "${YELLOW}9. Recent Container Logs (Last 10 lines)${NC}"
echo "---------------------------------------"

# Get logs from a few key services
for container in apa-vault apa-minio apa-postgres apa-svc-ingestion apa-svc-extract; do
    echo -e "\n${BLUE}=== $container ===${NC}"
    ssh ${REMOTE_USER}@${REMOTE_HOST} "docker logs $container --tail 10 2>&1" || echo "Container not found"
done

echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Verification Complete${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo -e "${YELLOW}Next Steps:${NC}"
echo "1. Check any failed services above"
echo "2. Review logs for errors: ssh ${REMOTE_USER}@${REMOTE_HOST} 'docker logs <container-name>'"
echo "3. Access Grafana: https://grafana.${DOMAIN}"
echo "4. Access Application: https://app.${DOMAIN}"
echo ""