# Multi-stage build for optimized Java containers
FROM eclipse-temurin:21-jdk-alpine AS builder

# Install native build tools
RUN apk add --no-cache \
    binutils \
    gcompat \
    upx

WORKDIR /build

# Copy dependency management files first for better caching
COPY pom.xml ./
COPY .mvn .mvn
COPY mvnw ./

# Download dependencies in separate layer
RUN ./mvnw dependency:go-offline -B

# Copy source and build
COPY src ./src
RUN ./mvnw clean package -DskipTests -B

# Create optimized JAR with dependencies
RUN java -Djarmode=layertools -jar target/order-service.jar extract

# Production image
FROM eclipse-temurin:21-jre-alpine AS production

# Create non-root user for security
RUN addgroup -g 1001 -S appgroup && \
    adduser -u 1001 -S appuser -G appgroup

# Install monitoring and debugging tools
RUN apk add --no-cache \
    curl \
    jattach \
    ttyd

# Optimize JVM for containers
ENV JAVA_OPTS="\
    -XX:+UseContainerSupport \
    -XX:InitialRAMPercentage=50.0 \
    -XX:MaxRAMPercentage=70.0 \
    -XX:+UseG1GC \
    -XX:+UseStringDeduplication \
    -XX:+OptimizeStringConcat \
    -Djava.security.egd=file:/dev/./urandom \
    -Dspring.jmx.enabled=false \
    "

# Application-specific optimizations
ENV APP_OPTS="\
    -Dserver.tomcat.threads.max=200 \
    -Dserver.tomcat.threads.min-spare=10 \
    -Dspring.jpa.hibernate.ddl-auto=none \
    -Dspring.jpa.open-in-view=false \
    "

WORKDIR /app

# Copy application layers for optimal caching
COPY --from=builder --chown=appuser:appgroup /build/dependencies/ ./
COPY --from=builder --chown=appuser:appgroup /build/spring-boot-loader/ ./
COPY --from=builder --chown=appuser:appgroup /build/snapshot-dependencies/ ./
COPY --from=builder --chown=appuser:appgroup /build/application/ ./

# Health check script
COPY --chown=appuser:appgroup scripts/healthcheck.sh ./
RUN chmod +x healthcheck.sh

USER appuser

EXPOSE 8080 8081

HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    CMD ./healthcheck.sh

# Use exec form to ensure proper signal handling
ENTRYPOINT ["java", "-cp", "BOOT-INF/classes:BOOT-INF/lib/*", "com.example.OrderServiceApplication"]

# GraalVM native image for ultra-fast startup
FROM ghcr.io/graalvm/graalvm-ce:ol8-java17 AS graalvm-builder

WORKDIR /build

# Install native-image
RUN gu install native-image

# Copy application
COPY pom.xml ./
COPY src ./src
COPY .mvn .mvn
COPY mvnw ./

# Build native image with optimizations
RUN ./mvnw package -Pnative \
    -Dspring.aot.enabled=true \
    -Dspring.native.buildtools.classpath.native-image.enabled=true

# Ultra-minimal runtime image
FROM gcr.io/distroless/base

COPY --from=graalvm-builder /build/target/order-service-native /order-service

USER 1001:1001

EXPOSE 8080

ENTRYPOINT ["/order-service"]

# Result: ~10MB image, <100ms startup time

# Production-ready Kubernetes deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
  labels:
    app: order-service
    version: v2.1.0
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 50%
      maxUnavailable: 25%
  selector:
    matchLabels:
      app: order-service
  template:
    metadata:
      labels:
        app: order-service
        version: v2.1.0
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8081"
        prometheus.io/path: "/actuator/prometheus"
    spec:
      serviceAccountName: order-service
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
        fsGroup: 1001
      
      # Resource management
      containers:
      - name: order-service
        image: order-service:2.1.0
        imagePullPolicy: IfNotPresent
        
        ports:
        - name: http
          containerPort: 8080
          protocol: TCP
        - name: management
          containerPort: 8081
          protocol: TCP
        
        # Resource limits and requests
        resources:
          requests:
            memory: "512Mi"
            cpu: "250m"
          limits:
            memory: "1Gi"
            cpu: "1000m"
        
        # Environment configuration
        env:
        - name: SPRING_PROFILES_ACTIVE
          value: "production"
        - name: JAVA_OPTS
          value: "-XX:MaxRAMPercentage=70.0 -XX:+UseG1GC"
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: order-service-secrets
              key: database-url
        
        # Health and readiness checks
        livenessProbe:
          httpGet:
            path: /actuator/health/liveness
            port: management
          initialDelaySeconds: 60
          periodSeconds: 30
          timeoutSeconds: 10
          failureThreshold: 3
        
        readinessProbe:
          httpGet:
            path: /actuator/health/readiness
            port: management
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        
        # Startup probe for slow-starting applications
        startupProbe:
          httpGet:
            path: /actuator/health/readiness
            port: management
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 30
        
        # Volume mounts
        volumeMounts:
        - name: config
          mountPath: /app/config
          readOnly: true
        - name: secrets
          mountPath: /app/secrets
          readOnly: true
        - name: tmp
          mountPath: /tmp
      
      # Volumes
      volumes:
      - name: config
        configMap:
          name: order-service-config
      - name: secrets
        secret:
          secretName: order-service-secrets
      - name: tmp
        emptyDir: {}
      
      # Pod distribution
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchLabels:
                  app: order-service
              topologyKey: kubernetes.io/hostname
      
      # Graceful shutdown
      terminationGracePeriodSeconds: 60

---
# Service configuration
apiVersion: v1
kind: Service
metadata:
  name: order-service
  labels:
    app: order-service
spec:
  type: ClusterIP
  ports:
  - name: http
    port: 80
    targetPort: http
    protocol: TCP
  selector:
    app: order-service

---
# Horizontal Pod Autoscaler
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: order-service-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "100"

// Application version and feature flag support
@Component
public class DeploymentManager {
    
    private final VersionInfo versionInfo;
    private final FeatureFlagService featureFlagService;
    
    @Value("${app.deployment.color:blue}")
    private String deploymentColor;
    
    @EventListener(ApplicationReadyEvent.class)
    public void registerDeployment() {
        DeploymentInfo deployment = DeploymentInfo.builder()
            .version(versionInfo.getVersion())
            .color(deploymentColor)
            .startTime(Instant.now())
            .health(HealthStatus.HEALTHY)
            .build();
            
        deploymentRegistry.register(deployment);
    }
    
    // Health endpoint for load balancer
    @GetMapping("/health/deployment")
    public ResponseEntity<Map<String, Object>> deploymentHealth() {
        boolean isHealthy = performHealthChecks();
        boolean isReady = featureFlagService.isEnabled("deployment.ready");
        
        Map<String, Object> health = Map.of(
            "status", isHealthy && isReady ? "UP" : "DOWN",
            "color", deploymentColor,
            "version", versionInfo.getVersion(),
            "ready", isReady,
            "checks", getDetailedHealthChecks()
        );
        
        return ResponseEntity.status(isHealthy && isReady ? 200 : 503)
            .body(health);
    }
    
    // Graceful feature migration
    public boolean shouldUseNewFeature(String featureName, String userId) {
        if (!featureFlagService.isEnabled(featureName)) {
            return false;
        }
        
        // Gradual rollout based on user ID hash
        int userHash = Math.abs(userId.hashCode());
        int rolloutPercentage = featureFlagService.getRolloutPercentage(featureName);
        
        return (userHash % 100) < rolloutPercentage;
    }
}

# Istio VirtualService for canary deployment
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: order-service-canary
spec:
  hosts:
  - order-service
  http:
  - match:
    - headers:
        x-canary-user:
          exact: "true"
    route:
    - destination:
        host: order-service
        subset: v2
      weight: 100
  - route:
    - destination:
        host: order-service
        subset: v1
      weight: 90
    - destination:
        host: order-service
        subset: v2
      weight: 10

---
# DestinationRule for traffic splitting
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: order-service
spec:
  host: order-service
  subsets:
  - name: v1
    labels:
      version: v1.0.0
  - name: v2
    labels:
      version: v2.0.0
    trafficPolicy:
      connectionPool:
        tcp:
          maxConnections: 10
        http:
          http1MaxPendingRequests: 10
          maxRequestsPerConnection: 2
      circuitBreaker:
        consecutiveErrors: 3
        interval: 30s
        baseEjectionTime: 30s

// Canary deployment health monitoring
@Component
public class CanaryAnalyzer {
    
    private final MeterRegistry meterRegistry;
    private final AlertManager alertManager;
    
    @Scheduled(fixedRate = 60000) // Every minute
    public void analyzeCanaryHealth() {
        CanaryMetrics v1Metrics = collectMetrics("v1");
        CanaryMetrics v2Metrics = collectMetrics("v2");
        
        CanaryAnalysisResult result = analyzeMetrics(v1Metrics, v2Metrics);
        
        if (result.shouldAbort()) {
            log.error("Canary deployment failed analysis: {}", result.getReason());
            abortCanaryDeployment();
            alertManager.sendAlert(AlertLevel.CRITICAL, 
                "Canary deployment aborted: " + result.getReason());
        } else if (result.shouldPromote()) {
            log.info("Canary deployment ready for promotion");
            promoteCanaryDeployment();
        }
    }
    
    private CanaryAnalysisResult analyzeMetrics(CanaryMetrics v1, CanaryMetrics v2) {
        // Error rate analysis
        if (v2.getErrorRate() > v1.getErrorRate() * 1.5) {
            return CanaryAnalysisResult.abort("Error rate too high");
        }
        
        // Response time analysis
        if (v2.getP95ResponseTime() > v1.getP95ResponseTime() * 1.2) {
            return CanaryAnalysisResult.abort("Response time degraded");
        }
        
        // Business metrics analysis
        if (v2.getConversionRate() < v1.getConversionRate() * 0.95) {
            return CanaryAnalysisResult.abort("Conversion rate dropped");
        }
        
        // Success criteria
        if (v2.getSuccessRate() >= 0.99 && v2.getSampleSize() >= 1000) {
            return CanaryAnalysisResult.promote("All metrics healthy");
        }
        
        return CanaryAnalysisResult.continue("Monitoring continues");
    }
}

// Business metrics instrumentation
@Component
public class BusinessMetricsCollector {
    
    private final MeterRegistry meterRegistry;
    private final Counter ordersCreated;
    private final Timer orderProcessingTime;
    private final Gauge activeOrders;
    private final DistributionSummary orderValue;
    
    public BusinessMetricsCollector(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
        
        this.ordersCreated = Counter.builder("business.orders.created")
            .description("Total number of orders created")
            .tag("service", "order-service")
            .register(meterRegistry);
            
        this.orderProcessingTime = Timer.builder("business.order.processing.time")
            .description("Time taken to process an order")
            .publishPercentiles(0.5, 0.75, 0.9, 0.95, 0.99)
            .register(meterRegistry);
            
        this.activeOrders = Gauge.builder("business.orders.active")
            .description("Number of orders being processed")
            .register(meterRegistry, this, BusinessMetricsCollector::getActiveOrderCount);
            
        this.orderValue = DistributionSummary.builder("business.order.value")
            .description("Distribution of order values")
            .baseUnit("USD")
            .publishPercentiles(0.5, 0.75, 0.9, 0.95, 0.99)
            .register(meterRegistry);
    }
    
    // Event-driven metrics collection
    @EventListener
    public void handleOrderCreated(OrderCreatedEvent event) {
        ordersCreated.increment(
            Tags.of(
                "customer.type", event.getCustomerType(),
                "channel", event.getChannel(),
                "product.category", event.getPrimaryCategory()
            )
        );
        
        orderValue.record(event.getTotalAmount().doubleValue());
    }
    
    @EventListener
    public void handleOrderProcessed(OrderProcessedEvent event) {
        Duration processingTime = Duration.between(
            event.getCreatedAt(), event.getProcessedAt());
        
        orderProcessingTime.record(processingTime);
    }
    
    // Custom metrics for SLIs
    public void recordSLIMetrics(String operation, boolean success, Duration duration) {
        Timer.builder("sli.operation.duration")
            .tag("operation", operation)
            .tag("success", String.valueOf(success))
            .register(meterRegistry)
            .record(duration);
            
        Counter.builder("sli.operation.total")
            .tag("operation", operation)
            .tag("result", success ? "success" : "failure")
            .register(meterRegistry)
            .increment();
    }
}

// SLO monitoring and alerting
@Component
public class SLOMonitor {
    
    private final MeterRegistry meterRegistry;
    private final AlertManager alertManager;
    
    // Define SLOs as code
    private final Map<String, SLO> slos = Map.of(
        "order.availability", SLO.builder()
            .target(0.999) // 99.9% availability
            .window(Duration.ofDays(30))
            .build(),
        "order.latency", SLO.builder()
            .target(0.95) // 95% of requests < 500ms
            .threshold(Duration.ofMillis(500))
            .window(Duration.ofDays(7))
            .build()
    );
    
    @Scheduled(fixedRate = 300000) // Every 5 minutes
    public void evaluateSLOs() {
        slos.forEach((name, slo) -> {
            SLOStatus status = evaluateSLO(name, slo);
            
            // Record SLO status as metric
            Gauge.builder("slo.status")
                .tag("slo", name)
                .register(meterRegistry, status, s -> s.getCurrentValue());
                
            // Alert if SLO is at risk
            if (status.isAtRisk()) {
                alertManager.sendAlert(AlertLevel.WARNING,
                    String.format("SLO %s at risk: %.3f (target: %.3f)",
                        name, status.getCurrentValue(), slo.getTarget()));
            }
            
            if (status.isBurning()) {
                alertManager.sendAlert(AlertLevel.CRITICAL,
                    String.format("SLO %s burning: %.3f (target: %.3f)",
                        name, status.getCurrentValue(), slo.getTarget()));
            }
        });
    }
}

// Advanced tracing with custom instrumentation
@Component
public class TracingService {
    
    private final Tracer tracer;
    private final MeterRegistry meterRegistry;
    
    public <T> T traceBusinessOperation(String operationName, 
                                       Map<String, String> businessContext,
                                       Supplier<T> operation) {
        
        Span span = tracer.spanBuilder(operationName)
            .setSpanKind(SpanKind.INTERNAL)
            .startSpan();
            
        // Add business context as attributes
        businessContext.forEach(span::setAttribute);
        
        // Add correlation IDs
        String correlationId = MDC.get("correlationId");
        if (correlationId != null) {
            span.setAttribute("correlation.id", correlationId);
        }
        
        try (Scope scope = span.makeCurrent()) {
            Timer.Sample sample = Timer.start(meterRegistry);
            
            T result = operation.get();
            
            // Record business events
            span.addEvent("business.operation.completed", 
                Attributes.of(AttributeKey.stringKey("result.type"), 
                             result.getClass().getSimpleName()));
            
            sample.stop(Timer.builder("business.operation.duration")
                .tag("operation", operationName)
                .register(meterRegistry));
                
            span.setStatus(StatusCode.OK);
            return result;
            
        } catch (Exception e) {
            span.recordException(e);
            span.setStatus(StatusCode.ERROR, e.getMessage());
            
            meterRegistry.counter("business.operation.errors",
                "operation", operationName,
                "error.type", e.getClass().getSimpleName())
                .increment();
                
            throw e;
        } finally {
            span.end();
        }
    }
    
    // Async operation tracing
    public <T> CompletableFuture<T> traceAsyncOperation(String operationName,
                                                       Supplier<CompletableFuture<T>> operation) {
        
        Span span = tracer.spanBuilder(operationName)
            .setSpanKind(SpanKind.INTERNAL)
            .startSpan();
            
        try (Scope scope = span.makeCurrent()) {
            Context currentContext = Context.current();
            
            return operation.get()
                .whenComplete((result, throwable) -> {
                    try (Scope asyncScope = currentContext.makeCurrent()) {
                        if (throwable != null) {
                            span.recordException(throwable);
                            span.setStatus(StatusCode.ERROR, throwable.getMessage());
                        } else {
                            span.setStatus(StatusCode.OK);
                        }
                    } finally {
                        span.end();
                    }
                });
        }
    }
}

// Correlation ID management
@Component
public class CorrelationContextManager {
    
    private static final String CORRELATION_ID_HEADER = "X-Correlation-ID";
    private static final String USER_ID_HEADER = "X-User-ID";
    private static final String SESSION_ID_HEADER = "X-Session-ID";
    
    @EventListener
    public void handleRequest(HttpServletRequest request) {
        // Extract or generate correlation ID
        String correlationId = Optional.ofNullable(request.getHeader(CORRELATION_ID_HEADER))
            .orElse(UUID.randomUUID().toString());
            
        String userId = request.getHeader(USER_ID_HEADER);
        String sessionId = request.getHeader(SESSION_ID_HEADER);
        
        // Set in MDC for logging
        MDC.put("correlationId", correlationId);
        MDC.put("userId", userId);
        MDC.put("sessionId", sessionId);
        
        // Set in OpenTelemetry context
        Span currentSpan = Span.current();
        currentSpan.setAttribute("correlation.id", correlationId);
        if (userId != null) {
            currentSpan.setAttribute("user.id", userId);
        }
        if (sessionId != null) {
            currentSpan.setAttribute("session.id", sessionId);
        }
    }
    
    @EventListener
    public void cleanupRequest(HttpServletRequestDestroyedEvent event) {
        MDC.clear();
    }
}

// Intelligent alerting with anomaly detection
@Component
public class AnomalyDetector {
    
    private final MeterRegistry meterRegistry;
    private final AlertManager alertManager;
    private final TimeSeriesAnalyzer analyzer;
    
    @Scheduled(fixedRate = 60000) // Every minute
    public void detectAnomalies() {
        detectErrorRateAnomalies();
        detectLatencyAnomalies();
        detectThroughputAnomalies();
        detectBusinessMetricAnomalies();
    }
    
    private void detectErrorRateAnomalies() {
        TimeSeries errorRateSeries = getMetricTimeSeries("http.server.requests", 
            Tags.of("status.class", "5xx"));
            
        AnomalyResult result = analyzer.detectAnomalies(errorRateSeries, 
            AnomalyDetectionConfig.builder()
                .algorithm(AnomalyAlgorithm.SEASONAL_ESD)
                .sensitivity(0.05)
                .seasonality(Duration.ofDays(7))
                .build());
                
        if (result.hasAnomalies()) {
            Alert alert = Alert.builder()
                .severity(AlertSeverity.HIGH)
                .title("Error Rate Anomaly Detected")
                .description(String.format(
                    "Error rate anomaly: current=%.3f, expected=%.3f±%.3f",
                    result.getCurrentValue(),
                    result.getExpectedValue(),
                    result.getStandardDeviation()))
                .tags(Map.of(
                    "metric", "error_rate",
                    "service", "order-service",
                    "anomaly.score", String.valueOf(result.getAnomalyScore())
                ))
                .build();
                
            alertManager.sendAlert(alert);
        }
    }
    
    // Predictive scaling based on patterns
    private void predictiveScaling() {
        TimeSeries requestSeries = getMetricTimeSeries("http.server.requests");
        
        Prediction prediction = analyzer.forecast(requestSeries, Duration.ofMinutes(30));
        
        if (prediction.getMaxValue() > getCurrentCapacity() * 0.8) {
            // Trigger preemptive scaling
            scalingManager.scaleOut(prediction.getRecommendedReplicas());
            
            alertManager.sendInfo("Predictive Scaling Triggered", 
                String.format("Scaling to %d replicas based on predicted load: %.0f req/min",
                    prediction.getRecommendedReplicas(),
                    prediction.getMaxValue()));
        }
    }
}

// Chaos engineering for resilience testing
@Component
@ConditionalOnProperty(name = "chaos.engineering.enabled", havingValue = "true")
public class ChaosEngineeringService {
    
    private final Random random = new SecureRandom();
    
    @EventListener
    public void chaosExperiment(OrderCreatedEvent event) {
        if (shouldRunChaosExperiment()) {
            ChaosExperiment experiment = selectRandomExperiment();
            runExperiment(experiment, event);
        }
    }
    
    private boolean shouldRunChaosExperiment() {
        // Run chaos experiments on 1% of traffic in production
        return random.nextDouble() < 0.01;
    }
    
    private void runExperiment(ChaosExperiment experiment, OrderCreatedEvent event) {
        switch (experiment) {
            case NETWORK_LATENCY -> injectNetworkLatency(Duration.ofMillis(500));
            case DATABASE_TIMEOUT -> simulateDatabaseTimeout();
            case MEMORY_PRESSURE -> createMemoryPressure();
            case CPU_SPIKE -> createCpuSpike();
            case SERVICE_UNAVAILABLE -> simulateServiceUnavailability();
        }
        
        // Record experiment for analysis
        meterRegistry.counter("chaos.experiments",
            "type", experiment.name(),
            "event.id", event.getOrderId())
            .increment();
    }
}

// Automated incident response
@Component
public class IncidentResponseAutomation {
    
    @EventListener
    public void handleHighErrorRate(HighErrorRateAlert alert) {
        IncidentResponse response = IncidentResponse.builder()
            .severity(alert.getSeverity())
            .service(alert.getService())
            .startTime(Instant.now())
            .build();
            
        // Automated diagnostic steps
        DiagnosticResult diagnostics = runDiagnostics(alert);
        response.addDiagnostics(diagnostics);
        
        // Automated mitigation steps
        if (diagnostics.suggestsMitigation()) {
            MitigationResult mitigation = attemptMitigation(diagnostics);
            response.addMitigation(mitigation);
            
            if (mitigation.isSuccessful()) {
                alertManager.sendInfo("Automated Mitigation Successful",
                    "Error rate returned to normal levels");
            }
        }
        
        // Create incident ticket if automation fails
        if (!response.isResolved()) {
            Incident incident = incidentManager.createIncident(
                IncidentSeverity.HIGH,
                "High Error Rate - Manual Intervention Required",
                response.getSummary()
            );
            
            // Page on-call engineer
            oncallManager.page(incident);
        }
    }
    
    private DiagnosticResult runDiagnostics(HighErrorRateAlert alert) {
        return DiagnosticRunner.builder()
            .addCheck("database.connectivity", this::checkDatabaseConnectivity)
            .addCheck("external.services", this::checkExternalServices)
            .addCheck("memory.usage", this::checkMemoryUsage)
            .addCheck("thread.pools", this::checkThreadPools)
            .addCheck("circuit.breakers", this::checkCircuitBreakers)
            .run();
    }
    
    private MitigationResult attemptMitigation(DiagnosticResult diagnostics) {
        MitigationStrategy strategy = determineMitigationStrategy(diagnostics);
        
        return switch (strategy) {
            case SCALE_OUT -> scaleOutService();
            case RESTART_PODS -> restartUnhealthyPods();
            case ENABLE_CIRCUIT_BREAKER -> enableCircuitBreaker();
            case REDIRECT_TRAFFIC -> redirectToHealthyRegion();
            case FALLBACK_MODE -> enableFallbackMode();
        };
    }
}

// Self-healing infrastructure
@Component
public class SelfHealingService {
    
    @Scheduled(fixedRate = 120000) // Every 2 minutes
    public void performHealthChecks() {
        List<PodHealth> unhealthyPods = getUnhealthyPods();
        
        for (PodHealth pod : unhealthyPods) {
            if (shouldAttemptHealing(pod)) {
                attemptHealing(pod);
            }
        }
    }
    
    private void attemptHealing(PodHealth pod) {
        HealingAction action = determineHealingAction(pod);
        
        switch (action) {
            case RESTART_POD -> {
                log.info("Restarting unhealthy pod: {}", pod.getName());
                kubernetesClient.deletePod(pod.getName());
                
                // Wait for new pod to be ready
                waitForPodReady(pod.getName(), Duration.ofMinutes(5));
                
                // Verify healing was successful
                if (isPodHealthy(pod.getName())) {
                    alertManager.sendInfo("Self-Healing Successful",
                        "Pod " + pod.getName() + " was automatically restarted and is now healthy");
                }
            }
            case SCALE_REPLACEMENT -> {
                // Scale up new pod before terminating unhealthy one
                scaleUp(1);
                waitForNewPodReady();
                kubernetesClient.deletePod(pod.getName());
            }
            case DRAIN_TRAFFIC -> {
                // Remove pod from load balancer
                removeFromService(pod.getName());
                scheduleDelayedRestart(pod.getName(), Duration.ofMinutes(10));
            }
        }
    }
}

// Resource usage analysis and optimization
@Component
public class CapacityPlanner {
    
    private final MeterRegistry meterRegistry;
    private final KubernetesClient kubernetesClient;
    
    @Scheduled(cron = "0 0 2 * * *") // Daily at 2 AM
    public void analyzeResourceUsage() {
        ResourceUsageAnalysis analysis = analyzeCurrentUsage();
        CapacityRecommendations recommendations = generateRecommendations(analysis);
        
        // Cost optimization opportunities
        List<CostOptimization> optimizations = identifyCostOptimizations(analysis);
        
        // Generate capacity planning report
        CapacityReport report = CapacityReport.builder()
            .analysis(analysis)
            .recommendations(recommendations)
            .costOptimizations(optimizations)
            .projectedSavings(calculateProjectedSavings(optimizations))
            .build();
            
        capacityReportService.save(report);
        
        // Auto-apply safe optimizations
        applySafeOptimizations(optimizations);
    }
    
    private ResourceUsageAnalysis analyzeCurrentUsage() {
        // Collect metrics over past 30 days
        TimeSeries cpuUsage = getMetricTimeSeries("container.cpu.usage", 30);
        TimeSeries memoryUsage = getMetricTimeSeries("container.memory.usage", 30);
        TimeSeries requestRate = getMetricTimeSeries("http.server.requests.rate", 30);
        
        return ResourceUsageAnalysis.builder()
            .cpuUtilization(cpuUsage.getStatistics())
            .memoryUtilization(memoryUsage.getStatistics())
            .requestPattern(requestRate.getPattern())
            .peakHours(identifyPeakHours(requestRate))
            .seasonality(identifySeasonality(requestRate))
            .growth(calculateGrowthRate(requestRate))
            .build();
    }
    
    private void applySafeOptimizations(List<CostOptimization> optimizations) {
        for (CostOptimization optimization : optimizations) {
            if (optimization.isSafe() && optimization.getConfidence() > 0.9) {
                switch (optimization.getType()) {
                    case REDUCE_REPLICA_COUNT -> {
                        if (isLowTrafficPeriod()) {
                            scaleDown(optimization.getRecommendedReplicas());
                        }
                    }
                    case ADJUST_RESOURCE_LIMITS -> {
                        updateResourceLimits(optimization.getResourceLimits());
                    }
                    case ENABLE_VERTICAL_SCALING -> {
                        enableVerticalPodAutoscaler(optimization.getVpaConfig());
                    }
                }
                
                log.info("Applied cost optimization: {} (estimated savings: ${})",
                    optimization.getDescription(),
                    optimization.getEstimatedMonthlySavings());
            }
        }
    }
}

// Cost monitoring and alerting
@Component
public class CostMonitor {
    
    @Scheduled(fixedRate = 3600000) // Every hour
    public void monitorCosts() {
        CurrentCosts costs = calculateCurrentCosts();
        CostBudget budget = getCurrentBudget();
        
        // Alert if approaching budget
        if (costs.getMonthToDate() > budget.getMonthly() * 0.8) {
            alertManager.sendAlert(AlertSeverity.WARNING,
                "Cost Budget Alert",
                String.format("Current month cost $%.2f is approaching budget $%.2f",
                    costs.getMonthToDate(), budget.getMonthly()));
        }
        
        // Detect cost spikes
        if (costs.getHourly() > costs.getAverageHourly() * 2) {
            alertManager.sendAlert(AlertSeverity.HIGH,
                "Cost Spike Detected",
                String.format("Hourly cost $%.2f is %.1fx higher than average $%.2f",
                    costs.getHourly(),
                    costs.getHourly() / costs.getAverageHourly(),
                    costs.getAverageHourly()));
        }
    }
}