Deployment Monitoring & Observability

Setup comprehensive deployment monitoring: $ARGUMENTS

Current Monitoring State

• Existing monitoring: !kubectl get pods -n monitoring 2>/dev/null || docker ps | grep -E "(prometheus|grafana|jaeger)" || echo "No monitoring detected"
• Health endpoints: !curl -s https://api.example.com/health 2>/dev/null | jq -r '.status // "Unknown"' || echo "Health endpoint needed"
• Metrics exposure: !curl -s https://api.example.com/metrics 2>/dev/null | head -5 || echo "Metrics endpoint needed"
• Log aggregation: !kubectl get pods -n logging 2>/dev/null || echo "Log aggregation setup needed"
• APM integration: Check for application performance monitoring setup

Task

Implement comprehensive monitoring and observability for deployments with real-time insights, alerting, and automated response capabilities.

Monitoring Architecture

1. Core Monitoring Stack

Prometheus Configuration

# prometheus-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
      
    rule_files:
      - "/etc/prometheus/rules/*.yml"
      
    scrape_configs:
      # Application metrics
      - job_name: 'myapp'
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            action: keep
            regex: true
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            regex: ([^:]+)(?::\d+)?;(\d+)
            replacement: $1:$2
            target_label: __address__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
            
      # Kubernetes cluster metrics
      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_phase]
            action: keep
            regex: Running
            
      # Node exporter for infrastructure metrics
      - job_name: 'node-exporter'
        kubernetes_sd_configs:
          - role: endpoints
        relabel_configs:
          - source_labels: [__meta_kubernetes_endpoints_name]
            action: keep
            regex: node-exporter
            
      # Deployment-specific metrics
      - job_name: 'deployment-metrics'
        static_configs:
          - targets: ['deployment-exporter:9090']
        metrics_path: /metrics
        scrape_interval: 30s

    alerting:
      alertmanagers:
        - static_configs:
            - targets: ['alertmanager:9093']

---
# Prometheus Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      serviceAccountName: prometheus
      containers:
      - name: prometheus
        image: prom/prometheus:v2.40.0
        args:
          - '--config.file=/etc/prometheus/prometheus.yml'
          - '--storage.tsdb.path=/prometheus'
          - '--web.console.libraries=/etc/prometheus/console_libraries'
          - '--web.console.templates=/etc/prometheus/consoles'
          - '--storage.tsdb.retention.time=30d'
          - '--web.enable-lifecycle'
          - '--web.enable-admin-api'
        ports:
        - containerPort: 9090
        volumeMounts:
        - name: prometheus-config
          mountPath: /etc/prometheus
        - name: prometheus-storage
          mountPath: /prometheus
        resources:
          requests:
            memory: "512Mi"
            cpu: "250m"
          limits:
            memory: "2Gi"
            cpu: "1000m"
      volumes:
      - name: prometheus-config
        configMap:
          name: prometheus-config
      - name: prometheus-storage
        persistentVolumeClaim:
          claimName: prometheus-pvc

Grafana Dashboard Configuration

# grafana-dashboard-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: deployment-dashboard
  namespace: monitoring
data:
  deployment-monitoring.json: |
    {
      "dashboard": {
        "id": null,
        "title": "Deployment Monitoring Dashboard",
        "tags": ["deployment", "monitoring"],
        "timezone": "browser",
        "panels": [
          {
            "id": 1,
            "title": "Deployment Status",
            "type": "stat",
            "targets": [
              {
                "expr": "up{job=\"myapp\"}",
                "legendFormat": "{{instance}}"
              }
            ],
            "fieldConfig": {
              "defaults": {
                "thresholds": {
                  "steps": [
                    {"color": "red", "value": 0},
                    {"color": "green", "value": 1}
                  ]
                }
              }
            }
          },
          {
            "id": 2,
            "title": "Request Rate",
            "type": "graph",
            "targets": [
              {
                "expr": "rate(http_requests_total[5m])",
                "legendFormat": "{{method}} {{status}}"
              }
            ]
          },
          {
            "id": 3,
            "title": "Error Rate",
            "type": "graph",
            "targets": [
              {
                "expr": "rate(http_requests_total{status=~\"5..\"}[5m]) / rate(http_requests_total[5m]) * 100",
                "legendFormat": "Error Rate %"
              }
            ]
          },
          {
            "id": 4,
            "title": "Response Time",
            "type": "graph",
            "targets": [
              {
                "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
                "legendFormat": "95th percentile"
              },
              {
                "expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))",
                "legendFormat": "50th percentile"
              }
            ]
          },
          {
            "id": 5,
            "title": "Pod Resource Usage",
            "type": "graph",
            "targets": [
              {
                "expr": "rate(container_cpu_usage_seconds_total{pod=~\"myapp-.*\"}[5m]) * 100",
                "legendFormat": "CPU Usage - {{pod}}"
              },
              {
                "expr": "container_memory_usage_bytes{pod=~\"myapp-.*\"} / 1024 / 1024",
                "legendFormat": "Memory Usage MB - {{pod}}"
              }
            ]
          },
          {
            "id": 6,
            "title": "Deployment Events",
            "type": "logs",
            "targets": [
              {
                "expr": "{job=\"kubernetes-events\"} |= \"myapp\"",
                "legendFormat": ""
              }
            ]
          }
        ],
        "time": {
          "from": "now-1h",
          "to": "now"
        },
        "refresh": "30s"
      }
    }

2. Application Health Monitoring

Health Check Implementation

// health-check.js - Application health endpoint
const express = require('express');
const { promisify } = require('util');

class HealthMonitor {
  constructor() {
    this.checks = new Map();
    this.status = 'healthy';
    this.lastCheck = new Date();
  }

  registerCheck(name, checkFunction, options = {}) {
    this.checks.set(name, {
      check: checkFunction,
      timeout: options.timeout || 5000,
      critical: options.critical || false,
      lastStatus: null,
      lastCheck: null,
      errorCount: 0
    });
  }

  async runHealthChecks() {
    const results = {};
    let overallHealthy = true;
    
    for (const [name, config] of this.checks) {
      try {
        const startTime = Date.now();
        const result = await Promise.race([
          config.check(),
          new Promise((_, reject) => 
            setTimeout(() => reject(new Error('Health check timeout')), config.timeout)
          )
        ]);
        
        const duration = Date.now() - startTime;
        
        results[name] = {
          status: 'healthy',
          duration,
          details: result,
          lastCheck: new Date().toISOString()
        };
        
        config.lastStatus = 'healthy';
        config.errorCount = 0;
      } catch (error) {
        results[name] = {
          status: 'unhealthy',
          error: error.message,
          lastCheck: new Date().toISOString()
        };
        
        config.lastStatus = 'unhealthy';
        config.errorCount++;
        
        if (config.critical) {
          overallHealthy = false;
        }
      }
      
      config.lastCheck = new Date();
    }
    
    this.status = overallHealthy ? 'healthy' : 'unhealthy';
    this.lastCheck = new Date();
    
    return {
      status: this.status,
      timestamp: this.lastCheck.toISOString(),
      checks: results,
      uptime: process.uptime(),
      version: process.env.APP_VERSION || 'unknown'
    };
  }

  setupEndpoints(app) {
    // Liveness probe - basic application health
    app.get('/health', async (req, res) => {
      const health = await this.runHealthChecks();
      const statusCode = health.status === 'healthy' ? 200 : 503;
      res.status(statusCode).json(health);
    });

    // Readiness probe - ready to receive traffic
    app.get('/ready', async (req, res) => {
      const health = await this.runHealthChecks();
      
      // Additional readiness checks
      const readinessChecks = {
        memoryUsage: process.memoryUsage().heapUsed / process.memoryUsage().heapTotal < 0.9,
        activeConnections: true, // Check active connections if applicable
      };
      
      const isReady = health.status === 'healthy' && 
                     Object.values(readinessChecks).every(check => check);
      
      res.status(isReady ? 200 : 503).json({
        ...health,
        ready: isReady,
        readinessChecks
      });
    });

    // Startup probe - application has started
    app.get('/startup', (req, res) => {
      res.status(200).json({
        status: 'started',
        timestamp: new Date().toISOString(),
        pid: process.pid,
        uptime: process.uptime()
      });
    });
  }
}

// Usage example
const healthMonitor = new HealthMonitor();

// Register health checks
healthMonitor.registerCheck('database', async () => {
  // Database connectivity check
  await db.query('SELECT 1');
  return { connected: true };
}, { critical: true, timeout: 3000 });

healthMonitor.registerCheck('redis', async () => {
  // Redis connectivity check
  await redis.ping();
  return { connected: true };
}, { critical: false, timeout: 2000 });

healthMonitor.registerCheck('external-api', async () => {
  // External service check
  const response = await fetch('https://api.external-service.com/health');
  return { status: response.status, healthy: response.ok };
}, { critical: false, timeout: 5000 });

module.exports = healthMonitor;

3. Custom Metrics and Instrumentation

Application Metrics

```javascript

// metrics.js - Application metrics collection

const promClient = require('prom-client');

class DeploymentMetrics {

constructor() {

// Default metrics

promClient.collectDefaultMetrics({

prefix: 'myapp_',

timeout: 5000,

});

// Custom deployment metrics

this.deploymentInfo = new promClient.Gauge({

name: 'myapp_deployment_info',

help: 'Deployment information',

labelNames: ['version', 'environment', 'commit_sha']

});

this.httpRequestsTotal = new promClient.Counter({

name: 'myapp_http_requests_total',

help: 'Tot