SpringBoot 监控与可观测性

┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
│     Metrics     │    │      Logs       │    │     Traces      │
│    （指标）      │    │    （日志）      │    │    （追踪）      │
├─────────────────┤    ├─────────────────┤    ├─────────────────┤
│ • 可聚合        │    │ • 离散事件      │    │ • 请求链路      │
│ • 计数/耗时/状态│    │ • 详细上下文    │    │ • 跨服务传播    │
│ • 适合告警      │    │ • 适合排查      │    │ • 适合定位      │
│ • 低开销        │    │ • 高存储成本    │    │ • 中等开销      │
└─────────────────┘    └─────────────────┘    └─────────────────┘
         │                      │                      │
         └──────────────────────┼──────────────────────┘
                                ▼
                    ┌─────────────────────┐
                    │   SpringBoot 3.x    │
                    │  Observability API  │
                    │  (Micrometer        │
                    │   Observation)      │
                    └─────────────────────┘

management:
  endpoints:
    web:
      exposure:
        include: health,info,metrics,prometheus  # 白名单
        # exclude: env,beans  # 黑名单
      base-path: /actuator  # 基础路径
  endpoint:
    health:
      show-details: when_authorized  # never / when_authorized / always
      show-components: always        # 显示各组件健康状态
      probes:
        enabled: true  # 启用 K8s 探针端点（/actuator/health/liveness, /actuator/health/readiness）

// Counter：单调递增的计数器
Counter requests = Counter.builder("http.requests")
    .tag("method", "GET")
    .tag("status", "200")
    .register(registry);
requests.increment();  // 只能增加，不能减少

// Timer：记录耗时和分位数
Timer timer = Timer.builder("http.request.duration")
    .tag("api", "order.create")
    .publishPercentiles(0.5, 0.95, 0.99)
    .register(registry);
timer.record(() -> { /* 业务逻辑 */ });

// Gauge：瞬时值，可增可减
Gauge.builder("jvm.memory.used", 
        () -> ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed())
    .tag("area", "heap")
    .register(registry);

// DistributionSummary：记录数值分布（不限于时间）
DistributionSummary summary = DistributionSummary.builder("order.amount")
    .publishPercentiles(0.5, 0.99)
    .register(registry);
summary.record(199.99);

你要记录什么？
├── 累计次数（请求数、错误数）
│   └── Counter（只增不减）
├── 耗时（接口响应时间）
│   └── Timer（自动记录 count、sum、max、分位数）
├── 瞬时值（内存、连接数、队列长度）
│   └── Gauge（回调式读取当前值）
└── 数值分布（订单金额、文件大小）
    └── DistributionSummary（记录数值的分位分布）

# HELP http_requests_total HTTP 请求总数
# TYPE http_requests_total counter
http_requests_total{method="GET",status="200",application="order-service"} 1024

# HELP http_request_duration_seconds HTTP 请求耗时
# TYPE http_request_duration_seconds summary
http_request_duration_seconds_count{api="order.create"} 500
http_request_duration_seconds_sum{api="order.create"} 45.2
http_request_duration_seconds_max{api="order.create"} 0.8
http_request_duration_seconds{api="order.create",quantile="0.5"} 0.05
http_request_duration_seconds{api="order.create",quantile="0.95"} 0.15
http_request_duration_seconds{api="order.create",quantile="0.99"} 0.35

// 统一的可观测性 API
Observation observation = Observation.start("order.create", registry);
try (Observation.Scope scope = observation.openScope()) {
    // 业务逻辑
    // 这里自动产生：
    // 1. Metric（Counter + Timer）
    // 2. Trace Span
    // 3. Log MDC（traceId, spanId）
    processOrder(order);
    observation.event(Observation.Event.of("order.processed"));
} catch (Exception e) {
    observation.error(e);
    throw e;
} finally {
    observation.stop();
}

// 端点定义（与技术无关，可被 JMX、Web、SSH 等多种方式暴露）
@Endpoint(id = "health")
public class HealthEndpoint {

    @ReadOperation
    public Health health() {
        // 聚合所有 HealthIndicator 的结果
    }
}

// Web 扩展（将端点适配为 HTTP 接口）
@EndpointWebExtension(endpoint = HealthEndpoint.class)
public class HealthEndpointWebExtension {

    @ReadOperation
    public WebEndpointResponse<Health> health(ApiVersion apiVersion) {
        Health health = delegate.health();
        // 根据健康状态返回对应的 HTTP 状态码
        int status = (health.getStatus() == Status.UP) ? 200 : 503;
        return new WebEndpointResponse<>(health, status);
    }
}

Endpoint（端点定义）
    │
    ├── JMX Extension → MBean 注册 → JMX 访问
    │
    └── Web Extension
            │
            ├── Servlet 环境 → 注册到 DispatcherServlet
            │
            └── Reactive 环境 → 注册到 RouterFunction

// HealthEndpoint 的聚合逻辑
public class HealthEndpoint {

    private final HealthContributorRegistry registry;

    public Health health() {
        // 遍历所有 HealthContributor，逐个检查
        Map<String, Health> components = new LinkedHashMap<>();
        for (Map.Entry<String, HealthContributor> entry : registry) {
            Health health = ((HealthIndicator) entry.getValue()).health();
            components.put(entry.getKey(), health);
        }
        // 整体状态 = 所有组件中最严重的状态
        Status status = StatusAggregator.getDefault()
            .getAggregateStatus(components.values());
        return Health.status(status)
            .withDetails(components)
            .build();
    }
}

management:
  endpoint:
    health:
      show-details: when_authorized
  health:
    db:
      enabled: true
      # SpringBoot 3.x 支持自定义超时
    defaults:
      enabled: true

@Component
public class SafeDataSourceHealthIndicator implements HealthIndicator {

    @Autowired
    private DataSource dataSource;

    @Override
    public Health health() {
        // 使用独立的轻量级连接，不占用连接池
        try (Connection conn = dataSource.getConnection()) {
            // 设置查询超时，避免阻塞
            Statement stmt = conn.createStatement();
            stmt.setQueryTimeout(1); // 1 秒超时
            stmt.execute("SELECT 1");
            return Health.up().build();
        } catch (Exception e) {
            return Health.down()
                .withDetail("error", e.getMessage())
                .build();
        }
    }
}

┌─────────────────────────────────────────────────────────────┐
│                     MeterRegistry                            │
│                    （指标注册中心）                            │
│                                                              │
│  ┌─────────────┐  ┌─────────────┐  ┌─────────────────────┐  │
│  │   Counter   │  │    Timer    │  │       Gauge         │  │
│  │             │  │             │  │                     │  │
│  │ _count      │  │ _count      │  │ （回调读取当前值）    │  │
│  │             │  │ _sum        │  │                     │  │
│  │             │  │ _max        │  │                     │  │
│  │             │  │ histogram   │  │                     │  │
│  └─────────────┘  └─────────────┘  └─────────────────────┘  │
│                                                              │
│  ┌──────────────────────────────────────────────────────┐   │
│  │              MeterFilter（标签过滤/转换）               │   │
│  │  • commonTags()：统一添加标签                          │   │
│  │  • ignoreTags()：忽略指定标签                          │   │
│  │  • maximumAllowableTags()：限制标签组合数               │   │
│  └──────────────────────────────────────────────────────┘   │
│                                                              │
└─────────────────────────────────────────────────────────────┘
                              │
                              ▼
              ┌───────────────────────────────┐
              │     PrometheusMeterRegistry   │
              │    （定时 scrape 时拉取数据）   │
              └───────────────────────────────┘

// 错误示例：将用户 ID 作为 tag
Counter.builder("user.requests")
    .tag("userId", userId)  // 每个用户一个时间序列！
    .register(registry);

// 正确：将高基数维度聚合为低基数分组
Counter.builder("user.requests")
    .tag("userType", getUserType(userId))  // 分组：VIP / NORMAL
    .tag("region", getRegion(userId))      // 分组：CN / US / EU
    .register(registry);

@Bean
public MeterRegistryCustomizer<MeterRegistry> limitTags() {
    return registry -> registry.config()
        .maximumAllowableTags("http.requests", "uri", 100, (k, v) -> "TOO_MANY");
}

// ObservationRegistry 注册多个 Handler
ObservationRegistry registry = ObservationRegistry.create();

// 1. Metrics Handler：产生 Counter 和 Timer
registry.observationConfig()
    .observationHandler(new DefaultMeterObservationHandler(meterRegistry));

// 2. Tracing Handler：产生 Trace Span
registry.observationConfig()
    .observationHandler(new DefaultTracingObservationHandler(tracer));

// 3. Logging Handler：向 MDC 注入 traceId
registry.observationConfig()
    .observationHandler(new LoggingObservationHandler());

Observation.start("order.create")
    │
    ├── DefaultMeterObservationHandler
    │       └── Timer.start("order.create")
    │
    ├── DefaultTracingObservationHandler
    │       └── Tracer.nextSpan().name("order.create").start()
    │
    └── LoggingObservationHandler
            └── MDC.put("traceId", span.context().traceId())
                MDC.put("spanId", span.context().spanId())

// 自动传播的字段（W3C TraceContext 标准）
// traceparent: 00-traceId-parentId-flags

// Baggage：业务自定义的传递字段
Baggage.create("userId").set(userId);
// 自动注入到下游请求的 HTTP Header 中

# 查看某个 logger 的当前级别
GET /actuator/loggers/com.example.order

# 动态调整为 DEBUG（无需重启）
POST /actuator/loggers/com.example.order
Content-Type: application/json

{"configuredLevel": "DEBUG"}

<!-- logback-spring.xml -->
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
    <includeContext>true</includeContext>
    <includeMdcKeyName>traceId</includeMdcKeyName>
    <includeMdcKeyName>spanId</includeMdcKeyName>
</encoder>

{
  "@timestamp": "2024-06-01T10:00:00.000Z",
  "level": "INFO",
  "logger_name": "com.example.order.OrderService",
  "message": "订单创建成功",
  "traceId": "abc123",
  "spanId": "def456",
  "thread_name": "http-nio-8080-exec-1"
}

@Component
public class RiskApiHealthIndicator implements HealthIndicator {

    @Autowired
    private RiskApiClient riskApiClient;

    @Override
    public Health health() {
        // 使用独立超时，避免阻塞整体 health 端点
        try {
            boolean healthy = riskApiClient.ping()
                .timeout(Duration.ofMillis(500))
                .block();
            if (healthy) {
                return Health.up()
                    .withDetail("api", "risk-service")
                    .withDetail("latency", "50ms")
                    .build();
            } else {
                return Health.down()
                    .withDetail("api", "risk-service")
                    .withDetail("reason", "ping returned false")
                    .build();
            }
        } catch (Exception e) {
            return Health.down()
                .withDetail("api", "risk-service")
                .withDetail("error", e.getMessage())
                .build();
        }
    }
}

@Component
public class OrderControllerMetrics {

    private final Timer orderCreateTimer;
    private final Counter orderCreateCounter;

    public OrderControllerMetrics(MeterRegistry registry) {
        this.orderCreateTimer = Timer.builder("order.create.duration")
            .description("订单创建接口耗时")
            .tag("api", "order.create")
            .publishPercentileHistogram()  // 生成 histogram，支持 Prometheus histogram_quantile
            .sla(Duration.ofMillis(100), Duration.ofMillis(500))
            .register(registry);

        this.orderCreateCounter = Counter.builder("order.create.count")
            .tag("api", "order.create")
            .register(registry);
    }

    public void record(Runnable action) {
        orderCreateCounter.increment();
        orderCreateTimer.record(action);
    }
}

@RestController
public class OrderController {

    @Autowired
    private OrderControllerMetrics metrics;

    @PostMapping("/orders")
    public Order createOrder(@RequestBody OrderRequest request) {
        return metrics.record(() -> {
            // 业务逻辑
            return orderService.create(request);
        });
    }
}

# prometheus.yml
scrape_configs:
  - job_name: 'spring-boot-apps'
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ['order-service:8080', 'user-service:8080']
    # 抓取间隔
    scrape_interval: 15s

# HTTP QPS
rate(http_server_requests_seconds_count[1m])

# HTTP P99 延迟
histogram_quantile(0.99,
  rate(http_server_requests_seconds_bucket[5m])
)

# JVM 堆内存使用率
jvm_memory_used_bytes{area="heap"} / jvm_memory_max_bytes{area="heap"}

# 错误率
rate(http_server_requests_seconds_count{status=~"5.."}[5m])
  /
rate(http_server_requests_seconds_count[5m])

# alerts.yml
groups:
  - name: spring-boot-alerts
    rules:
      # 高错误率告警
      - alert: HighErrorRate
        expr: |
          rate(http_server_requests_seconds_count{status=~"5.."}[5m])
            / rate(http_server_requests_seconds_count[5m]) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "服务 {{ $labels.application }} 错误率超过 5%"

      # 高延迟告警
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            rate(http_server_requests_seconds_bucket[5m])
          ) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "服务 {{ $labels.application }} P99 延迟超过 1s"

      # JVM 内存告警
      - alert: JvmMemoryHigh
        expr: |
          jvm_memory_used_bytes{area="heap"}
            / jvm_memory_max_bytes{area="heap"} > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "服务 {{ $labels.application }} JVM 堆内存使用率超过 85%"

@Service
public class OrderService {

    @Autowired
    private ObservationRegistry observationRegistry;

    @Autowired
    private PaymentClient paymentClient;

    public Order createOrder(OrderRequest request) {
        return Observation.createNotStarted("order.create", observationRegistry)
            .contextualName("创建订单")
            .lowCardinalityKeyValue("order.type", request.getType())
            .highCardinalityKeyValue("order.userId", request.getUserId())
            .observe(() -> {
                // 1. 创建订单
                Order order = saveOrder(request);

                // 2. 调用支付服务（trace 自动传播）
                PaymentResult result = paymentClient.charge(order);

                // 3. 更新订单状态
                order.setStatus(result.isSuccess() ? "PAID" : "FAILED");
                return updateOrder(order);
            });
    }
}

<!-- logback-spring.xml -->
<pattern>
    %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36}
    [traceId=%X{traceId},spanId=%X{spanId}]
    - %msg%n
</pattern>

2024-06-01 10:00:00.123 [http-nio-8080-exec-1] INFO c.e.o.OrderService
[traceId=abc123,spanId=def456]
- 订单创建成功，orderId=ORDER_20240601_001

# deployment.yml
spec:
  template:
    spec:
      containers:
        - name: app
          livenessProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 10
            failureThreshold: 3  # 3 次失败才重启
          readinessProbe:
            httpGet:
              path: /actuator/health/readiness
              port: 8080
            initialDelaySeconds: 5
            periodSeconds: 5
            failureThreshold: 3
          startupProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8080
            initialDelaySeconds: 10
            periodSeconds: 5
            failureThreshold: 30  # 给慢启动应用最多 150 秒

management:
  endpoint:
    health:
      probes:
        enabled: true
      group:
        liveness:
          include: ping  # 只包含 ping，不检查外部依赖
        readiness:
          include: db, redis, diskSpace  # 检查外部依赖