埋点最佳实践

# 错误：直接读 Counter 瞬时值，重启归零后曲线会断崖
http_requests_total
 
# 正确：读每秒速率
rate(http_requests_total[5m])
 
# 想知道「累计了多少」时用 increase()
increase(http_requests_total[1h])

# 当前内存使用量（直接读就好）
go_memstats_heap_inuse_bytes
 
# 看它随时间的峰值
max_over_time(go_memstats_heap_inuse_bytes[1h])
 
# 错误：对 Gauge 用 rate 是概念错误
rate(go_memstats_heap_inuse_bytes[5m])

# 整体 P99 延迟（跨所有实例聚合，结果仍正确）
histogram_quantile(
  0.99,
  sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
)

# 好：带命名空间、带单位后缀、snake_case
http_server_request_duration_seconds
myapp_cache_evictions_total
node_filesystem_avail_bytes
 
# 坏：没有单位、没有命名空间、含义模糊
http_request_time
requestCount
myMetric1

# RED 三件套的标准查询
# Rate
sum by (service) (rate(http_requests_total[5m]))
 
# Errors（错误率 = 错误速率 / 总速率）
sum by (service) (rate(http_requests_total{status="500"}[5m]))
/
sum by (service) (rate(http_requests_total[5m]))
 
# Duration（P99）
histogram_quantile(0.99,
  sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
)

# USE 在 node_exporter 上的典型映射
# Utilization：CPU 使用率
1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))
 
# Saturation：运行队列长度（CPU 供不应求的程度）
node_load1  # 或 node_procs_running
 
# Errors：磁盘 I/O 错误
increase(node_disk_io_errors_total[5m])

package metrics
 
import (
    "net/http"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)
 
// 1. 创建指标（用 MustRegister 注册到默认 Registry）
var (
    httpRequestsTotal = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Namespace: "myapp",
            Subsystem: "http",
            Name:      "requests_total",
            Help:      "Total number of HTTP requests.",
        },
        []string{"method", "code"}, // 低基数标签
    )
    httpRequestDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Namespace: "myapp",
            Subsystem: "http",
            Name:      "request_duration_seconds",
            Help:      "HTTP request latency in seconds.",
            Buckets:   []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
        },
        []string{"method"},
    )
)
 
func init() {
    prometheus.MustRegister(httpRequestsTotal, httpRequestDuration)
}
 
// 2. 在处理路径上记录
func Handle(w http.ResponseWriter, r *http.Request) {
    timer := prometheus.NewTimer(httpRequestDuration.WithLabelValues(r.Method))
    defer timer.ObserveDuration()
    defer func() {
        httpRequestsTotal.WithLabelValues(r.Method, "200").Inc()
    }()
    // ... 业务逻辑
}
 
// 3. 暴露 /metrics
func Handler() http.Handler {
    return promhttp.Handler()
}

import io.prometheus.client.Counter;
import io.prometheus.client.Histogram;
import io.prometheus.client.exporter.HTTPServer;
 
public class Metrics {
    // 1. 定义指标
    static final Counter httpRequests = Counter.build()
            .name("myapp_http_requests_total")
            .help("Total HTTP requests.")
            .labelNames("method", "code")
            .register();
 
    static final Histogram httpDuration = Histogram.build()
            .name("myapp_http_request_duration_seconds")
            .help("HTTP request latency in seconds.")
            .buckets(0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
            .labelNames("method")
            .register();
 
    public static void handle(String method) {
        // 2. 记录
        Histogram.Timer timer = httpDuration.labels(method).startTimer();
        try {
            httpRequests.labels(method, "200").inc();
        } finally {
            timer.observeDuration();
        }
    }
 
    // 3. 暴露 /metrics
    public static void main(String[] args) throws Exception {
        new HTTPServer(9100); // 默认暴露 /metrics
    }
}

from prometheus_client import Counter, Histogram, start_http_server
 
# 1. 定义指标
http_requests_total = Counter(
    "myapp_http_requests_total",
    "Total HTTP requests.",
    ["method", "code"],
)
http_request_duration_seconds = Histogram(
    "myapp_http_request_duration_seconds",
    "HTTP request latency in seconds.",
    ["method"],
    buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10),
)
 
def handle(method: str) -> None:
    # 2. 记录
    with http_request_duration_seconds.labels(method).time():
        http_requests_total.labels(method, "200").inc()
 
# 3. 暴露 /metrics（独立端口，或用 make_asgi_app 挂到现有 app）
if __name__ == "__main__":
    start_http_server(9100)

# HELP myapp_pending_jobs gauge of jobs waiting in queue
# TYPE myapp_pending_jobs gauge
myapp_pending_jobs{queue="orders"} 42
 
# HELP myapp_jobs_processed_total number of jobs processed
# TYPE myapp_jobs_processed_total counter
myapp_jobs_processed_total{queue="orders"} 1387
 
# HELP myapp_job_duration_seconds job processing latency
# TYPE myapp_job_duration_seconds histogram
myapp_job_duration_seconds_bucket{queue="orders",le="0.1"} 120
myapp_job_duration_seconds_bucket{queue="orders",le="0.5"} 350
myapp_job_duration_seconds_bucket{queue="orders",le="1"} 410
myapp_job_duration_seconds_bucket{queue="orders",le="+Inf"} 420
myapp_job_duration_seconds_sum{queue="orders"} 312.7
myapp_job_duration_seconds_count{queue="orders"} 420

from http.server import BaseHTTPRequestHandler, HTTPServer
from prometheus_client import generate_latest, REGISTRY
 
class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path != "/metrics":
            self.send_response(404); self.end_headers(); return
        # generate_latest 会按规范输出 HELP/TYPE/样本
        data = generate_latest(REGISTRY)
        self.send_response(200)
        self.send_header("Content-Type", "text/plain; version=0.0.4")
        self.end_headers()
        self.wfile.write(data)
 
HTTPServer(("0.0.0.0", 9100), Handler).serve_forever()