错误预算耗尽还在催发版：用三级门禁把可靠性变成产品决策的硬约束

# 错误预算周报（2026-09-08 ~ 2026-09-14）

## 总览

| 服务 | SLO 目标 | 剩余预算 | 燃烧速率 | 状态 | 行动建议 |
|------|----------|----------|----------|------|----------|
| 订单服务 | 99.9% | 34% | 1.2x | 🟡 黄区 | 降级为每日1次发布 |
| 支付网关 | 99.95% | 78% | 0.3x | 🟢 绿区 | 正常发布 |
| 用户中心 | 99.9% | 8% | 4.5x | 🔴 红区 | 冻结非紧急变更 |
| 搜索引擎 | 99.5% | 92% | 0.1x | 🟢 绿区 | 可承接大促压测 |

## 重点风险

**用户中心**：近 24 小时错误率 0.12%，超 SLO 目标 0.1%。按当前速率，
剩余预算将在 4 小时内耗尽。根因：周三上线的头像压缩功能内存泄漏，
已回滚但 Pod 仍在 OOM 重启。建议：阻止一切非 P0 修复类变更。

## 本周决策记录

- [x] 周二：订单服务预算 45%，允许上线 v2.3.1
- [x] 周三：用户中心预算 15%，叫停头像功能 v2 上线
- [ ] 周五：搜索引擎预算充足，批准大促前全链路压测

package main

import (
    "fmt"
    "math"
)

// BudgetImpact 将错误预算消耗翻译成业务影响
type BudgetImpact struct {
    ServiceName     string
    SLOTarget       float64 // 如 0.999
    ActualAvailability float64 // 如 0.9982
    AvgQPS          float64
    AvgOrderValue   float64 // 平均订单金额
    WindowDays      int     // SLO 窗口天数
}

// CalculateImpact 计算预算消耗的业务影响
func (b *BudgetImpact) CalculateImpact() string {
    // 总请求数
    totalRequests := b.AvgQPS * 86400 * float64(b.WindowDays)
    
    // 错误请求数 = 总请求 * (1 - 实际可用性)
    errorRequests := totalRequests * (1 - b.ActualAvailability)
    
    // 错误预算总额度 = 总请求 * (1 - SLO 目标)
    totalBudget := totalRequests * (1 - b.SLOTarget)
    
    // 已消耗预算百分比
    consumedPercent := (errorRequests / totalBudget) * 100
    if consumedPercent > 100 {
        consumedPercent = 100
    }
    
    // 估算损失（假设错误请求中有 30% 会转化为直接订单损失）
    lostOrders := errorRequests * 0.3
    lostRevenue := lostOrders * b.AvgOrderValue
    
    return fmt.Sprintf(
        "[%s] 预算消耗 %.1f%% | 影响 %d 个请求 | 估算损失 %d 笔订单 / ¥%.0f\n"+
        "按当前速率，剩余预算将在 %.1f 小时内耗尽",
        b.ServiceName,
        consumedPercent,
        int(errorRequests),
        int(lostOrders),
        lostRevenue,
        b.estimateDepletionTime(consumedPercent),
    )
}

// estimateDepletionTime 基于当前消耗速率估算预算耗尽时间
func (b *BudgetImpact) estimateDepletionTime(consumedPercent float64) float64 {
    if consumedPercent >= 100 {
        return 0
    }
    // 简化估算：假设消耗速率线性
    // 实际应基于近 1h 和 6h 的燃烧速率
    remainingPercent := 100 - consumedPercent
    // 假设当前燃烧速率（实际从监控获取）
    burnRate := 2.0 // 2x 燃烧速率
    hoursToDeplete := (remainingPercent / 100) * float64(b.WindowDays) * 24 / burnRate
    return math.Round(hoursToDeplete*10) / 10
}

func main() {
    impact := BudgetImpact{
        ServiceName:         "用户中心",
        SLOTarget:          0.999,
        ActualAvailability: 0.9988,
        AvgQPS:             3500,
        AvgOrderValue:       85.0,
        WindowDays:          30,
    }
    fmt.Println(impact.CalculateImpact())
    // 输出: [用户中心] 预算消耗 80.0% | 影响 302400 个请求 | 估算损失 90720 笔订单 / ¥7706400
    //        按当前速率，剩余预算将在 72.0 小时内耗尽
}

package main

import (
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "os"
)

// BudgetStatus 错误预算状态
type BudgetStatus struct {
    ServiceName      string  `json:"service_name"`
    SLOTarget       float64 `json:"slo_target"`
    RemainingBudget float64 `json:"remaining_budget"` // 0-100
    BurnRate        float64 `json:"burn_rate"`        // 当前燃烧速率
    BudgetWindow    string  `json:"budget_window"`    // 如 "30d"
}

// GateDecision 发布门禁决策
type GateDecision struct {
    Allowed   bool     `json:"allowed"`
    Level     string   `json:"level"`     // green / yellow / red
    Reason    string   `json:"reason"`
    Actions   []string `json:"actions"`   // 建议行动
}

// CheckBudgetGate 检查错误预算门禁
func CheckBudgetGate(status *BudgetStatus, changeType string) *GateDecision {
    level := getLevel(status.RemainingBudget, status.BurnRate)
    
    switch level {
    case "green":
        return &GateDecision{
            Allowed: true,
            Level:   "green",
            Reason:  fmt.Sprintf("预算充足（剩余 %.1f%%），允许发布", status.RemainingBudget),
            Actions: []string{"按正常流程发布"},
        }
        
    case "yellow":
        // 黄区：允许修复类变更，功能变更需要 SRE 评审
        if changeType == "fix" || changeType == "security" {
            return &GateDecision{
                Allowed: true,
                Level:   "yellow",
                Reason:  fmt.Sprintf("预算偏低（剩余 %.1f%%），但修复类变更允许通过", status.RemainingBudget),
                Actions: []string{"必须先灰度 30 分钟", "SRE 需在评审记录上签字"},
            }
        }
        return &GateDecision{
            Allowed: false,
            Level:   "yellow",
            Reason:  fmt.Sprintf("预算偏低（剩余 %.1f%%），功能变更需 SRE 评审 + 灰度发布", status.RemainingBudget),
            Actions: []string{"联系值班 SRE 评审", "灰度发布至少 30 分钟", "准备回滚方案"},
        }
        
    case "red":
        // 红区：只允许 P0 修复
        if changeType == "p0_fix" || changeType == "security" {
            return &GateDecision{
                Allowed: true,
                Level:   "red",
                Reason:  fmt.Sprintf("预算严重不足（剩余 %.1f%%），仅允许 P0 修复", status.RemainingBudget),
                Actions: []string{"必须 SRE Lead 签字", "变更后立即验证", "准备好热回滚"},
            }
        }
        return &GateDecision{
            Allowed: false,
            Level:   "red",
            Reason:  fmt.Sprintf("预算耗尽风险（剩余 %.1f%%，燃烧速率 %.1fx），冻结一切非紧急变更", 
                status.RemainingBudget, status.BurnRate),
            Actions: []string{"冻结发布", "全员投入稳定性修复", "恢复至黄区后重新评审"},
        }
    }
    
    return &GateDecision{Allowed: false, Level: "unknown", Reason: "无法判断预算级别"}
}

// getLevel 根据剩余预算和燃烧速率判定级别
func getLevel(remaining, burnRate float64) string {
    if remaining < 25 || (remaining < 50 && burnRate > 2.0) {
        return "red"
    }
    if remaining < 50 || burnRate > 1.0 {
        return "yellow"
    }
    return "green"
}

// IntegrateWithCI 集成到 CI/CD 流水线
func IntegrateWithCI(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
        http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
        return
    }
    
    var req struct {
        ServiceName string `json:"service_name"`
        ChangeType  string `json:"change_type"` // feature / fix / security / p0_fix
    }
    
    if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
        http.Error(w, err.Error(), http.StatusBadRequest)
        return
    }
    
    // 从监控平台获取预算状态（实际调用 Prometheus API）
    status := fetchBudgetStatus(req.ServiceName)
    
    decision := CheckBudgetGate(status, req.ChangeType)
    
    w.Header().Set("Content-Type", "application/json")
    json.NewEncoder(w).Encode(decision)
    
    if !decision.Allowed {
        log.Printf("[GATE] 发布被拦截: service=%s level=%s reason=%s",
            req.ServiceName, decision.Level, decision.Reason)
    }
}

// fetchBudgetStatus 从 Prometheus 查询预算状态
func fetchBudgetStatus(serviceName string) *BudgetStatus {
    // 实际实现中调用 Prometheus HTTP API
    // 这里返回模拟数据
    return &BudgetStatus{
        ServiceName:      serviceName,
        SLOTarget:        0.999,
        RemainingBudget:  42.5,
        BurnRate:         1.8,
        BudgetWindow:     "30d",
    }
}

func main() {
    http.HandleFunc("/gate", IntegrateWithCI)
    
    port := os.Getenv("GATE_PORT")
    if port == "" {
        port = "8080"
    }
    
    log.Printf("Error budget gate listening on :%s", port)
    log.Fatal(http.ListenAndServe(":"+port, nil))
}

燃烧速率 = 实际错误率 / SLO 允许的错误率

示例：
SLO = 99.9%（允许错误率 0.1%）
当前实际错误率 = 0.6%
燃烧速率 = 0.6% / 0.1% = 6x

含义：按当前速率，30 天窗口的预算将在 5 天内耗尽（30 / 6 = 5）

# Prometheus SLO 告警规则：多窗口燃烧速率
# 参考：https://sre.google/workbook/alerting-on-slo-error-budget/

groups:
  - name: slo_burn_rate
    rules:
      # 页面告警：1 小时窗口 + 5 分钟窗口
      # 燃烧速率 14.4x = 30 天预算在 2 天内耗尽
      - alert: SLOBurnRateFastPage
        expr: |
          (
            rate(http_requests_total{code=~"5..",service="user-center"}[5m]) 
            / 
            (rate(http_requests_total{service="user-center"}[5m]))
            >
            14.4 * (1 - 0.999)
          )
          and
          (
            rate(http_requests_total{code=~"5..",service="user-center"}[1h]) 
            / 
            (rate(http_requests_total{service="user-center"}[1h]))
            >
            14.4 * (1 - 0.999)
          )          
        for: 2m
        labels:
          severity: page
          service: user-center
        annotations:
          summary: "SLO 燃烧速率告警（快速）"
          description: "近 5 分钟和 1 小时窗口的错误率均超 SLO 14.4 倍，预计 2 天内预算耗尽"

      # 工单告警：6 小时窗口 + 30 分钟窗口
      # 燃烧速率 6x = 30 天预算在 5 天内耗尽
      - alert: SLOBurnRateSlowTicket
        expr: |
          (
            rate(http_requests_total{code=~"5..",service="user-center"}[30m]) 
            / 
            (rate(http_requests_total{service="user-center"}[30m]))
            >
            6 * (1 - 0.999)
          )
          and
          (
            rate(http_requests_total{code=~"5..",service="user-center"}[6h]) 
            / 
            (rate(http_requests_total{service="user-center"}[6h]))
            >
            6 * (1 - 0.999)
          )          
        for: 15m
        labels:
          severity: ticket
          service: user-center
        annotations:
          summary: "SLO 燃烧速率告警（慢速）"
          description: "近 30 分钟和 6 小时窗口错误率均超 SLO 6 倍，预计 5 天内预算耗尽"

      # 预算耗尽告警：3 天窗口
      # 燃烧速率 1x = 持续超 SLO，预算正在稳定消耗
      - alert: SLOBurnRateSustained
        expr: |
          rate(http_requests_total{code=~"5..",service="user-center"}[3d]) 
          / 
          (rate(http_requests_total{service="user-center"}[3d]))
          >
          1 * (1 - 0.999)          
        for: 1h
        labels:
          severity: ticket
          service: user-center
        annotations:
          summary: "SLO 持续燃烧"
          description: "近 3 天错误率持续超 SLO 目标，预算正在稳定消耗"

// BudgetWindowReview SLO 窗口结束时的自动复盘
type BudgetWindowReview struct {
    ServiceName      string
    WindowStart      time.Time
    WindowEnd        time.Time
    BudgetConsumed   float64 // 消耗百分比，可能 > 100
    TopContributors  []struct {
        Incident     string
        BudgetImpact float64 // 该事件消耗的预算百分比
        RootCause    string
    }
    ActionItems      []string
}

// GenerateReview 生成窗口复盘报告
func GenerateReview(serviceName string, windowDays int) *BudgetWindowReview {
    review := &BudgetWindowReview{
        ServiceName: serviceName,
        WindowStart: time.Now().AddDate(0, 0, -windowDays),
        WindowEnd:   time.Now(),
    }
    
    // 从事件管理系统拉取窗口内所有事件
    // 从 Prometheus 查询每个事件的预算影响
    // 按消耗量排序，取 Top 3
    // 生成行动项
    
    return review
}

{
  "title": "Error Budget Remaining",
  "type": "gauge",
  "datasource": "Prometheus",
  "targets": [
    {
      "expr": "1 - (\n  sum(rate(http_requests_total{code=~\"5..\",service=\"$service\"}[$__range]))\n  /\n  sum(rate(http_requests_total{service=\"$service\"}[$__range]))\n) / (1 - 0.999)\n* 100",
      "legendFormat": "Remaining Budget %",
      "refId": "A"
    }
  ],
  "fieldConfig": {
    "defaults": {
      "min": 0,
      "max": 100,
      "thresholds": {
        "steps": [
          {"color": "red", "value": 0},
          {"color": "yellow", "value": 25},
          {"color": "green", "value": 50}
        ]
      },
      "unit": "percent"
    }
  },
  "options": {
    "reduceOptions": {"calcs": ["lastNotNull"]}
  }
}

# Grafana Annotations 配置（通过 API 注入）
# 每次 CI/CD 发布时自动创建 annotation
annotations:
  - name: "Deploy: user-center v2.3.1"
    time: "2026-09-10T14:30:00Z"
    tags: ["deploy", "user-center"]
    text: "Release v2.3.1 - 拼车优惠叠加功能"