Skip to content

可观测性与监控 ​

构建透明、可追踪的 Agent 系统


📋 可观测性概述 ​

可观测性是指通过系统产生的数据来理解系统内部状态的能力。对于 AI Agent 系统,可观测性至关重要。

三大支柱:

  • ✅ 日志(Logging):记录事件和状态变化
  • ✅ 指标(Metrics):量化系统性能和行为
  • ✅ 追踪(Tracing):追踪请求的完整生命周期

核心价值:

  • 📊 性能监控:实时了解系统运行状态
  • 🔍 问题诊断:快速定位和解决问题
  • 🎯 优化决策:基于数据进行系统优化
  • 🛡️ 安全审计:记录所有操作和访问

📝 日志系统 ​

日志架构 ​

┌─────────────────┐     ┌─────────────────┐     ┌─────────────────┐
│   Agent         │     │   Tool          │     │   Workflow      │
│   Logs          │ →   │   Logs          │ →   │   Logs          │
└─────────────────┘     └─────────────────┘     └─────────────────┘
         │                      │                      │
         └──────────────────────┴──────────────────────┘
                                 │
                                 ▼
                      ┌─────────────────┐
                      │   日志收集器    │
                      │  (Collector)    │
                      └─────────────────┘
                                 │
                                 ▼
                      ┌─────────────────┐
                      │   日志存储      │
                      │  (Elasticsearch)│
                      └─────────────────┘
                                 │
                                 ▼
                      ┌─────────────────┐
                      │   日志查询      │
                      │  (Kibana)       │
                      └─────────────────┘

结构化日志 ​

python
import logging
import json
from datetime import datetime

class StructuredLogger:
    """结构化日志记录器"""
    
    def __init__(self, name: str):
        self.logger = logging.getLogger(name)
        self.logger.setLevel(logging.INFO)
        
        # 创建JSON格式化器
        formatter = logging.Formatter(
            "%(asctime)s %(levelname)s %(message)s"
        )
        
        # 添加控制台处理器
        console_handler = logging.StreamHandler()
        console_handler.setFormatter(formatter)
        self.logger.addHandler(console_handler)
    
    def log(self, level: str, message: str, **kwargs):
        """记录结构化日志"""
        log_entry = {
            "timestamp": datetime.now().isoformat(),
            "level": level.upper(),
            "message": message,
            **kwargs
        }
        
        log_str = json.dumps(log_entry, ensure_ascii=False)
        
        if level == "debug":
            self.logger.debug(log_str)
        elif level == "info":
            self.logger.info(log_str)
        elif level == "warning":
            self.logger.warning(log_str)
        elif level == "error":
            self.logger.error(log_str)
        elif level == "critical":
            self.logger.critical(log_str)

# 使用结构化日志
logger = StructuredLogger("agent")
logger.log("info", "Agent启动", agent_id="agent_001", version="1.0.0")
logger.log("error", "工具调用失败", tool_name="web_search", error="网络超时")

Agent 操作日志 ​

python
class AgentLogger:
    """Agent操作日志"""
    
    def __init__(self):
        self.logger = StructuredLogger("agent")
    
    def log_agent_start(self, agent_id: str, task: str):
        """记录Agent启动"""
        self.logger.log("info", "Agent启动", agent_id=agent_id, task=task)
    
    def log_tool_call(self, agent_id: str, tool_name: str, params: dict):
        """记录工具调用"""
        self.logger.log(
            "info",
            "工具调用",
            agent_id=agent_id,
            tool_name=tool_name,
            params=params
        )
    
    def log_tool_result(self, agent_id: str, tool_name: str, success: bool, execution_time: float):
        """记录工具结果"""
        self.logger.log(
            "info" if success else "error",
            "工具执行完成",
            agent_id=agent_id,
            tool_name=tool_name,
            success=success,
            execution_time=execution_time
        )
    
    def log_agent_completion(self, agent_id: str, task: str, success: bool, output: str):
        """记录Agent完成"""
        self.logger.log(
            "info" if success else "error",
            "Agent完成任务",
            agent_id=agent_id,
            task=task,
            success=success,
            output_length=len(output)
        )

📊 指标系统 ​

指标类型 ​

类型说明示例
计数器累加值请求总数、错误数
仪表盘当前值内存使用、活跃连接数
直方图分布统计请求延迟分布
摘要百分位数P50、P95、P99延迟

使用 Prometheus ​

python
from prometheus_client import Counter, Histogram, Gauge, start_http_server

# 定义指标
REQUEST_COUNT = Counter(
    'agent_request_count',
    'Total agent requests',
    ['agent_id', 'task_type']
)

REQUEST_LATENCY = Histogram(
    'agent_request_latency_seconds',
    'Agent request latency',
    ['agent_id', 'task_type']
)

ERROR_COUNT = Counter(
    'agent_error_count',
    'Total agent errors',
    ['agent_id', 'error_type']
)

ACTIVE_AGENTS = Gauge(
    'agent_active_count',
    'Number of active agents'
)

TOKEN_USAGE = Counter(
    'agent_token_usage',
    'Total tokens used by agents',
    ['agent_id', 'model']
)

# 启动监控服务器
start_http_server(8000)

# 使用指标
def process_request(agent_id, task_type):
    """处理请求"""
    REQUEST_COUNT.labels(agent_id=agent_id, task_type=task_type).inc()
    
    with REQUEST_LATENCY.labels(agent_id=agent_id, task_type=task_type).time():
        # 处理请求逻辑
        pass
    
    ACTIVE_AGENTS.dec()

自定义指标收集 ​

python
class MetricsCollector:
    """指标收集器"""
    
    def __init__(self):
        self.metrics = {
            "total_requests": 0,
            "successful_requests": 0,
            "failed_requests": 0,
            "total_tokens": 0,
            "avg_latency": 0,
            "max_latency": 0,
            "min_latency": float("inf")
        }
        self.latencies = []
    
    def record_request(self, success: bool, latency: float, tokens: int = 0):
        """记录请求指标"""
        self.metrics["total_requests"] += 1
        self.metrics["total_tokens"] += tokens
        
        if success:
            self.metrics["successful_requests"] += 1
        else:
            self.metrics["failed_requests"] += 1
        
        self.latencies.append(latency)
        self.metrics["max_latency"] = max(self.metrics["max_latency"], latency)
        self.metrics["min_latency"] = min(self.metrics["min_latency"], latency)
        self.metrics["avg_latency"] = sum(self.latencies) / len(self.latencies)
    
    def get_report(self) -> dict:
        """生成指标报告"""
        return {
            "总请求数": self.metrics["total_requests"],
            "成功请求数": self.metrics["successful_requests"],
            "失败请求数": self.metrics["failed_requests"],
            "成功率": f"{self.metrics['successful_requests'] / max(self.metrics['total_requests'], 1) * 100:.2f}%",
            "总Token消耗": self.metrics["total_tokens"],
            "平均延迟": f"{self.metrics['avg_latency']:.2f}s",
            "最大延迟": f"{self.metrics['max_latency']:.2f}s",
            "最小延迟": f"{self.metrics['min_latency']:.2f}s"
        }

🔍 追踪系统 ​

分布式追踪 ​

python
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import ConsoleSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# 配置追踪器
trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)

# 添加控制台导出器
span_processor = BatchSpanProcessor(ConsoleSpanExporter())
trace.get_tracer_provider().add_span_processor(span_processor)

# 使用追踪
with tracer.start_as_current_span("agent_task") as span:
    span.set_attribute("agent_id", "agent_001")
    span.set_attribute("task", "搜索2026年AI趋势")
    
    with tracer.start_as_current_span("tool_call") as child_span:
        child_span.set_attribute("tool_name", "web_search")
        child_span.set_attribute("query", "2026年AI趋势")
        
        # 执行工具调用
        pass
    
    span.set_attribute("success", True)

Agent 执行追踪 ​

python
class AgentTracer:
    """Agent执行追踪器"""
    
    def __init__(self, tracer):
        self.tracer = tracer
        self.spans = []
    
    def start_task(self, task_id: str, task_description: str):
        """开始任务追踪"""
        span = self.tracer.start_as_current_span(f"task_{task_id}")
        span.set_attribute("task_description", task_description)
        self.spans.append(span)
        return span
    
    def start_step(self, step_name: str, parent_span=None):
        """开始步骤追踪"""
        span = self.tracer.start_as_current_span(step_name)
        if parent_span:
            span.set_parent(parent_span.get_context())
        self.spans.append(span)
        return span
    
    def end_span(self, span, success: bool, result=None):
        """结束追踪"""
        span.set_attribute("success", success)
        if result:
            span.set_attribute("result", str(result)[:100])
        span.end()
    
    def end_task(self, success: bool, output=None):
        """结束任务"""
        if self.spans:
            task_span = self.spans[0]
            task_span.set_attribute("success", success)
            if output:
                task_span.set_attribute("output", str(output)[:200])
            task_span.end()

📈 监控仪表盘 ​

Grafana 配置 ​

yaml
# dashboard.yaml
apiVersion: 1
providers:
  - name: 'Agent监控'
    folder: 'Agent'
    type: file
    options:
      path: /etc/grafana/provisioning/dashboards/

# agent_dashboard.json
{
  "title": "Agent监控仪表盘",
  "panels": [
    {
      "title": "请求数",
      "type": "graph",
      "targets": [
        {
          "expr": "sum(rate(agent_request_count[5m]))",
          "legendFormat": "请求数/秒"
        }
      ]
    },
    {
      "title": "成功率",
      "type": "singlestat",
      "targets": [
        {
          "expr": "sum(agent_successful_requests) / sum(agent_request_count) * 100",
          "legendFormat": "成功率"
        }
      ]
    },
    {
      "title": "延迟分布",
      "type": "graph",
      "targets": [
        {
          "expr": "histogram_quantile(0.5, sum(rate(agent_request_latency_bucket[5m])) by (le))",
          "legendFormat": "P50"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(agent_request_latency_bucket[5m])) by (le))",
          "legendFormat": "P95"
        }
      ]
    },
    {
      "title": "Token消耗",
      "type": "graph",
      "targets": [
        {
          "expr": "sum(rate(agent_token_usage[5m]))",
          "legendFormat": "Token/秒"
        }
      ]
    }
  ]
}

⚠️ 告警系统 ​

告警规则 ​

yaml
# prometheus_rules.yaml
groups:
  - name: agent_alerts
    rules:
      - alert: AgentHighErrorRate
        expr: sum(rate(agent_error_count[5m])) / sum(rate(agent_request_count[5m])) > 0.1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Agent错误率超过10%"
          description: "最近5分钟错误率为 {{ $value * 100 }}%"
      
      - alert: AgentHighLatency
        expr: histogram_quantile(0.95, sum(rate(agent_request_latency_bucket[5m])) by (le)) > 5
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Agent P95延迟超过5秒"
          description: "当前P95延迟为 {{ $value }} 秒"
      
      - alert: AgentTokenOverBudget
        expr: sum(agent_token_usage) > 1000000
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: "Agent Token消耗超过预算"
          description: "当前Token消耗为 {{ $value }}"
      
      - alert: AgentDown
        expr: sum(agent_active_count) == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "所有Agent离线"
          description: "当前没有活跃的Agent"

告警通知 ​

python
import smtplib
from email.mime.text import MIMEText

class AlertNotifier:
    """告警通知器"""
    
    def __init__(self, smtp_config):
        self.smtp_config = smtp_config
    
    def send_email_alert(self, alert):
        """发送邮件告警"""
        msg = MIMEText(alert["description"])
        msg["Subject"] = f"【告警】{alert['summary']}"
        msg["From"] = self.smtp_config["from"]
        msg["To"] = ", ".join(self.smtp_config["to"])
        
        with smtplib.SMTP(self.smtp_config["server"], self.smtp_config["port"]) as server:
            server.starttls()
            server.login(self.smtp_config["user"], self.smtp_config["password"])
            server.send_message(msg)
    
    def send_slack_alert(self, alert):
        """发送Slack告警"""
        import requests
        
        payload = {
            "text": f"⚠️ *{alert['summary']}*\n\n{alert['description']}"
        }
        
        response = requests.post(
            self.smtp_config["slack_webhook"],
            json=payload
        )
        return response.status_code

🔗 快速导航 ​

章节内容
项目概述可观测性核心概念
日志系统结构化日志与收集
指标系统性能指标与监控
追踪系统分布式追踪与链路
告警系统告警规则与通知