可观测性与监控
构建透明、可追踪的 Agent 系统
📋 可观测性概述
可观测性是指通过系统产生的数据来理解系统内部状态的能力。对于 AI Agent 系统,可观测性至关重要。
三大支柱:
- ✅ 日志(Logging):记录事件和状态变化
- ✅ 指标(Metrics):量化系统性能和行为
- ✅ 追踪(Tracing):追踪请求的完整生命周期
核心价值:
- 📊 性能监控:实时了解系统运行状态
- 🔍 问题诊断:快速定位和解决问题
- 🎯 优化决策:基于数据进行系统优化
- 🛡️ 安全审计:记录所有操作和访问
📝 日志系统
日志架构
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ Agent │ │ Tool │ │ Workflow │
│ Logs │ → │ Logs │ → │ Logs │
└─────────────────┘ └─────────────────┘ └─────────────────┘
│ │ │
└──────────────────────┴──────────────────────┘
│
▼
┌─────────────────┐
│ 日志收集器 │
│ (Collector) │
└─────────────────┘
│
▼
┌─────────────────┐
│ 日志存储 │
│ (Elasticsearch)│
└─────────────────┘
│
▼
┌─────────────────┐
│ 日志查询 │
│ (Kibana) │
└─────────────────┘结构化日志
python
import logging
import json
from datetime import datetime
class StructuredLogger:
"""结构化日志记录器"""
def __init__(self, name: str):
self.logger = logging.getLogger(name)
self.logger.setLevel(logging.INFO)
# 创建JSON格式化器
formatter = logging.Formatter(
"%(asctime)s %(levelname)s %(message)s"
)
# 添加控制台处理器
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
self.logger.addHandler(console_handler)
def log(self, level: str, message: str, **kwargs):
"""记录结构化日志"""
log_entry = {
"timestamp": datetime.now().isoformat(),
"level": level.upper(),
"message": message,
**kwargs
}
log_str = json.dumps(log_entry, ensure_ascii=False)
if level == "debug":
self.logger.debug(log_str)
elif level == "info":
self.logger.info(log_str)
elif level == "warning":
self.logger.warning(log_str)
elif level == "error":
self.logger.error(log_str)
elif level == "critical":
self.logger.critical(log_str)
# 使用结构化日志
logger = StructuredLogger("agent")
logger.log("info", "Agent启动", agent_id="agent_001", version="1.0.0")
logger.log("error", "工具调用失败", tool_name="web_search", error="网络超时")Agent 操作日志
python
class AgentLogger:
"""Agent操作日志"""
def __init__(self):
self.logger = StructuredLogger("agent")
def log_agent_start(self, agent_id: str, task: str):
"""记录Agent启动"""
self.logger.log("info", "Agent启动", agent_id=agent_id, task=task)
def log_tool_call(self, agent_id: str, tool_name: str, params: dict):
"""记录工具调用"""
self.logger.log(
"info",
"工具调用",
agent_id=agent_id,
tool_name=tool_name,
params=params
)
def log_tool_result(self, agent_id: str, tool_name: str, success: bool, execution_time: float):
"""记录工具结果"""
self.logger.log(
"info" if success else "error",
"工具执行完成",
agent_id=agent_id,
tool_name=tool_name,
success=success,
execution_time=execution_time
)
def log_agent_completion(self, agent_id: str, task: str, success: bool, output: str):
"""记录Agent完成"""
self.logger.log(
"info" if success else "error",
"Agent完成任务",
agent_id=agent_id,
task=task,
success=success,
output_length=len(output)
)📊 指标系统
指标类型
| 类型 | 说明 | 示例 |
|---|---|---|
| 计数器 | 累加值 | 请求总数、错误数 |
| 仪表盘 | 当前值 | 内存使用、活跃连接数 |
| 直方图 | 分布统计 | 请求延迟分布 |
| 摘要 | 百分位数 | P50、P95、P99延迟 |
使用 Prometheus
python
from prometheus_client import Counter, Histogram, Gauge, start_http_server
# 定义指标
REQUEST_COUNT = Counter(
'agent_request_count',
'Total agent requests',
['agent_id', 'task_type']
)
REQUEST_LATENCY = Histogram(
'agent_request_latency_seconds',
'Agent request latency',
['agent_id', 'task_type']
)
ERROR_COUNT = Counter(
'agent_error_count',
'Total agent errors',
['agent_id', 'error_type']
)
ACTIVE_AGENTS = Gauge(
'agent_active_count',
'Number of active agents'
)
TOKEN_USAGE = Counter(
'agent_token_usage',
'Total tokens used by agents',
['agent_id', 'model']
)
# 启动监控服务器
start_http_server(8000)
# 使用指标
def process_request(agent_id, task_type):
"""处理请求"""
REQUEST_COUNT.labels(agent_id=agent_id, task_type=task_type).inc()
with REQUEST_LATENCY.labels(agent_id=agent_id, task_type=task_type).time():
# 处理请求逻辑
pass
ACTIVE_AGENTS.dec()自定义指标收集
python
class MetricsCollector:
"""指标收集器"""
def __init__(self):
self.metrics = {
"total_requests": 0,
"successful_requests": 0,
"failed_requests": 0,
"total_tokens": 0,
"avg_latency": 0,
"max_latency": 0,
"min_latency": float("inf")
}
self.latencies = []
def record_request(self, success: bool, latency: float, tokens: int = 0):
"""记录请求指标"""
self.metrics["total_requests"] += 1
self.metrics["total_tokens"] += tokens
if success:
self.metrics["successful_requests"] += 1
else:
self.metrics["failed_requests"] += 1
self.latencies.append(latency)
self.metrics["max_latency"] = max(self.metrics["max_latency"], latency)
self.metrics["min_latency"] = min(self.metrics["min_latency"], latency)
self.metrics["avg_latency"] = sum(self.latencies) / len(self.latencies)
def get_report(self) -> dict:
"""生成指标报告"""
return {
"总请求数": self.metrics["total_requests"],
"成功请求数": self.metrics["successful_requests"],
"失败请求数": self.metrics["failed_requests"],
"成功率": f"{self.metrics['successful_requests'] / max(self.metrics['total_requests'], 1) * 100:.2f}%",
"总Token消耗": self.metrics["total_tokens"],
"平均延迟": f"{self.metrics['avg_latency']:.2f}s",
"最大延迟": f"{self.metrics['max_latency']:.2f}s",
"最小延迟": f"{self.metrics['min_latency']:.2f}s"
}🔍 追踪系统
分布式追踪
python
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import ConsoleSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor
# 配置追踪器
trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)
# 添加控制台导出器
span_processor = BatchSpanProcessor(ConsoleSpanExporter())
trace.get_tracer_provider().add_span_processor(span_processor)
# 使用追踪
with tracer.start_as_current_span("agent_task") as span:
span.set_attribute("agent_id", "agent_001")
span.set_attribute("task", "搜索2026年AI趋势")
with tracer.start_as_current_span("tool_call") as child_span:
child_span.set_attribute("tool_name", "web_search")
child_span.set_attribute("query", "2026年AI趋势")
# 执行工具调用
pass
span.set_attribute("success", True)Agent 执行追踪
python
class AgentTracer:
"""Agent执行追踪器"""
def __init__(self, tracer):
self.tracer = tracer
self.spans = []
def start_task(self, task_id: str, task_description: str):
"""开始任务追踪"""
span = self.tracer.start_as_current_span(f"task_{task_id}")
span.set_attribute("task_description", task_description)
self.spans.append(span)
return span
def start_step(self, step_name: str, parent_span=None):
"""开始步骤追踪"""
span = self.tracer.start_as_current_span(step_name)
if parent_span:
span.set_parent(parent_span.get_context())
self.spans.append(span)
return span
def end_span(self, span, success: bool, result=None):
"""结束追踪"""
span.set_attribute("success", success)
if result:
span.set_attribute("result", str(result)[:100])
span.end()
def end_task(self, success: bool, output=None):
"""结束任务"""
if self.spans:
task_span = self.spans[0]
task_span.set_attribute("success", success)
if output:
task_span.set_attribute("output", str(output)[:200])
task_span.end()📈 监控仪表盘
Grafana 配置
yaml
# dashboard.yaml
apiVersion: 1
providers:
- name: 'Agent监控'
folder: 'Agent'
type: file
options:
path: /etc/grafana/provisioning/dashboards/
# agent_dashboard.json
{
"title": "Agent监控仪表盘",
"panels": [
{
"title": "请求数",
"type": "graph",
"targets": [
{
"expr": "sum(rate(agent_request_count[5m]))",
"legendFormat": "请求数/秒"
}
]
},
{
"title": "成功率",
"type": "singlestat",
"targets": [
{
"expr": "sum(agent_successful_requests) / sum(agent_request_count) * 100",
"legendFormat": "成功率"
}
]
},
{
"title": "延迟分布",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.5, sum(rate(agent_request_latency_bucket[5m])) by (le))",
"legendFormat": "P50"
},
{
"expr": "histogram_quantile(0.95, sum(rate(agent_request_latency_bucket[5m])) by (le))",
"legendFormat": "P95"
}
]
},
{
"title": "Token消耗",
"type": "graph",
"targets": [
{
"expr": "sum(rate(agent_token_usage[5m]))",
"legendFormat": "Token/秒"
}
]
}
]
}⚠️ 告警系统
告警规则
yaml
# prometheus_rules.yaml
groups:
- name: agent_alerts
rules:
- alert: AgentHighErrorRate
expr: sum(rate(agent_error_count[5m])) / sum(rate(agent_request_count[5m])) > 0.1
for: 1m
labels:
severity: critical
annotations:
summary: "Agent错误率超过10%"
description: "最近5分钟错误率为 {{ $value * 100 }}%"
- alert: AgentHighLatency
expr: histogram_quantile(0.95, sum(rate(agent_request_latency_bucket[5m])) by (le)) > 5
for: 1m
labels:
severity: warning
annotations:
summary: "Agent P95延迟超过5秒"
description: "当前P95延迟为 {{ $value }} 秒"
- alert: AgentTokenOverBudget
expr: sum(agent_token_usage) > 1000000
for: 1h
labels:
severity: critical
annotations:
summary: "Agent Token消耗超过预算"
description: "当前Token消耗为 {{ $value }}"
- alert: AgentDown
expr: sum(agent_active_count) == 0
for: 5m
labels:
severity: critical
annotations:
summary: "所有Agent离线"
description: "当前没有活跃的Agent"告警通知
python
import smtplib
from email.mime.text import MIMEText
class AlertNotifier:
"""告警通知器"""
def __init__(self, smtp_config):
self.smtp_config = smtp_config
def send_email_alert(self, alert):
"""发送邮件告警"""
msg = MIMEText(alert["description"])
msg["Subject"] = f"【告警】{alert['summary']}"
msg["From"] = self.smtp_config["from"]
msg["To"] = ", ".join(self.smtp_config["to"])
with smtplib.SMTP(self.smtp_config["server"], self.smtp_config["port"]) as server:
server.starttls()
server.login(self.smtp_config["user"], self.smtp_config["password"])
server.send_message(msg)
def send_slack_alert(self, alert):
"""发送Slack告警"""
import requests
payload = {
"text": f"⚠️ *{alert['summary']}*\n\n{alert['description']}"
}
response = requests.post(
self.smtp_config["slack_webhook"],
json=payload
)
return response.status_code🔗 快速导航
| 章节 | 内容 |
|---|---|
| 项目概述 | 可观测性核心概念 |
| 日志系统 | 结构化日志与收集 |
| 指标系统 | 性能指标与监控 |
| 追踪系统 | 分布式追踪与链路 |
| 告警系统 | 告警规则与通知 |