feat: 四层架构全面增强

安全与稳定性:
- 移除硬编码 API Key,改用 .env + 环境变量
- LLM 调用统一重试机制(指数退避,3 次重试,处理 429/5xx/超时)
- 中文字体检测增强(CJK 关键词兜底 + 无字体时英文 fallback)
- 缺失 API Key 给出友好提示而非崩溃

分析能力提升:
- 异常检测新增 z-score 检测(标准差>2 标记异常)
- 新增变异系数 CV 检测(数据波动性)
- 新增零值/缺失检测
- 上下文管理器升级为关键词语义匹配(替代简单取最近 2 条)

用户体验:
- 报告自动保存为 Markdown(reports/ 目录)
- 新增 export 命令导出查询结果为 CSV
- 新增 reports 命令查看已保存报告
- CLI 支持 readline 命令历史(方向键翻阅)
- CSV 导入工具重写:自动列名映射、容错处理、dry-run 模式
- 新增 .env.example 配置模板
This commit is contained in:
openclaw
2026-03-31 14:39:17 +08:00
parent b7a27b12bd
commit e8f8e2f1ba
14 changed files with 588 additions and 115 deletions

View File

@@ -5,7 +5,7 @@ import json
from typing import Any
from core.config import LLM_CONFIG
from core.utils import get_llm_client, extract_json_array
from core.utils import get_llm_client, llm_chat, extract_json_array
from layers.explorer import ExplorationStep
@@ -68,15 +68,14 @@ class InsightEngine:
return []
history = self._build_history(steps)
response = self.client.chat.completions.create(
model=self.model,
content = llm_chat(
self.client, self.model,
messages=[
{"role": "system", "content": INSIGHT_SYSTEM},
{"role": "user", "content": f"## 用户问题\n{question}\n\n## 探索历史\n{history}\n\n请分析以上数据,输出异常和洞察。"},
],
temperature=0.3, max_tokens=2048,
)
content = response.choices[0].message.content.strip()
return [Insight(d) for d in extract_json_array(content)]
def format_insights(self, insights: list[Insight]) -> str:
@@ -109,9 +108,9 @@ class InsightEngine:
def quick_detect(steps: list[ExplorationStep]) -> list[str]:
"""基于规则的快速异常检测,不调 LLM"""
"""基于规则的快速异常检测(零 LLM 成本)"""
alerts = []
seen = set() # 去重
seen = set()
for step in steps:
if not step.success or not step.rows:
@@ -119,22 +118,21 @@ def quick_detect(steps: list[ExplorationStep]) -> list[str]:
for col in step.columns:
vals = [r.get(col) for r in step.rows if isinstance(r.get(col), (int, float))]
if not vals:
if len(vals) < 2:
continue
col_lower = col.lower()
# 占比列:某个分组占比过高
# ── 占比列:集中度过高 ──
if col_lower in ("pct", "percent", "percentage", "占比"):
for v in vals:
if v > 50:
key = f"pct_{step.purpose}"
if key not in seen:
seen.add(key)
alerts.append(f"⚠️ {step.purpose} 中某个分组占比 {v}%,集中度过高")
break
max_pct = max(vals)
if max_pct > 50:
key = f"pct_{step.purpose}"
if key not in seen:
seen.add(key)
alerts.append(f"⚠️ {step.purpose}: 最高占比 {max_pct}%,集中度过高")
# 计数列:极值差异
# ── 计数列:极值差异 ──
if col_lower in ("count", "cnt", "n", "total", "order_count") and len(vals) >= 3:
avg = sum(vals) / len(vals)
if avg > 0:
@@ -143,6 +141,47 @@ def quick_detect(steps: list[ExplorationStep]) -> list[str]:
key = f"count_{step.purpose}"
if key not in seen:
seen.add(key)
alerts.append(f"⚠️ {step.purpose} 最大值是均值的 {ratio:.1f}")
alerts.append(f"⚠️ {step.purpose}: 最大值是均值的 {ratio:.1f}")
# ── Z-Score 异常检测 ──
if len(vals) >= 5 and col_lower not in ("id", "year", "month"):
mean = sum(vals) / len(vals)
variance = sum((v - mean) ** 2 for v in vals) / len(vals)
std = variance ** 0.5
if std > 0:
outliers = [(i, v) for i, v in enumerate(vals) if abs(v - mean) / std > 2]
if outliers:
key = f"zscore_{step.purpose}_{col}"
if key not in seen:
seen.add(key)
outlier_desc = ", ".join(f"{v:.1f}" for _, v in outliers[:3])
alerts.append(
f"⚠️ {step.purpose}{col}」发现 {len(outliers)} 个异常值 "
f"(均值={mean:.1f}, σ={std:.1f}, 异常值={outlier_desc})"
)
# ── 离散度检测(变异系数 CV──
if len(vals) >= 3 and col_lower not in ("id", "year", "month"):
mean = sum(vals) / len(vals)
if mean != 0:
variance = sum((v - mean) ** 2 for v in vals) / len(vals)
std = variance ** 0.5
cv = std / abs(mean)
if cv > 1.0:
key = f"cv_{step.purpose}_{col}"
if key not in seen:
seen.add(key)
alerts.append(f"⚠️ {step.purpose}{col}」离散度高 (CV={cv:.2f}),数据波动大")
# ── 零值/缺失检测 ──
if col_lower in ("count", "cnt", "total", "amount", "sum", "关闭时长"):
zero_count = sum(1 for v in vals if v == 0)
if zero_count > 0 and zero_count < len(vals):
pct = zero_count / len(vals) * 100
if pct > 10:
key = f"zero_{step.purpose}_{col}"
if key not in seen:
seen.add(key)
alerts.append(f"⚠️ {step.purpose}{col}」有 {zero_count} 个零值 ({pct:.0f}%)")
return alerts