修复 P0:根因判定、退服率口径、影响评估与告警关联量化
- 根因判定改为三级(衍生关系/时间窗/疑似)并带置信度;小区退服告警 ±2 分钟匹配; 衍生告警按父告警归类;优先选随业务恢复清除的告警;时间窗按历史流水 95% 分位数标定 - 退服率按日历统计周期计算;站点数分母支持站名关键字;重复退服率除以全部站点; 月均按统计周期自然月数;导出与看板同一口径 - 影响评估新增受影响用户·小时与 500 米邻站覆盖补偿 - 新增告警—退服关联量化(支持度/置信度/提升度)及 /api/association,根因页展示 - 新增 tests/(扰动数据集留出检验、指标口径检验)与 bench.py;旧库启动时自动迁移 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f8d1ccc526
commit
e9616af34d
10 files changed
+869
-119
No files matched your search
@@ -0,0 +1,173 @@
|
||||
"""测试公共工具:在 tests/_work 下的临时数据库里加载数据集并调用 core 的分析逻辑,不影响 data/outage.db。"""
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
|
||||
PROJECT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, PROJECT)
|
||||
WORK = os.path.join(PROJECT, 'tests', '_work')
|
||||
os.makedirs(WORK, exist_ok=True)
|
||||
os.environ['OUTAGE_DB'] = os.path.join(WORK, 'app.db') # 必须在 import core 之前设置
|
||||
|
||||
import pandas as pd # noqa: E402
|
||||
from werkzeug.security import generate_password_hash # noqa: E402
|
||||
|
||||
import core # noqa: E402
|
||||
import data_gen # noqa: E402
|
||||
import importer # noqa: E402
|
||||
|
||||
FAILED = []
|
||||
|
||||
|
||||
def check(name, ok, detail=''):
|
||||
"""打印一条检查结果;失败的记入 FAILED,脚本结束时以非 0 退出码返回。"""
|
||||
print(f" [{'通过' if ok else '失败'}] {name} {detail}")
|
||||
if not ok:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
def finish():
|
||||
print('\n全部检查通过' if not FAILED else f'\n{len(FAILED)} 项未通过:' + '、'.join(FAILED))
|
||||
sys.exit(1 if FAILED else 0)
|
||||
|
||||
|
||||
def use_db(name):
|
||||
"""切换到 tests/_work/<name>.db(删除重建)。"""
|
||||
path = os.path.join(WORK, f'{name}.db')
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
core.DB_PATH = path
|
||||
core.data_changed()
|
||||
with core.db() as conn:
|
||||
conn.executescript(core.SCHEMA)
|
||||
for scene, values in core.SCENE_DEFAULT.items():
|
||||
conn.execute('INSERT INTO scenes VALUES(?,?,?,?,?)', (scene, *values))
|
||||
conn.execute("INSERT INTO users(username, password, role, created_at) VALUES('admin', ?, 'admin', '')",
|
||||
(generate_password_hash('admin123'),)) # 测试库的接口测试账号
|
||||
|
||||
|
||||
def load(name, sites, alarms, outages, calibrate_until=None):
|
||||
"""建库并写入 sites / alarms / outages(全文本 DataFrame),做根因分析,返回 full_frame()。
|
||||
calibrate_until 指定时,时间窗只用该日期之前的流水标定(留出检验,避免用测试数据标定)。"""
|
||||
use_db(name)
|
||||
with core.db() as conn:
|
||||
sites.to_sql('sites', conn, if_exists='append', index=False)
|
||||
core.save_alarms(alarms.astype(object).where(alarms.notna(), None))
|
||||
good, _ = importer.validate_outages(outages)
|
||||
core.save_outages(good)
|
||||
if calibrate_until:
|
||||
core.calibrate_windows(until=calibrate_until)
|
||||
core.analyze_all(recalibrate=False)
|
||||
else:
|
||||
core.analyze_all()
|
||||
core.data_changed()
|
||||
return core.full_frame()
|
||||
|
||||
|
||||
def read_dir(directory):
|
||||
def read(filename):
|
||||
return pd.read_csv(os.path.join(directory, filename), encoding='utf-8-sig', dtype=str)
|
||||
return read('sites.csv'), read('data_alarm.csv'), read('data_outage.csv')
|
||||
|
||||
|
||||
def sample_data():
|
||||
return read_dir(os.path.join(PROJECT, 'data'))
|
||||
|
||||
|
||||
def generate(name, lead=None, rows=6000, seed=2026):
|
||||
"""用 data_gen 生成一份数据到 tests/_work/gen_<name>;lead 可覆盖告警提前量分布。"""
|
||||
out = os.path.join(WORK, 'gen_' + name)
|
||||
saved = dict(data_gen.LEAD)
|
||||
if lead is not None:
|
||||
data_gen.LEAD.clear()
|
||||
data_gen.LEAD.update(lead)
|
||||
try:
|
||||
data_gen.generate(n=rows, seed=seed, out=out)
|
||||
finally:
|
||||
data_gen.LEAD.clear()
|
||||
data_gen.LEAD.update(saved)
|
||||
return read_dir(out)
|
||||
|
||||
|
||||
def score(df):
|
||||
"""以生成器真值(cause_category)计算总体准确率与各类 precision / recall。"""
|
||||
d = df[df['cause_category'].isin(core.CAUSES)]
|
||||
if d.empty:
|
||||
return 0.0, []
|
||||
accuracy = (d['cause_category'] == d['root_cause']).mean() * 100
|
||||
rows = []
|
||||
for cause in core.CAUSES:
|
||||
tp = ((d['cause_category'] == cause) & (d['root_cause'] == cause)).sum()
|
||||
precision = tp / max((d['root_cause'] == cause).sum(), 1) * 100
|
||||
recall = tp / max((d['cause_category'] == cause).sum(), 1) * 100
|
||||
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0
|
||||
rows.append((cause, round(precision, 1), round(recall, 1), round(f1, 1)))
|
||||
return round(accuracy, 2), rows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 扰动工具(固定随机种子,结果可复现)
|
||||
rng = random.Random(7)
|
||||
|
||||
|
||||
def root_alarm_mask(alarms, outages):
|
||||
"""每条退服的“真根因告警”:同站、清除时间 = 退服恢复时间、类别 = 上报原因、本身不是衍生告警。"""
|
||||
key = set(zip(outages['ne_id'], outages['end_time'], outages['cause_category']))
|
||||
parent = alarms['parent_alarm_id'].fillna('')
|
||||
return pd.Series([(n, c, cat) in key and p == ''
|
||||
for n, c, cat, p in zip(alarms['ne_id'], alarms['clear_time'], alarms['category'], parent)],
|
||||
index=alarms.index)
|
||||
|
||||
|
||||
def no_parent(alarms, keep=0.0):
|
||||
"""去掉父告警关系(keep 为保留比例)。"""
|
||||
alarms = alarms.copy()
|
||||
alarms['parent_alarm_id'] = [p if rng.random() < keep else '' for p in alarms['parent_alarm_id'].fillna('')]
|
||||
return alarms
|
||||
|
||||
|
||||
def drop_root(alarms, outages, fraction):
|
||||
"""随机删掉一部分真根因告警(网管漏报 / 动环未接入)。"""
|
||||
mask = root_alarm_mask(alarms, outages)
|
||||
drop = [i for i in alarms.index[mask] if rng.random() < fraction]
|
||||
return alarms.drop(drop)
|
||||
|
||||
|
||||
def skew_cell_alarm(alarms, max_seconds=120):
|
||||
"""“小区退服”告警时刻比工单开始时间晚 5~max_seconds 秒(两套系统时钟不同步)。"""
|
||||
alarms = alarms.copy()
|
||||
is_cell = alarms['category'] == '小区退服'
|
||||
times = pd.to_datetime(alarms.loc[is_cell, 'start_time'])
|
||||
times = times + pd.to_timedelta([rng.randint(5, max_seconds) for _ in range(len(times))], unit='s')
|
||||
alarms.loc[is_cell, 'start_time'] = times.dt.strftime(core.TIME_FMT)
|
||||
return alarms
|
||||
|
||||
|
||||
def jitter_clear(alarms, outages, max_minutes=10):
|
||||
"""真根因告警的清除时刻与业务恢复不再完全相同,随机偏移 ±max_minutes 分钟。"""
|
||||
alarms = alarms.copy()
|
||||
mask = root_alarm_mask(alarms, outages)
|
||||
times = pd.to_datetime(alarms.loc[mask, 'clear_time'])
|
||||
times = times + pd.to_timedelta([rng.randint(-max_minutes * 60, max_minutes * 60) for _ in range(len(times))],
|
||||
unit='s')
|
||||
alarms.loc[mask, 'clear_time'] = times.dt.strftime(core.TIME_FMT)
|
||||
return alarms
|
||||
|
||||
|
||||
def add_noise(alarms, outages, fraction):
|
||||
"""一部分退服开始前 0~12 分钟出现一条无关的短时告警(与 data_gen 的干扰告警同分布)。"""
|
||||
rows = []
|
||||
for ne_id, start in zip(outages['ne_id'], pd.to_datetime(outages['start_time'])):
|
||||
if rng.random() < fraction:
|
||||
category = rng.choice(['传输中断', '设备硬件', '天馈故障', '机房环境'])
|
||||
begin = start - pd.Timedelta(minutes=rng.randint(0, 12))
|
||||
end = begin + pd.Timedelta(minutes=rng.randint(2, 30))
|
||||
rows.append(dict(alarm_id=f'NZ{len(rows):07d}', ne_id=ne_id, alarm_code='', alarm_name='干扰' + category,
|
||||
alarm_level='次要', category=category, start_time=begin.strftime(core.TIME_FMT),
|
||||
clear_time=end.strftime(core.TIME_FMT), duration_s='', cleared='是',
|
||||
parent_alarm_id=''))
|
||||
return pd.concat([alarms, pd.DataFrame(rows)], ignore_index=True)
|
||||
|
||||
|
||||
# 比 data_gen 默认更宽的告警提前量(传输/设备也可能提前半小时,停电后备可达 7 小时)
|
||||
WIDE_LEAD = {'电源故障': (10, 420), '机房环境': (10, 240), '割接施工': (0, 45), '天馈故障': (0, 45),
|
||||
'传输中断': (0, 30), '设备硬件': (0, 30)}
|
||||
@@ -0,0 +1,87 @@
|
||||
"""考核指标口径、影响评估与关联量化检验(仓库样例数据)。
|
||||
|
||||
运行:python tests/test_kpi.py
|
||||
"""
|
||||
import calendar
|
||||
|
||||
from harness import check, core, finish, load, sample_data # 先导入 harness:它把数据库指向 tests/_work
|
||||
|
||||
import app as webapp # noqa: E402
|
||||
|
||||
s, a, o = sample_data()
|
||||
full = load('kpi', s, a, o)
|
||||
N = core.site_count()
|
||||
DAYS = 273 # 样例数据 2026-01-01 ~ 2026-09-30
|
||||
|
||||
|
||||
def calendar_rate(df, n_sites, days):
|
||||
return round(df['duration_min'].sum() / (n_sites * days * 1440) * 100, 4)
|
||||
|
||||
|
||||
print('== 1. 统计周期')
|
||||
minutes, months = core.period()
|
||||
check('全量周期 = 273 天、9 个月', (round(minutes / 1440), months) == (DAYS, 9), f'({minutes / 1440} 天, {months} 月)')
|
||||
check('全量退服率按日历周期', core.kpis(full, N)['outage_rate'] == calendar_rate(full, N, DAYS))
|
||||
for m in ['2026-02', '2026-07']:
|
||||
f = {'month': m}
|
||||
days = calendar.monthrange(*map(int, m.split('-')))[1]
|
||||
k = core.kpis(core.frame(f), N, *core.period(f))
|
||||
check(f'{m} 月退服率按 {days} 天', k['outage_rate'] == calendar_rate(core.frame(f), N, days), f'({k["outage_rate"]}%)')
|
||||
|
||||
print('\n== 2. 稀疏筛选')
|
||||
counts = full.groupby('ne_id').size().sort_values()
|
||||
ne = counts.index[0]
|
||||
f = {'site': ne}
|
||||
df = core.frame(f)
|
||||
k = core.kpis(df, core.site_count(f), *core.period(f))
|
||||
check('单站筛选分母为 1 个站', k['sites'] == 1, f'({k["sites"]})')
|
||||
check('单站退服率按 273 天', k['outage_rate'] == calendar_rate(df, 1, DAYS), f'({k["outage_rate"]}%)')
|
||||
day = df['start_time'].iloc[0][:10]
|
||||
f = {'site': ne, 'date_from': day, 'date_to': day}
|
||||
df = core.frame(f)
|
||||
k = core.kpis(df, core.site_count(f), *core.period(f))
|
||||
check('单站单日退服率按 1 站 × 1 天', k['outage_rate'] == calendar_rate(df, 1, 1), f'({k["outage_rate"]}%)')
|
||||
|
||||
print('\n== 3. 月均口径:单独查看一个站与全量查看判定一致')
|
||||
whole = core.site_stats(full, months).set_index('ne_id')
|
||||
flips = 0
|
||||
for ne in whole.index:
|
||||
f = {'site': ne}
|
||||
one = core.site_stats(core.frame(f), core.period(f)[1]).set_index('ne_id').loc[ne]
|
||||
flips += (bool(one['long']) != bool(whole.loc[ne, 'long'])) + (bool(one['repeat']) != bool(whole.loc[ne, 'repeat']))
|
||||
check('超长 / 高频判定翻转数为 0', flips == 0, f'({flips})')
|
||||
check('重复退服率分母为全部站点', core.kpis(full, N)['repeat_rate'] == round(whole['repeat'].sum() / N * 100, 2))
|
||||
|
||||
print('\n== 4. 导出与看板同一口径')
|
||||
client = webapp.app.test_client()
|
||||
client.post('/login', json={'username': 'admin', 'password': 'admin123'})
|
||||
for scene in core.SCENES:
|
||||
dash = client.get('/api/dashboard', query_string={'scene': scene}).get_json()['kpi']['outage_rate']
|
||||
f = {'scene': scene}
|
||||
exported = core.kpis(core.frame(f), core.site_count(f), *core.period(f))['outage_rate']
|
||||
check(f'{scene} 看板 = 导出', dash == exported, f'({dash}% / {exported}%)')
|
||||
response = client.get('/api/export/stats', query_string={'scene': '商圈'})
|
||||
check('导出接口可用', response.status_code == 200)
|
||||
|
||||
print('\n== 5. 影响评估')
|
||||
corr = full['user_hours'].corr(full['duration_min'])
|
||||
check('受影响用户·小时与时长正相关(修复前受影响用户与时长 r = 0.03)', corr > 0.5, f'(r = {corr:.3f})')
|
||||
expected = full['affected_users'] * full['duration_min'] / 60
|
||||
relative = ((full['user_hours'] - expected).abs() / expected.clip(lower=1)).max()
|
||||
check('用户·小时 = 受影响用户 × 时长', relative < 0.02, f'(最大相对误差 {relative:.4f},来自用户数取整)')
|
||||
check('覆盖补偿系数在 [0.4, 1]', full['cover_factor'].between(core.COVER_FLOOR, 1).all(),
|
||||
f'(均值 {full["cover_factor"].mean():.2f})')
|
||||
isolated = core.site_cover()
|
||||
check('有邻站的站补偿系数 < 1', min(isolated.values()) < 1)
|
||||
print(f' 用户·小时合计 {core.kpis(full, N)["user_h"]:,.0f},覆盖补偿系数分布 '
|
||||
f'{full.drop_duplicates("ne_id")["cover_factor"].value_counts().sort_index().to_dict()}')
|
||||
|
||||
print('\n== 6. 告警—退服关联量化')
|
||||
rows = core.association()
|
||||
for r in rows:
|
||||
print(f' {r["category"]:<6} 窗口 {r["window_min"]:>6} 分 告警 {r["alarms"]:>5} 支持度 {r["support"]:>6}% '
|
||||
f'置信度 {r["confidence"]:>6}% 提升度 {r["lift"]:>6}')
|
||||
check('6 类告警均有关联指标', len(rows) == 6 and all(r['alarms'] > 0 for r in rows))
|
||||
check('各类提升度 > 1(告警出现后退服概率高于随机)', all(r['lift'] > 1 for r in rows))
|
||||
check('关联接口可用', client.get('/api/association').status_code == 200)
|
||||
finish()
|
||||
@@ -0,0 +1,54 @@
|
||||
"""根因判定准确率检验:基线 + 6 组扰动数据集,时间窗只用 1~6 月流水标定,在 7~9 月(留出集)上计分。
|
||||
|
||||
运行:python tests/test_rootcause.py
|
||||
"""
|
||||
from harness import (WIDE_LEAD, add_noise, check, core, drop_root, finish, generate, jitter_clear, load, no_parent,
|
||||
sample_data, score, skew_cell_alarm)
|
||||
|
||||
SPLIT = '2026-07-01' # 之前的流水用于标定时间窗,之后的用于计分
|
||||
|
||||
s, a, o = sample_data()
|
||||
sw, aw, ow = generate('wide', lead=WIDE_LEAD)
|
||||
|
||||
# (名称, 数据, 准确率下限);下限为 None 的只报告不判定——根因告警缺失时,缺失的那部分本来就无法从告警判出
|
||||
SCENARIOS = [
|
||||
('S0 基线(仓库样例数据)', (s, a, o), 90),
|
||||
('S1 去掉父告警(只靠时间窗)', (s, no_parent(a), o), 90),
|
||||
('S2 小区退服告警时刻偏 5-120 秒', (s, skew_cell_alarm(a), o), 90),
|
||||
('S3 S1 + 20% 根因告警缺失', (s, drop_root(no_parent(a), o, .2), o), None),
|
||||
('S4 S1 + 30% 退服前干扰告警', (s, add_noise(no_parent(a), o, .3), o), 90),
|
||||
('S5 S1 + 告警提前量变宽', (sw, no_parent(aw), ow), 90),
|
||||
('S6 综合(S5 + 时刻偏移 + 30% 父告警 + 清除时刻 ±10 分 + 15% 缺失 + 25% 干扰)',
|
||||
# 先删根因告警再扰动清除时刻:drop_root 靠“清除时间 = 恢复时间”识别根因告警
|
||||
(sw, add_noise(jitter_clear(skew_cell_alarm(drop_root(no_parent(aw, keep=.3), ow, .15)), ow), ow, .25), ow),
|
||||
None),
|
||||
]
|
||||
|
||||
summary = []
|
||||
for name, (sites, alarms, outages), floor in SCENARIOS:
|
||||
df = load('rootcause', sites, alarms, outages, calibrate_until=SPLIT)
|
||||
test = df[df['start_time'] >= SPLIT]
|
||||
accuracy, rows = score(test)
|
||||
other = (test['root_cause'] == '其他').mean() * 100
|
||||
print(f'\n== {name}\n 留出集 {len(test)} 条,准确率 {accuracy}%,判为“其他” {other:.1f}%')
|
||||
print(' 类别 precision recall F1')
|
||||
for cause, precision, recall, f1 in rows:
|
||||
print(f' {cause:<8} {precision:>7} {recall:>7} {f1:>6}')
|
||||
confidence = test.groupby('confidence').apply(lambda x: (x['cause_category'] == x['root_cause']).mean() * 100,
|
||||
include_groups=False).round(1)
|
||||
print(' 各置信度准确率:', confidence.to_dict(), ' 条数:', test['confidence'].value_counts().to_dict())
|
||||
print(' 时间窗:', {w['category']: w['window_min'] for w in core.window_table()})
|
||||
summary.append((name, accuracy))
|
||||
if floor is not None:
|
||||
check(f'{name.split()[0]} 准确率 ≥ {floor}%', accuracy >= floor, f'({accuracy}%)')
|
||||
else:
|
||||
# 缺失场景的理论上限:真根因告警仍在的比例;判定应接近它,且“高/中”置信度的判定仍应可靠
|
||||
reliable = test[test['confidence'].isin(['高', '中'])]
|
||||
rel_acc = (reliable['cause_category'] == reliable['root_cause']).mean() * 100
|
||||
check(f'{name.split()[0]} 高/中置信度判定准确率 ≥ 90%', rel_acc >= 90, f'({rel_acc:.1f}%,'
|
||||
f'占 {len(reliable) / len(test) * 100:.0f}%)')
|
||||
|
||||
print('\n汇总(留出集准确率):')
|
||||
for name, accuracy in summary:
|
||||
print(f' {accuracy:>6}% {name}')
|
||||
finish()
|
||||
Reference in new issue
Block a user