"""核心层:数据库、入库、根因关联分析、考核指标核算。字段口径与 data_gen.py(data.py 第 18 题)一致。 从上到下依次是: 1. 配置常量(场景、根因、时间窗规则、处置建议) 2. 数据库基础操作与明细缓存 3. 建库与首次导入样例数据 4. 入库(退服流水、告警、站点) 5. 根因关联分析 6. 指标核算(明细、站点统计、考核指标) 文件导入的读取与校验在 importer.py。 """ import os import sqlite3 from contextlib import contextmanager from datetime import datetime import numpy as np import pandas as pd from werkzeug.security import generate_password_hash # ================================================================ 1. 配置常量 BASE = os.path.dirname(os.path.abspath(__file__)) DATA = os.path.join(BASE, 'data') DB_PATH = os.environ.get('OUTAGE_DB', os.path.join(DATA, 'outage.db')) # 线上部署可指向持久化目录 TIME_FMT = '%Y-%m-%d %H:%M:%S' SCENES = ['商圈', '高校', '交通枢纽', '城中村', '工业园区'] SITE_TYPES = ['宏站', '室分', '分布式皮基站'] CAUSES = ['传输中断', '电源故障', '设备硬件', '天馈故障', '机房环境', '割接施工', '其他'] KEY_SCENES = ['商圈', '高校', '交通枢纽'] # 高话务关键场景,断站分级时从严 # 场景默认阈值:超长退服时长(分钟)、高频退服次数(次/月)、单小区平均在线用户、单小区每小时话务(GB) SCENE_DEFAULT = { '商圈': (120, 4, 320, 18), '高校': (120, 4, 420, 22), '交通枢纽': (90, 3, 380, 20), '城中村': (240, 6, 160, 9), '工业园区': (180, 5, 120, 7), } # 时间窗规则:(告警类别, 判定根因, 告警最多早于退服多少分钟),按优先级从高到低排列。 # 割接操作最先排除;停电有蓄电池后备、机房高温有过热保护延时,所以窗口放宽; # 停电往往同时引起传输告警,因为电源优先级更高,这种情况归为电源故障。 RULES = [ ('割接施工', '割接施工', 30), ('电源故障', '电源故障', 300), ('机房环境', '机房环境', 180), ('传输中断', '传输中断', 15), ('天馈故障', '天馈故障', 30), ('设备硬件', '设备硬件', 15), ] ADVICE = { '电源故障': ['核查市电及开关电源状态,必要时调度油机发电', '检测蓄电池组容量,老化电池纳入更换计划', '评估加装智能电表与远程动环监控'], '传输中断': ['联系传输专业排查光缆/光模块,定位断点', '核查 SCTP 偶联与 NG 链路配置', '重要站点规划传输双路由保护'], '设备硬件': ['远程复位 BBU/AAU,观察告警是否清除', '安排备件上站更换故障单板或射频单元', '同批次设备开展隐患排查'], '天馈故障': ['测试驻波比,排查馈线接头与防水', '检查天线是否被遮挡或破坏', '雨季前开展天馈防水专项整治'], '机房环境': ['上站检修空调,临时加装排风', '排查漏水点并做好设备防护', '接入动环温湿度与水浸监控'], '割接施工': ['核实割接/闭塞是否已报备并在窗口内', '督促割接完成后及时解闭塞并验证业务', '施工前联系光缆路由交底,防止误碰'], '其他': ['调取网管日志与现场照片综合研判', '派单现场复核,联系业主确认是否断电', '持续观察,重复发生升级处理'], } LEVEL_TIP = {'重要': '一级(2小时内响应,优先闭环)', '主要': '二级(4小时内响应)', '次要': '三级(24小时内处理)'} # ================================================================ 2. 数据库基础操作 @contextmanager def db(): """ 打开数据库连接,用法:with db() as conn: ... 正常结束时提交、出错时回滚,最后一定关闭连接。sqlite3 自带的 with 只负责提交/回滚,不会关闭连接, 所以这里统一包一层,全文件都通过它访问数据库。 """ conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row try: with conn: yield conn finally: conn.close() def query(sql, args=()): """执行查询,返回字典列表。""" with db() as conn: rows = conn.execute(sql, args).fetchall() return [dict(row) for row in rows] def execute(sql, args=(), changes_data=True): """执行一条写语句,返回新行 id。changes_data=True 表示改动了业务数据,需要清空明细缓存。""" with db() as conn: new_id = conn.execute(sql, args).lastrowid if changes_data: data_changed() return new_id def log(user, action, detail=''): """记录操作日志。日志不影响统计结果,所以不清空明细缓存。""" now = datetime.now().strftime(TIME_FMT) execute('INSERT INTO logs(username, action, detail, ts) VALUES(?,?,?,?)', (user, action, detail, now), changes_data=False) # 明细缓存:未筛选的退服明细(含派生字段)计算一次后保存在内存里,各接口在它上面做筛选; # 任何写入业务数据的操作都要调用 data_changed(),下次访问时重新计算。 _frame_cache = None def data_changed(): global _frame_cache _frame_cache = None # ================================================================ 3. 建库与样例数据 SCHEMA = """ CREATE TABLE IF NOT EXISTS users(id INTEGER PRIMARY KEY, username TEXT UNIQUE, password TEXT, role TEXT DEFAULT 'user', created_at TEXT); CREATE TABLE IF NOT EXISTS scenes(scene TEXT PRIMARY KEY, long_min INTEGER, freq_count INTEGER, users_per_cell INTEGER, gb_per_cell_h REAL); CREATE TABLE IF NOT EXISTS sites(ne_id TEXT PRIMARY KEY, site_name TEXT, district TEXT, scene TEXT, site_type TEXT, vendor TEXT, cells INTEGER, lng REAL, lat REAL); CREATE TABLE IF NOT EXISTS outages(id INTEGER PRIMARY KEY, record_id TEXT, ne_id TEXT, cell_id TEXT, start_time TEXT, end_time TEXT, duration_min REAL, reported_min REAL, cause_category TEXT, cause_detail TEXT, cause_code TEXT, ticket_id TEXT, root_cause TEXT, evidence TEXT, status TEXT DEFAULT '待处理', handler TEXT, remark TEXT); CREATE INDEX IF NOT EXISTS ix_out_ne ON outages(ne_id, start_time); CREATE TABLE IF NOT EXISTS alarms(alarm_id TEXT PRIMARY KEY, ne_id TEXT, alarm_code TEXT, alarm_name TEXT, alarm_level TEXT, category TEXT, start_time TEXT, clear_time TEXT, duration_s INTEGER, cleared TEXT, parent_alarm_id TEXT); CREATE INDEX IF NOT EXISTS ix_alarm_ne ON alarms(ne_id, start_time); CREATE TABLE IF NOT EXISTS logs(id INTEGER PRIMARY KEY, username TEXT, action TEXT, detail TEXT, ts TEXT); """ def init_db(): """启动时调用:建表;数据库文件是第一次创建时,再写入默认账号、场景阈值,并导入 data/ 下的样例数据。""" is_first_run = not os.path.exists(DB_PATH) os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) with db() as conn: conn.executescript(SCHEMA) if not is_first_run: return now = datetime.now().strftime(TIME_FMT) with db() as conn: conn.execute('INSERT INTO users(username,password,role,created_at) VALUES(?,?,?,?)', ('admin', generate_password_hash('admin123'), 'admin', now)) conn.execute('INSERT INTO users(username,password,role,created_at) VALUES(?,?,?,?)', ('user', generate_password_hash('user123'), 'user', now)) for scene, (long_min, freq_count, users_per_cell, gb_per_cell_h) in SCENE_DEFAULT.items(): conn.execute('INSERT INTO scenes VALUES(?,?,?,?,?)', (scene, long_min, freq_count, users_per_cell, gb_per_cell_h)) # 样例数据缺失时现场生成(约 1 秒) if not os.path.exists(os.path.join(DATA, 'data_outage.csv')): import data_gen data_gen.generate(out=DATA) sites = pd.read_csv(os.path.join(DATA, 'sites.csv'), encoding='utf-8-sig', dtype=str) alarms = pd.read_csv(os.path.join(DATA, 'data_alarm.csv'), encoding='utf-8-sig', dtype=str) outages = pd.read_csv(os.path.join(DATA, 'data_outage.csv'), encoding='utf-8-sig', dtype=str) with db() as conn: sites.to_sql('sites', conn, if_exists='append', index=False) save_alarms(alarms) # 样例退服流水与用户上传的文件走同一套校验(时长重算、上报时长核对)。 # importer 依赖本模块,所以在函数内导入,避免两个模块互相导入时出错。 import importer good_outages, _ = importer.validate_outages(outages) save_outages(good_outages) analyze_all() # ================================================================ 4. 入库 def save_outages(good): """合格退服流水入库,返回新记录的 id 列表(用于随后只对新记录做根因分析)。""" with db() as conn: last_id = conn.execute('SELECT COALESCE(MAX(id), 0) FROM outages').fetchone()[0] good.to_sql('outages', conn, if_exists='append', index=False) data_changed() new_rows = query('SELECT id FROM outages WHERE id > ?', (last_id,)) return [row['id'] for row in new_rows] def save_alarms(df): """告警入库。告警编号相同的覆盖旧记录(重复导入同一份网管导出不会产生重复告警)。""" with db() as conn: df.to_sql('alarms_tmp', conn, if_exists='replace', index=False) conn.execute('INSERT OR REPLACE INTO alarms SELECT * FROM alarms_tmp') conn.execute('DROP TABLE alarms_tmp') data_changed() return len(df) def save_sites(df): """ 工参站点入库,返回 (新增数, 更新数)。 已存在的基站只更新名称、片区、站型、厂家、小区数、经纬度,保留管理员在页面上设置过的场景标签。 """ added = 0 updated = 0 with db() as conn: for site in df.to_dict('records'): values = (site['ne_id'], site['site_name'], site['district'], site['scene'], site['site_type'], site['vendor'], site['cells'], site['lng'], site['lat']) conn.execute('INSERT INTO sites(ne_id, site_name, district, scene, site_type, vendor, cells, lng, lat) ' 'VALUES(?,?,?,?,?,?,?,?,?) ' 'ON CONFLICT(ne_id) DO UPDATE SET site_name=excluded.site_name, district=excluded.district, ' 'site_type=excluded.site_type, vendor=excluded.vendor, cells=excluded.cells, ' 'lng=excluded.lng, lat=excluded.lat', values) if site['action'] == '新增': added += 1 else: updated += 1 data_changed() return added, updated # ================================================================ 5. 根因关联分析 def analyze(outages): """ 两级根因判定。outages 需含 id, ne_id, start_time 三列,返回 [(根因, 证据, id), ...]。 第一级 衍生关系:同站、同一时刻的“小区退服”告警如果带 parent_alarm_id,直接取父告警的类别 (网管已经给出因果链,最可靠); 第二级 时间窗: 否则回溯同站在退服前仍未清除的告警,按 RULES 的优先级取第一个命中的类别; 都没有命中判为“其他”。 """ if outages.empty: return [] # 只取涉及到的基站的告警,避免全表读取 ne_ids = tuple(outages['ne_id'].dropna().unique()) placeholders = ','.join('?' * len(ne_ids)) with db() as conn: alarms = pd.read_sql(f'SELECT alarm_id, ne_id, alarm_name, category, start_time, clear_time, parent_alarm_id ' f'FROM alarms WHERE ne_id IN ({placeholders})', conn, params=ne_ids, parse_dates=['start_time', 'clear_time']) alarms = alarms.rename(columns={'start_time': 'alarm_time'}) outages = outages[['id', 'ne_id', 'start_time']].copy() outages['start_time'] = pd.to_datetime(outages['start_time']) results = {} # 退服 id -> (根因, 证据) # ---- 第一级:衍生关系。找到带父告警的“小区退服”告警,再查出父告警的类别 child_alarms = alarms[(alarms['category'] == '小区退服') & alarms['parent_alarm_id'].fillna('').ne('')] child_alarms = child_alarms[['ne_id', 'alarm_time', 'parent_alarm_id']].rename(columns={'alarm_time': 'start_time'}) parent_alarms = alarms.drop(columns=['ne_id', 'parent_alarm_id']) child_with_parent = child_alarms.merge(parent_alarms, left_on='parent_alarm_id', right_on='alarm_id') matched = outages.merge(child_with_parent, on=['ne_id', 'start_time']).drop_duplicates('id') for row in matched.itertuples(): if row.category in CAUSES: evidence = f'衍生关系:{row.alarm_name}@{row.alarm_time:%m-%d %H:%M}' results[row.id] = (row.category, evidence) # ---- 第二级:时间窗。把剩余退服与同站的根因类告警两两配对,逐条规则打标 remaining = outages[~outages['id'].isin(results)] pairs = remaining.merge(alarms[alarms['category'] != '小区退服'], on='ne_id') minutes_before = (pairs['start_time'] - pairs['alarm_time']).dt.total_seconds() / 60 # 告警在退服时仍未清除(允许 1 分钟误差)才可能是原因 still_active = pairs['clear_time'].isna() | (pairs['clear_time'] >= pairs['start_time'] - pd.Timedelta(minutes=1)) pairs['rank'] = 99 # 命中规则的序号,越小优先级越高;99 表示没有命中 for rank, (category, _, window) in enumerate(RULES): hit = ((pairs['category'] == category) & (minutes_before <= window) & (minutes_before >= -5) & still_active & (pairs['rank'] > rank)) pairs.loc[hit, 'rank'] = rank # 每条退服取优先级最高、时间最早的一条告警作为根因 best = pairs[pairs['rank'] < 99].sort_values(['id', 'rank', 'alarm_time']).drop_duplicates('id') for row in best.itertuples(): evidence = f'时间窗:{row.alarm_name}@{row.alarm_time:%m-%d %H:%M}' results[row.id] = (RULES[row.rank][1], evidence) output = [] for outage_id in outages['id']: cause, evidence = results.get(outage_id, ('其他', '时间窗内无关联告警')) output.append((cause, evidence, int(outage_id))) return output def analyze_all(ids=None): """对指定 id(不传则全部)的退服流水重新做根因分析并写回数据库,返回分析条数。""" sql = 'SELECT id, ne_id, start_time FROM outages' if ids: sql += ' WHERE id IN (' + ','.join(str(i) for i in ids) + ')' with db() as conn: outages = pd.read_sql(sql, conn) results = analyze(outages) with db() as conn: conn.executemany('UPDATE outages SET root_cause=?, evidence=? WHERE id=?', results) data_changed() return len(results) # ================================================================ 6. 指标核算 def full_frame(): """ 全部退服流水 + 站点信息 + 场景阈值,并计算派生字段: is_long 是否超长、dur_diff 上报时长差错、affected_users 受影响用户、loss_gb 业务损失、level 断站分级。 结果缓存到数据变化为止(见 data_changed)。 """ global _frame_cache if _frame_cache is not None: return _frame_cache with db() as conn: df = pd.read_sql('SELECT o.*, s.site_name, s.district AS region, s.scene, s.site_type, s.vendor, s.cells, ' 'c.long_min, c.freq_count, c.users_per_cell, c.gb_per_cell_h FROM outages o ' 'LEFT JOIN sites s ON o.ne_id = s.ne_id LEFT JOIN scenes c ON s.scene = c.scene', conn) df['site_name'] = df['site_name'].fillna(df['ne_id']) # 站点库里没有的站,用站号代替站名 df['start'] = pd.to_datetime(df['start_time']) df['month'] = df['start_time'].str[:7] long_min = df['long_min'].fillna(120) df['is_long'] = df['duration_min'] >= long_min df['dur_diff'] = (df['reported_min'] - df['duration_min']).abs() > 1 # 人工上报时长与系统计算相差超过 1 分钟 # 影响评估:受影响用户 = 小区数 × 单小区在线用户 × 忙闲时系数;业务损失 = 小区数 × 单小区话务 × 系数 × 时长 busy_factor = np.where(df['start'].dt.hour.between(9, 22), 1.0, 0.35) # 9-22 点为忙时 cells = df['cells'].fillna(3).astype(float) df['affected_users'] = (cells * df['users_per_cell'].fillna(200) * busy_factor).round().astype(int) df['loss_gb'] = (cells * df['gb_per_cell_h'].fillna(10) * busy_factor * df['duration_min'] / 60).round(2) # 断站分级(按整列一次计算,比逐行判断快几十倍): # 重要:超长,且是关键场景或时长超过阈值 2 倍 # 主要:超长;或关键场景下退服 ≥ 30 分钟 # 次要:其余 key_scene = df['scene'].isin(KEY_SCENES) is_important = df['is_long'] & (key_scene | (df['duration_min'] >= 2 * long_min)) is_major = df['is_long'] | (key_scene & (df['duration_min'] >= 30)) df['level'] = np.select([is_important, is_major], ['重要', '主要'], default='次要') _frame_cache = df return df def frame(filters=None): """按筛选条件(字典)取退服明细。返回独立副本,调用方可以随意修改而不影响缓存。""" filters = filters or {} df = full_frame() for column in ('scene', 'region', 'site_type', 'vendor', 'root_cause', 'status', 'cause_category'): if filters.get(column): df = df[df[column] == filters[column]] keyword = filters.get('site') if keyword: name_hit = df['site_name'].str.contains(keyword, regex=False) id_hit = df['ne_id'].str.contains(keyword, regex=False) df = df[name_hit | id_hit] if filters.get('month'): df = df[df['month'] == filters['month']] if filters.get('date_from'): df = df[df['start_time'] >= filters['date_from']] if filters.get('date_to'): df = df[df['start_time'] <= filters['date_to'] + ' 23:59:59'] return df.copy() def site_count(filters=None): """满足站点类筛选条件(场景、片区、站型、厂家)的站点数,用作退服率的分母。""" filters = filters or {} column_of = {'scene': 'scene', 'region': 'district', 'site_type': 'site_type', 'vendor': 'vendor'} conditions = [] args = [] for key, column in column_of.items(): if filters.get(key): conditions.append(f'{column} = ?') args.append(filters[key]) sql = 'SELECT COUNT(*) AS n FROM sites' if conditions: sql += ' WHERE ' + ' AND '.join(conditions) return query(sql, args)[0]['n'] def kpis(df, n_sites=None): """ 运维考核指标。 退服率 = 总退服时长 ÷(站点数 × 统计时长);网络可用率 = 1 − 退服率 根因判定准确率 = 上报原因与系统判定一致的比例(只统计上报原因已填写的记录) """ if not n_sites: n_sites = site_count() if df.empty: return dict(count=0, total_h=0, avg_min=0, outage_rate=0, availability=100, long_ratio=0, repeat_rate=0, users=0, loss_gb=0, sites=n_sites, acc=0, dur_fix=0, closed=0) total_minutes = df['duration_min'].sum() period_minutes = max((df['start'].max() - df['start'].min()).total_seconds() / 60, 1) outage_rate = total_minutes / (n_sites * period_minutes) * 100 stats = site_stats(df) known = df[df['cause_category'].isin(CAUSES) & df['root_cause'].notna()] if len(known): accuracy = round((known['cause_category'] == known['root_cause']).mean() * 100, 2) else: accuracy = 0 return dict( count=len(df), total_h=round(total_minutes / 60, 1), avg_min=round(df['duration_min'].mean(), 1), outage_rate=round(outage_rate, 4), availability=round(100 - outage_rate, 4), long_ratio=round(df['is_long'].mean() * 100, 2), repeat_rate=round(stats['repeat'].mean() * 100, 2), users=int(df['affected_users'].sum()), loss_gb=round(df['loss_gb'].sum(), 1), sites=n_sites, acc=accuracy, dur_fix=int(df['dur_diff'].sum()), closed=round((df['status'] == '已闭环').mean() * 100, 2), ) def site_stats(df): """ 站点维度统计,按累计退服时长从大到小排序。 超长退服站点 long:月均超长退服 ≥ 1 次 高频重复站点 repeat:月均退服次数 ≥ 场景次数阈值 """ if df.empty: return pd.DataFrame(columns=['site_name', 'count', 'total_min', 'long', 'repeat']) stats = df.groupby(['ne_id', 'site_name', 'region', 'scene', 'site_type'], dropna=False).agg( count=('id', 'size'), total_min=('duration_min', 'sum'), avg_min=('duration_min', 'mean'), max_min=('duration_min', 'max'), long_count=('is_long', 'sum'), users=('affected_users', 'sum'), long_min=('long_min', 'first'), freq_count=('freq_count', 'first'), ).reset_index() # 主要根因:每站出现次数最多的根因;次数相同时取名称排序靠前的(与 pandas 的 mode 结果一致) cause_counts = df.groupby(['ne_id', 'root_cause']).size().rename('n').reset_index() cause_counts = cause_counts.sort_values(['ne_id', 'n', 'root_cause'], ascending=[True, False, True]) top_cause = cause_counts.drop_duplicates('ne_id').set_index('ne_id')['root_cause'] stats['top_cause'] = stats['ne_id'].map(top_cause).fillna('') months = max(df['month'].nunique(), 1) stats['long'] = stats['long_count'] / months >= 1 stats['repeat'] = stats['count'] / months >= stats['freq_count'].fillna(4) stats['total_min'] = stats['total_min'].round(1) stats['avg_min'] = stats['avg_min'].round(1) return stats.sort_values('total_min', ascending=False) def advice(cause, level): """处置建议文本:响应时限(按断站分级)+ 该根因的三条处置措施。""" measures = ADVICE.get(cause or '其他', ADVICE['其他']) return f"【{LEVEL_TIP.get(level, '')}】" + ';'.join(measures)