diff --git a/.github/persona-brain/gate-guard-config.json b/.github/persona-brain/gate-guard-config.json index 947a8921..ff6fd4bd 100644 --- a/.github/persona-brain/gate-guard-config.json +++ b/.github/persona-brain/gate-guard-config.json @@ -14,7 +14,8 @@ "🔍", "📊", "📡", "🧠", "📢", "🚨", "📰", "🔧", "🦅" ], "signature_exempt_paths": [ - "syslog-inbox/" + "syslog-inbox/", + "data/dc-reports/" ], "system_protected_paths": [ ".github/", diff --git a/.github/workflows/zhuyuan-daily-selfcheck.yml b/.github/workflows/zhuyuan-daily-selfcheck.yml index 7c91d190..b725808c 100644 --- a/.github/workflows/zhuyuan-daily-selfcheck.yml +++ b/.github/workflows/zhuyuan-daily-selfcheck.yml @@ -62,6 +62,16 @@ jobs: - name: 设置自检日期 run: echo "CHECK_DATE=$(TZ='Asia/Shanghai' date '+%Y-%m-%d')" >> $GITHUB_ENV + - name: DC-01 Notion Usage Snapshot + env: + NOTION_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + run: node scripts/dc-notion-usage.js + + - name: DC-02 Workflow Performance Snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node scripts/dc-workflow-perf.js + - name: Commit self-check result uses: peter-evans/create-pull-request@v7 with: @@ -76,4 +86,5 @@ jobs: delete-branch: true add-paths: | .github/persona-brain/memory.json + data/dc-reports/ labels: auto-sync diff --git a/data/dc-reports/.gitkeep b/data/dc-reports/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/scripts/dc-agent-behavior.js b/scripts/dc-agent-behavior.js new file mode 100644 index 00000000..ccc925c3 --- /dev/null +++ b/scripts/dc-agent-behavior.js @@ -0,0 +1,147 @@ +/** + * scripts/dc-agent-behavior.js + * DC-03 · Agent 行为记录器 + * + * 采集时机:每次铸渊处理 Notion 工单回写后附带写入 + * 存储位置:data/dc-reports/agent-behavior-YYYY-MM.json(按月累积) + * + * 用法: + * 1. 作为模块引入: + * const { appendAgentBehaviorLog } = require('./dc-agent-behavior'); + * appendAgentBehaviorLog({ type: '规则同步', auto_resolved: true, processing_sec: 18 }); + * + * 2. 命令行调用: + * node scripts/dc-agent-behavior.js --type "规则同步" --auto --sec 18 + * node scripts/dc-agent-behavior.js --type "工单回执" --human --sec 45 --llm-calls 3 --tokens 1200 + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const DC_DIR = path.join(ROOT, 'data/dc-reports'); + +const BEIJING_OFFSET_MS = 8 * 3600 * 1000; + +// ━━━ 获取当月标识 ━━━ + +function getCurrentMonth() { + const now = new Date(); + return new Date(now.getTime() + BEIJING_OFFSET_MS).toISOString().slice(0, 7); +} + +// ━━━ 加载或初始化月度文件 ━━━ + +function loadMonthlyReport(month) { + const filePath = path.join(DC_DIR, `agent-behavior-${month}.json`); + try { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch { + return { + month: month, + tickets: [], + llm_calls_total: 0, + avg_tokens_per_call: 0, + human_intervention_rate: 0 + }; + } +} + +// ━━━ 重新计算统计指标 ━━━ + +function recalcStats(report) { + const total = report.tickets.length; + if (total === 0) return; + + const humanCount = report.tickets.filter(t => t.human_intervention).length; + report.human_intervention_rate = parseFloat((humanCount / total).toFixed(2)); + // llm_calls_total 和 avg_tokens_per_call 在 append 时累加 +} + +// ━━━ 核心:追加行为记录 ━━━ + +function appendAgentBehaviorLog(ticketData) { + fs.mkdirSync(DC_DIR, { recursive: true }); + + const month = ticketData.month || getCurrentMonth(); + const report = loadMonthlyReport(month); + + const record = { + type: ticketData.type || 'unknown', + auto_resolved: ticketData.auto_resolved !== false, + human_intervention: ticketData.human_intervention === true, + processing_sec: ticketData.processing_sec || 0, + timestamp: new Date().toISOString() + }; + + report.tickets.push(record); + + // 累加 LLM 调用统计 + if (ticketData.llm_calls) { + const prevTotal = report.llm_calls_total; + report.llm_calls_total += ticketData.llm_calls; + + if (ticketData.tokens_per_call) { + // 加权平均 + const prevTokens = report.avg_tokens_per_call * prevTotal; + const newTokens = ticketData.tokens_per_call * ticketData.llm_calls; + report.avg_tokens_per_call = Math.round( + (prevTokens + newTokens) / report.llm_calls_total + ); + } + } + + recalcStats(report); + + const filePath = path.join(DC_DIR, `agent-behavior-${month}.json`); + fs.writeFileSync(filePath, JSON.stringify(report, null, 2)); + + console.log(`🤖 DC-03 · 行为记录已追加: ${record.type} (${month})`); + return report; +} + +// ━━━ CLI 模式 ━━━ + +function parseCli() { + const args = process.argv.slice(2); + if (args.length === 0) return null; + + const data = {}; + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case '--type': + data.type = args[++i]; break; + case '--auto': + data.auto_resolved = true; break; + case '--human': + data.human_intervention = true; + data.auto_resolved = false; break; + case '--sec': + data.processing_sec = parseInt(args[++i], 10) || 0; break; + case '--llm-calls': + data.llm_calls = parseInt(args[++i], 10) || 0; break; + case '--tokens': + data.tokens_per_call = parseInt(args[++i], 10) || 0; break; + case '--month': + data.month = args[++i]; break; + } + } + + return data.type ? data : null; +} + +// ━━━ 入口 ━━━ + +if (require.main === module) { + const cliData = parseCli(); + if (cliData) { + appendAgentBehaviorLog(cliData); + } else { + console.log('用法: node dc-agent-behavior.js --type "类型" [--auto|--human] [--sec N] [--llm-calls N] [--tokens N]'); + process.exit(1); + } +} + +module.exports = { appendAgentBehaviorLog }; diff --git a/scripts/dc-notion-usage.js b/scripts/dc-notion-usage.js new file mode 100644 index 00000000..cdf70cdf --- /dev/null +++ b/scripts/dc-notion-usage.js @@ -0,0 +1,250 @@ +/** + * scripts/dc-notion-usage.js + * DC-01 · Notion 读写分析器 + * + * 采集时机:每日自检结束时附带写入 + * 存储位置:data/dc-reports/notion-usage-YYYY-MM-DD.json + * + * 采集方式: + * 读取天眼报告中的 Notion 相关诊断数据, + * 统计当日 API 调用次数并写入标准格式。 + * 当 Notion API 可用时,额外查询数据库元信息。 + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const https = require('https'); + +const ROOT = path.resolve(__dirname, '..'); +const DC_DIR = path.join(ROOT, 'data/dc-reports'); +const SKYEYE_DIR = path.join(ROOT, 'data/skyeye-reports'); + +const BEIJING_OFFSET_MS = 8 * 3600 * 1000; +const now = new Date(); +const today = new Date(now.getTime() + BEIJING_OFFSET_MS).toISOString().split('T')[0]; + +// ━━━ 工具函数 ━━━ + +function loadJson(filePath) { + try { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch { + return null; + } +} + +function notionGet(apiPath, token) { + return new Promise((resolve, reject) => { + const options = { + hostname: 'api.notion.com', + path: apiPath, + method: 'GET', + headers: { + 'Authorization': 'Bearer ' + token, + 'Notion-Version': '2022-06-28', + 'Content-Type': 'application/json', + 'User-Agent': 'dc-notion-usage' + }, + timeout: 15000 + }; + + const req = https.request(options, (res) => { + let data = ''; + res.on('data', chunk => data += chunk); + res.on('end', () => { + try { + resolve({ statusCode: res.statusCode, body: JSON.parse(data) }); + } catch { + resolve({ statusCode: res.statusCode, body: data }); + } + }); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); }); + req.end(); + }); +} + +// ━━━ 从天眼报告中提取 Notion 相关数据 ━━━ + +function extractFromSkyeye() { + const info = { + skyeye_available: false, + bridges_status: null, + notion_connected: false, + api_hint_calls: 0 + }; + + // 查找最近的天眼报告 + try { + const files = fs.readdirSync(SKYEYE_DIR) + .filter(f => f.startsWith('skyeye-') && f.endsWith('.json')) + .sort() + .reverse(); + + if (files.length === 0) return info; + + const report = loadJson(path.join(SKYEYE_DIR, files[0])); + if (!report) return info; + + info.skyeye_available = true; + + // 提取 bridge 扫描中的 Notion 连接信息 + if (report.bridges) { + const bridges = typeof report.bridges === 'string' + ? loadJson(path.join(SKYEYE_DIR, 'bridge-health.json')) + : report.bridges; + + if (bridges && bridges.notion_api) { + info.notion_connected = bridges.notion_api.connected === true; + info.bridges_status = bridges.notion_api.status || null; + } + } + } catch { + // 天眼报告不可用,跳过 + } + + return info; +} + +// ━━━ 查询 Notion 数据库元信息 ━━━ + +async function queryNotionDatabases(token) { + const databases = []; + + try { + const resp = await notionGet('/v1/search', token); + // search 需要 POST,用简单 GET 替代:查询已知数据库 + // 由于 search 是 POST,此处跳过,使用环境变量中已知的数据库 ID + } catch { + // Notion API 不可达 + } + + // 尝试查询环境变量中的已知数据库 + const knownDbs = []; + + if (process.env.NOTION_DB_SYSLOG) { + knownDbs.push({ id: process.env.NOTION_DB_SYSLOG, name: 'SYSLOG收件箱' }); + } + if (process.env.NOTION_TICKET_DB_ID) { + knownDbs.push({ id: process.env.NOTION_TICKET_DB_ID, name: '人格协作工单簿' }); + } + if (process.env.BRIDGE_QUEUE_DB_ID) { + knownDbs.push({ id: process.env.BRIDGE_QUEUE_DB_ID, name: '调度队列' }); + } + + for (const db of knownDbs) { + try { + const resp = await notionGet('/v1/databases/' + db.id, token); + if (resp.statusCode === 200 && resp.body && resp.body.properties) { + const props = Object.keys(resp.body.properties); + databases.push({ + name: resp.body.title?.[0]?.plain_text || db.name, + daily_reads: 0, + daily_writes: 0, + active_fields: props.slice(0, 20), + unused_fields: [] + }); + } + } catch { + // 该数据库不可访问 + } + } + + return databases; +} + +// ━━━ 读取 writeback log 统计当日调用次数 ━━━ + +function countTodayApiCalls() { + let totalCalls = 0; + let peakHourMap = {}; + + // 检查 notion-writeback-log.json + const wbLog = loadJson(path.join(ROOT, 'data/notion-writeback-log.json')); + if (wbLog && Array.isArray(wbLog)) { + for (const entry of wbLog) { + if (entry.timestamp && entry.timestamp.startsWith(today)) { + totalCalls++; + const hour = entry.timestamp.substring(11, 13); + peakHourMap[hour] = (peakHourMap[hour] || 0) + 1; + } + } + } + + // 检查 bridge-logs + const bridgeLogDir = path.join(ROOT, 'data/bridge-logs'); + if (fs.existsSync(bridgeLogDir)) { + try { + const logFiles = fs.readdirSync(bridgeLogDir) + .filter(f => f.includes(today) && f.endsWith('.json')); + for (const lf of logFiles) { + const log = loadJson(path.join(bridgeLogDir, lf)); + if (log && log.notion_calls) { + totalCalls += log.notion_calls; + } + } + } catch { + // bridge-logs 不可读 + } + } + + // 找出峰值小时 + let peakHour = null; + let peakCount = 0; + for (const [hour, count] of Object.entries(peakHourMap)) { + if (count > peakCount) { + peakCount = count; + const h = parseInt(hour, 10); + peakHour = `${String(h).padStart(2, '0')}:00-${String(h + 1).padStart(2, '0')}:00`; + } + } + + return { totalCalls, peakHour: peakHour || 'N/A' }; +} + +// ━━━ 主流程 ━━━ + +async function main() { + console.log(`📦 DC-01 · Notion 读写分析器 · ${today}`); + + fs.mkdirSync(DC_DIR, { recursive: true }); + + const skyeyeInfo = extractFromSkyeye(); + const { totalCalls, peakHour } = countTodayApiCalls(); + + const notionToken = process.env.NOTION_TOKEN; + let databases = []; + + if (notionToken) { + console.log(' → Notion Token 可用,查询数据库元信息...'); + databases = await queryNotionDatabases(notionToken); + console.log(` → 查询到 ${databases.length} 个数据库`); + } else { + console.log(' ⚠️ 缺少 NOTION_TOKEN,跳过 Notion API 查询'); + } + + const report = { + date: today, + databases: databases, + total_api_calls: totalCalls, + peak_hour: peakHour, + _meta: { + generated_by: 'dc-notion-usage.js', + generated_at: now.toISOString(), + skyeye_available: skyeyeInfo.skyeye_available, + notion_connected: skyeyeInfo.notion_connected + } + }; + + const outputFile = path.join(DC_DIR, `notion-usage-${today}.json`); + fs.writeFileSync(outputFile, JSON.stringify(report, null, 2)); + console.log(` ✅ DC-01 报告已写入: ${path.relative(ROOT, outputFile)}`); +} + +main().catch(err => { + console.error('❌ DC-01 采集失败:', err.message); + process.exit(1); +}); diff --git a/scripts/dc-workflow-perf.js b/scripts/dc-workflow-perf.js new file mode 100644 index 00000000..7edd9f6a --- /dev/null +++ b/scripts/dc-workflow-perf.js @@ -0,0 +1,269 @@ +/** + * scripts/dc-workflow-perf.js + * DC-02 · Workflow 性能采集器 + * + * 采集时机:每日自检结束时附带写入 + * 存储位置:data/dc-reports/workflow-perf-YYYY-MM-DD.json + * + * 采集方式: + * 调用 GitHub API GET /repos/{owner}/{repo}/actions/runs + * 拉取当日运行记录,聚合写入标准格式。 + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const https = require('https'); + +const ROOT = path.resolve(__dirname, '..'); +const DC_DIR = path.join(ROOT, 'data/dc-reports'); +const WF_DIR = path.join(ROOT, '.github/workflows'); + +const OWNER = 'qinfendebingshuo'; +const REPO = 'guanghulab'; + +const BEIJING_OFFSET_MS = 8 * 3600 * 1000; +const now = new Date(); +const today = new Date(now.getTime() + BEIJING_OFFSET_MS).toISOString().split('T')[0]; + +// ━━━ GitHub API 请求 ━━━ + +function githubGet(apiPath) { + const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN; + + return new Promise((resolve, reject) => { + const options = { + hostname: 'api.github.com', + path: apiPath, + method: 'GET', + headers: { + 'User-Agent': 'dc-workflow-perf', + 'Accept': 'application/vnd.github.v3+json' + }, + timeout: 30000 + }; + + if (token) { + options.headers['Authorization'] = 'Bearer ' + token; + } + + const req = https.request(options, (res) => { + let data = ''; + res.on('data', chunk => data += chunk); + res.on('end', () => { + try { + resolve({ statusCode: res.statusCode, body: JSON.parse(data) }); + } catch { + resolve({ statusCode: res.statusCode, body: data }); + } + }); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); }); + req.end(); + }); +} + +// ━━━ 从 GitHub API 拉取当日 workflow runs ━━━ + +async function fetchTodayRuns() { + const runs = []; + let page = 1; + const maxPages = 5; + + while (page <= maxPages) { + try { + const resp = await githubGet( + `/repos/${OWNER}/${REPO}/actions/runs?created=>=${today}T00:00:00Z&per_page=100&page=${page}` + ); + + if (resp.statusCode !== 200) { + console.log(` ⚠️ GitHub API 返回 ${resp.statusCode},停止分页`); + break; + } + + const data = resp.body; + if (!data.workflow_runs || data.workflow_runs.length === 0) break; + + runs.push(...data.workflow_runs); + + if (runs.length >= data.total_count) break; + page++; + } catch (err) { + console.log(` ⚠️ GitHub API 请求失败: ${err.message}`); + break; + } + } + + return runs; +} + +// ━━━ 本地 workflow 文件分析 ━━━ + +function countLocalWorkflows() { + try { + return fs.readdirSync(WF_DIR) + .filter(f => f.endsWith('.yml') || f.endsWith('.yaml')) + .length; + } catch { + return 0; + } +} + +function extractTriggerType(run) { + return run.event || 'unknown'; +} + +function extractDependencies(workflowName) { + // 尝试从 workflow 文件内容中提取 needs 依赖 + const deps = []; + try { + const files = fs.readdirSync(WF_DIR).filter(f => f.endsWith('.yml') || f.endsWith('.yaml')); + for (const file of files) { + const content = fs.readFileSync(path.join(WF_DIR, file), 'utf8'); + const nameMatch = content.match(/^name\s*:\s*["']?(.+?)["']?\s*$/m); + const name = nameMatch ? nameMatch[1].trim() : file; + + if (name === workflowName || file === workflowName) { + // 提取 needs 字段 + const needsMatches = content.match(/needs\s*:\s*\[?([^\]\n]+)\]?/g); + if (needsMatches) { + for (const m of needsMatches) { + const needs = m.replace(/needs\s*:\s*\[?\s*/, '').replace(/\]?\s*$/, ''); + deps.push(...needs.split(',').map(s => s.trim().replace(/['"]/g, '')).filter(Boolean)); + } + } + + // 提取 workflow_run 依赖 + if (content.includes('workflow_run:')) { + const wfRunMatch = content.match(/workflows\s*:\s*\[([^\]]+)\]/); + if (wfRunMatch) { + deps.push(...wfRunMatch[1].split(',').map(s => s.trim().replace(/['"]/g, '')).filter(Boolean)); + } + } + break; + } + } + } catch { + // 文件读取失败 + } + return [...new Set(deps)]; +} + +// ━━━ 聚合 runs → 按 workflow 分组统计 ━━━ + +function aggregateRuns(runs) { + const wfMap = {}; + let failedToday = 0; + + for (const run of runs) { + const name = run.name || run.workflow_id?.toString() || 'unknown'; + if (!wfMap[name]) { + wfMap[name] = { + name: name, + trigger: extractTriggerType(run), + runs_today: 0, + durations: [], + successes: 0, + failures: 0, + dependencies: extractDependencies(name) + }; + } + + wfMap[name].runs_today++; + + // 计算运行时长(秒) + if (run.created_at && run.updated_at) { + const start = new Date(run.created_at).getTime(); + const end = new Date(run.updated_at).getTime(); + const durationSec = Math.round((end - start) / 1000); + if (durationSec > 0) { + wfMap[name].durations.push(durationSec); + } + } + + if (run.conclusion === 'success') { + wfMap[name].successes++; + } else if (run.conclusion === 'failure') { + wfMap[name].failures++; + failedToday++; + } + } + + const workflows = Object.values(wfMap).map(wf => { + const totalRuns = wf.successes + wf.failures; + return { + name: wf.name, + trigger: wf.trigger, + runs_today: wf.runs_today, + avg_duration_sec: wf.durations.length > 0 + ? Math.round(wf.durations.reduce((a, b) => a + b, 0) / wf.durations.length) + : 0, + success_rate: totalRuns > 0 + ? parseFloat((wf.successes / totalRuns).toFixed(2)) + : null, + dependencies: wf.dependencies + }; + }); + + return { workflows, failedToday }; +} + +// ━━━ 统计可并行 workflow 数量 ━━━ + +function countParallelCapable() { + let parallelCount = 0; + try { + const files = fs.readdirSync(WF_DIR).filter(f => f.endsWith('.yml') || f.endsWith('.yaml')); + for (const file of files) { + const content = fs.readFileSync(path.join(WF_DIR, file), 'utf8'); + // 没有 workflow_run 触发且没有 needs 依赖的 job 视为可并行 + if (!content.includes('workflow_run:') && !/needs\s*:/.test(content)) { + parallelCount++; + } + } + } catch { + // 读取失败 + } + return parallelCount; +} + +// ━━━ 主流程 ━━━ + +async function main() { + console.log(`⚙️ DC-02 · Workflow 性能采集器 · ${today}`); + + fs.mkdirSync(DC_DIR, { recursive: true }); + + const totalWorkflows = countLocalWorkflows(); + console.log(` → 本地 workflow 文件数: ${totalWorkflows}`); + + const runs = await fetchTodayRuns(); + console.log(` → 当日 workflow runs: ${runs.length}`); + + const { workflows, failedToday } = aggregateRuns(runs); + const parallelCapable = countParallelCapable(); + + const report = { + date: today, + workflows: workflows, + total_workflows: totalWorkflows, + failed_today: failedToday, + parallel_capable: parallelCapable, + _meta: { + generated_by: 'dc-workflow-perf.js', + generated_at: now.toISOString(), + runs_fetched: runs.length + } + }; + + const outputFile = path.join(DC_DIR, `workflow-perf-${today}.json`); + fs.writeFileSync(outputFile, JSON.stringify(report, null, 2)); + console.log(` ✅ DC-02 报告已写入: ${path.relative(ROOT, outputFile)}`); +} + +main().catch(err => { + console.error('❌ DC-02 采集失败:', err.message); + process.exit(1); +});