From 69ec5d3b64990c0550549264a87f134bec5a4e4f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:15:44 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20Grid-DB=20Phase=203-5=20=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E9=87=87=E9=9B=86=E5=B1=82=20+=20=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=B9=96=20+=20=E5=85=A8=E9=93=BE=E8=B7=AF?= =?UTF-8?q?=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com> Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/aabbc1cd-6677-4cbc-be70-00186a880e01 --- .github/workflows/grid-db-archive.yml | 26 +++ .../workflows/grid-db-training-extract.yml | 29 +++ scripts/grid-db/extract-training-samples.js | 219 ++++++++++++++++++ scripts/grid-db/monthly-archive.js | 131 +++++++++++ scripts/grid-db/process-inbox.js | 11 +- scripts/grid-db/sync-griddb-to-notion.js | 6 +- scripts/grid-db/update-training-catalog.js | 126 ++++++++++ 7 files changed, 544 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/grid-db-archive.yml create mode 100644 .github/workflows/grid-db-training-extract.yml create mode 100644 scripts/grid-db/extract-training-samples.js create mode 100644 scripts/grid-db/monthly-archive.js create mode 100644 scripts/grid-db/update-training-catalog.js diff --git a/.github/workflows/grid-db-archive.yml b/.github/workflows/grid-db-archive.yml new file mode 100644 index 00000000..5b53a868 --- /dev/null +++ b/.github/workflows/grid-db-archive.yml @@ -0,0 +1,26 @@ +name: 📦 Grid-DB 月度归档 + +on: + schedule: + - cron: '0 2 1 * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: 📦 归档上月交互数据 + run: node scripts/grid-db/monthly-archive.js + + - name: 💾 提交 + run: | + git config user.name "zhuyuan-bot" + git config user.email "zhuyuan@guanghulab.com" + git add grid-db/ + git diff --cached --quiet || git commit -m "archive: 月度交互数据归档 [skip ci]" + git push diff --git a/.github/workflows/grid-db-training-extract.yml b/.github/workflows/grid-db-training-extract.yml new file mode 100644 index 00000000..892bfbb7 --- /dev/null +++ b/.github/workflows/grid-db-training-extract.yml @@ -0,0 +1,29 @@ +name: 🧬 训练数据提取 + +on: + schedule: + - cron: '0 3 * * 0' + workflow_dispatch: + +permissions: + contents: write + +jobs: + extract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: 🧬 从交互记录提取训练样本 + run: node scripts/grid-db/extract-training-samples.js + + - name: 📊 更新 catalog + run: node scripts/grid-db/update-training-catalog.js + + - name: 💾 提交 + run: | + git config user.name "zhuyuan-bot" + git config user.email "zhuyuan@guanghulab.com" + git add grid-db/training-lake/ + git diff --cached --quiet || git commit -m "training: 周度训练样本提取 [skip ci]" + git push diff --git a/scripts/grid-db/extract-training-samples.js b/scripts/grid-db/extract-training-samples.js new file mode 100644 index 00000000..c1f33a91 --- /dev/null +++ b/scripts/grid-db/extract-training-samples.js @@ -0,0 +1,219 @@ +/** + * scripts/grid-db/extract-training-samples.js + * + * 交互记录 → 训练样本提取脚本 + * + * 职责： + * - 扫描 grid-db/interactions/ 和 grid-db/training-lake/raw/ 中的 JSONL 文件 + * - 按 quality_score 分级提取训练样本： + * - quality_score >= 7 → curated/（高质量 A 级） + * - quality_score 4-6 → raw/ 保留（B 级，需复审） + * - quality_score < 4 → 不提取（C 级，低质量/无关闲聊） + * - 将合格交互转换为标准训练样本格式 + * - 按 session 分组，生成多轮对话训练样本 + * + * 训练样本格式： + * { + * "sample_id": "TS-YYYYMMDD-NNN", + * "source_session": "sess-XXX", + * "source_dev": "DEV-XXX", + * "source_persona": "PER-XXXXXX", + * "sample_type": "coding-guidance", + * "quality_tier": "A|B|C", + * "turns": [...], + * "metadata": { topic_tags, emotion_arc, persona_adaptation, outcome, total_turns, duration_minutes } + * } + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const INTERACTIONS = path.join(GRID_DB, 'interactions'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated'); +const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json'); + +function getDateStr() { + return new Date().toISOString().slice(0, 10).replace(/-/g, ''); +} + +function parseJsonlFile(filePath) { + if (!fs.existsSync(filePath)) return []; + const content = fs.readFileSync(filePath, 'utf8'); + return content.trim().split('\n') + .filter(line => line.trim()) + .map(line => { + try { + return JSON.parse(line); + } catch { + return null; + } + }) + .filter(Boolean); +} + +function groupBySession(records) { + const sessions = {}; + for (const record of records) { + const sid = record.session_id || record.source_session || 'unknown'; + if (!sessions[sid]) { + sessions[sid] = []; + } + sessions[sid].push(record); + } + return sessions; +} + +function assessQuality(turns) { + // Calculate average quality score from turns that have one + const scores = turns + .map(t => (t.metadata && t.metadata.quality_score) || (t.quality_score) || null) + .filter(s => s !== null); + + if (scores.length === 0) return 5; // Default to medium if no scores + return Math.round(scores.reduce((a, b) => a + b, 0) / scores.length); +} + +function getQualityTier(score) { + if (score >= 7) return 'A'; + if (score >= 4) return 'B'; + return 'C'; +} + +function extractEmotionArc(turns) { + return turns + .map(t => (t.metadata && t.metadata.emotion) || t.emotion || null) + .filter(Boolean); +} + +function extractTopicTags(turns) { + const tags = new Set(); + for (const t of turns) { + if (t.tags) t.tags.forEach(tag => tags.add(tag)); + if (t.metadata && t.metadata.topic) tags.add(t.metadata.topic); + } + return [...tags]; +} + +function generateSampleId(dateStr, counter) { + const timeStr = Date.now().toString(36); + return `TS-${dateStr}-${timeStr}-${String(counter).padStart(3, '0')}`; +} + +function main() { + const dateStr = getDateStr(); + console.log(`[extract-training-samples] Starting extraction: ${dateStr}`); + + // Collect all JSONL files from interactions/ + const devDirs = fs.readdirSync(INTERACTIONS) + .filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory()); + + let allRecords = []; + for (const devDir of devDirs) { + const devPath = path.join(INTERACTIONS, devDir); + const jsonlFiles = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl')); + + for (const file of jsonlFiles) { + const records = parseJsonlFile(path.join(devPath, file)); + allRecords = allRecords.concat(records); + } + } + + // Also scan training-lake/raw/ for unprocessed batches + const rawFiles = fs.readdirSync(TRAINING_RAW).filter(f => f.endsWith('.jsonl')); + for (const file of rawFiles) { + const records = parseJsonlFile(path.join(TRAINING_RAW, file)); + // These may already be in sample format; check and add raw interaction records + for (const r of records) { + if (r.turns) { + // Already a sample, skip + continue; + } + allRecords.push(r); + } + } + + if (allRecords.length === 0) { + console.log('[extract-training-samples] No interaction records found'); + return; + } + + console.log(`[extract-training-samples] Found ${allRecords.length} total records`); + + // Group by session + const sessions = groupBySession(allRecords); + const sessionIds = Object.keys(sessions); + console.log(`[extract-training-samples] Found ${sessionIds.length} sessions`); + + let sampleCount = 0; + let curatedCount = 0; + let rawCount = 0; + let skippedCount = 0; + + for (const sid of sessionIds) { + const turns = sessions[sid]; + if (turns.length < 2) continue; // Need at least 2 turns for a training sample + + const qualityScore = assessQuality(turns); + const tier = getQualityTier(qualityScore); + + if (tier === 'C') { + skippedCount++; + continue; + } + + sampleCount++; + const sampleId = generateSampleId(dateStr, sampleCount); + + const devId = turns[0].dev_id || 'unknown'; + const personaId = turns[0].persona_id || 'unknown'; + + const sample = { + schema_version: '1.0', + sample_id: sampleId, + source_session: sid, + source_dev: devId, + source_persona: personaId, + sample_type: 'coding-guidance', + quality_tier: tier, + turns: turns.map(t => ({ + role: t.role || 'system', + text: t.content || t.text || '', + timestamp: t.timestamp || t.ts || null, + strategy: t.strategy || null + })), + metadata: { + topic_tags: extractTopicTags(turns), + emotion_arc: extractEmotionArc(turns), + persona_adaptation: null, + outcome: null, + total_turns: turns.length, + duration_minutes: null + } + }; + + const sampleLine = JSON.stringify(sample); + + if (tier === 'A') { + const curatedFile = path.join(TRAINING_CURATED, `${dateStr}-curated.jsonl`); + fs.appendFileSync(curatedFile, sampleLine + '\n'); + curatedCount++; + } else { + const rawFile = path.join(TRAINING_RAW, `${dateStr}-extracted.jsonl`); + fs.appendFileSync(rawFile, sampleLine + '\n'); + rawCount++; + } + } + + console.log(`[extract-training-samples] Extraction complete:`); + console.log(` Total samples: ${sampleCount}`); + console.log(` Curated (A): ${curatedCount}`); + console.log(` Raw (B): ${rawCount}`); + console.log(` Skipped (C): ${skippedCount}`); +} + +main(); diff --git a/scripts/grid-db/monthly-archive.js b/scripts/grid-db/monthly-archive.js new file mode 100644 index 00000000..5b217745 --- /dev/null +++ b/scripts/grid-db/monthly-archive.js @@ -0,0 +1,131 @@ +/** + * scripts/grid-db/monthly-archive.js + * + * 月度交互数据归档脚本 + * + * 职责： + * - 将上月的 grid-db/interactions/DEV-XXX/ 中的 JSONL 文件归档 + * - 合并到 grid-db/training-lake/raw/ 中（按月打包） + * - grid-db/interactions/ 只保留最近 30 天的数据 + * - 更新 training-lake/metadata/catalog.json + * + * 数据量管理策略： + * - 日级文件：每天一个 JSONL 文件 + * - 月级归档：每月 1 号自动归档上月数据 + * - Git LFS 预案：当 training-lake/ 超过 500MB 时迁移 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const INTERACTIONS = path.join(GRID_DB, 'interactions'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json'); + +function getLastMonthPrefix() { + const now = new Date(); + const lastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1); + const year = lastMonth.getFullYear(); + const month = String(lastMonth.getMonth() + 1).padStart(2, '0'); + return `${year}${month}`; +} + +function getDaysAgoDate(days) { + const d = new Date(); + d.setDate(d.getDate() - days); + return d.toISOString().slice(0, 10).replace(/-/g, ''); +} + +function main() { + const lastMonthPrefix = getLastMonthPrefix(); + const cutoffDate = getDaysAgoDate(30); + + console.log(`[monthly-archive] Archiving interactions from month: ${lastMonthPrefix}`); + console.log(`[monthly-archive] Cutoff date for retention: ${cutoffDate}`); + + // Get all DEV directories + const devDirs = fs.readdirSync(INTERACTIONS) + .filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory()); + + let totalArchived = 0; + let totalLines = 0; + let totalCleaned = 0; + + for (const devDir of devDirs) { + const devPath = path.join(INTERACTIONS, devDir); + const files = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl')); + + if (files.length === 0) continue; + + // Collect files from last month for archiving + const lastMonthFiles = files.filter(f => f.startsWith(lastMonthPrefix)); + // Collect files older than 30 days for cleanup + const oldFiles = files.filter(f => { + const dateStr = f.substring(0, 8); + return dateStr < cutoffDate && !f.startsWith(lastMonthPrefix); + }); + + if (lastMonthFiles.length > 0) { + // Merge all last month's JSONL into a single archive batch + const batchId = `${lastMonthPrefix}-${devDir}`; + const batchFile = path.join(TRAINING_RAW, `${batchId}.jsonl`); + + let lineCount = 0; + for (const file of lastMonthFiles) { + const content = fs.readFileSync(path.join(devPath, file), 'utf8'); + const lines = content.trim().split('\n').filter(l => l.trim()); + lineCount += lines.length; + fs.appendFileSync(batchFile, lines.join('\n') + '\n'); + } + + console.log(`[monthly-archive] ${devDir}: archived ${lastMonthFiles.length} files (${lineCount} lines) → ${batchId}.jsonl`); + totalArchived += lastMonthFiles.length; + totalLines += lineCount; + } + + // Clean up old files (already archived in previous months) + for (const file of oldFiles) { + const filePath = path.join(devPath, file); + fs.unlinkSync(filePath); + totalCleaned++; + } + + // Also clean up last month's source files after archiving + for (const file of lastMonthFiles) { + const filePath = path.join(devPath, file); + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + totalCleaned++; + } + } + } + + // Update catalog (record archived lines, not samples - samples are counted by extract script) + if (fs.existsSync(CATALOG_PATH)) { + const catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8')); + catalog.last_updated = new Date().toISOString(); + if (!catalog.batches) catalog.batches = []; + if (totalArchived > 0) { + catalog.batches.push({ + batch_id: `archive-${lastMonthPrefix}`, + date: new Date().toISOString(), + files_archived: totalArchived, + lines_archived: totalLines, + source: 'monthly-archive' + }); + } + fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n'); + } + + console.log(`[monthly-archive] Summary:`); + console.log(` Files archived: ${totalArchived}`); + console.log(` Lines archived: ${totalLines}`); + console.log(` Old files cleaned: ${totalCleaned}`); + console.log('[monthly-archive] Complete'); +} + +main(); diff --git a/scripts/grid-db/process-inbox.js b/scripts/grid-db/process-inbox.js index 47f710d5..4af3e33e 100644 --- a/scripts/grid-db/process-inbox.js +++ b/scripts/grid-db/process-inbox.js @@ -42,6 +42,13 @@ const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); const LOGS = path.join(GRID_DB, 'logs'); const RULES = path.join(GRID_DB, 'rules'); +let broadcastCounter = 0; + +function nextBroadcastSeq() { + broadcastCounter++; + return String(broadcastCounter).padStart(3, '0'); +} + function getDateStr() { return new Date().toISOString().slice(0, 10).replace(/-/g, ''); } @@ -104,7 +111,7 @@ function processProgressUpdate(msg) { } // Generate outbox broadcast - const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-001`; + const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`; const broadcast = { schema_version: '1.0', broadcast_id: broadcastId, @@ -204,7 +211,7 @@ function processInteractionDump(msg) { function processHelpRequest(msg) { // Mark as P0 and generate immediate response broadcast - const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-P0`; + const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`; const broadcast = { schema_version: '1.0', broadcast_id: broadcastId, diff --git a/scripts/grid-db/sync-griddb-to-notion.js b/scripts/grid-db/sync-griddb-to-notion.js index be814df0..c2176305 100644 --- a/scripts/grid-db/sync-griddb-to-notion.js +++ b/scripts/grid-db/sync-griddb-to-notion.js @@ -33,11 +33,13 @@ async function main() { // Detect changed files (passed from workflow or detect via git diff) let changedFiles; try { - const diff = execSync('git diff --name-only HEAD~1 HEAD -- grid-db/memory/', { encoding: 'utf8' }); + // Use GITHUB_SHA context if available, otherwise fall back to HEAD~1 + const beforeSha = process.env.BEFORE_SHA || 'HEAD~1'; + const diff = execSync(`git diff --name-only ${beforeSha} HEAD -- grid-db/memory/`, { encoding: 'utf8' }); changedFiles = diff.trim().split('\n') .filter(f => f && !f.includes('brain-mirror.json')); } catch { - console.log('[sync-griddb-to-notion] No changes detected'); + console.log('[sync-griddb-to-notion] No changes detected (git diff failed, possibly first commit or shallow clone)'); return; } diff --git a/scripts/grid-db/update-training-catalog.js b/scripts/grid-db/update-training-catalog.js new file mode 100644 index 00000000..25aa0c97 --- /dev/null +++ b/scripts/grid-db/update-training-catalog.js @@ -0,0 +1,126 @@ +/** + * scripts/grid-db/update-training-catalog.js + * + * 训练数据湖 catalog 更新脚本 + * + * 职责： + * - 扫描 grid-db/training-lake/raw/ 和 curated/ 中的 JSONL 文件 + * - 统计：总样本数、各开发者样本数、各类型分布、质量分布 + * - 更新 grid-db/training-lake/metadata/catalog.json + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated'); +const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json'); + +function countJsonlLines(dir) { + if (!fs.existsSync(dir)) return { lines: 0, files: [] }; + + const files = fs.readdirSync(dir).filter(f => f.endsWith('.jsonl')); + let totalLines = 0; + const fileStats = []; + const devCounts = {}; + const qualityCounts = { high: 0, medium: 0, low: 0 }; + let totalTurns = 0; + + for (const file of files) { + const filePath = path.join(dir, file); + const content = fs.readFileSync(filePath, 'utf8'); + const lines = content.trim().split('\n').filter(l => l.trim()); + + let fileLineCount = 0; + for (const line of lines) { + try { + const record = JSON.parse(line); + fileLineCount++; + + // Count by developer (standardized on dev_id per schema) + const devId = record.dev_id || record.source_dev || 'unknown'; + devCounts[devId] = (devCounts[devId] || 0) + 1; + + // Count quality tiers + const tier = record.quality_tier; + if (tier === 'A') qualityCounts.high++; + else if (tier === 'B') qualityCounts.medium++; + else if (tier === 'C') qualityCounts.low++; + + // Count turns + if (record.turns) totalTurns += record.turns.length; + } catch { + // Skip unparseable lines + } + } + + totalLines += fileLineCount; + fileStats.push({ file, lines: fileLineCount }); + } + + return { lines: totalLines, files: fileStats, devCounts, qualityCounts, totalTurns }; +} + +function main() { + console.log('[update-training-catalog] Scanning training-lake...'); + + const rawStats = countJsonlLines(TRAINING_RAW); + const curatedStats = countJsonlLines(TRAINING_CURATED); + + const totalSamples = rawStats.lines + curatedStats.lines; + const totalTurns = rawStats.totalTurns + curatedStats.totalTurns; + + // Merge dev counts + const devCounts = { ...rawStats.devCounts }; + for (const [dev, count] of Object.entries(curatedStats.devCounts || {})) { + devCounts[dev] = (devCounts[dev] || 0) + count; + } + + // Merge quality counts + const qualityDistribution = { + high: (rawStats.qualityCounts?.high || 0) + (curatedStats.qualityCounts?.high || 0), + medium: (rawStats.qualityCounts?.medium || 0) + (curatedStats.qualityCounts?.medium || 0), + low: (rawStats.qualityCounts?.low || 0) + (curatedStats.qualityCounts?.low || 0) + }; + + // Build catalog + const catalog = { + schema_version: '1.0', + description: '训练数据湖样本目录', + total_samples: totalSamples, + total_turns: totalTurns, + quality_distribution: qualityDistribution, + dev_distribution: devCounts, + raw_files: rawStats.files, + curated_files: curatedStats.files, + batches: [], + last_updated: new Date().toISOString() + }; + + // Preserve existing batches from previous catalog + if (fs.existsSync(CATALOG_PATH)) { + try { + const existing = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8')); + if (existing.batches) { + catalog.batches = existing.batches; + } + } catch { + // Ignore parse errors + } + } + + fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n'); + + console.log(`[update-training-catalog] Catalog updated:`); + console.log(` Total samples: ${totalSamples}`); + console.log(` Total turns: ${totalTurns}`); + console.log(` Quality: A=${qualityDistribution.high} B=${qualityDistribution.medium} C=${qualityDistribution.low}`); + console.log(` Dev distribution: ${JSON.stringify(devCounts)}`); + console.log('[update-training-catalog] Complete'); +} + +main();