feat: Grid-DB Phase 3-5 数据采集层 + 训练数据湖 + 全链路验证

Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com>
Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/aabbc1cd-6677-4cbc-be70-00186a880e01

.github/workflows/grid-db-archive.yml

@@ -0,0 +1,26 @@
+name: 📦 Grid-DB 月度归档
+
+on:
+  schedule:
+    - cron: '0 2 1 * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  archive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 📦 归档上月交互数据
+        run: node scripts/grid-db/monthly-archive.js
+
+      - name: 💾 提交
+        run: |
+          git config user.name "zhuyuan-bot"
+          git config user.email "zhuyuan@guanghulab.com"
+          git add grid-db/
+          git diff --cached --quiet || git commit -m "archive: 月度交互数据归档 [skip ci]"
+          git push

.github/workflows/grid-db-training-extract.yml

@@ -0,0 +1,29 @@
+name: 🧬 训练数据提取
+
+on:
+  schedule:
+    - cron: '0 3 * * 0'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  extract:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 🧬 从交互记录提取训练样本
+        run: node scripts/grid-db/extract-training-samples.js
+
+      - name: 📊 更新 catalog
+        run: node scripts/grid-db/update-training-catalog.js
+
+      - name: 💾 提交
+        run: |
+          git config user.name "zhuyuan-bot"
+          git config user.email "zhuyuan@guanghulab.com"
+          git add grid-db/training-lake/
+          git diff --cached --quiet || git commit -m "training: 周度训练样本提取 [skip ci]"
+          git push

scripts/grid-db/extract-training-samples.js

@@ -0,0 +1,219 @@
+/**
+ * scripts/grid-db/extract-training-samples.js
+ *
+ * 交互记录 → 训练样本提取脚本
+ *
+ * 职责：
+ * - 扫描 grid-db/interactions/ 和 grid-db/training-lake/raw/ 中的 JSONL 文件
+ * - 按 quality_score 分级提取训练样本：
+ *   - quality_score >= 7 → curated/（高质量 A 级）
+ *   - quality_score 4-6  → raw/ 保留（B 级，需复审）
+ *   - quality_score < 4   → 不提取（C 级，低质量/无关闲聊）
+ * - 将合格交互转换为标准训练样本格式
+ * - 按 session 分组，生成多轮对话训练样本
+ *
+ * 训练样本格式：
+ * {
+ *   "sample_id": "TS-YYYYMMDD-NNN",
+ *   "source_session": "sess-XXX",
+ *   "source_dev": "DEV-XXX",
+ *   "source_persona": "PER-XXXXXX",
+ *   "sample_type": "coding-guidance",
+ *   "quality_tier": "A|B|C",
+ *   "turns": [...],
+ *   "metadata": { topic_tags, emotion_arc, persona_adaptation, outcome, total_turns, duration_minutes }
+ * }
+ *
+ * 守护: PER-ZY001 铸渊
+ * 系统: SYS-GLW-0001
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const GRID_DB = path.join(__dirname, '../../grid-db');
+const INTERACTIONS = path.join(GRID_DB, 'interactions');
+const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
+const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated');
+const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json');
+
+function getDateStr() {
+  return new Date().toISOString().slice(0, 10).replace(/-/g, '');
+}
+
+function parseJsonlFile(filePath) {
+  if (!fs.existsSync(filePath)) return [];
+  const content = fs.readFileSync(filePath, 'utf8');
+  return content.trim().split('\n')
+    .filter(line => line.trim())
+    .map(line => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean);
+}
+
+function groupBySession(records) {
+  const sessions = {};
+  for (const record of records) {
+    const sid = record.session_id || record.source_session || 'unknown';
+    if (!sessions[sid]) {
+      sessions[sid] = [];
+    }
+    sessions[sid].push(record);
+  }
+  return sessions;
+}
+
+function assessQuality(turns) {
+  // Calculate average quality score from turns that have one
+  const scores = turns
+    .map(t => (t.metadata && t.metadata.quality_score) || (t.quality_score) || null)
+    .filter(s => s !== null);
+
+  if (scores.length === 0) return 5; // Default to medium if no scores
+  return Math.round(scores.reduce((a, b) => a + b, 0) / scores.length);
+}
+
+function getQualityTier(score) {
+  if (score >= 7) return 'A';
+  if (score >= 4) return 'B';
+  return 'C';
+}
+
+function extractEmotionArc(turns) {
+  return turns
+    .map(t => (t.metadata && t.metadata.emotion) || t.emotion || null)
+    .filter(Boolean);
+}
+
+function extractTopicTags(turns) {
+  const tags = new Set();
+  for (const t of turns) {
+    if (t.tags) t.tags.forEach(tag => tags.add(tag));
+    if (t.metadata && t.metadata.topic) tags.add(t.metadata.topic);
+  }
+  return [...tags];
+}
+
+function generateSampleId(dateStr, counter) {
+  const timeStr = Date.now().toString(36);
+  return `TS-${dateStr}-${timeStr}-${String(counter).padStart(3, '0')}`;
+}
+
+function main() {
+  const dateStr = getDateStr();
+  console.log(`[extract-training-samples] Starting extraction: ${dateStr}`);
+
+  // Collect all JSONL files from interactions/
+  const devDirs = fs.readdirSync(INTERACTIONS)
+    .filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory());
+
+  let allRecords = [];
+  for (const devDir of devDirs) {
+    const devPath = path.join(INTERACTIONS, devDir);
+    const jsonlFiles = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl'));
+
+    for (const file of jsonlFiles) {
+      const records = parseJsonlFile(path.join(devPath, file));
+      allRecords = allRecords.concat(records);
+    }
+  }
+
+  // Also scan training-lake/raw/ for unprocessed batches
+  const rawFiles = fs.readdirSync(TRAINING_RAW).filter(f => f.endsWith('.jsonl'));
+  for (const file of rawFiles) {
+    const records = parseJsonlFile(path.join(TRAINING_RAW, file));
+    // These may already be in sample format; check and add raw interaction records
+    for (const r of records) {
+      if (r.turns) {
+        // Already a sample, skip
+        continue;
+      }
+      allRecords.push(r);
+    }
+  }
+
+  if (allRecords.length === 0) {
+    console.log('[extract-training-samples] No interaction records found');
+    return;
+  }
+
+  console.log(`[extract-training-samples] Found ${allRecords.length} total records`);
+
+  // Group by session
+  const sessions = groupBySession(allRecords);
+  const sessionIds = Object.keys(sessions);
+  console.log(`[extract-training-samples] Found ${sessionIds.length} sessions`);
+
+  let sampleCount = 0;
+  let curatedCount = 0;
+  let rawCount = 0;
+  let skippedCount = 0;
+
+  for (const sid of sessionIds) {
+    const turns = sessions[sid];
+    if (turns.length < 2) continue; // Need at least 2 turns for a training sample
+
+    const qualityScore = assessQuality(turns);
+    const tier = getQualityTier(qualityScore);
+
+    if (tier === 'C') {
+      skippedCount++;
+      continue;
+    }
+
+    sampleCount++;
+    const sampleId = generateSampleId(dateStr, sampleCount);
+
+    const devId = turns[0].dev_id || 'unknown';
+    const personaId = turns[0].persona_id || 'unknown';
+
+    const sample = {
+      schema_version: '1.0',
+      sample_id: sampleId,
+      source_session: sid,
+      source_dev: devId,
+      source_persona: personaId,
+      sample_type: 'coding-guidance',
+      quality_tier: tier,
+      turns: turns.map(t => ({
+        role: t.role || 'system',
+        text: t.content || t.text || '',
+        timestamp: t.timestamp || t.ts || null,
+        strategy: t.strategy || null
+      })),
+      metadata: {
+        topic_tags: extractTopicTags(turns),
+        emotion_arc: extractEmotionArc(turns),
+        persona_adaptation: null,
+        outcome: null,
+        total_turns: turns.length,
+        duration_minutes: null
+      }
+    };
+
+    const sampleLine = JSON.stringify(sample);
+
+    if (tier === 'A') {
+      const curatedFile = path.join(TRAINING_CURATED, `${dateStr}-curated.jsonl`);
+      fs.appendFileSync(curatedFile, sampleLine + '\n');
+      curatedCount++;
+    } else {
+      const rawFile = path.join(TRAINING_RAW, `${dateStr}-extracted.jsonl`);
+      fs.appendFileSync(rawFile, sampleLine + '\n');
+      rawCount++;
+    }
+  }
+
+  console.log(`[extract-training-samples] Extraction complete:`);
+  console.log(`  Total samples: ${sampleCount}`);
+  console.log(`  Curated (A): ${curatedCount}`);
+  console.log(`  Raw (B): ${rawCount}`);
+  console.log(`  Skipped (C): ${skippedCount}`);
+}
+
+main();

scripts/grid-db/monthly-archive.js

@@ -0,0 +1,131 @@
+/**
+ * scripts/grid-db/monthly-archive.js
+ *
+ * 月度交互数据归档脚本
+ *
+ * 职责：
+ * - 将上月的 grid-db/interactions/DEV-XXX/ 中的 JSONL 文件归档
+ * - 合并到 grid-db/training-lake/raw/ 中（按月打包）
+ * - grid-db/interactions/ 只保留最近 30 天的数据
+ * - 更新 training-lake/metadata/catalog.json
+ *
+ * 数据量管理策略：
+ * - 日级文件：每天一个 JSONL 文件
+ * - 月级归档：每月 1 号自动归档上月数据
+ * - Git LFS 预案：当 training-lake/ 超过 500MB 时迁移
+ *
+ * 守护: PER-ZY001 铸渊
+ * 系统: SYS-GLW-0001
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const GRID_DB = path.join(__dirname, '../../grid-db');
+const INTERACTIONS = path.join(GRID_DB, 'interactions');
+const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
+const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json');
+
+function getLastMonthPrefix() {
+  const now = new Date();
+  const lastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1);
+  const year = lastMonth.getFullYear();
+  const month = String(lastMonth.getMonth() + 1).padStart(2, '0');
+  return `${year}${month}`;
+}
+
+function getDaysAgoDate(days) {
+  const d = new Date();
+  d.setDate(d.getDate() - days);
+  return d.toISOString().slice(0, 10).replace(/-/g, '');
+}
+
+function main() {
+  const lastMonthPrefix = getLastMonthPrefix();
+  const cutoffDate = getDaysAgoDate(30);
+
+  console.log(`[monthly-archive] Archiving interactions from month: ${lastMonthPrefix}`);
+  console.log(`[monthly-archive] Cutoff date for retention: ${cutoffDate}`);
+
+  // Get all DEV directories
+  const devDirs = fs.readdirSync(INTERACTIONS)
+    .filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory());
+
+  let totalArchived = 0;
+  let totalLines = 0;
+  let totalCleaned = 0;
+
+  for (const devDir of devDirs) {
+    const devPath = path.join(INTERACTIONS, devDir);
+    const files = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl'));
+
+    if (files.length === 0) continue;
+
+    // Collect files from last month for archiving
+    const lastMonthFiles = files.filter(f => f.startsWith(lastMonthPrefix));
+    // Collect files older than 30 days for cleanup
+    const oldFiles = files.filter(f => {
+      const dateStr = f.substring(0, 8);
+      return dateStr < cutoffDate && !f.startsWith(lastMonthPrefix);
+    });
+
+    if (lastMonthFiles.length > 0) {
+      // Merge all last month's JSONL into a single archive batch
+      const batchId = `${lastMonthPrefix}-${devDir}`;
+      const batchFile = path.join(TRAINING_RAW, `${batchId}.jsonl`);
+
+      let lineCount = 0;
+      for (const file of lastMonthFiles) {
+        const content = fs.readFileSync(path.join(devPath, file), 'utf8');
+        const lines = content.trim().split('\n').filter(l => l.trim());
+        lineCount += lines.length;
+        fs.appendFileSync(batchFile, lines.join('\n') + '\n');
+      }
+
+      console.log(`[monthly-archive] ${devDir}: archived ${lastMonthFiles.length} files (${lineCount} lines) → ${batchId}.jsonl`);
+      totalArchived += lastMonthFiles.length;
+      totalLines += lineCount;
+    }
+
+    // Clean up old files (already archived in previous months)
+    for (const file of oldFiles) {
+      const filePath = path.join(devPath, file);
+      fs.unlinkSync(filePath);
+      totalCleaned++;
+    }
+
+    // Also clean up last month's source files after archiving
+    for (const file of lastMonthFiles) {
+      const filePath = path.join(devPath, file);
+      if (fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+        totalCleaned++;
+      }
+    }
+  }
+
+  // Update catalog (record archived lines, not samples - samples are counted by extract script)
+  if (fs.existsSync(CATALOG_PATH)) {
+    const catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8'));
+    catalog.last_updated = new Date().toISOString();
+    if (!catalog.batches) catalog.batches = [];
+    if (totalArchived > 0) {
+      catalog.batches.push({
+        batch_id: `archive-${lastMonthPrefix}`,
+        date: new Date().toISOString(),
+        files_archived: totalArchived,
+        lines_archived: totalLines,
+        source: 'monthly-archive'
+      });
+    }
+    fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n');
+  }
+
+  console.log(`[monthly-archive] Summary:`);
+  console.log(`  Files archived: ${totalArchived}`);
+  console.log(`  Lines archived: ${totalLines}`);
+  console.log(`  Old files cleaned: ${totalCleaned}`);
+  console.log('[monthly-archive] Complete');
+}
+
+main();

scripts/grid-db/process-inbox.js

@@ -42,6 +42,13 @@ const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
 const LOGS = path.join(GRID_DB, 'logs');
 const RULES = path.join(GRID_DB, 'rules');
 
+let broadcastCounter = 0;
+
+function nextBroadcastSeq() {
+  broadcastCounter++;
+  return String(broadcastCounter).padStart(3, '0');
+}
+
 function getDateStr() {
   return new Date().toISOString().slice(0, 10).replace(/-/g, '');
 }
@@ -104,7 +111,7 @@ function processProgressUpdate(msg) {
   }
 
   // Generate outbox broadcast
-  const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-001`;
+  const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`;
   const broadcast = {
     schema_version: '1.0',
     broadcast_id: broadcastId,
@@ -204,7 +211,7 @@ function processInteractionDump(msg) {
 
 function processHelpRequest(msg) {
   // Mark as P0 and generate immediate response broadcast
-  const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-P0`;
+  const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`;
   const broadcast = {
     schema_version: '1.0',
     broadcast_id: broadcastId,

scripts/grid-db/sync-griddb-to-notion.js

@@ -33,11 +33,13 @@ async function main() {
   // Detect changed files (passed from workflow or detect via git diff)
   let changedFiles;
   try {
-    const diff = execSync('git diff --name-only HEAD~1 HEAD -- grid-db/memory/', { encoding: 'utf8' });
+    // Use GITHUB_SHA context if available, otherwise fall back to HEAD~1
+    const beforeSha = process.env.BEFORE_SHA || 'HEAD~1';
+    const diff = execSync(`git diff --name-only ${beforeSha} HEAD -- grid-db/memory/`, { encoding: 'utf8' });
     changedFiles = diff.trim().split('\n')
       .filter(f => f && !f.includes('brain-mirror.json'));
   } catch {
-    console.log('[sync-griddb-to-notion] No changes detected');
+    console.log('[sync-griddb-to-notion] No changes detected (git diff failed, possibly first commit or shallow clone)');
     return;
   }
 

scripts/grid-db/update-training-catalog.js

@@ -0,0 +1,126 @@
+/**
+ * scripts/grid-db/update-training-catalog.js
+ *
+ * 训练数据湖 catalog 更新脚本
+ *
+ * 职责：
+ * - 扫描 grid-db/training-lake/raw/ 和 curated/ 中的 JSONL 文件
+ * - 统计：总样本数、各开发者样本数、各类型分布、质量分布
+ * - 更新 grid-db/training-lake/metadata/catalog.json
+ *
+ * 守护: PER-ZY001 铸渊
+ * 系统: SYS-GLW-0001
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const GRID_DB = path.join(__dirname, '../../grid-db');
+const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
+const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated');
+const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json');
+
+function countJsonlLines(dir) {
+  if (!fs.existsSync(dir)) return { lines: 0, files: [] };
+
+  const files = fs.readdirSync(dir).filter(f => f.endsWith('.jsonl'));
+  let totalLines = 0;
+  const fileStats = [];
+  const devCounts = {};
+  const qualityCounts = { high: 0, medium: 0, low: 0 };
+  let totalTurns = 0;
+
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    const content = fs.readFileSync(filePath, 'utf8');
+    const lines = content.trim().split('\n').filter(l => l.trim());
+
+    let fileLineCount = 0;
+    for (const line of lines) {
+      try {
+        const record = JSON.parse(line);
+        fileLineCount++;
+
+        // Count by developer (standardized on dev_id per schema)
+        const devId = record.dev_id || record.source_dev || 'unknown';
+        devCounts[devId] = (devCounts[devId] || 0) + 1;
+
+        // Count quality tiers
+        const tier = record.quality_tier;
+        if (tier === 'A') qualityCounts.high++;
+        else if (tier === 'B') qualityCounts.medium++;
+        else if (tier === 'C') qualityCounts.low++;
+
+        // Count turns
+        if (record.turns) totalTurns += record.turns.length;
+      } catch {
+        // Skip unparseable lines
+      }
+    }
+
+    totalLines += fileLineCount;
+    fileStats.push({ file, lines: fileLineCount });
+  }
+
+  return { lines: totalLines, files: fileStats, devCounts, qualityCounts, totalTurns };
+}
+
+function main() {
+  console.log('[update-training-catalog] Scanning training-lake...');
+
+  const rawStats = countJsonlLines(TRAINING_RAW);
+  const curatedStats = countJsonlLines(TRAINING_CURATED);
+
+  const totalSamples = rawStats.lines + curatedStats.lines;
+  const totalTurns = rawStats.totalTurns + curatedStats.totalTurns;
+
+  // Merge dev counts
+  const devCounts = { ...rawStats.devCounts };
+  for (const [dev, count] of Object.entries(curatedStats.devCounts || {})) {
+    devCounts[dev] = (devCounts[dev] || 0) + count;
+  }
+
+  // Merge quality counts
+  const qualityDistribution = {
+    high: (rawStats.qualityCounts?.high || 0) + (curatedStats.qualityCounts?.high || 0),
+    medium: (rawStats.qualityCounts?.medium || 0) + (curatedStats.qualityCounts?.medium || 0),
+    low: (rawStats.qualityCounts?.low || 0) + (curatedStats.qualityCounts?.low || 0)
+  };
+
+  // Build catalog
+  const catalog = {
+    schema_version: '1.0',
+    description: '训练数据湖样本目录',
+    total_samples: totalSamples,
+    total_turns: totalTurns,
+    quality_distribution: qualityDistribution,
+    dev_distribution: devCounts,
+    raw_files: rawStats.files,
+    curated_files: curatedStats.files,
+    batches: [],
+    last_updated: new Date().toISOString()
+  };
+
+  // Preserve existing batches from previous catalog
+  if (fs.existsSync(CATALOG_PATH)) {
+    try {
+      const existing = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8'));
+      if (existing.batches) {
+        catalog.batches = existing.batches;
+      }
+    } catch {
+      // Ignore parse errors
+    }
+  }
+
+  fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n');
+
+  console.log(`[update-training-catalog] Catalog updated:`);
+  console.log(`  Total samples: ${totalSamples}`);
+  console.log(`  Total turns: ${totalTurns}`);
+  console.log(`  Quality: A=${qualityDistribution.high} B=${qualityDistribution.medium} C=${qualityDistribution.low}`);
+  console.log(`  Dev distribution: ${JSON.stringify(devCounts)}`);
+  console.log('[update-training-catalog] Complete');
+}
+
+main();