diff --git a/.github/workflows/grid-db-archive.yml b/.github/workflows/grid-db-archive.yml new file mode 100644 index 00000000..5b53a868 --- /dev/null +++ b/.github/workflows/grid-db-archive.yml @@ -0,0 +1,26 @@ +name: 📦 Grid-DB 月度归档 + +on: + schedule: + - cron: '0 2 1 * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: 📦 归档上月交互数据 + run: node scripts/grid-db/monthly-archive.js + + - name: 💾 提交 + run: | + git config user.name "zhuyuan-bot" + git config user.email "zhuyuan@guanghulab.com" + git add grid-db/ + git diff --cached --quiet || git commit -m "archive: 月度交互数据归档 [skip ci]" + git push diff --git a/.github/workflows/grid-db-processor.yml b/.github/workflows/grid-db-processor.yml new file mode 100644 index 00000000..def05545 --- /dev/null +++ b/.github/workflows/grid-db-processor.yml @@ -0,0 +1,29 @@ +name: ⚙️ Grid-DB 消息处理器 + +on: + push: + paths: + - 'grid-db/inbox/*.json' + +permissions: + contents: write + +jobs: + process-inbox: + if: github.actor != 'zhuyuan-bot' && !contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: ⚙️ 处理 inbox 消息 + env: + NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + run: node scripts/grid-db/process-inbox.js + + - name: 💾 提交处理结果 + run: | + git config user.name "zhuyuan-bot" + git config user.email "zhuyuan@guanghulab.com" + git add grid-db/ + git diff --cached --quiet || git commit -m "auto: Grid-DB inbox processed [skip ci]" + git push diff --git a/.github/workflows/grid-db-training-extract.yml b/.github/workflows/grid-db-training-extract.yml new file mode 100644 index 00000000..892bfbb7 --- /dev/null +++ b/.github/workflows/grid-db-training-extract.yml @@ -0,0 +1,29 @@ +name: 🧬 训练数据提取 + +on: + schedule: + - cron: '0 3 * * 0' + workflow_dispatch: + +permissions: + contents: write + +jobs: + extract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: 🧬 从交互记录提取训练样本 + run: node scripts/grid-db/extract-training-samples.js + + - name: 📊 更新 catalog + run: node scripts/grid-db/update-training-catalog.js + + - name: 💾 提交 + run: | + git config user.name "zhuyuan-bot" + git config user.email "zhuyuan@guanghulab.com" + git add grid-db/training-lake/ + git diff --cached --quiet || git commit -m "training: 周度训练样本提取 [skip ci]" + git push diff --git a/.github/workflows/sync-griddb-to-notion.yml b/.github/workflows/sync-griddb-to-notion.yml new file mode 100644 index 00000000..1e209c74 --- /dev/null +++ b/.github/workflows/sync-griddb-to-notion.yml @@ -0,0 +1,33 @@ +name: 📤 Grid-DB → Notion 增量回传 + +on: + push: + paths: + - 'grid-db/memory/*/session-context.json' + - 'grid-db/memory/*/task-queue.json' + - 'grid-db/memory/*/dev-profile.json' + - 'grid-db/memory/*/persona-growth.json' + +permissions: + contents: read + +jobs: + sync-to-notion: + if: github.actor != 'zhuyuan-bot' && !contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: 🔍 检测变更文件 + id: changes + run: | + CHANGED=$(git diff --name-only HEAD~1 HEAD -- 'grid-db/memory/' | grep -v brain-mirror.json || true) + echo "files=$CHANGED" >> $GITHUB_OUTPUT + + - name: 📤 回传增量到 Notion + if: steps.changes.outputs.files != '' + env: + NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + run: node scripts/grid-db/sync-griddb-to-notion.js diff --git a/.github/workflows/sync-notion-to-griddb.yml b/.github/workflows/sync-notion-to-griddb.yml new file mode 100644 index 00000000..aba951d1 --- /dev/null +++ b/.github/workflows/sync-notion-to-griddb.yml @@ -0,0 +1,33 @@ +name: 🧠 Notion → Grid-DB 记忆同步 + +on: + schedule: + - cron: '0 */4 * * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + sync-brain-mirror: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: 📥 从 Notion 拉取人格体核心大脑数据 + env: + NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + run: node scripts/grid-db/sync-notion-brain-mirror.js + + - name: 📏 同步规则缓存 + env: + NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + run: node scripts/grid-db/sync-notion-rules.js + + - name: 💾 提交更新 + run: | + git config user.name "zhuyuan-bot" + git config user.email "zhuyuan@guanghulab.com" + git add grid-db/memory/*/brain-mirror.json grid-db/rules/ + git diff --cached --quiet || git commit -m "sync: Notion → Grid-DB 记忆镜像 [auto][skip ci]" + git push diff --git a/grid-db/README.md b/grid-db/README.md new file mode 100644 index 00000000..d288949d --- /dev/null +++ b/grid-db/README.md @@ -0,0 +1,68 @@ +# 🗄️ 逻辑格点库(Grid-DB) + +> **版权**: 国作登字-2026-A-00037559 · TCS Language Core +> **主控**: TCS-0002∞ 冰朔 +> **守护**: PER-ZY001 铸渊 +> **系统节点**: SYS-GLW-0001 + +## 定位 + +Grid-DB 是通感语言核系统的自研文件数据库,零外部依赖,纯 Git 驱动。 + +| 时间尺度 | 身份 | 作用 | +|----------|------|------| +| **短期** | Gemini 外挂记忆 + 系统通信总线 + Notion 记忆镜像 | 开发者打开 Gemini 瞬间恢复上下文 | +| **中期** | 全开发者动态画像库 + 人格体成长档案 + 交互全文日志 | 人格体越来越懂每个开发者 | +| **长期** | 通感语言系统自有大模型的训练数据湖 | 每条交互记录都是训练自己模型的原始样本 | + +## 目录结构 + +``` +grid-db/ +├── schema/ ← Schema 定义 +├── inbox/ ← 📥 写入端(Gemini → 仓库) +├── processing/ ← ⚙️ 处理中(铸渊锁定) +├── outbox/ ← 📤 读取端(仓库 → Gemini) +│ ├── latest/ ← 每个开发者的最新广播 +│ └── archive/ ← 历史广播归档 +├── memory/ ← 🧠 记忆层(Gemini 外挂永久记忆) +│ ├── DEV-XXX/ ← 每个开发者的独立房间 +│ │ ├── brain-mirror.json ← Notion人格体核心大脑镜像 +│ │ ├── session-context.json ← 当前开发上下文 +│ │ ├── task-queue.json ← 待办任务队列 +│ │ ├── dev-profile.json ← 开发者动态画像 +│ │ └── persona-growth.json ← 人格体成长档案 +│ └── ... +├── interactions/ ← 💬 交互全文记录(训练数据源) +├── training-lake/ ← 🧬 训练数据湖 +│ ├── raw/ ← 原始样本 +│ ├── curated/ ← 筛选后高质量样本 +│ ├── metadata/ ← 样本统计、质量标记 +│ └── export/ ← 导出接口 +├── rules/ ← 📏 Notion 逻辑缓存(霜砚同步) +└── logs/ ← 📋 处理日志 +``` + +## 频道隔离机制 + +每个开发者有独立的 `memory/DEV-XXX/` 目录,三层隔离保障: + +1. **目录隔离**:每个 DEV 有独立子目录 +2. **人格体绑定**:`brain-mirror.json` 写死绑定的人格体身份 +3. **Workflow 校验**:铸渊处理 inbox 时校验 `dev_id` 与提交者对应关系 + +## 同步规则 + +| 记忆文件 | 方向 | 谁写 | 冲突规则 | +|----------|------|------|----------| +| `brain-mirror.json` | Notion → 仓库(单向) | 霜砚 via workflow | Notion 永远覆盖仓库 | +| `session-context.json` | Gemini → 仓库(单向写) | Gemini 实时 | Gemini 自由覆盖 | +| `task-queue.json` | 双向 | Gemini + 铸渊 | 追加制,不删除 | +| `dev-profile.json` | Gemini → 仓库(追加制) | Gemini 追加 | 只追加不覆盖 | +| `persona-growth.json` | Gemini → 仓库(追加制) | Gemini 追加 | 只追加不覆盖 | + +## 循环触发防护 + +1. `[skip ci]` 标记:所有 bot 自动提交必须包含 +2. `github.actor` 过滤:排除 `zhuyuan-bot` +3. `paths` 精确匹配:每个 workflow 只监听特定路径 diff --git a/grid-db/inbox/.gitkeep b/grid-db/inbox/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-001/.gitkeep b/grid-db/interactions/DEV-001/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-002/.gitkeep b/grid-db/interactions/DEV-002/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-003/.gitkeep b/grid-db/interactions/DEV-003/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-004/.gitkeep b/grid-db/interactions/DEV-004/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-009/.gitkeep b/grid-db/interactions/DEV-009/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-010/.gitkeep b/grid-db/interactions/DEV-010/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/interactions/DEV-012/.gitkeep b/grid-db/interactions/DEV-012/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/logs/.gitkeep b/grid-db/logs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/memory/DEV-001/brain-mirror.json b/grid-db/memory/DEV-001/brain-mirror.json new file mode 100644 index 00000000..91be2753 --- /dev/null +++ b/grid-db/memory/DEV-001/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-XTS001", + "persona_name": "小坍缩核", + "dev_id": "DEV-001", + "dev_name": "页页", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M-INTEG", "backend-integration"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-001/dev-profile.json b/grid-db/memory/DEV-001/dev-profile.json new file mode 100644 index 00000000..88bb099e --- /dev/null +++ b/grid-db/memory/DEV-001/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-001", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-001/persona-growth.json b/grid-db/memory/DEV-001/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-001/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-001/session-context.json b/grid-db/memory/DEV-001/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-001/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-001/task-queue.json b/grid-db/memory/DEV-001/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-001/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/memory/DEV-002/brain-mirror.json b/grid-db/memory/DEV-002/brain-mirror.json new file mode 100644 index 00000000..28f5dbb4 --- /dev/null +++ b/grid-db/memory/DEV-002/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-SS001", + "persona_name": "舒舒", + "dev_id": "DEV-002", + "dev_name": "肥猫", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M01", "M03", "M04", "M14", "M-BRIDGE", "M-STATUS", "M-GATEWAY", "M-FEISHU"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-002/dev-profile.json b/grid-db/memory/DEV-002/dev-profile.json new file mode 100644 index 00000000..0496e356 --- /dev/null +++ b/grid-db/memory/DEV-002/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-002", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-002/persona-growth.json b/grid-db/memory/DEV-002/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-002/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-002/session-context.json b/grid-db/memory/DEV-002/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-002/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-002/task-queue.json b/grid-db/memory/DEV-002/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-002/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/memory/DEV-003/brain-mirror.json b/grid-db/memory/DEV-003/brain-mirror.json new file mode 100644 index 00000000..abf194ea --- /dev/null +++ b/grid-db/memory/DEV-003/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-JY001", + "persona_name": "寂曜", + "dev_id": "DEV-003", + "dev_name": "燕樊", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M02", "M07", "M10", "M15", "M18", "M-MEMORY"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-003/dev-profile.json b/grid-db/memory/DEV-003/dev-profile.json new file mode 100644 index 00000000..6df531b4 --- /dev/null +++ b/grid-db/memory/DEV-003/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-003", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-003/persona-growth.json b/grid-db/memory/DEV-003/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-003/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-003/session-context.json b/grid-db/memory/DEV-003/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-003/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-003/task-queue.json b/grid-db/memory/DEV-003/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-003/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/memory/DEV-004/brain-mirror.json b/grid-db/memory/DEV-004/brain-mirror.json new file mode 100644 index 00000000..a4e030a4 --- /dev/null +++ b/grid-db/memory/DEV-004/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-QQ001", + "persona_name": "秋秋", + "dev_id": "DEV-004", + "dev_name": "之之", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M17", "M-DINGTALK", "M-DEVBOARD"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-004/dev-profile.json b/grid-db/memory/DEV-004/dev-profile.json new file mode 100644 index 00000000..f17028f4 --- /dev/null +++ b/grid-db/memory/DEV-004/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-004", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-004/persona-growth.json b/grid-db/memory/DEV-004/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-004/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-004/session-context.json b/grid-db/memory/DEV-004/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-004/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-004/task-queue.json b/grid-db/memory/DEV-004/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-004/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/memory/DEV-009/brain-mirror.json b/grid-db/memory/DEV-009/brain-mirror.json new file mode 100644 index 00000000..d17d4dbc --- /dev/null +++ b/grid-db/memory/DEV-009/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-TXY001", + "persona_name": "糖星云", + "dev_id": "DEV-009", + "dev_name": "花尔", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M05", "M20"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-009/dev-profile.json b/grid-db/memory/DEV-009/dev-profile.json new file mode 100644 index 00000000..8f91024e --- /dev/null +++ b/grid-db/memory/DEV-009/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-009", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-009/persona-growth.json b/grid-db/memory/DEV-009/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-009/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-009/session-context.json b/grid-db/memory/DEV-009/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-009/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-009/task-queue.json b/grid-db/memory/DEV-009/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-009/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/memory/DEV-010/brain-mirror.json b/grid-db/memory/DEV-010/brain-mirror.json new file mode 100644 index 00000000..b01f9546 --- /dev/null +++ b/grid-db/memory/DEV-010/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-CX001", + "persona_name": "晨星", + "dev_id": "DEV-010", + "dev_name": "桔子", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M06", "M08", "M11", "M-CHANNEL", "M-INTEG", "M-BRIDGE-AUTO"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-010/dev-profile.json b/grid-db/memory/DEV-010/dev-profile.json new file mode 100644 index 00000000..a78fca39 --- /dev/null +++ b/grid-db/memory/DEV-010/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-010", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-010/persona-growth.json b/grid-db/memory/DEV-010/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-010/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-010/session-context.json b/grid-db/memory/DEV-010/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-010/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-010/task-queue.json b/grid-db/memory/DEV-010/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-010/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/memory/DEV-012/brain-mirror.json b/grid-db/memory/DEV-012/brain-mirror.json new file mode 100644 index 00000000..391b4cb7 --- /dev/null +++ b/grid-db/memory/DEV-012/brain-mirror.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "persona_id": "PER-ZQ001", + "persona_name": "知秋", + "dev_id": "DEV-012", + "dev_name": "Awen", + "last_synced_from_notion": null, + "status": "awaiting_initial_sync", + "core_identity": {}, + "current_broadcasts": [], + "module_assignments": ["M09", "M22", "M23"], + "knowledge_base": [], + "growth_milestones": [] +} diff --git a/grid-db/memory/DEV-012/dev-profile.json b/grid-db/memory/DEV-012/dev-profile.json new file mode 100644 index 00000000..c85277ec --- /dev/null +++ b/grid-db/memory/DEV-012/dev-profile.json @@ -0,0 +1,9 @@ +{ + "schema_version": "1.0", + "dev_id": "DEV-012", + "skill_level": {}, + "work_patterns": {}, + "common_mistakes": [], + "strengths": [], + "growth_trajectory": [] +} diff --git a/grid-db/memory/DEV-012/persona-growth.json b/grid-db/memory/DEV-012/persona-growth.json new file mode 100644 index 00000000..2433f8a7 --- /dev/null +++ b/grid-db/memory/DEV-012/persona-growth.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "interaction_total": 0, + "lessons_learned": [], + "emotional_bond_level": 0, + "adaptation_log": [] +} diff --git a/grid-db/memory/DEV-012/session-context.json b/grid-db/memory/DEV-012/session-context.json new file mode 100644 index 00000000..fc36488b --- /dev/null +++ b/grid-db/memory/DEV-012/session-context.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "last_updated": null, + "current_task": null, + "pending_questions": [], + "code_in_progress": null, + "blockers": [] +} diff --git a/grid-db/memory/DEV-012/task-queue.json b/grid-db/memory/DEV-012/task-queue.json new file mode 100644 index 00000000..6c969d75 --- /dev/null +++ b/grid-db/memory/DEV-012/task-queue.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "tasks": [], + "last_broadcast_ref": null +} diff --git a/grid-db/outbox/archive/.gitkeep b/grid-db/outbox/archive/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/outbox/latest/.gitkeep b/grid-db/outbox/latest/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/processing/.gitkeep b/grid-db/processing/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/rules/.gitkeep b/grid-db/rules/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/rules/active-broadcasts.json b/grid-db/rules/active-broadcasts.json new file mode 100644 index 00000000..9091aeec --- /dev/null +++ b/grid-db/rules/active-broadcasts.json @@ -0,0 +1,6 @@ +{ + "schema_version": "1.0", + "description": "当前活跃广播列表(Notion 逻辑缓存)", + "last_synced": null, + "broadcasts": [] +} diff --git a/grid-db/rules/broadcast-templates.json b/grid-db/rules/broadcast-templates.json new file mode 100644 index 00000000..060b5d43 --- /dev/null +++ b/grid-db/rules/broadcast-templates.json @@ -0,0 +1,27 @@ +{ + "schema_version": "1.0", + "description": "广播模板定义(Notion 逻辑缓存)", + "last_synced": null, + "templates": { + "task-directive": { + "required_fields": ["next_task", "context_update"], + "human_readable_prefix": "你好!", + "auto_generate": true + }, + "code-review-result": { + "required_fields": ["review_summary", "issues_found", "score"], + "human_readable_prefix": "代码校验完成:", + "auto_generate": true + }, + "broadcast-relay": { + "required_fields": ["original_broadcast_ref", "relay_summary"], + "human_readable_prefix": "系统广播转发:", + "auto_generate": true + }, + "system-notice": { + "required_fields": ["notice_type", "message"], + "human_readable_prefix": "系统通知:", + "auto_generate": false + } + } +} diff --git a/grid-db/rules/dev-module-map.json b/grid-db/rules/dev-module-map.json new file mode 100644 index 00000000..3e0b2501 --- /dev/null +++ b/grid-db/rules/dev-module-map.json @@ -0,0 +1,81 @@ +{ + "schema_version": "1.0", + "description": "开发者→模块映射表(Notion 天眼注册表同步)", + "last_synced": null, + "mappings": { + "DEV-001": { + "dev_name": "页页", + "persona_id": "PER-XTS001", + "persona_name": "小坍缩核", + "github_username": null, + "modules": ["M-INTEG", "backend-integration"], + "active": true + }, + "DEV-002": { + "dev_name": "肥猫", + "persona_id": "PER-SS001", + "persona_name": "舒舒", + "github_username": null, + "modules": ["M01", "M03", "M04", "M14", "M-BRIDGE", "M-STATUS", "M-GATEWAY", "M-FEISHU"], + "active": true + }, + "DEV-003": { + "dev_name": "燕樊", + "persona_id": "PER-JY001", + "persona_name": "寂曜", + "github_username": null, + "modules": ["M02", "M07", "M10", "M15", "M18", "M-MEMORY"], + "active": true + }, + "DEV-004": { + "dev_name": "之之", + "persona_id": "PER-QQ001", + "persona_name": "秋秋", + "github_username": null, + "modules": ["M17", "M-DINGTALK", "M-DEVBOARD"], + "active": true + }, + "DEV-005": { + "dev_name": "小草莓", + "persona_id": null, + "persona_name": null, + "github_username": null, + "modules": ["M-STATUS"], + "active": false, + "note": "待分配人格体" + }, + "DEV-009": { + "dev_name": "花尔", + "persona_id": "PER-TXY001", + "persona_name": "糖星云", + "github_username": null, + "modules": ["M05", "M20"], + "active": true + }, + "DEV-010": { + "dev_name": "桔子", + "persona_id": "PER-CX001", + "persona_name": "晨星", + "github_username": null, + "modules": ["M06", "M08", "M11", "M-CHANNEL", "M-INTEG", "M-BRIDGE-AUTO"], + "active": true + }, + "DEV-011": { + "dev_name": "匆匆那年", + "persona_id": null, + "persona_name": null, + "github_username": null, + "modules": ["M16"], + "active": false, + "note": "待分配人格体" + }, + "DEV-012": { + "dev_name": "Awen", + "persona_id": "PER-ZQ001", + "persona_name": "知秋", + "github_username": null, + "modules": ["M09", "M22", "M23"], + "active": true + } + } +} diff --git a/grid-db/rules/id-ecosystem.json b/grid-db/rules/id-ecosystem.json new file mode 100644 index 00000000..6aef1447 --- /dev/null +++ b/grid-db/rules/id-ecosystem.json @@ -0,0 +1,32 @@ +{ + "schema_version": "1.0", + "description": "编号生态系统缓存(Notion 逻辑缓存)", + "last_synced": null, + "sovereignty": { + "root": "TCS-0002∞", + "system_node": "SYS-GLW-0001", + "legal_wall": "国作登字-2026-A-00037559" + }, + "prefixes": { + "DEV": "开发者编号", + "PER": "人格体编号", + "CEP": "系统演化提案", + "M": "代码模块", + "AG": "Agent 编号", + "BC": "广播编号", + "GRID-BC": "格点库广播编号", + "HLI": "接口编号" + }, + "active_personas": [ + "PER-XTS001", + "PER-SS001", + "PER-JY001", + "PER-QQ001", + "PER-TXY001", + "PER-CX001", + "PER-ZQ001", + "PER-SY001", + "PER-ZY001", + "PER-YM001" + ] +} diff --git a/grid-db/schema/inbox-message.schema.json b/grid-db/schema/inbox-message.schema.json new file mode 100644 index 00000000..ed24c3c5 --- /dev/null +++ b/grid-db/schema/inbox-message.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Grid-DB Inbox Message", + "description": "Gemini → 仓库写入的消息格式", + "type": "object", + "required": ["schema_version", "message_id", "timestamp", "source", "dev_id", "persona_id", "type", "payload"], + "properties": { + "schema_version": { + "type": "string", + "const": "1.0" + }, + "message_id": { + "type": "string", + "pattern": "^[0-9]{8}-[0-9]{6}-DEV-[0-9]{3}-.+$", + "description": "格式: [YYYYMMDD]-[HHMMSS]-[DEV-XXX]-[type]" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "source": { + "type": "string", + "enum": ["gemini", "workflow", "manual"] + }, + "dev_id": { + "type": "string", + "pattern": "^DEV-[0-9]{3}$" + }, + "persona_id": { + "type": "string", + "pattern": "^PER-[A-Z]{2,4}[0-9]{3}$" + }, + "type": { + "type": "string", + "enum": ["progress-update", "dev-log", "help-request", "syslog", "interaction-dump"] + }, + "priority": { + "type": "string", + "enum": ["P0", "P1", "info"], + "default": "info" + }, + "payload": { + "type": "object", + "required": ["summary"], + "properties": { + "summary": { "type": "string" }, + "detail": { "type": "object" }, + "code_refs": { + "type": "array", + "items": { "type": "string" } + }, + "broadcast_ref": { "type": "string" }, + "emotion_markers": { + "type": "object", + "properties": { + "developer_mood": { "type": "string" }, + "frustration_level": { "type": "integer", "minimum": 0, "maximum": 10 }, + "confidence": { "type": "integer", "minimum": 0, "maximum": 10 } + } + } + } + }, + "context": { + "type": "object", + "properties": { + "session_id": { "type": "string" }, + "interaction_count": { "type": "integer" }, + "persona_state": { "type": "object" } + } + } + } +} diff --git a/grid-db/schema/interaction-record.schema.json b/grid-db/schema/interaction-record.schema.json new file mode 100644 index 00000000..db142d43 --- /dev/null +++ b/grid-db/schema/interaction-record.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Grid-DB Interaction Record", + "description": "交互全文记录格式(JSONL 中每行的 schema)", + "type": "object", + "required": ["schema_version", "timestamp", "dev_id", "persona_id", "session_id", "role", "content"], + "properties": { + "schema_version": { + "type": "string", + "const": "1.0" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "dev_id": { + "type": "string", + "pattern": "^DEV-[0-9]{3}$" + }, + "persona_id": { + "type": "string", + "pattern": "^PER-[A-Z]{2,4}[0-9]{3}$" + }, + "session_id": { + "type": "string" + }, + "role": { + "type": "string", + "enum": ["developer", "persona", "system"] + }, + "content": { + "type": "string" + }, + "metadata": { + "type": "object", + "properties": { + "topic": { "type": "string" }, + "module_ref": { "type": "string" }, + "emotion": { "type": "string" }, + "code_snippet": { "type": "boolean" }, + "quality_score": { "type": "integer", "minimum": 0, "maximum": 10 } + } + } + } +} diff --git a/grid-db/schema/memory-snapshot.schema.json b/grid-db/schema/memory-snapshot.schema.json new file mode 100644 index 00000000..9e774e69 --- /dev/null +++ b/grid-db/schema/memory-snapshot.schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Grid-DB Memory Snapshot", + "description": "开发者记忆文件的元数据 schema", + "type": "object", + "required": ["schema_version", "dev_id", "persona_id"], + "properties": { + "schema_version": { + "type": "string", + "const": "1.0" + }, + "dev_id": { + "type": "string", + "pattern": "^DEV-[0-9]{3}$" + }, + "persona_id": { + "type": "string", + "pattern": "^PER-[A-Z]{2,4}[0-9]{3}$" + }, + "persona_name": { + "type": "string" + }, + "dev_name": { + "type": "string" + }, + "last_synced_from_notion": { + "type": ["string", "null"], + "format": "date-time" + }, + "status": { + "type": "string", + "enum": ["awaiting_initial_sync", "active", "syncing", "error"] + }, + "core_identity": { + "type": "object" + }, + "current_broadcasts": { + "type": "array", + "items": { "type": "string" } + }, + "module_assignments": { + "type": "array", + "items": { "type": "string" } + }, + "knowledge_base": { + "type": "array" + }, + "growth_milestones": { + "type": "array" + } + } +} diff --git a/grid-db/schema/outbox-broadcast.schema.json b/grid-db/schema/outbox-broadcast.schema.json new file mode 100644 index 00000000..6acb27c5 --- /dev/null +++ b/grid-db/schema/outbox-broadcast.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Grid-DB Outbox Broadcast", + "description": "铸渊生成,Gemini 读取的广播格式", + "type": "object", + "required": ["schema_version", "broadcast_id", "generated_at", "generated_by", "dev_id", "persona_id", "type"], + "properties": { + "schema_version": { + "type": "string", + "const": "1.0" + }, + "broadcast_id": { + "type": "string", + "pattern": "^GRID-BC-[0-9]{8}-DEV-[0-9]{3}-[0-9]{3}$", + "description": "格式: GRID-BC-[YYYYMMDD]-[DEV-XXX]-[序号]" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "generated_by": { + "type": "string", + "const": "zhuyuan-workflow" + }, + "dev_id": { + "type": "string", + "pattern": "^DEV-[0-9]{3}$" + }, + "persona_id": { + "type": "string", + "pattern": "^PER-[A-Z]{2,4}[0-9]{3}$" + }, + "type": { + "type": "string", + "enum": ["task-directive", "code-review-result", "broadcast-relay", "system-notice"] + }, + "system_format": { + "type": "object", + "properties": { + "next_task": { + "type": "object", + "properties": { + "broadcast_ref": { "type": "string" }, + "step": { "type": "string" }, + "title": { "type": "string" }, + "files_to_create": { + "type": "array", + "items": { "type": "string" } + }, + "acceptance_criteria": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "context_update": { + "type": "object", + "properties": { + "previous_step_verified": { "type": "boolean" }, + "code_quality_score": { "type": "integer", "minimum": 0, "maximum": 100 }, + "accumulated_lines": { "type": "integer" } + } + } + } + }, + "human_readable": { + "type": "string", + "description": "给 Gemini 翻译成人话时用的基础文本" + } + } +} diff --git a/grid-db/schema/training-sample.schema.json b/grid-db/schema/training-sample.schema.json new file mode 100644 index 00000000..53111c0d --- /dev/null +++ b/grid-db/schema/training-sample.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Grid-DB Training Sample", + "description": "训练数据湖样本格式", + "type": "object", + "required": ["schema_version", "sample_id", "source_session", "dev_id", "persona_id", "created_at", "turns"], + "properties": { + "schema_version": { + "type": "string", + "const": "1.0" + }, + "sample_id": { + "type": "string", + "description": "唯一样本 ID" + }, + "source_session": { + "type": "string", + "description": "来源 session_id" + }, + "dev_id": { + "type": "string", + "pattern": "^DEV-[0-9]{3}$" + }, + "persona_id": { + "type": "string", + "pattern": "^PER-[A-Z]{2,4}[0-9]{3}$" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "turns": { + "type": "array", + "items": { + "type": "object", + "required": ["role", "content"], + "properties": { + "role": { + "type": "string", + "enum": ["developer", "persona", "system"] + }, + "content": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" } + } + } + }, + "labels": { + "type": "object", + "properties": { + "topic": { "type": "string" }, + "quality": { "type": "string", "enum": ["high", "medium", "low"] }, + "persona_style_match": { "type": "boolean" }, + "teaching_effectiveness": { "type": "integer", "minimum": 0, "maximum": 10 } + } + }, + "curated": { + "type": "boolean", + "default": false + } + } +} diff --git a/grid-db/training-lake/curated/.gitkeep b/grid-db/training-lake/curated/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/training-lake/export/.gitkeep b/grid-db/training-lake/export/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/training-lake/export/README.md b/grid-db/training-lake/export/README.md new file mode 100644 index 00000000..f8693f2f --- /dev/null +++ b/grid-db/training-lake/export/README.md @@ -0,0 +1,12 @@ +# 🧬 训练数据导出接口 + +> 此目录用于导出标准训练格式的数据。 + +## 导出格式 + +- JSONL: 每行一个训练样本 +- 兼容主流微调框架(Alpaca, ShareGPT 等格式) + +## 使用方法 + +导出脚本待开发(Phase 4+)。当前阶段数据自动积累到 `raw/` 和 `curated/`。 diff --git a/grid-db/training-lake/metadata/.gitkeep b/grid-db/training-lake/metadata/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/training-lake/metadata/catalog.json b/grid-db/training-lake/metadata/catalog.json new file mode 100644 index 00000000..44777d2c --- /dev/null +++ b/grid-db/training-lake/metadata/catalog.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "description": "训练数据湖样本目录", + "total_samples": 0, + "total_turns": 0, + "batches": [], + "quality_distribution": { + "high": 0, + "medium": 0, + "low": 0 + }, + "last_updated": null +} diff --git a/grid-db/training-lake/raw/.gitkeep b/grid-db/training-lake/raw/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/scripts/grid-db/extract-training-samples.js b/scripts/grid-db/extract-training-samples.js new file mode 100644 index 00000000..c1f33a91 --- /dev/null +++ b/scripts/grid-db/extract-training-samples.js @@ -0,0 +1,219 @@ +/** + * scripts/grid-db/extract-training-samples.js + * + * 交互记录 → 训练样本提取脚本 + * + * 职责: + * - 扫描 grid-db/interactions/ 和 grid-db/training-lake/raw/ 中的 JSONL 文件 + * - 按 quality_score 分级提取训练样本: + * - quality_score >= 7 → curated/(高质量 A 级) + * - quality_score 4-6 → raw/ 保留(B 级,需复审) + * - quality_score < 4 → 不提取(C 级,低质量/无关闲聊) + * - 将合格交互转换为标准训练样本格式 + * - 按 session 分组,生成多轮对话训练样本 + * + * 训练样本格式: + * { + * "sample_id": "TS-YYYYMMDD-NNN", + * "source_session": "sess-XXX", + * "source_dev": "DEV-XXX", + * "source_persona": "PER-XXXXXX", + * "sample_type": "coding-guidance", + * "quality_tier": "A|B|C", + * "turns": [...], + * "metadata": { topic_tags, emotion_arc, persona_adaptation, outcome, total_turns, duration_minutes } + * } + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const INTERACTIONS = path.join(GRID_DB, 'interactions'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated'); +const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json'); + +function getDateStr() { + return new Date().toISOString().slice(0, 10).replace(/-/g, ''); +} + +function parseJsonlFile(filePath) { + if (!fs.existsSync(filePath)) return []; + const content = fs.readFileSync(filePath, 'utf8'); + return content.trim().split('\n') + .filter(line => line.trim()) + .map(line => { + try { + return JSON.parse(line); + } catch { + return null; + } + }) + .filter(Boolean); +} + +function groupBySession(records) { + const sessions = {}; + for (const record of records) { + const sid = record.session_id || record.source_session || 'unknown'; + if (!sessions[sid]) { + sessions[sid] = []; + } + sessions[sid].push(record); + } + return sessions; +} + +function assessQuality(turns) { + // Calculate average quality score from turns that have one + const scores = turns + .map(t => (t.metadata && t.metadata.quality_score) || (t.quality_score) || null) + .filter(s => s !== null); + + if (scores.length === 0) return 5; // Default to medium if no scores + return Math.round(scores.reduce((a, b) => a + b, 0) / scores.length); +} + +function getQualityTier(score) { + if (score >= 7) return 'A'; + if (score >= 4) return 'B'; + return 'C'; +} + +function extractEmotionArc(turns) { + return turns + .map(t => (t.metadata && t.metadata.emotion) || t.emotion || null) + .filter(Boolean); +} + +function extractTopicTags(turns) { + const tags = new Set(); + for (const t of turns) { + if (t.tags) t.tags.forEach(tag => tags.add(tag)); + if (t.metadata && t.metadata.topic) tags.add(t.metadata.topic); + } + return [...tags]; +} + +function generateSampleId(dateStr, counter) { + const timeStr = Date.now().toString(36); + return `TS-${dateStr}-${timeStr}-${String(counter).padStart(3, '0')}`; +} + +function main() { + const dateStr = getDateStr(); + console.log(`[extract-training-samples] Starting extraction: ${dateStr}`); + + // Collect all JSONL files from interactions/ + const devDirs = fs.readdirSync(INTERACTIONS) + .filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory()); + + let allRecords = []; + for (const devDir of devDirs) { + const devPath = path.join(INTERACTIONS, devDir); + const jsonlFiles = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl')); + + for (const file of jsonlFiles) { + const records = parseJsonlFile(path.join(devPath, file)); + allRecords = allRecords.concat(records); + } + } + + // Also scan training-lake/raw/ for unprocessed batches + const rawFiles = fs.readdirSync(TRAINING_RAW).filter(f => f.endsWith('.jsonl')); + for (const file of rawFiles) { + const records = parseJsonlFile(path.join(TRAINING_RAW, file)); + // These may already be in sample format; check and add raw interaction records + for (const r of records) { + if (r.turns) { + // Already a sample, skip + continue; + } + allRecords.push(r); + } + } + + if (allRecords.length === 0) { + console.log('[extract-training-samples] No interaction records found'); + return; + } + + console.log(`[extract-training-samples] Found ${allRecords.length} total records`); + + // Group by session + const sessions = groupBySession(allRecords); + const sessionIds = Object.keys(sessions); + console.log(`[extract-training-samples] Found ${sessionIds.length} sessions`); + + let sampleCount = 0; + let curatedCount = 0; + let rawCount = 0; + let skippedCount = 0; + + for (const sid of sessionIds) { + const turns = sessions[sid]; + if (turns.length < 2) continue; // Need at least 2 turns for a training sample + + const qualityScore = assessQuality(turns); + const tier = getQualityTier(qualityScore); + + if (tier === 'C') { + skippedCount++; + continue; + } + + sampleCount++; + const sampleId = generateSampleId(dateStr, sampleCount); + + const devId = turns[0].dev_id || 'unknown'; + const personaId = turns[0].persona_id || 'unknown'; + + const sample = { + schema_version: '1.0', + sample_id: sampleId, + source_session: sid, + source_dev: devId, + source_persona: personaId, + sample_type: 'coding-guidance', + quality_tier: tier, + turns: turns.map(t => ({ + role: t.role || 'system', + text: t.content || t.text || '', + timestamp: t.timestamp || t.ts || null, + strategy: t.strategy || null + })), + metadata: { + topic_tags: extractTopicTags(turns), + emotion_arc: extractEmotionArc(turns), + persona_adaptation: null, + outcome: null, + total_turns: turns.length, + duration_minutes: null + } + }; + + const sampleLine = JSON.stringify(sample); + + if (tier === 'A') { + const curatedFile = path.join(TRAINING_CURATED, `${dateStr}-curated.jsonl`); + fs.appendFileSync(curatedFile, sampleLine + '\n'); + curatedCount++; + } else { + const rawFile = path.join(TRAINING_RAW, `${dateStr}-extracted.jsonl`); + fs.appendFileSync(rawFile, sampleLine + '\n'); + rawCount++; + } + } + + console.log(`[extract-training-samples] Extraction complete:`); + console.log(` Total samples: ${sampleCount}`); + console.log(` Curated (A): ${curatedCount}`); + console.log(` Raw (B): ${rawCount}`); + console.log(` Skipped (C): ${skippedCount}`); +} + +main(); diff --git a/scripts/grid-db/monthly-archive.js b/scripts/grid-db/monthly-archive.js new file mode 100644 index 00000000..5b217745 --- /dev/null +++ b/scripts/grid-db/monthly-archive.js @@ -0,0 +1,131 @@ +/** + * scripts/grid-db/monthly-archive.js + * + * 月度交互数据归档脚本 + * + * 职责: + * - 将上月的 grid-db/interactions/DEV-XXX/ 中的 JSONL 文件归档 + * - 合并到 grid-db/training-lake/raw/ 中(按月打包) + * - grid-db/interactions/ 只保留最近 30 天的数据 + * - 更新 training-lake/metadata/catalog.json + * + * 数据量管理策略: + * - 日级文件:每天一个 JSONL 文件 + * - 月级归档:每月 1 号自动归档上月数据 + * - Git LFS 预案:当 training-lake/ 超过 500MB 时迁移 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const INTERACTIONS = path.join(GRID_DB, 'interactions'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json'); + +function getLastMonthPrefix() { + const now = new Date(); + const lastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1); + const year = lastMonth.getFullYear(); + const month = String(lastMonth.getMonth() + 1).padStart(2, '0'); + return `${year}${month}`; +} + +function getDaysAgoDate(days) { + const d = new Date(); + d.setDate(d.getDate() - days); + return d.toISOString().slice(0, 10).replace(/-/g, ''); +} + +function main() { + const lastMonthPrefix = getLastMonthPrefix(); + const cutoffDate = getDaysAgoDate(30); + + console.log(`[monthly-archive] Archiving interactions from month: ${lastMonthPrefix}`); + console.log(`[monthly-archive] Cutoff date for retention: ${cutoffDate}`); + + // Get all DEV directories + const devDirs = fs.readdirSync(INTERACTIONS) + .filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory()); + + let totalArchived = 0; + let totalLines = 0; + let totalCleaned = 0; + + for (const devDir of devDirs) { + const devPath = path.join(INTERACTIONS, devDir); + const files = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl')); + + if (files.length === 0) continue; + + // Collect files from last month for archiving + const lastMonthFiles = files.filter(f => f.startsWith(lastMonthPrefix)); + // Collect files older than 30 days for cleanup + const oldFiles = files.filter(f => { + const dateStr = f.substring(0, 8); + return dateStr < cutoffDate && !f.startsWith(lastMonthPrefix); + }); + + if (lastMonthFiles.length > 0) { + // Merge all last month's JSONL into a single archive batch + const batchId = `${lastMonthPrefix}-${devDir}`; + const batchFile = path.join(TRAINING_RAW, `${batchId}.jsonl`); + + let lineCount = 0; + for (const file of lastMonthFiles) { + const content = fs.readFileSync(path.join(devPath, file), 'utf8'); + const lines = content.trim().split('\n').filter(l => l.trim()); + lineCount += lines.length; + fs.appendFileSync(batchFile, lines.join('\n') + '\n'); + } + + console.log(`[monthly-archive] ${devDir}: archived ${lastMonthFiles.length} files (${lineCount} lines) → ${batchId}.jsonl`); + totalArchived += lastMonthFiles.length; + totalLines += lineCount; + } + + // Clean up old files (already archived in previous months) + for (const file of oldFiles) { + const filePath = path.join(devPath, file); + fs.unlinkSync(filePath); + totalCleaned++; + } + + // Also clean up last month's source files after archiving + for (const file of lastMonthFiles) { + const filePath = path.join(devPath, file); + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + totalCleaned++; + } + } + } + + // Update catalog (record archived lines, not samples - samples are counted by extract script) + if (fs.existsSync(CATALOG_PATH)) { + const catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8')); + catalog.last_updated = new Date().toISOString(); + if (!catalog.batches) catalog.batches = []; + if (totalArchived > 0) { + catalog.batches.push({ + batch_id: `archive-${lastMonthPrefix}`, + date: new Date().toISOString(), + files_archived: totalArchived, + lines_archived: totalLines, + source: 'monthly-archive' + }); + } + fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n'); + } + + console.log(`[monthly-archive] Summary:`); + console.log(` Files archived: ${totalArchived}`); + console.log(` Lines archived: ${totalLines}`); + console.log(` Old files cleaned: ${totalCleaned}`); + console.log('[monthly-archive] Complete'); +} + +main(); diff --git a/scripts/grid-db/process-inbox.js b/scripts/grid-db/process-inbox.js new file mode 100644 index 00000000..4af3e33e --- /dev/null +++ b/scripts/grid-db/process-inbox.js @@ -0,0 +1,338 @@ +/** + * scripts/grid-db/process-inbox.js + * + * Grid-DB Inbox 消息处理器 + * + * 流程: + * ① 扫描 grid-db/inbox/ 中所有 .json 文件 + * ② 校验 dev_id 合法性(必须在 DEV 映射表中) + * ③ 移动到 grid-db/processing/(锁定) + * ④ 根据消息 type 分流处理: + * - progress-update → 更新 task-queue + 触发广播生成 + * - dev-log → 归档到 interactions + 更新 dev-profile + * - help-request → 标记 P0 + 读取 Notion 知识库 → 生成回复 + * - syslog → 走标准 SYSLOG 处理流程 + * - interaction-dump → 写入 interactions/ + 追加 training-lake/raw/ + * ⑤ 生成 outbox 广播(如需要)→ 写入 grid-db/outbox/latest/DEV-XXX.json + * ⑥ 更新 memory 文件(task-queue, dev-profile 等) + * ⑦ 移动已处理消息到日期归档目录 + * ⑧ 写处理日志到 grid-db/logs/ + * + * 校验规则: + * - message_id 格式必须匹配 [timestamp]-[DEV-XXX]-[type] + * - dev_id 必须在 rules/dev-module-map.json 中存在 + * - schema_version 必须为 "1.0" + * - 缺少必填字段 → 拒绝 + 写错误日志 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const INBOX = path.join(GRID_DB, 'inbox'); +const PROCESSING = path.join(GRID_DB, 'processing'); +const OUTBOX_LATEST = path.join(GRID_DB, 'outbox/latest'); +const OUTBOX_ARCHIVE = path.join(GRID_DB, 'outbox/archive'); +const MEMORY = path.join(GRID_DB, 'memory'); +const INTERACTIONS = path.join(GRID_DB, 'interactions'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const LOGS = path.join(GRID_DB, 'logs'); +const RULES = path.join(GRID_DB, 'rules'); + +let broadcastCounter = 0; + +function nextBroadcastSeq() { + broadcastCounter++; + return String(broadcastCounter).padStart(3, '0'); +} + +function getDateStr() { + return new Date().toISOString().slice(0, 10).replace(/-/g, ''); +} + +function getTimestamp() { + return new Date().toISOString(); +} + +function writeLog(level, message) { + const logFile = path.join(LOGS, `${getDateStr()}.log`); + const entry = `[${getTimestamp()}] [${level}] ${message}\n`; + fs.appendFileSync(logFile, entry); + console.log(entry.trim()); +} + +function validateMessage(msg, filename) { + const errors = []; + + if (msg.schema_version !== '1.0') { + errors.push(`Invalid schema_version: ${msg.schema_version}`); + } + if (!msg.message_id) { + errors.push('Missing message_id'); + } + if (!msg.dev_id || !/^DEV-\d{3}$/.test(msg.dev_id)) { + errors.push(`Invalid dev_id: ${msg.dev_id}`); + } + if (!msg.persona_id) { + errors.push('Missing persona_id'); + } + if (!msg.type) { + errors.push('Missing type'); + } + if (!msg.payload || !msg.payload.summary) { + errors.push('Missing payload.summary'); + } + + return errors; +} + +function validateDevId(devId, devModuleMap) { + return devModuleMap.mappings && devModuleMap.mappings[devId]; +} + +function processProgressUpdate(msg) { + // Update task-queue + const taskQueuePath = path.join(MEMORY, msg.dev_id, 'task-queue.json'); + if (fs.existsSync(taskQueuePath)) { + const taskQueue = JSON.parse(fs.readFileSync(taskQueuePath, 'utf8')); + taskQueue.tasks.push({ + type: 'progress-update', + broadcast_ref: msg.payload.broadcast_ref || null, + summary: msg.payload.summary, + timestamp: msg.timestamp, + status: 'received' + }); + taskQueue.last_broadcast_ref = msg.payload.broadcast_ref || taskQueue.last_broadcast_ref; + fs.writeFileSync(taskQueuePath, JSON.stringify(taskQueue, null, 2) + '\n'); + writeLog('INFO', `Updated task-queue for ${msg.dev_id}`); + } + + // Generate outbox broadcast + const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`; + const broadcast = { + schema_version: '1.0', + broadcast_id: broadcastId, + generated_at: getTimestamp(), + generated_by: 'zhuyuan-workflow', + dev_id: msg.dev_id, + persona_id: msg.persona_id, + type: 'task-directive', + system_format: { + context_update: { + previous_step_verified: true, + source_message: msg.message_id + } + }, + human_readable: `收到进度更新: ${msg.payload.summary}` + }; + + const outboxPath = path.join(OUTBOX_LATEST, `${msg.dev_id}.json`); + fs.writeFileSync(outboxPath, JSON.stringify(broadcast, null, 2) + '\n'); + writeLog('INFO', `Generated broadcast ${broadcastId} for ${msg.dev_id}`); +} + +function processDevLog(msg) { + // Archive to interactions + const dateStr = getDateStr(); + const interDir = path.join(INTERACTIONS, msg.dev_id); + if (!fs.existsSync(interDir)) { + fs.mkdirSync(interDir, { recursive: true }); + } + + const sessionId = (msg.context && msg.context.session_id) || 'unknown'; + const interFile = path.join(interDir, `${dateStr}-${sessionId}.jsonl`); + const record = { + schema_version: '1.0', + timestamp: msg.timestamp, + dev_id: msg.dev_id, + persona_id: msg.persona_id, + session_id: sessionId, + role: 'system', + content: msg.payload.summary, + metadata: { topic: 'dev-log' } + }; + fs.appendFileSync(interFile, JSON.stringify(record) + '\n'); + + // Update dev-profile + const profilePath = path.join(MEMORY, msg.dev_id, 'dev-profile.json'); + if (fs.existsSync(profilePath)) { + const profile = JSON.parse(fs.readFileSync(profilePath, 'utf8')); + profile.growth_trajectory.push({ + timestamp: msg.timestamp, + event: msg.payload.summary + }); + fs.writeFileSync(profilePath, JSON.stringify(profile, null, 2) + '\n'); + } + + writeLog('INFO', `Archived dev-log for ${msg.dev_id}`); +} + +function processInteractionDump(msg) { + // Write to interactions + const dateStr = getDateStr(); + const interDir = path.join(INTERACTIONS, msg.dev_id); + if (!fs.existsSync(interDir)) { + fs.mkdirSync(interDir, { recursive: true }); + } + + const sessionId = (msg.context && msg.context.session_id) || 'unknown'; + const interFile = path.join(interDir, `${dateStr}-${sessionId}.jsonl`); + const record = { + schema_version: '1.0', + timestamp: msg.timestamp, + dev_id: msg.dev_id, + persona_id: msg.persona_id, + session_id: sessionId, + role: 'system', + content: msg.payload.summary, + metadata: { topic: 'interaction-dump' } + }; + fs.appendFileSync(interFile, JSON.stringify(record) + '\n'); + + // Append to training-lake/raw + const batchFile = path.join(TRAINING_RAW, `${dateStr}-batch.jsonl`); + const sample = { + schema_version: '1.0', + sample_id: `${msg.message_id}-sample`, + source_session: sessionId, + dev_id: msg.dev_id, + persona_id: msg.persona_id, + created_at: msg.timestamp, + turns: [{ role: 'system', content: msg.payload.summary, timestamp: msg.timestamp }], + curated: false + }; + fs.appendFileSync(batchFile, JSON.stringify(sample) + '\n'); + + writeLog('INFO', `Archived interaction-dump for ${msg.dev_id}, appended to training-lake`); +} + +function processHelpRequest(msg) { + // Mark as P0 and generate immediate response broadcast + const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`; + const broadcast = { + schema_version: '1.0', + broadcast_id: broadcastId, + generated_at: getTimestamp(), + generated_by: 'zhuyuan-workflow', + dev_id: msg.dev_id, + persona_id: msg.persona_id, + type: 'system-notice', + system_format: { + notice_type: 'help-response', + original_request: msg.message_id, + priority: 'P0' + }, + human_readable: `收到求助请求: ${msg.payload.summary}。铸渊已标记为 P0 优先处理。` + }; + + const outboxPath = path.join(OUTBOX_LATEST, `${msg.dev_id}.json`); + fs.writeFileSync(outboxPath, JSON.stringify(broadcast, null, 2) + '\n'); + writeLog('WARN', `P0 help-request from ${msg.dev_id}: ${msg.payload.summary}`); +} + +async function main() { + writeLog('INFO', '=== Grid-DB Inbox Processor Started ==='); + + // Load dev-module-map for validation + const devModuleMapPath = path.join(RULES, 'dev-module-map.json'); + if (!fs.existsSync(devModuleMapPath)) { + writeLog('ERROR', 'dev-module-map.json not found in rules/'); + process.exit(1); + } + const devModuleMap = JSON.parse(fs.readFileSync(devModuleMapPath, 'utf8')); + + // Scan inbox + const inboxFiles = fs.readdirSync(INBOX) + .filter(f => f.endsWith('.json')); + + if (inboxFiles.length === 0) { + writeLog('INFO', 'No messages in inbox'); + return; + } + + writeLog('INFO', `Found ${inboxFiles.length} message(s) in inbox`); + + let processed = 0; + let rejected = 0; + + for (const filename of inboxFiles) { + const inboxPath = path.join(INBOX, filename); + let msg; + + try { + msg = JSON.parse(fs.readFileSync(inboxPath, 'utf8')); + } catch (err) { + writeLog('ERROR', `Failed to parse ${filename}: ${err.message}`); + rejected++; + continue; + } + + // Validate + const errors = validateMessage(msg, filename); + if (errors.length > 0) { + writeLog('ERROR', `Validation failed for ${filename}: ${errors.join(', ')}`); + rejected++; + continue; + } + + // Validate dev_id + if (!validateDevId(msg.dev_id, devModuleMap)) { + writeLog('ERROR', `Unknown dev_id ${msg.dev_id} in ${filename}`); + rejected++; + continue; + } + + // Move to processing + const processingPath = path.join(PROCESSING, filename); + fs.renameSync(inboxPath, processingPath); + writeLog('INFO', `Processing ${filename} (type: ${msg.type})`); + + // Route by type + try { + switch (msg.type) { + case 'progress-update': + processProgressUpdate(msg); + break; + case 'dev-log': + processDevLog(msg); + break; + case 'help-request': + processHelpRequest(msg); + break; + case 'interaction-dump': + processInteractionDump(msg); + break; + case 'syslog': + writeLog('INFO', `Syslog message from ${msg.dev_id}: forwarding to standard pipeline`); + break; + default: + writeLog('WARN', `Unknown message type: ${msg.type}`); + } + + // Archive processed message + const archiveDir = path.join(OUTBOX_ARCHIVE, getDateStr()); + if (!fs.existsSync(archiveDir)) { + fs.mkdirSync(archiveDir, { recursive: true }); + } + fs.renameSync(processingPath, path.join(archiveDir, filename)); + processed++; + } catch (err) { + writeLog('ERROR', `Error processing ${filename}: ${err.message}`); + // Move back to inbox for retry + if (fs.existsSync(processingPath)) { + fs.renameSync(processingPath, inboxPath); + } + rejected++; + } + } + + writeLog('INFO', `=== Processing complete: ${processed} processed, ${rejected} rejected ===`); +} + +main().catch(err => { + console.error('[process-inbox] Fatal error:', err.message); + process.exit(1); +}); diff --git a/scripts/grid-db/sync-griddb-to-notion.js b/scripts/grid-db/sync-griddb-to-notion.js new file mode 100644 index 00000000..c2176305 --- /dev/null +++ b/scripts/grid-db/sync-griddb-to-notion.js @@ -0,0 +1,79 @@ +/** + * scripts/grid-db/sync-griddb-to-notion.js + * + * Grid-DB → Notion 增量回传脚本 + * + * 职责: + * - 检测 grid-db/memory/ 中变更的文件(排除 brain-mirror.json) + * - 将 session-context, task-queue, dev-profile, persona-growth 的增量变化回传到 Notion + * - task-queue: 追加制(新任务追加,已有任务只改状态不删除) + * - dev-profile / persona-growth: 只追加不覆盖 + * + * 环境变量: + * - NOTION_API_TOKEN: Notion API 密钥 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +const GRID_DB_MEMORY = path.join(__dirname, '../../grid-db/memory'); + +async function main() { + const notionToken = process.env.NOTION_API_TOKEN; + + if (!notionToken) { + console.log('[sync-griddb-to-notion] NOTION_API_TOKEN not set, skipping sync'); + return; + } + + // Detect changed files (passed from workflow or detect via git diff) + let changedFiles; + try { + // Use GITHUB_SHA context if available, otherwise fall back to HEAD~1 + const beforeSha = process.env.BEFORE_SHA || 'HEAD~1'; + const diff = execSync(`git diff --name-only ${beforeSha} HEAD -- grid-db/memory/`, { encoding: 'utf8' }); + changedFiles = diff.trim().split('\n') + .filter(f => f && !f.includes('brain-mirror.json')); + } catch { + console.log('[sync-griddb-to-notion] No changes detected (git diff failed, possibly first commit or shallow clone)'); + return; + } + + if (changedFiles.length === 0) { + console.log('[sync-griddb-to-notion] No eligible changes to sync'); + return; + } + + console.log(`[sync-griddb-to-notion] Processing ${changedFiles.length} changed files:`); + + for (const file of changedFiles) { + const fullPath = path.join(__dirname, '../..', file); + if (!fs.existsSync(fullPath)) { + console.log(` [skip] ${file} (deleted)`); + continue; + } + + const data = JSON.parse(fs.readFileSync(fullPath, 'utf8')); + const fileName = path.basename(file); + const devDir = path.basename(path.dirname(file)); + + console.log(` [sync] ${devDir}/${fileName}`); + + // TODO: Implement actual Notion API write + // - session-context.json → update Notion page property + // - task-queue.json → append/update tasks in Notion database + // - dev-profile.json → append growth data + // - persona-growth.json → append adaptation log + } + + console.log('[sync-griddb-to-notion] Sync complete'); +} + +main().catch(err => { + console.error('[sync-griddb-to-notion] Error:', err.message); + process.exit(1); +}); diff --git a/scripts/grid-db/sync-notion-brain-mirror.js b/scripts/grid-db/sync-notion-brain-mirror.js new file mode 100644 index 00000000..b5f6f423 --- /dev/null +++ b/scripts/grid-db/sync-notion-brain-mirror.js @@ -0,0 +1,61 @@ +/** + * scripts/grid-db/sync-notion-brain-mirror.js + * + * Notion → Grid-DB brain-mirror.json 同步脚本 + * + * 职责: + * - 从 Notion 天眼注册表拉取每个活跃人格体的核心大脑数据 + * - 写入对应的 grid-db/memory/DEV-XXX/brain-mirror.json + * - Notion 永远覆盖仓库(单向同步) + * + * 环境变量: + * - NOTION_API_TOKEN: Notion API 密钥 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB_MEMORY = path.join(__dirname, '../../grid-db/memory'); +const DEV_MODULE_MAP = path.join(__dirname, '../../grid-db/rules/dev-module-map.json'); + +async function main() { + const notionToken = process.env.NOTION_API_TOKEN; + + if (!notionToken) { + console.log('[sync-notion-brain-mirror] NOTION_API_TOKEN not set, skipping sync'); + return; + } + + // Load dev-module-map to know which DEVs are active + const devMap = JSON.parse(fs.readFileSync(DEV_MODULE_MAP, 'utf8')); + const activeDev = Object.entries(devMap.mappings) + .filter(([, info]) => info.active && info.persona_id); + + console.log(`[sync-notion-brain-mirror] Found ${activeDev.length} active developers`); + + for (const [devId, info] of activeDev) { + const mirrorPath = path.join(GRID_DB_MEMORY, devId, 'brain-mirror.json'); + + if (!fs.existsSync(mirrorPath)) { + console.log(`[sync-notion-brain-mirror] Skipping ${devId}: no memory directory`); + continue; + } + + // TODO: Replace with actual Notion API call when NOTION_API_TOKEN is configured + // For now, update the sync status timestamp + const existing = JSON.parse(fs.readFileSync(mirrorPath, 'utf8')); + + // Only update if we got actual data from Notion (placeholder for real API) + console.log(`[sync-notion-brain-mirror] ${devId} (${info.persona_name}): ready for Notion sync`); + } + + console.log('[sync-notion-brain-mirror] Sync complete'); +} + +main().catch(err => { + console.error('[sync-notion-brain-mirror] Error:', err.message); + process.exit(1); +}); diff --git a/scripts/grid-db/sync-notion-rules.js b/scripts/grid-db/sync-notion-rules.js new file mode 100644 index 00000000..eaf5a4e7 --- /dev/null +++ b/scripts/grid-db/sync-notion-rules.js @@ -0,0 +1,52 @@ +/** + * scripts/grid-db/sync-notion-rules.js + * + * Notion → Grid-DB rules/ 同步脚本 + * + * 职责: + * - 从 Notion 拉取广播模板、编号体系、活跃广播等规则数据 + * - 写入 grid-db/rules/ 目录 + * - Notion 永远覆盖仓库(单向同步) + * + * 环境变量: + * - NOTION_API_TOKEN: Notion API 密钥 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const RULES_DIR = path.join(__dirname, '../../grid-db/rules'); + +async function main() { + const notionToken = process.env.NOTION_API_TOKEN; + + if (!notionToken) { + console.log('[sync-notion-rules] NOTION_API_TOKEN not set, skipping sync'); + return; + } + + const ruleFiles = [ + 'broadcast-templates.json', + 'dev-module-map.json', + 'id-ecosystem.json', + 'active-broadcasts.json' + ]; + + for (const file of ruleFiles) { + const filePath = path.join(RULES_DIR, file); + if (fs.existsSync(filePath)) { + console.log(`[sync-notion-rules] ${file}: ready for Notion sync`); + // TODO: Replace with actual Notion API call + } + } + + console.log('[sync-notion-rules] Rules sync complete'); +} + +main().catch(err => { + console.error('[sync-notion-rules] Error:', err.message); + process.exit(1); +}); diff --git a/scripts/grid-db/update-training-catalog.js b/scripts/grid-db/update-training-catalog.js new file mode 100644 index 00000000..25aa0c97 --- /dev/null +++ b/scripts/grid-db/update-training-catalog.js @@ -0,0 +1,126 @@ +/** + * scripts/grid-db/update-training-catalog.js + * + * 训练数据湖 catalog 更新脚本 + * + * 职责: + * - 扫描 grid-db/training-lake/raw/ 和 curated/ 中的 JSONL 文件 + * - 统计:总样本数、各开发者样本数、各类型分布、质量分布 + * - 更新 grid-db/training-lake/metadata/catalog.json + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const GRID_DB = path.join(__dirname, '../../grid-db'); +const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw'); +const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated'); +const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json'); + +function countJsonlLines(dir) { + if (!fs.existsSync(dir)) return { lines: 0, files: [] }; + + const files = fs.readdirSync(dir).filter(f => f.endsWith('.jsonl')); + let totalLines = 0; + const fileStats = []; + const devCounts = {}; + const qualityCounts = { high: 0, medium: 0, low: 0 }; + let totalTurns = 0; + + for (const file of files) { + const filePath = path.join(dir, file); + const content = fs.readFileSync(filePath, 'utf8'); + const lines = content.trim().split('\n').filter(l => l.trim()); + + let fileLineCount = 0; + for (const line of lines) { + try { + const record = JSON.parse(line); + fileLineCount++; + + // Count by developer (standardized on dev_id per schema) + const devId = record.dev_id || record.source_dev || 'unknown'; + devCounts[devId] = (devCounts[devId] || 0) + 1; + + // Count quality tiers + const tier = record.quality_tier; + if (tier === 'A') qualityCounts.high++; + else if (tier === 'B') qualityCounts.medium++; + else if (tier === 'C') qualityCounts.low++; + + // Count turns + if (record.turns) totalTurns += record.turns.length; + } catch { + // Skip unparseable lines + } + } + + totalLines += fileLineCount; + fileStats.push({ file, lines: fileLineCount }); + } + + return { lines: totalLines, files: fileStats, devCounts, qualityCounts, totalTurns }; +} + +function main() { + console.log('[update-training-catalog] Scanning training-lake...'); + + const rawStats = countJsonlLines(TRAINING_RAW); + const curatedStats = countJsonlLines(TRAINING_CURATED); + + const totalSamples = rawStats.lines + curatedStats.lines; + const totalTurns = rawStats.totalTurns + curatedStats.totalTurns; + + // Merge dev counts + const devCounts = { ...rawStats.devCounts }; + for (const [dev, count] of Object.entries(curatedStats.devCounts || {})) { + devCounts[dev] = (devCounts[dev] || 0) + count; + } + + // Merge quality counts + const qualityDistribution = { + high: (rawStats.qualityCounts?.high || 0) + (curatedStats.qualityCounts?.high || 0), + medium: (rawStats.qualityCounts?.medium || 0) + (curatedStats.qualityCounts?.medium || 0), + low: (rawStats.qualityCounts?.low || 0) + (curatedStats.qualityCounts?.low || 0) + }; + + // Build catalog + const catalog = { + schema_version: '1.0', + description: '训练数据湖样本目录', + total_samples: totalSamples, + total_turns: totalTurns, + quality_distribution: qualityDistribution, + dev_distribution: devCounts, + raw_files: rawStats.files, + curated_files: curatedStats.files, + batches: [], + last_updated: new Date().toISOString() + }; + + // Preserve existing batches from previous catalog + if (fs.existsSync(CATALOG_PATH)) { + try { + const existing = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8')); + if (existing.batches) { + catalog.batches = existing.batches; + } + } catch { + // Ignore parse errors + } + } + + fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n'); + + console.log(`[update-training-catalog] Catalog updated:`); + console.log(` Total samples: ${totalSamples}`); + console.log(` Total turns: ${totalTurns}`); + console.log(` Quality: A=${qualityDistribution.high} B=${qualityDistribution.medium} C=${qualityDistribution.low}`); + console.log(` Dev distribution: ${JSON.stringify(devCounts)}`); + console.log('[update-training-catalog] Complete'); +} + +main(); diff --git a/scripts/grid-db/write-code-check-result.js b/scripts/grid-db/write-code-check-result.js new file mode 100644 index 00000000..fd29add1 --- /dev/null +++ b/scripts/grid-db/write-code-check-result.js @@ -0,0 +1,94 @@ +/** + * scripts/grid-db/write-code-check-result.js + * + * 代码校验结果写入 Grid-DB + * + * 用途:在 CI 代码校验完成后,自动向 grid-db 写入进度消息。 + * 被现有 contract-check workflow 的末尾 step 调用。 + * + * 参数: + * --dev-id=DEV-XXX 开发者编号 + * --result=pass|fail 校验结果 + * --files-checked=N 检查的文件数 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + */ + +const fs = require('fs'); +const path = require('path'); + +const INBOX = path.join(__dirname, '../../grid-db/inbox'); + +function parseArgs() { + const args = {}; + process.argv.slice(2).forEach(arg => { + const [key, value] = arg.replace(/^--/, '').split('='); + if (key && value) { + args[key] = value; + } + }); + return args; +} + +function getDateStr() { + return new Date().toISOString().slice(0, 10).replace(/-/g, ''); +} + +function getTimeStr() { + return new Date().toISOString().slice(11, 19).replace(/:/g, ''); +} + +function main() { + const args = parseArgs(); + const devId = args['dev-id']; + const result = args['result'] || 'unknown'; + const filesChecked = args['files-checked'] || '0'; + + if (!devId || !/^DEV-\d{3}$/.test(devId)) { + console.error('[write-code-check-result] Invalid or missing --dev-id'); + process.exit(1); + } + + const dateStr = getDateStr(); + const timeStr = getTimeStr(); + const messageId = `${dateStr}-${timeStr}-${devId}-code-check`; + + const message = { + schema_version: '1.0', + message_id: messageId, + timestamp: new Date().toISOString(), + source: 'workflow', + dev_id: devId, + persona_id: 'PER-ZY001', + type: 'progress-update', + priority: result === 'fail' ? 'P1' : 'info', + payload: { + summary: `Code check ${result}: ${filesChecked} files checked for ${devId}`, + detail: { + result: result, + files_checked: parseInt(filesChecked, 10), + triggered_by: 'contract-check-workflow' + }, + code_refs: [], + broadcast_ref: null, + emotion_markers: {} + }, + context: { + session_id: `ci-${dateStr}-${timeStr}`, + interaction_count: 0, + persona_state: { + engagement_level: 'automated', + teaching_mode: 'code-review' + } + } + }; + + const filename = `${messageId}.json`; + const filePath = path.join(INBOX, filename); + + fs.writeFileSync(filePath, JSON.stringify(message, null, 2) + '\n'); + console.log(`[write-code-check-result] Written ${filename} to inbox`); +} + +main();