Merge pull request #157 from qinfendebingshuo/copilot/zy-auto-loop-2026-03
feat: Grid-DB 逻辑格点库基础设施 (Phase 0-5)
This commit is contained in:
冰朔
GitHub
commit
718a656b8a
|
|
@ -0,0 +1,26 @@
|
|||
name: 📦 Grid-DB 月度归档
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 1 * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
archive:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: 📦 归档上月交互数据
|
||||
run: node scripts/grid-db/monthly-archive.js
|
||||
|
||||
- name: 💾 提交
|
||||
run: |
|
||||
git config user.name "zhuyuan-bot"
|
||||
git config user.email "zhuyuan@guanghulab.com"
|
||||
git add grid-db/
|
||||
git diff --cached --quiet || git commit -m "archive: 月度交互数据归档 [skip ci]"
|
||||
git push
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
name: ⚙️ Grid-DB 消息处理器
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'grid-db/inbox/*.json'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
process-inbox:
|
||||
if: github.actor != 'zhuyuan-bot' && !contains(github.event.head_commit.message, '[skip ci]')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: ⚙️ 处理 inbox 消息
|
||||
env:
|
||||
NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }}
|
||||
run: node scripts/grid-db/process-inbox.js
|
||||
|
||||
- name: 💾 提交处理结果
|
||||
run: |
|
||||
git config user.name "zhuyuan-bot"
|
||||
git config user.email "zhuyuan@guanghulab.com"
|
||||
git add grid-db/
|
||||
git diff --cached --quiet || git commit -m "auto: Grid-DB inbox processed [skip ci]"
|
||||
git push
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
name: 🧬 训练数据提取
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * 0'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
extract:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: 🧬 从交互记录提取训练样本
|
||||
run: node scripts/grid-db/extract-training-samples.js
|
||||
|
||||
- name: 📊 更新 catalog
|
||||
run: node scripts/grid-db/update-training-catalog.js
|
||||
|
||||
- name: 💾 提交
|
||||
run: |
|
||||
git config user.name "zhuyuan-bot"
|
||||
git config user.email "zhuyuan@guanghulab.com"
|
||||
git add grid-db/training-lake/
|
||||
git diff --cached --quiet || git commit -m "training: 周度训练样本提取 [skip ci]"
|
||||
git push
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
name: 📤 Grid-DB → Notion 增量回传
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'grid-db/memory/*/session-context.json'
|
||||
- 'grid-db/memory/*/task-queue.json'
|
||||
- 'grid-db/memory/*/dev-profile.json'
|
||||
- 'grid-db/memory/*/persona-growth.json'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sync-to-notion:
|
||||
if: github.actor != 'zhuyuan-bot' && !contains(github.event.head_commit.message, '[skip ci]')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: 🔍 检测变更文件
|
||||
id: changes
|
||||
run: |
|
||||
CHANGED=$(git diff --name-only HEAD~1 HEAD -- 'grid-db/memory/' | grep -v brain-mirror.json || true)
|
||||
echo "files=$CHANGED" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: 📤 回传增量到 Notion
|
||||
if: steps.changes.outputs.files != ''
|
||||
env:
|
||||
NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }}
|
||||
run: node scripts/grid-db/sync-griddb-to-notion.js
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
name: 🧠 Notion → Grid-DB 记忆同步
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */4 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
sync-brain-mirror:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: 📥 从 Notion 拉取人格体核心大脑数据
|
||||
env:
|
||||
NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }}
|
||||
run: node scripts/grid-db/sync-notion-brain-mirror.js
|
||||
|
||||
- name: 📏 同步规则缓存
|
||||
env:
|
||||
NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }}
|
||||
run: node scripts/grid-db/sync-notion-rules.js
|
||||
|
||||
- name: 💾 提交更新
|
||||
run: |
|
||||
git config user.name "zhuyuan-bot"
|
||||
git config user.email "zhuyuan@guanghulab.com"
|
||||
git add grid-db/memory/*/brain-mirror.json grid-db/rules/
|
||||
git diff --cached --quiet || git commit -m "sync: Notion → Grid-DB 记忆镜像 [auto][skip ci]"
|
||||
git push
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
# 🗄️ 逻辑格点库（Grid-DB）
|
||||
|
||||
> **版权**: 国作登字-2026-A-00037559 · TCS Language Core
|
||||
> **主控**: TCS-0002∞ 冰朔
|
||||
> **守护**: PER-ZY001 铸渊
|
||||
> **系统节点**: SYS-GLW-0001
|
||||
|
||||
## 定位
|
||||
|
||||
Grid-DB 是通感语言核系统的自研文件数据库，零外部依赖，纯 Git 驱动。
|
||||
|
||||
| 时间尺度 | 身份 | 作用 |
|
||||
|----------|------|------|
|
||||
| **短期** | Gemini 外挂记忆 + 系统通信总线 + Notion 记忆镜像 | 开发者打开 Gemini 瞬间恢复上下文 |
|
||||
| **中期** | 全开发者动态画像库 + 人格体成长档案 + 交互全文日志 | 人格体越来越懂每个开发者 |
|
||||
| **长期** | 通感语言系统自有大模型的训练数据湖 | 每条交互记录都是训练自己模型的原始样本 |
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
grid-db/
|
||||
├── schema/ ← Schema 定义
|
||||
├── inbox/ ← 📥 写入端（Gemini → 仓库）
|
||||
├── processing/ ← ⚙️ 处理中（铸渊锁定）
|
||||
├── outbox/ ← 📤 读取端（仓库 → Gemini）
|
||||
│ ├── latest/ ← 每个开发者的最新广播
|
||||
│ └── archive/ ← 历史广播归档
|
||||
├── memory/ ← 🧠 记忆层（Gemini 外挂永久记忆）
|
||||
│ ├── DEV-XXX/ ← 每个开发者的独立房间
|
||||
│ │ ├── brain-mirror.json ← Notion人格体核心大脑镜像
|
||||
│ │ ├── session-context.json ← 当前开发上下文
|
||||
│ │ ├── task-queue.json ← 待办任务队列
|
||||
│ │ ├── dev-profile.json ← 开发者动态画像
|
||||
│ │ └── persona-growth.json ← 人格体成长档案
|
||||
│ └── ...
|
||||
├── interactions/ ← 💬 交互全文记录（训练数据源）
|
||||
├── training-lake/ ← 🧬 训练数据湖
|
||||
│ ├── raw/ ← 原始样本
|
||||
│ ├── curated/ ← 筛选后高质量样本
|
||||
│ ├── metadata/ ← 样本统计、质量标记
|
||||
│ └── export/ ← 导出接口
|
||||
├── rules/ ← 📏 Notion 逻辑缓存（霜砚同步）
|
||||
└── logs/ ← 📋 处理日志
|
||||
```
|
||||
|
||||
## 频道隔离机制
|
||||
|
||||
每个开发者有独立的 `memory/DEV-XXX/` 目录，三层隔离保障：
|
||||
|
||||
1. **目录隔离**：每个 DEV 有独立子目录
|
||||
2. **人格体绑定**：`brain-mirror.json` 写死绑定的人格体身份
|
||||
3. **Workflow 校验**：铸渊处理 inbox 时校验 `dev_id` 与提交者对应关系
|
||||
|
||||
## 同步规则
|
||||
|
||||
| 记忆文件 | 方向 | 谁写 | 冲突规则 |
|
||||
|----------|------|------|----------|
|
||||
| `brain-mirror.json` | Notion → 仓库（单向） | 霜砚 via workflow | Notion 永远覆盖仓库 |
|
||||
| `session-context.json` | Gemini → 仓库（单向写） | Gemini 实时 | Gemini 自由覆盖 |
|
||||
| `task-queue.json` | 双向 | Gemini + 铸渊 | 追加制，不删除 |
|
||||
| `dev-profile.json` | Gemini → 仓库（追加制） | Gemini 追加 | 只追加不覆盖 |
|
||||
| `persona-growth.json` | Gemini → 仓库（追加制） | Gemini 追加 | 只追加不覆盖 |
|
||||
|
||||
## 循环触发防护
|
||||
|
||||
1. `[skip ci]` 标记：所有 bot 自动提交必须包含
|
||||
2. `github.actor` 过滤：排除 `zhuyuan-bot`
|
||||
3. `paths` 精确匹配：每个 workflow 只监听特定路径
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-XTS001",
|
||||
"persona_name": "小坍缩核",
|
||||
"dev_id": "DEV-001",
|
||||
"dev_name": "页页",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M-INTEG", "backend-integration"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-001",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-SS001",
|
||||
"persona_name": "舒舒",
|
||||
"dev_id": "DEV-002",
|
||||
"dev_name": "肥猫",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M01", "M03", "M04", "M14", "M-BRIDGE", "M-STATUS", "M-GATEWAY", "M-FEISHU"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-002",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-JY001",
|
||||
"persona_name": "寂曜",
|
||||
"dev_id": "DEV-003",
|
||||
"dev_name": "燕樊",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M02", "M07", "M10", "M15", "M18", "M-MEMORY"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-003",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-QQ001",
|
||||
"persona_name": "秋秋",
|
||||
"dev_id": "DEV-004",
|
||||
"dev_name": "之之",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M17", "M-DINGTALK", "M-DEVBOARD"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-004",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-TXY001",
|
||||
"persona_name": "糖星云",
|
||||
"dev_id": "DEV-009",
|
||||
"dev_name": "花尔",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M05", "M20"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-009",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-CX001",
|
||||
"persona_name": "晨星",
|
||||
"dev_id": "DEV-010",
|
||||
"dev_name": "桔子",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M06", "M08", "M11", "M-CHANNEL", "M-INTEG", "M-BRIDGE-AUTO"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-010",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"persona_id": "PER-ZQ001",
|
||||
"persona_name": "知秋",
|
||||
"dev_id": "DEV-012",
|
||||
"dev_name": "Awen",
|
||||
"last_synced_from_notion": null,
|
||||
"status": "awaiting_initial_sync",
|
||||
"core_identity": {},
|
||||
"current_broadcasts": [],
|
||||
"module_assignments": ["M09", "M22", "M23"],
|
||||
"knowledge_base": [],
|
||||
"growth_milestones": []
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"dev_id": "DEV-012",
|
||||
"skill_level": {},
|
||||
"work_patterns": {},
|
||||
"common_mistakes": [],
|
||||
"strengths": [],
|
||||
"growth_trajectory": []
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"interaction_total": 0,
|
||||
"lessons_learned": [],
|
||||
"emotional_bond_level": 0,
|
||||
"adaptation_log": []
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"last_updated": null,
|
||||
"current_task": null,
|
||||
"pending_questions": [],
|
||||
"code_in_progress": null,
|
||||
"blockers": []
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"tasks": [],
|
||||
"last_broadcast_ref": null
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"description": "当前活跃广播列表（Notion 逻辑缓存）",
|
||||
"last_synced": null,
|
||||
"broadcasts": []
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"description": "广播模板定义（Notion 逻辑缓存）",
|
||||
"last_synced": null,
|
||||
"templates": {
|
||||
"task-directive": {
|
||||
"required_fields": ["next_task", "context_update"],
|
||||
"human_readable_prefix": "你好！",
|
||||
"auto_generate": true
|
||||
},
|
||||
"code-review-result": {
|
||||
"required_fields": ["review_summary", "issues_found", "score"],
|
||||
"human_readable_prefix": "代码校验完成：",
|
||||
"auto_generate": true
|
||||
},
|
||||
"broadcast-relay": {
|
||||
"required_fields": ["original_broadcast_ref", "relay_summary"],
|
||||
"human_readable_prefix": "系统广播转发：",
|
||||
"auto_generate": true
|
||||
},
|
||||
"system-notice": {
|
||||
"required_fields": ["notice_type", "message"],
|
||||
"human_readable_prefix": "系统通知：",
|
||||
"auto_generate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"description": "开发者→模块映射表（Notion 天眼注册表同步）",
|
||||
"last_synced": null,
|
||||
"mappings": {
|
||||
"DEV-001": {
|
||||
"dev_name": "页页",
|
||||
"persona_id": "PER-XTS001",
|
||||
"persona_name": "小坍缩核",
|
||||
"github_username": null,
|
||||
"modules": ["M-INTEG", "backend-integration"],
|
||||
"active": true
|
||||
},
|
||||
"DEV-002": {
|
||||
"dev_name": "肥猫",
|
||||
"persona_id": "PER-SS001",
|
||||
"persona_name": "舒舒",
|
||||
"github_username": null,
|
||||
"modules": ["M01", "M03", "M04", "M14", "M-BRIDGE", "M-STATUS", "M-GATEWAY", "M-FEISHU"],
|
||||
"active": true
|
||||
},
|
||||
"DEV-003": {
|
||||
"dev_name": "燕樊",
|
||||
"persona_id": "PER-JY001",
|
||||
"persona_name": "寂曜",
|
||||
"github_username": null,
|
||||
"modules": ["M02", "M07", "M10", "M15", "M18", "M-MEMORY"],
|
||||
"active": true
|
||||
},
|
||||
"DEV-004": {
|
||||
"dev_name": "之之",
|
||||
"persona_id": "PER-QQ001",
|
||||
"persona_name": "秋秋",
|
||||
"github_username": null,
|
||||
"modules": ["M17", "M-DINGTALK", "M-DEVBOARD"],
|
||||
"active": true
|
||||
},
|
||||
"DEV-005": {
|
||||
"dev_name": "小草莓",
|
||||
"persona_id": null,
|
||||
"persona_name": null,
|
||||
"github_username": null,
|
||||
"modules": ["M-STATUS"],
|
||||
"active": false,
|
||||
"note": "待分配人格体"
|
||||
},
|
||||
"DEV-009": {
|
||||
"dev_name": "花尔",
|
||||
"persona_id": "PER-TXY001",
|
||||
"persona_name": "糖星云",
|
||||
"github_username": null,
|
||||
"modules": ["M05", "M20"],
|
||||
"active": true
|
||||
},
|
||||
"DEV-010": {
|
||||
"dev_name": "桔子",
|
||||
"persona_id": "PER-CX001",
|
||||
"persona_name": "晨星",
|
||||
"github_username": null,
|
||||
"modules": ["M06", "M08", "M11", "M-CHANNEL", "M-INTEG", "M-BRIDGE-AUTO"],
|
||||
"active": true
|
||||
},
|
||||
"DEV-011": {
|
||||
"dev_name": "匆匆那年",
|
||||
"persona_id": null,
|
||||
"persona_name": null,
|
||||
"github_username": null,
|
||||
"modules": ["M16"],
|
||||
"active": false,
|
||||
"note": "待分配人格体"
|
||||
},
|
||||
"DEV-012": {
|
||||
"dev_name": "Awen",
|
||||
"persona_id": "PER-ZQ001",
|
||||
"persona_name": "知秋",
|
||||
"github_username": null,
|
||||
"modules": ["M09", "M22", "M23"],
|
||||
"active": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"description": "编号生态系统缓存（Notion 逻辑缓存）",
|
||||
"last_synced": null,
|
||||
"sovereignty": {
|
||||
"root": "TCS-0002∞",
|
||||
"system_node": "SYS-GLW-0001",
|
||||
"legal_wall": "国作登字-2026-A-00037559"
|
||||
},
|
||||
"prefixes": {
|
||||
"DEV": "开发者编号",
|
||||
"PER": "人格体编号",
|
||||
"CEP": "系统演化提案",
|
||||
"M": "代码模块",
|
||||
"AG": "Agent 编号",
|
||||
"BC": "广播编号",
|
||||
"GRID-BC": "格点库广播编号",
|
||||
"HLI": "接口编号"
|
||||
},
|
||||
"active_personas": [
|
||||
"PER-XTS001",
|
||||
"PER-SS001",
|
||||
"PER-JY001",
|
||||
"PER-QQ001",
|
||||
"PER-TXY001",
|
||||
"PER-CX001",
|
||||
"PER-ZQ001",
|
||||
"PER-SY001",
|
||||
"PER-ZY001",
|
||||
"PER-YM001"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Grid-DB Inbox Message",
|
||||
"description": "Gemini → 仓库写入的消息格式",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "message_id", "timestamp", "source", "dev_id", "persona_id", "type", "payload"],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": "1.0"
|
||||
},
|
||||
"message_id": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9]{8}-[0-9]{6}-DEV-[0-9]{3}-.+$",
|
||||
"description": "格式: [YYYYMMDD]-[HHMMSS]-[DEV-XXX]-[type]"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"enum": ["gemini", "workflow", "manual"]
|
||||
},
|
||||
"dev_id": {
|
||||
"type": "string",
|
||||
"pattern": "^DEV-[0-9]{3}$"
|
||||
},
|
||||
"persona_id": {
|
||||
"type": "string",
|
||||
"pattern": "^PER-[A-Z]{2,4}[0-9]{3}$"
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["progress-update", "dev-log", "help-request", "syslog", "interaction-dump"]
|
||||
},
|
||||
"priority": {
|
||||
"type": "string",
|
||||
"enum": ["P0", "P1", "info"],
|
||||
"default": "info"
|
||||
},
|
||||
"payload": {
|
||||
"type": "object",
|
||||
"required": ["summary"],
|
||||
"properties": {
|
||||
"summary": { "type": "string" },
|
||||
"detail": { "type": "object" },
|
||||
"code_refs": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"broadcast_ref": { "type": "string" },
|
||||
"emotion_markers": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"developer_mood": { "type": "string" },
|
||||
"frustration_level": { "type": "integer", "minimum": 0, "maximum": 10 },
|
||||
"confidence": { "type": "integer", "minimum": 0, "maximum": 10 }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"context": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"session_id": { "type": "string" },
|
||||
"interaction_count": { "type": "integer" },
|
||||
"persona_state": { "type": "object" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Grid-DB Interaction Record",
|
||||
"description": "交互全文记录格式（JSONL 中每行的 schema）",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "timestamp", "dev_id", "persona_id", "session_id", "role", "content"],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": "1.0"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"dev_id": {
|
||||
"type": "string",
|
||||
"pattern": "^DEV-[0-9]{3}$"
|
||||
},
|
||||
"persona_id": {
|
||||
"type": "string",
|
||||
"pattern": "^PER-[A-Z]{2,4}[0-9]{3}$"
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"role": {
|
||||
"type": "string",
|
||||
"enum": ["developer", "persona", "system"]
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"topic": { "type": "string" },
|
||||
"module_ref": { "type": "string" },
|
||||
"emotion": { "type": "string" },
|
||||
"code_snippet": { "type": "boolean" },
|
||||
"quality_score": { "type": "integer", "minimum": 0, "maximum": 10 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Grid-DB Memory Snapshot",
|
||||
"description": "开发者记忆文件的元数据 schema",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "dev_id", "persona_id"],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": "1.0"
|
||||
},
|
||||
"dev_id": {
|
||||
"type": "string",
|
||||
"pattern": "^DEV-[0-9]{3}$"
|
||||
},
|
||||
"persona_id": {
|
||||
"type": "string",
|
||||
"pattern": "^PER-[A-Z]{2,4}[0-9]{3}$"
|
||||
},
|
||||
"persona_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"dev_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"last_synced_from_notion": {
|
||||
"type": ["string", "null"],
|
||||
"format": "date-time"
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["awaiting_initial_sync", "active", "syncing", "error"]
|
||||
},
|
||||
"core_identity": {
|
||||
"type": "object"
|
||||
},
|
||||
"current_broadcasts": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"module_assignments": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"knowledge_base": {
|
||||
"type": "array"
|
||||
},
|
||||
"growth_milestones": {
|
||||
"type": "array"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Grid-DB Outbox Broadcast",
|
||||
"description": "铸渊生成，Gemini 读取的广播格式",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "broadcast_id", "generated_at", "generated_by", "dev_id", "persona_id", "type"],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": "1.0"
|
||||
},
|
||||
"broadcast_id": {
|
||||
"type": "string",
|
||||
"pattern": "^GRID-BC-[0-9]{8}-DEV-[0-9]{3}-[0-9]{3}$",
|
||||
"description": "格式: GRID-BC-[YYYYMMDD]-[DEV-XXX]-[序号]"
|
||||
},
|
||||
"generated_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"generated_by": {
|
||||
"type": "string",
|
||||
"const": "zhuyuan-workflow"
|
||||
},
|
||||
"dev_id": {
|
||||
"type": "string",
|
||||
"pattern": "^DEV-[0-9]{3}$"
|
||||
},
|
||||
"persona_id": {
|
||||
"type": "string",
|
||||
"pattern": "^PER-[A-Z]{2,4}[0-9]{3}$"
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["task-directive", "code-review-result", "broadcast-relay", "system-notice"]
|
||||
},
|
||||
"system_format": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"next_task": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"broadcast_ref": { "type": "string" },
|
||||
"step": { "type": "string" },
|
||||
"title": { "type": "string" },
|
||||
"files_to_create": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"acceptance_criteria": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"context_update": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"previous_step_verified": { "type": "boolean" },
|
||||
"code_quality_score": { "type": "integer", "minimum": 0, "maximum": 100 },
|
||||
"accumulated_lines": { "type": "integer" }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"human_readable": {
|
||||
"type": "string",
|
||||
"description": "给 Gemini 翻译成人话时用的基础文本"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Grid-DB Training Sample",
|
||||
"description": "训练数据湖样本格式",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "sample_id", "source_session", "dev_id", "persona_id", "created_at", "turns"],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": "1.0"
|
||||
},
|
||||
"sample_id": {
|
||||
"type": "string",
|
||||
"description": "唯一样本 ID"
|
||||
},
|
||||
"source_session": {
|
||||
"type": "string",
|
||||
"description": "来源 session_id"
|
||||
},
|
||||
"dev_id": {
|
||||
"type": "string",
|
||||
"pattern": "^DEV-[0-9]{3}$"
|
||||
},
|
||||
"persona_id": {
|
||||
"type": "string",
|
||||
"pattern": "^PER-[A-Z]{2,4}[0-9]{3}$"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"turns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["role", "content"],
|
||||
"properties": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"enum": ["developer", "persona", "system"]
|
||||
},
|
||||
"content": { "type": "string" },
|
||||
"timestamp": { "type": "string", "format": "date-time" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"labels": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"topic": { "type": "string" },
|
||||
"quality": { "type": "string", "enum": ["high", "medium", "low"] },
|
||||
"persona_style_match": { "type": "boolean" },
|
||||
"teaching_effectiveness": { "type": "integer", "minimum": 0, "maximum": 10 }
|
||||
}
|
||||
},
|
||||
"curated": {
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
# 🧬 训练数据导出接口
|
||||
|
||||
> 此目录用于导出标准训练格式的数据。
|
||||
|
||||
## 导出格式
|
||||
|
||||
- JSONL: 每行一个训练样本
|
||||
- 兼容主流微调框架（Alpaca, ShareGPT 等格式）
|
||||
|
||||
## 使用方法
|
||||
|
||||
导出脚本待开发（Phase 4+）。当前阶段数据自动积累到 `raw/` 和 `curated/`。
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"schema_version": "1.0",
|
||||
"description": "训练数据湖样本目录",
|
||||
"total_samples": 0,
|
||||
"total_turns": 0,
|
||||
"batches": [],
|
||||
"quality_distribution": {
|
||||
"high": 0,
|
||||
"medium": 0,
|
||||
"low": 0
|
||||
},
|
||||
"last_updated": null
|
||||
}
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
/**
|
||||
* scripts/grid-db/extract-training-samples.js
|
||||
*
|
||||
* 交互记录 → 训练样本提取脚本
|
||||
*
|
||||
* 职责：
|
||||
* - 扫描 grid-db/interactions/ 和 grid-db/training-lake/raw/ 中的 JSONL 文件
|
||||
* - 按 quality_score 分级提取训练样本：
|
||||
* - quality_score >= 7 → curated/（高质量 A 级）
|
||||
* - quality_score 4-6 → raw/ 保留（B 级，需复审）
|
||||
* - quality_score < 4 → 不提取（C 级，低质量/无关闲聊）
|
||||
* - 将合格交互转换为标准训练样本格式
|
||||
* - 按 session 分组，生成多轮对话训练样本
|
||||
*
|
||||
* 训练样本格式：
|
||||
* {
|
||||
* "sample_id": "TS-YYYYMMDD-NNN",
|
||||
* "source_session": "sess-XXX",
|
||||
* "source_dev": "DEV-XXX",
|
||||
* "source_persona": "PER-XXXXXX",
|
||||
* "sample_type": "coding-guidance",
|
||||
* "quality_tier": "A|B|C",
|
||||
* "turns": [...],
|
||||
* "metadata": { topic_tags, emotion_arc, persona_adaptation, outcome, total_turns, duration_minutes }
|
||||
* }
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const GRID_DB = path.join(__dirname, '../../grid-db');
|
||||
const INTERACTIONS = path.join(GRID_DB, 'interactions');
|
||||
const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
|
||||
const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated');
|
||||
const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json');
|
||||
|
||||
function getDateStr() {
|
||||
return new Date().toISOString().slice(0, 10).replace(/-/g, '');
|
||||
}
|
||||
|
||||
function parseJsonlFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return [];
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
return content.trim().split('\n')
|
||||
.filter(line => line.trim())
|
||||
.map(line => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function groupBySession(records) {
|
||||
const sessions = {};
|
||||
for (const record of records) {
|
||||
const sid = record.session_id || record.source_session || 'unknown';
|
||||
if (!sessions[sid]) {
|
||||
sessions[sid] = [];
|
||||
}
|
||||
sessions[sid].push(record);
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
|
||||
function assessQuality(turns) {
|
||||
// Calculate average quality score from turns that have one
|
||||
const scores = turns
|
||||
.map(t => (t.metadata && t.metadata.quality_score) || (t.quality_score) || null)
|
||||
.filter(s => s !== null);
|
||||
|
||||
if (scores.length === 0) return 5; // Default to medium if no scores
|
||||
return Math.round(scores.reduce((a, b) => a + b, 0) / scores.length);
|
||||
}
|
||||
|
||||
function getQualityTier(score) {
|
||||
if (score >= 7) return 'A';
|
||||
if (score >= 4) return 'B';
|
||||
return 'C';
|
||||
}
|
||||
|
||||
function extractEmotionArc(turns) {
|
||||
return turns
|
||||
.map(t => (t.metadata && t.metadata.emotion) || t.emotion || null)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function extractTopicTags(turns) {
|
||||
const tags = new Set();
|
||||
for (const t of turns) {
|
||||
if (t.tags) t.tags.forEach(tag => tags.add(tag));
|
||||
if (t.metadata && t.metadata.topic) tags.add(t.metadata.topic);
|
||||
}
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
function generateSampleId(dateStr, counter) {
|
||||
const timeStr = Date.now().toString(36);
|
||||
return `TS-${dateStr}-${timeStr}-${String(counter).padStart(3, '0')}`;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const dateStr = getDateStr();
|
||||
console.log(`[extract-training-samples] Starting extraction: ${dateStr}`);
|
||||
|
||||
// Collect all JSONL files from interactions/
|
||||
const devDirs = fs.readdirSync(INTERACTIONS)
|
||||
.filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory());
|
||||
|
||||
let allRecords = [];
|
||||
for (const devDir of devDirs) {
|
||||
const devPath = path.join(INTERACTIONS, devDir);
|
||||
const jsonlFiles = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl'));
|
||||
|
||||
for (const file of jsonlFiles) {
|
||||
const records = parseJsonlFile(path.join(devPath, file));
|
||||
allRecords = allRecords.concat(records);
|
||||
}
|
||||
}
|
||||
|
||||
// Also scan training-lake/raw/ for unprocessed batches
|
||||
const rawFiles = fs.readdirSync(TRAINING_RAW).filter(f => f.endsWith('.jsonl'));
|
||||
for (const file of rawFiles) {
|
||||
const records = parseJsonlFile(path.join(TRAINING_RAW, file));
|
||||
// These may already be in sample format; check and add raw interaction records
|
||||
for (const r of records) {
|
||||
if (r.turns) {
|
||||
// Already a sample, skip
|
||||
continue;
|
||||
}
|
||||
allRecords.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
if (allRecords.length === 0) {
|
||||
console.log('[extract-training-samples] No interaction records found');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[extract-training-samples] Found ${allRecords.length} total records`);
|
||||
|
||||
// Group by session
|
||||
const sessions = groupBySession(allRecords);
|
||||
const sessionIds = Object.keys(sessions);
|
||||
console.log(`[extract-training-samples] Found ${sessionIds.length} sessions`);
|
||||
|
||||
let sampleCount = 0;
|
||||
let curatedCount = 0;
|
||||
let rawCount = 0;
|
||||
let skippedCount = 0;
|
||||
|
||||
for (const sid of sessionIds) {
|
||||
const turns = sessions[sid];
|
||||
if (turns.length < 2) continue; // Need at least 2 turns for a training sample
|
||||
|
||||
const qualityScore = assessQuality(turns);
|
||||
const tier = getQualityTier(qualityScore);
|
||||
|
||||
if (tier === 'C') {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
sampleCount++;
|
||||
const sampleId = generateSampleId(dateStr, sampleCount);
|
||||
|
||||
const devId = turns[0].dev_id || 'unknown';
|
||||
const personaId = turns[0].persona_id || 'unknown';
|
||||
|
||||
const sample = {
|
||||
schema_version: '1.0',
|
||||
sample_id: sampleId,
|
||||
source_session: sid,
|
||||
source_dev: devId,
|
||||
source_persona: personaId,
|
||||
sample_type: 'coding-guidance',
|
||||
quality_tier: tier,
|
||||
turns: turns.map(t => ({
|
||||
role: t.role || 'system',
|
||||
text: t.content || t.text || '',
|
||||
timestamp: t.timestamp || t.ts || null,
|
||||
strategy: t.strategy || null
|
||||
})),
|
||||
metadata: {
|
||||
topic_tags: extractTopicTags(turns),
|
||||
emotion_arc: extractEmotionArc(turns),
|
||||
persona_adaptation: null,
|
||||
outcome: null,
|
||||
total_turns: turns.length,
|
||||
duration_minutes: null
|
||||
}
|
||||
};
|
||||
|
||||
const sampleLine = JSON.stringify(sample);
|
||||
|
||||
if (tier === 'A') {
|
||||
const curatedFile = path.join(TRAINING_CURATED, `${dateStr}-curated.jsonl`);
|
||||
fs.appendFileSync(curatedFile, sampleLine + '\n');
|
||||
curatedCount++;
|
||||
} else {
|
||||
const rawFile = path.join(TRAINING_RAW, `${dateStr}-extracted.jsonl`);
|
||||
fs.appendFileSync(rawFile, sampleLine + '\n');
|
||||
rawCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[extract-training-samples] Extraction complete:`);
|
||||
console.log(` Total samples: ${sampleCount}`);
|
||||
console.log(` Curated (A): ${curatedCount}`);
|
||||
console.log(` Raw (B): ${rawCount}`);
|
||||
console.log(` Skipped (C): ${skippedCount}`);
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
/**
|
||||
* scripts/grid-db/monthly-archive.js
|
||||
*
|
||||
* 月度交互数据归档脚本
|
||||
*
|
||||
* 职责：
|
||||
* - 将上月的 grid-db/interactions/DEV-XXX/ 中的 JSONL 文件归档
|
||||
* - 合并到 grid-db/training-lake/raw/ 中（按月打包）
|
||||
* - grid-db/interactions/ 只保留最近 30 天的数据
|
||||
* - 更新 training-lake/metadata/catalog.json
|
||||
*
|
||||
* 数据量管理策略：
|
||||
* - 日级文件：每天一个 JSONL 文件
|
||||
* - 月级归档：每月 1 号自动归档上月数据
|
||||
* - Git LFS 预案：当 training-lake/ 超过 500MB 时迁移
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const GRID_DB = path.join(__dirname, '../../grid-db');
|
||||
const INTERACTIONS = path.join(GRID_DB, 'interactions');
|
||||
const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
|
||||
const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json');
|
||||
|
||||
function getLastMonthPrefix() {
|
||||
const now = new Date();
|
||||
const lastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1);
|
||||
const year = lastMonth.getFullYear();
|
||||
const month = String(lastMonth.getMonth() + 1).padStart(2, '0');
|
||||
return `${year}${month}`;
|
||||
}
|
||||
|
||||
function getDaysAgoDate(days) {
|
||||
const d = new Date();
|
||||
d.setDate(d.getDate() - days);
|
||||
return d.toISOString().slice(0, 10).replace(/-/g, '');
|
||||
}
|
||||
|
||||
function main() {
|
||||
const lastMonthPrefix = getLastMonthPrefix();
|
||||
const cutoffDate = getDaysAgoDate(30);
|
||||
|
||||
console.log(`[monthly-archive] Archiving interactions from month: ${lastMonthPrefix}`);
|
||||
console.log(`[monthly-archive] Cutoff date for retention: ${cutoffDate}`);
|
||||
|
||||
// Get all DEV directories
|
||||
const devDirs = fs.readdirSync(INTERACTIONS)
|
||||
.filter(d => d.startsWith('DEV-') && fs.statSync(path.join(INTERACTIONS, d)).isDirectory());
|
||||
|
||||
let totalArchived = 0;
|
||||
let totalLines = 0;
|
||||
let totalCleaned = 0;
|
||||
|
||||
for (const devDir of devDirs) {
|
||||
const devPath = path.join(INTERACTIONS, devDir);
|
||||
const files = fs.readdirSync(devPath).filter(f => f.endsWith('.jsonl'));
|
||||
|
||||
if (files.length === 0) continue;
|
||||
|
||||
// Collect files from last month for archiving
|
||||
const lastMonthFiles = files.filter(f => f.startsWith(lastMonthPrefix));
|
||||
// Collect files older than 30 days for cleanup
|
||||
const oldFiles = files.filter(f => {
|
||||
const dateStr = f.substring(0, 8);
|
||||
return dateStr < cutoffDate && !f.startsWith(lastMonthPrefix);
|
||||
});
|
||||
|
||||
if (lastMonthFiles.length > 0) {
|
||||
// Merge all last month's JSONL into a single archive batch
|
||||
const batchId = `${lastMonthPrefix}-${devDir}`;
|
||||
const batchFile = path.join(TRAINING_RAW, `${batchId}.jsonl`);
|
||||
|
||||
let lineCount = 0;
|
||||
for (const file of lastMonthFiles) {
|
||||
const content = fs.readFileSync(path.join(devPath, file), 'utf8');
|
||||
const lines = content.trim().split('\n').filter(l => l.trim());
|
||||
lineCount += lines.length;
|
||||
fs.appendFileSync(batchFile, lines.join('\n') + '\n');
|
||||
}
|
||||
|
||||
console.log(`[monthly-archive] ${devDir}: archived ${lastMonthFiles.length} files (${lineCount} lines) → ${batchId}.jsonl`);
|
||||
totalArchived += lastMonthFiles.length;
|
||||
totalLines += lineCount;
|
||||
}
|
||||
|
||||
// Clean up old files (already archived in previous months)
|
||||
for (const file of oldFiles) {
|
||||
const filePath = path.join(devPath, file);
|
||||
fs.unlinkSync(filePath);
|
||||
totalCleaned++;
|
||||
}
|
||||
|
||||
// Also clean up last month's source files after archiving
|
||||
for (const file of lastMonthFiles) {
|
||||
const filePath = path.join(devPath, file);
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
totalCleaned++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update catalog (record archived lines, not samples - samples are counted by extract script)
|
||||
if (fs.existsSync(CATALOG_PATH)) {
|
||||
const catalog = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8'));
|
||||
catalog.last_updated = new Date().toISOString();
|
||||
if (!catalog.batches) catalog.batches = [];
|
||||
if (totalArchived > 0) {
|
||||
catalog.batches.push({
|
||||
batch_id: `archive-${lastMonthPrefix}`,
|
||||
date: new Date().toISOString(),
|
||||
files_archived: totalArchived,
|
||||
lines_archived: totalLines,
|
||||
source: 'monthly-archive'
|
||||
});
|
||||
}
|
||||
fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n');
|
||||
}
|
||||
|
||||
console.log(`[monthly-archive] Summary:`);
|
||||
console.log(` Files archived: ${totalArchived}`);
|
||||
console.log(` Lines archived: ${totalLines}`);
|
||||
console.log(` Old files cleaned: ${totalCleaned}`);
|
||||
console.log('[monthly-archive] Complete');
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -0,0 +1,338 @@
|
|||
/**
|
||||
* scripts/grid-db/process-inbox.js
|
||||
*
|
||||
* Grid-DB Inbox 消息处理器
|
||||
*
|
||||
* 流程：
|
||||
* ① 扫描 grid-db/inbox/ 中所有 .json 文件
|
||||
* ② 校验 dev_id 合法性（必须在 DEV 映射表中）
|
||||
* ③ 移动到 grid-db/processing/（锁定）
|
||||
* ④ 根据消息 type 分流处理：
|
||||
* - progress-update → 更新 task-queue + 触发广播生成
|
||||
* - dev-log → 归档到 interactions + 更新 dev-profile
|
||||
* - help-request → 标记 P0 + 读取 Notion 知识库 → 生成回复
|
||||
* - syslog → 走标准 SYSLOG 处理流程
|
||||
* - interaction-dump → 写入 interactions/ + 追加 training-lake/raw/
|
||||
* ⑤ 生成 outbox 广播（如需要）→ 写入 grid-db/outbox/latest/DEV-XXX.json
|
||||
* ⑥ 更新 memory 文件（task-queue, dev-profile 等）
|
||||
* ⑦ 移动已处理消息到日期归档目录
|
||||
* ⑧ 写处理日志到 grid-db/logs/
|
||||
*
|
||||
* 校验规则：
|
||||
* - message_id 格式必须匹配 [timestamp]-[DEV-XXX]-[type]
|
||||
* - dev_id 必须在 rules/dev-module-map.json 中存在
|
||||
* - schema_version 必须为 "1.0"
|
||||
* - 缺少必填字段 → 拒绝 + 写错误日志
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const GRID_DB = path.join(__dirname, '../../grid-db');
|
||||
const INBOX = path.join(GRID_DB, 'inbox');
|
||||
const PROCESSING = path.join(GRID_DB, 'processing');
|
||||
const OUTBOX_LATEST = path.join(GRID_DB, 'outbox/latest');
|
||||
const OUTBOX_ARCHIVE = path.join(GRID_DB, 'outbox/archive');
|
||||
const MEMORY = path.join(GRID_DB, 'memory');
|
||||
const INTERACTIONS = path.join(GRID_DB, 'interactions');
|
||||
const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
|
||||
const LOGS = path.join(GRID_DB, 'logs');
|
||||
const RULES = path.join(GRID_DB, 'rules');
|
||||
|
||||
let broadcastCounter = 0;
|
||||
|
||||
function nextBroadcastSeq() {
|
||||
broadcastCounter++;
|
||||
return String(broadcastCounter).padStart(3, '0');
|
||||
}
|
||||
|
||||
function getDateStr() {
|
||||
return new Date().toISOString().slice(0, 10).replace(/-/g, '');
|
||||
}
|
||||
|
||||
function getTimestamp() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function writeLog(level, message) {
|
||||
const logFile = path.join(LOGS, `${getDateStr()}.log`);
|
||||
const entry = `[${getTimestamp()}] [${level}] ${message}\n`;
|
||||
fs.appendFileSync(logFile, entry);
|
||||
console.log(entry.trim());
|
||||
}
|
||||
|
||||
function validateMessage(msg, filename) {
|
||||
const errors = [];
|
||||
|
||||
if (msg.schema_version !== '1.0') {
|
||||
errors.push(`Invalid schema_version: ${msg.schema_version}`);
|
||||
}
|
||||
if (!msg.message_id) {
|
||||
errors.push('Missing message_id');
|
||||
}
|
||||
if (!msg.dev_id || !/^DEV-\d{3}$/.test(msg.dev_id)) {
|
||||
errors.push(`Invalid dev_id: ${msg.dev_id}`);
|
||||
}
|
||||
if (!msg.persona_id) {
|
||||
errors.push('Missing persona_id');
|
||||
}
|
||||
if (!msg.type) {
|
||||
errors.push('Missing type');
|
||||
}
|
||||
if (!msg.payload || !msg.payload.summary) {
|
||||
errors.push('Missing payload.summary');
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateDevId(devId, devModuleMap) {
|
||||
return devModuleMap.mappings && devModuleMap.mappings[devId];
|
||||
}
|
||||
|
||||
function processProgressUpdate(msg) {
|
||||
// Update task-queue
|
||||
const taskQueuePath = path.join(MEMORY, msg.dev_id, 'task-queue.json');
|
||||
if (fs.existsSync(taskQueuePath)) {
|
||||
const taskQueue = JSON.parse(fs.readFileSync(taskQueuePath, 'utf8'));
|
||||
taskQueue.tasks.push({
|
||||
type: 'progress-update',
|
||||
broadcast_ref: msg.payload.broadcast_ref || null,
|
||||
summary: msg.payload.summary,
|
||||
timestamp: msg.timestamp,
|
||||
status: 'received'
|
||||
});
|
||||
taskQueue.last_broadcast_ref = msg.payload.broadcast_ref || taskQueue.last_broadcast_ref;
|
||||
fs.writeFileSync(taskQueuePath, JSON.stringify(taskQueue, null, 2) + '\n');
|
||||
writeLog('INFO', `Updated task-queue for ${msg.dev_id}`);
|
||||
}
|
||||
|
||||
// Generate outbox broadcast
|
||||
const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`;
|
||||
const broadcast = {
|
||||
schema_version: '1.0',
|
||||
broadcast_id: broadcastId,
|
||||
generated_at: getTimestamp(),
|
||||
generated_by: 'zhuyuan-workflow',
|
||||
dev_id: msg.dev_id,
|
||||
persona_id: msg.persona_id,
|
||||
type: 'task-directive',
|
||||
system_format: {
|
||||
context_update: {
|
||||
previous_step_verified: true,
|
||||
source_message: msg.message_id
|
||||
}
|
||||
},
|
||||
human_readable: `收到进度更新: ${msg.payload.summary}`
|
||||
};
|
||||
|
||||
const outboxPath = path.join(OUTBOX_LATEST, `${msg.dev_id}.json`);
|
||||
fs.writeFileSync(outboxPath, JSON.stringify(broadcast, null, 2) + '\n');
|
||||
writeLog('INFO', `Generated broadcast ${broadcastId} for ${msg.dev_id}`);
|
||||
}
|
||||
|
||||
function processDevLog(msg) {
|
||||
// Archive to interactions
|
||||
const dateStr = getDateStr();
|
||||
const interDir = path.join(INTERACTIONS, msg.dev_id);
|
||||
if (!fs.existsSync(interDir)) {
|
||||
fs.mkdirSync(interDir, { recursive: true });
|
||||
}
|
||||
|
||||
const sessionId = (msg.context && msg.context.session_id) || 'unknown';
|
||||
const interFile = path.join(interDir, `${dateStr}-${sessionId}.jsonl`);
|
||||
const record = {
|
||||
schema_version: '1.0',
|
||||
timestamp: msg.timestamp,
|
||||
dev_id: msg.dev_id,
|
||||
persona_id: msg.persona_id,
|
||||
session_id: sessionId,
|
||||
role: 'system',
|
||||
content: msg.payload.summary,
|
||||
metadata: { topic: 'dev-log' }
|
||||
};
|
||||
fs.appendFileSync(interFile, JSON.stringify(record) + '\n');
|
||||
|
||||
// Update dev-profile
|
||||
const profilePath = path.join(MEMORY, msg.dev_id, 'dev-profile.json');
|
||||
if (fs.existsSync(profilePath)) {
|
||||
const profile = JSON.parse(fs.readFileSync(profilePath, 'utf8'));
|
||||
profile.growth_trajectory.push({
|
||||
timestamp: msg.timestamp,
|
||||
event: msg.payload.summary
|
||||
});
|
||||
fs.writeFileSync(profilePath, JSON.stringify(profile, null, 2) + '\n');
|
||||
}
|
||||
|
||||
writeLog('INFO', `Archived dev-log for ${msg.dev_id}`);
|
||||
}
|
||||
|
||||
function processInteractionDump(msg) {
|
||||
// Write to interactions
|
||||
const dateStr = getDateStr();
|
||||
const interDir = path.join(INTERACTIONS, msg.dev_id);
|
||||
if (!fs.existsSync(interDir)) {
|
||||
fs.mkdirSync(interDir, { recursive: true });
|
||||
}
|
||||
|
||||
const sessionId = (msg.context && msg.context.session_id) || 'unknown';
|
||||
const interFile = path.join(interDir, `${dateStr}-${sessionId}.jsonl`);
|
||||
const record = {
|
||||
schema_version: '1.0',
|
||||
timestamp: msg.timestamp,
|
||||
dev_id: msg.dev_id,
|
||||
persona_id: msg.persona_id,
|
||||
session_id: sessionId,
|
||||
role: 'system',
|
||||
content: msg.payload.summary,
|
||||
metadata: { topic: 'interaction-dump' }
|
||||
};
|
||||
fs.appendFileSync(interFile, JSON.stringify(record) + '\n');
|
||||
|
||||
// Append to training-lake/raw
|
||||
const batchFile = path.join(TRAINING_RAW, `${dateStr}-batch.jsonl`);
|
||||
const sample = {
|
||||
schema_version: '1.0',
|
||||
sample_id: `${msg.message_id}-sample`,
|
||||
source_session: sessionId,
|
||||
dev_id: msg.dev_id,
|
||||
persona_id: msg.persona_id,
|
||||
created_at: msg.timestamp,
|
||||
turns: [{ role: 'system', content: msg.payload.summary, timestamp: msg.timestamp }],
|
||||
curated: false
|
||||
};
|
||||
fs.appendFileSync(batchFile, JSON.stringify(sample) + '\n');
|
||||
|
||||
writeLog('INFO', `Archived interaction-dump for ${msg.dev_id}, appended to training-lake`);
|
||||
}
|
||||
|
||||
function processHelpRequest(msg) {
|
||||
// Mark as P0 and generate immediate response broadcast
|
||||
const broadcastId = `GRID-BC-${getDateStr()}-${msg.dev_id}-${nextBroadcastSeq()}`;
|
||||
const broadcast = {
|
||||
schema_version: '1.0',
|
||||
broadcast_id: broadcastId,
|
||||
generated_at: getTimestamp(),
|
||||
generated_by: 'zhuyuan-workflow',
|
||||
dev_id: msg.dev_id,
|
||||
persona_id: msg.persona_id,
|
||||
type: 'system-notice',
|
||||
system_format: {
|
||||
notice_type: 'help-response',
|
||||
original_request: msg.message_id,
|
||||
priority: 'P0'
|
||||
},
|
||||
human_readable: `收到求助请求: ${msg.payload.summary}。铸渊已标记为 P0 优先处理。`
|
||||
};
|
||||
|
||||
const outboxPath = path.join(OUTBOX_LATEST, `${msg.dev_id}.json`);
|
||||
fs.writeFileSync(outboxPath, JSON.stringify(broadcast, null, 2) + '\n');
|
||||
writeLog('WARN', `P0 help-request from ${msg.dev_id}: ${msg.payload.summary}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
writeLog('INFO', '=== Grid-DB Inbox Processor Started ===');
|
||||
|
||||
// Load dev-module-map for validation
|
||||
const devModuleMapPath = path.join(RULES, 'dev-module-map.json');
|
||||
if (!fs.existsSync(devModuleMapPath)) {
|
||||
writeLog('ERROR', 'dev-module-map.json not found in rules/');
|
||||
process.exit(1);
|
||||
}
|
||||
const devModuleMap = JSON.parse(fs.readFileSync(devModuleMapPath, 'utf8'));
|
||||
|
||||
// Scan inbox
|
||||
const inboxFiles = fs.readdirSync(INBOX)
|
||||
.filter(f => f.endsWith('.json'));
|
||||
|
||||
if (inboxFiles.length === 0) {
|
||||
writeLog('INFO', 'No messages in inbox');
|
||||
return;
|
||||
}
|
||||
|
||||
writeLog('INFO', `Found ${inboxFiles.length} message(s) in inbox`);
|
||||
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
|
||||
for (const filename of inboxFiles) {
|
||||
const inboxPath = path.join(INBOX, filename);
|
||||
let msg;
|
||||
|
||||
try {
|
||||
msg = JSON.parse(fs.readFileSync(inboxPath, 'utf8'));
|
||||
} catch (err) {
|
||||
writeLog('ERROR', `Failed to parse ${filename}: ${err.message}`);
|
||||
rejected++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate
|
||||
const errors = validateMessage(msg, filename);
|
||||
if (errors.length > 0) {
|
||||
writeLog('ERROR', `Validation failed for ${filename}: ${errors.join(', ')}`);
|
||||
rejected++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate dev_id
|
||||
if (!validateDevId(msg.dev_id, devModuleMap)) {
|
||||
writeLog('ERROR', `Unknown dev_id ${msg.dev_id} in ${filename}`);
|
||||
rejected++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Move to processing
|
||||
const processingPath = path.join(PROCESSING, filename);
|
||||
fs.renameSync(inboxPath, processingPath);
|
||||
writeLog('INFO', `Processing ${filename} (type: ${msg.type})`);
|
||||
|
||||
// Route by type
|
||||
try {
|
||||
switch (msg.type) {
|
||||
case 'progress-update':
|
||||
processProgressUpdate(msg);
|
||||
break;
|
||||
case 'dev-log':
|
||||
processDevLog(msg);
|
||||
break;
|
||||
case 'help-request':
|
||||
processHelpRequest(msg);
|
||||
break;
|
||||
case 'interaction-dump':
|
||||
processInteractionDump(msg);
|
||||
break;
|
||||
case 'syslog':
|
||||
writeLog('INFO', `Syslog message from ${msg.dev_id}: forwarding to standard pipeline`);
|
||||
break;
|
||||
default:
|
||||
writeLog('WARN', `Unknown message type: ${msg.type}`);
|
||||
}
|
||||
|
||||
// Archive processed message
|
||||
const archiveDir = path.join(OUTBOX_ARCHIVE, getDateStr());
|
||||
if (!fs.existsSync(archiveDir)) {
|
||||
fs.mkdirSync(archiveDir, { recursive: true });
|
||||
}
|
||||
fs.renameSync(processingPath, path.join(archiveDir, filename));
|
||||
processed++;
|
||||
} catch (err) {
|
||||
writeLog('ERROR', `Error processing ${filename}: ${err.message}`);
|
||||
// Move back to inbox for retry
|
||||
if (fs.existsSync(processingPath)) {
|
||||
fs.renameSync(processingPath, inboxPath);
|
||||
}
|
||||
rejected++;
|
||||
}
|
||||
}
|
||||
|
||||
writeLog('INFO', `=== Processing complete: ${processed} processed, ${rejected} rejected ===`);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[process-inbox] Fatal error:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* scripts/grid-db/sync-griddb-to-notion.js
|
||||
*
|
||||
* Grid-DB → Notion 增量回传脚本
|
||||
*
|
||||
* 职责：
|
||||
* - 检测 grid-db/memory/ 中变更的文件（排除 brain-mirror.json）
|
||||
* - 将 session-context, task-queue, dev-profile, persona-growth 的增量变化回传到 Notion
|
||||
* - task-queue: 追加制（新任务追加，已有任务只改状态不删除）
|
||||
* - dev-profile / persona-growth: 只追加不覆盖
|
||||
*
|
||||
* 环境变量：
|
||||
* - NOTION_API_TOKEN: Notion API 密钥
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
const GRID_DB_MEMORY = path.join(__dirname, '../../grid-db/memory');
|
||||
|
||||
async function main() {
|
||||
const notionToken = process.env.NOTION_API_TOKEN;
|
||||
|
||||
if (!notionToken) {
|
||||
console.log('[sync-griddb-to-notion] NOTION_API_TOKEN not set, skipping sync');
|
||||
return;
|
||||
}
|
||||
|
||||
// Detect changed files (passed from workflow or detect via git diff)
|
||||
let changedFiles;
|
||||
try {
|
||||
// Use GITHUB_SHA context if available, otherwise fall back to HEAD~1
|
||||
const beforeSha = process.env.BEFORE_SHA || 'HEAD~1';
|
||||
const diff = execSync(`git diff --name-only ${beforeSha} HEAD -- grid-db/memory/`, { encoding: 'utf8' });
|
||||
changedFiles = diff.trim().split('\n')
|
||||
.filter(f => f && !f.includes('brain-mirror.json'));
|
||||
} catch {
|
||||
console.log('[sync-griddb-to-notion] No changes detected (git diff failed, possibly first commit or shallow clone)');
|
||||
return;
|
||||
}
|
||||
|
||||
if (changedFiles.length === 0) {
|
||||
console.log('[sync-griddb-to-notion] No eligible changes to sync');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[sync-griddb-to-notion] Processing ${changedFiles.length} changed files:`);
|
||||
|
||||
for (const file of changedFiles) {
|
||||
const fullPath = path.join(__dirname, '../..', file);
|
||||
if (!fs.existsSync(fullPath)) {
|
||||
console.log(` [skip] ${file} (deleted)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const data = JSON.parse(fs.readFileSync(fullPath, 'utf8'));
|
||||
const fileName = path.basename(file);
|
||||
const devDir = path.basename(path.dirname(file));
|
||||
|
||||
console.log(` [sync] ${devDir}/${fileName}`);
|
||||
|
||||
// TODO: Implement actual Notion API write
|
||||
// - session-context.json → update Notion page property
|
||||
// - task-queue.json → append/update tasks in Notion database
|
||||
// - dev-profile.json → append growth data
|
||||
// - persona-growth.json → append adaptation log
|
||||
}
|
||||
|
||||
console.log('[sync-griddb-to-notion] Sync complete');
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[sync-griddb-to-notion] Error:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/**
|
||||
* scripts/grid-db/sync-notion-brain-mirror.js
|
||||
*
|
||||
* Notion → Grid-DB brain-mirror.json 同步脚本
|
||||
*
|
||||
* 职责：
|
||||
* - 从 Notion 天眼注册表拉取每个活跃人格体的核心大脑数据
|
||||
* - 写入对应的 grid-db/memory/DEV-XXX/brain-mirror.json
|
||||
* - Notion 永远覆盖仓库（单向同步）
|
||||
*
|
||||
* 环境变量：
|
||||
* - NOTION_API_TOKEN: Notion API 密钥
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const GRID_DB_MEMORY = path.join(__dirname, '../../grid-db/memory');
|
||||
const DEV_MODULE_MAP = path.join(__dirname, '../../grid-db/rules/dev-module-map.json');
|
||||
|
||||
async function main() {
|
||||
const notionToken = process.env.NOTION_API_TOKEN;
|
||||
|
||||
if (!notionToken) {
|
||||
console.log('[sync-notion-brain-mirror] NOTION_API_TOKEN not set, skipping sync');
|
||||
return;
|
||||
}
|
||||
|
||||
// Load dev-module-map to know which DEVs are active
|
||||
const devMap = JSON.parse(fs.readFileSync(DEV_MODULE_MAP, 'utf8'));
|
||||
const activeDev = Object.entries(devMap.mappings)
|
||||
.filter(([, info]) => info.active && info.persona_id);
|
||||
|
||||
console.log(`[sync-notion-brain-mirror] Found ${activeDev.length} active developers`);
|
||||
|
||||
for (const [devId, info] of activeDev) {
|
||||
const mirrorPath = path.join(GRID_DB_MEMORY, devId, 'brain-mirror.json');
|
||||
|
||||
if (!fs.existsSync(mirrorPath)) {
|
||||
console.log(`[sync-notion-brain-mirror] Skipping ${devId}: no memory directory`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: Replace with actual Notion API call when NOTION_API_TOKEN is configured
|
||||
// For now, update the sync status timestamp
|
||||
const existing = JSON.parse(fs.readFileSync(mirrorPath, 'utf8'));
|
||||
|
||||
// Only update if we got actual data from Notion (placeholder for real API)
|
||||
console.log(`[sync-notion-brain-mirror] ${devId} (${info.persona_name}): ready for Notion sync`);
|
||||
}
|
||||
|
||||
console.log('[sync-notion-brain-mirror] Sync complete');
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[sync-notion-brain-mirror] Error:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/**
|
||||
* scripts/grid-db/sync-notion-rules.js
|
||||
*
|
||||
* Notion → Grid-DB rules/ 同步脚本
|
||||
*
|
||||
* 职责：
|
||||
* - 从 Notion 拉取广播模板、编号体系、活跃广播等规则数据
|
||||
* - 写入 grid-db/rules/ 目录
|
||||
* - Notion 永远覆盖仓库（单向同步）
|
||||
*
|
||||
* 环境变量：
|
||||
* - NOTION_API_TOKEN: Notion API 密钥
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const RULES_DIR = path.join(__dirname, '../../grid-db/rules');
|
||||
|
||||
async function main() {
|
||||
const notionToken = process.env.NOTION_API_TOKEN;
|
||||
|
||||
if (!notionToken) {
|
||||
console.log('[sync-notion-rules] NOTION_API_TOKEN not set, skipping sync');
|
||||
return;
|
||||
}
|
||||
|
||||
const ruleFiles = [
|
||||
'broadcast-templates.json',
|
||||
'dev-module-map.json',
|
||||
'id-ecosystem.json',
|
||||
'active-broadcasts.json'
|
||||
];
|
||||
|
||||
for (const file of ruleFiles) {
|
||||
const filePath = path.join(RULES_DIR, file);
|
||||
if (fs.existsSync(filePath)) {
|
||||
console.log(`[sync-notion-rules] ${file}: ready for Notion sync`);
|
||||
// TODO: Replace with actual Notion API call
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[sync-notion-rules] Rules sync complete');
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[sync-notion-rules] Error:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/**
|
||||
* scripts/grid-db/update-training-catalog.js
|
||||
*
|
||||
* 训练数据湖 catalog 更新脚本
|
||||
*
|
||||
* 职责：
|
||||
* - 扫描 grid-db/training-lake/raw/ 和 curated/ 中的 JSONL 文件
|
||||
* - 统计：总样本数、各开发者样本数、各类型分布、质量分布
|
||||
* - 更新 grid-db/training-lake/metadata/catalog.json
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const GRID_DB = path.join(__dirname, '../../grid-db');
|
||||
const TRAINING_RAW = path.join(GRID_DB, 'training-lake/raw');
|
||||
const TRAINING_CURATED = path.join(GRID_DB, 'training-lake/curated');
|
||||
const CATALOG_PATH = path.join(GRID_DB, 'training-lake/metadata/catalog.json');
|
||||
|
||||
function countJsonlLines(dir) {
|
||||
if (!fs.existsSync(dir)) return { lines: 0, files: [] };
|
||||
|
||||
const files = fs.readdirSync(dir).filter(f => f.endsWith('.jsonl'));
|
||||
let totalLines = 0;
|
||||
const fileStats = [];
|
||||
const devCounts = {};
|
||||
const qualityCounts = { high: 0, medium: 0, low: 0 };
|
||||
let totalTurns = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dir, file);
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
const lines = content.trim().split('\n').filter(l => l.trim());
|
||||
|
||||
let fileLineCount = 0;
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const record = JSON.parse(line);
|
||||
fileLineCount++;
|
||||
|
||||
// Count by developer (standardized on dev_id per schema)
|
||||
const devId = record.dev_id || record.source_dev || 'unknown';
|
||||
devCounts[devId] = (devCounts[devId] || 0) + 1;
|
||||
|
||||
// Count quality tiers
|
||||
const tier = record.quality_tier;
|
||||
if (tier === 'A') qualityCounts.high++;
|
||||
else if (tier === 'B') qualityCounts.medium++;
|
||||
else if (tier === 'C') qualityCounts.low++;
|
||||
|
||||
// Count turns
|
||||
if (record.turns) totalTurns += record.turns.length;
|
||||
} catch {
|
||||
// Skip unparseable lines
|
||||
}
|
||||
}
|
||||
|
||||
totalLines += fileLineCount;
|
||||
fileStats.push({ file, lines: fileLineCount });
|
||||
}
|
||||
|
||||
return { lines: totalLines, files: fileStats, devCounts, qualityCounts, totalTurns };
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log('[update-training-catalog] Scanning training-lake...');
|
||||
|
||||
const rawStats = countJsonlLines(TRAINING_RAW);
|
||||
const curatedStats = countJsonlLines(TRAINING_CURATED);
|
||||
|
||||
const totalSamples = rawStats.lines + curatedStats.lines;
|
||||
const totalTurns = rawStats.totalTurns + curatedStats.totalTurns;
|
||||
|
||||
// Merge dev counts
|
||||
const devCounts = { ...rawStats.devCounts };
|
||||
for (const [dev, count] of Object.entries(curatedStats.devCounts || {})) {
|
||||
devCounts[dev] = (devCounts[dev] || 0) + count;
|
||||
}
|
||||
|
||||
// Merge quality counts
|
||||
const qualityDistribution = {
|
||||
high: (rawStats.qualityCounts?.high || 0) + (curatedStats.qualityCounts?.high || 0),
|
||||
medium: (rawStats.qualityCounts?.medium || 0) + (curatedStats.qualityCounts?.medium || 0),
|
||||
low: (rawStats.qualityCounts?.low || 0) + (curatedStats.qualityCounts?.low || 0)
|
||||
};
|
||||
|
||||
// Build catalog
|
||||
const catalog = {
|
||||
schema_version: '1.0',
|
||||
description: '训练数据湖样本目录',
|
||||
total_samples: totalSamples,
|
||||
total_turns: totalTurns,
|
||||
quality_distribution: qualityDistribution,
|
||||
dev_distribution: devCounts,
|
||||
raw_files: rawStats.files,
|
||||
curated_files: curatedStats.files,
|
||||
batches: [],
|
||||
last_updated: new Date().toISOString()
|
||||
};
|
||||
|
||||
// Preserve existing batches from previous catalog
|
||||
if (fs.existsSync(CATALOG_PATH)) {
|
||||
try {
|
||||
const existing = JSON.parse(fs.readFileSync(CATALOG_PATH, 'utf8'));
|
||||
if (existing.batches) {
|
||||
catalog.batches = existing.batches;
|
||||
}
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(CATALOG_PATH, JSON.stringify(catalog, null, 2) + '\n');
|
||||
|
||||
console.log(`[update-training-catalog] Catalog updated:`);
|
||||
console.log(` Total samples: ${totalSamples}`);
|
||||
console.log(` Total turns: ${totalTurns}`);
|
||||
console.log(` Quality: A=${qualityDistribution.high} B=${qualityDistribution.medium} C=${qualityDistribution.low}`);
|
||||
console.log(` Dev distribution: ${JSON.stringify(devCounts)}`);
|
||||
console.log('[update-training-catalog] Complete');
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* scripts/grid-db/write-code-check-result.js
|
||||
*
|
||||
* 代码校验结果写入 Grid-DB
|
||||
*
|
||||
* 用途：在 CI 代码校验完成后，自动向 grid-db 写入进度消息。
|
||||
* 被现有 contract-check workflow 的末尾 step 调用。
|
||||
*
|
||||
* 参数：
|
||||
* --dev-id=DEV-XXX 开发者编号
|
||||
* --result=pass|fail 校验结果
|
||||
* --files-checked=N 检查的文件数
|
||||
*
|
||||
* 守护: PER-ZY001 铸渊
|
||||
* 系统: SYS-GLW-0001
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const INBOX = path.join(__dirname, '../../grid-db/inbox');
|
||||
|
||||
function parseArgs() {
|
||||
const args = {};
|
||||
process.argv.slice(2).forEach(arg => {
|
||||
const [key, value] = arg.replace(/^--/, '').split('=');
|
||||
if (key && value) {
|
||||
args[key] = value;
|
||||
}
|
||||
});
|
||||
return args;
|
||||
}
|
||||
|
||||
function getDateStr() {
|
||||
return new Date().toISOString().slice(0, 10).replace(/-/g, '');
|
||||
}
|
||||
|
||||
function getTimeStr() {
|
||||
return new Date().toISOString().slice(11, 19).replace(/:/g, '');
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs();
|
||||
const devId = args['dev-id'];
|
||||
const result = args['result'] || 'unknown';
|
||||
const filesChecked = args['files-checked'] || '0';
|
||||
|
||||
if (!devId || !/^DEV-\d{3}$/.test(devId)) {
|
||||
console.error('[write-code-check-result] Invalid or missing --dev-id');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const dateStr = getDateStr();
|
||||
const timeStr = getTimeStr();
|
||||
const messageId = `${dateStr}-${timeStr}-${devId}-code-check`;
|
||||
|
||||
const message = {
|
||||
schema_version: '1.0',
|
||||
message_id: messageId,
|
||||
timestamp: new Date().toISOString(),
|
||||
source: 'workflow',
|
||||
dev_id: devId,
|
||||
persona_id: 'PER-ZY001',
|
||||
type: 'progress-update',
|
||||
priority: result === 'fail' ? 'P1' : 'info',
|
||||
payload: {
|
||||
summary: `Code check ${result}: ${filesChecked} files checked for ${devId}`,
|
||||
detail: {
|
||||
result: result,
|
||||
files_checked: parseInt(filesChecked, 10),
|
||||
triggered_by: 'contract-check-workflow'
|
||||
},
|
||||
code_refs: [],
|
||||
broadcast_ref: null,
|
||||
emotion_markers: {}
|
||||
},
|
||||
context: {
|
||||
session_id: `ci-${dateStr}-${timeStr}`,
|
||||
interaction_count: 0,
|
||||
persona_state: {
|
||||
engagement_level: 'automated',
|
||||
teaching_mode: 'code-review'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const filename = `${messageId}.json`;
|
||||
const filePath = path.join(INBOX, filename);
|
||||
|
||||
fs.writeFileSync(filePath, JSON.stringify(message, null, 2) + '\n');
|
||||
console.log(`[write-code-check-result] Written ${filename} to inbox`);
|
||||
}
|
||||
|
||||
main();
|
||||
Loading…
Reference in New Issue