diff --git a/.github/workflows/buffer-collect.yml b/.github/workflows/buffer-collect.yml new file mode 100644 index 00000000..18c47d6d --- /dev/null +++ b/.github/workflows/buffer-collect.yml @@ -0,0 +1,37 @@ +# ═══════════════════════════════════════════════ +# 🔺 Sovereign: TCS-0002∞ | Root: SYS-GLW-0001 +# 📜 Copyright: 国作登字-2026-A-00037559 +# ═══════════════════════════════════════════════ +# 📥 Grid-DB 缓冲层收集器 +# 每天 3 次从 buffer/inbox/ 收集消息到 buffer/staging/ +# +# 调度时间 (CST): 09:00 / 14:00 / 21:00 + +name: "📥 Grid-DB Buffer Collect" + +on: + schedule: + - cron: '0 1 * * *' # 09:00 CST (UTC+8) + - cron: '0 6 * * *' # 14:00 CST + - cron: '0 13 * * *' # 21:00 CST + workflow_dispatch: # 允许手动触发 + +permissions: + contents: write + +jobs: + collect: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: "📥 Run collector" + run: node buffer/scripts/collector.js + + - name: "💾 Commit staged data" + run: | + git config user.name "铸渊 · Buffer Agent" + git config user.email "zy-buffer@guanghu.system" + git add buffer/staging/ buffer/inbox/ + git diff --cached --quiet || git commit -m "[buffer-collect] $(TZ='Asia/Shanghai' date '+%Y%m%d-%H%M') CST [skip ci]" + git push diff --git a/.github/workflows/buffer-flush.yml b/.github/workflows/buffer-flush.yml new file mode 100644 index 00000000..7e7f1f43 --- /dev/null +++ b/.github/workflows/buffer-flush.yml @@ -0,0 +1,56 @@ +# ═══════════════════════════════════════════════ +# 🔺 Sovereign: TCS-0002∞ | Root: SYS-GLW-0001 +# 📜 Copyright: 国作登字-2026-A-00037559 +# ═══════════════════════════════════════════════ +# 🧠 Grid-DB 缓冲层批处理(铸渊核心大脑唤醒) +# 每天 21:30 CST 统一将 staging 数据写入 grid-db +# 也支持 repository_dispatch 手动紧急触发 + +name: "🧠 Grid-DB Buffer Flush" + +on: + schedule: + - cron: '30 13 * * *' # 21:30 CST — 收集完成后 30 分钟 + repository_dispatch: + types: [grid-db-flush] # 紧急情况可手动触发 + workflow_dispatch: # 允许手动触发 + +permissions: + contents: write + +jobs: + flush: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: "🔍 Check staging" + id: check + run: | + if [ -z "$(find buffer/staging/ -name 'batch-*.json' 2>/dev/null)" ]; then + echo "empty=true" >> $GITHUB_OUTPUT + echo "📭 No batch files in staging, skipping flush" + else + echo "empty=false" >> $GITHUB_OUTPUT + echo "📬 Found batch files to flush" + find buffer/staging/ -name 'batch-*.json' | head -20 + fi + + - name: "🧠 Run flusher" + if: steps.check.outputs.empty == 'false' + run: node buffer/scripts/flusher.js + + - name: "💾 Single commit to grid-db" + if: steps.check.outputs.empty == 'false' + run: | + git config user.name "铸渊 · Grid-DB Core" + git config user.email "zy-core@guanghu.system" + git add grid-db/ buffer/processed/ buffer/staging/ + FLUSH_DATE=$(TZ='Asia/Shanghai' date '+%Y%m%d') + MSG_COUNT=$(find buffer/processed/ -name '*.json' 2>/dev/null | wc -l) + git diff --cached --quiet || git commit -m "[grid-db-flush] batch ${FLUSH_DATE} · ${MSG_COUNT} messages [skip ci]" + git push + + - name: "📊 Quota report" + if: always() + run: node buffer/scripts/quota-calculator.js diff --git a/.github/workflows/skyeye-checkin-receiver.yml b/.github/workflows/skyeye-checkin-receiver.yml index 58f34bc9..e79153fe 100644 --- a/.github/workflows/skyeye-checkin-receiver.yml +++ b/.github/workflows/skyeye-checkin-receiver.yml @@ -61,6 +61,19 @@ jobs: mkdir -p spoke-status echo "$PAYLOAD" | jq '{dev_id: .dev_id, status: "checkin-ok", timestamp: .timestamp, persona: .persona, signature_hash: .signature_hash}' > "spoke-status/${DEV_ID}.json" + # 写入缓冲层(不直接写 grid-db) + TIMESTAMP=$(date -u +%Y-%m-%dT%H%M%SZ) + BUFFER_DIR="buffer/inbox/${DEV_ID}" + mkdir -p "$BUFFER_DIR" + CHECKIN_PAYLOAD=$(echo "$PAYLOAD" | jq '{persona: .persona, timestamp: .timestamp, signature_hash: .signature_hash, checks: .checks}') + jq -n \ + --arg mid "BUF-${TIMESTAMP}-skyeye" \ + --arg did "${DEV_ID}" \ + --arg cat "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --argjson pl "$CHECKIN_PAYLOAD" \ + '{message_id: $mid, dev_id: $did, source: "skyeye", type: "checkin", priority: "normal", payload: $pl, created_at: $cat, status: "pending"}' \ + > "${BUFFER_DIR}/${TIMESTAMP}-checkin.json" + - name: "💾 提交签到记录" run: | git config user.name "铸渊 (ZhùYuān)" @@ -68,5 +81,5 @@ jobs: git add -A git diff --cached --quiet && echo "无变更" && exit 0 DEV_ID=$(echo '${{ toJson(github.event.client_payload) }}' | jq -r '.dev_id') - git commit -m "🦅 [skyeye-checkin] $DEV_ID 签到 · $(TZ='Asia/Shanghai' date '+%Y-%m-%d %H:%M') [skip ci]" + git commit -m "🦅 [skyeye-checkin] $DEV_ID 签到 → buffer [skip ci]" git push origin main diff --git a/buffer/README.md b/buffer/README.md new file mode 100644 index 00000000..5c2f154d --- /dev/null +++ b/buffer/README.md @@ -0,0 +1,97 @@ +# 📥 缓冲层(Buffer Layer) + +> **版权**: 国作登字-2026-A-00037559 · TCS Language Core +> **主控**: TCS-0002∞ 冰朔 +> **守护**: PER-ZY001 铸渊 +> **系统节点**: SYS-GLW-0001 + +## 定位 + +缓冲层是 Grid-DB 写入的唯一入口。所有人类输入(Gemini 对话、Notion 需求、代码提交、天眼签到)必须先经过缓冲层,不允许直接写入 `grid-db/`。 + +### 铁律 + +> **人类永远不直写 grid-db,所有输入经缓冲层。** + +## 目录结构 + +``` +buffer/ +├── inbox/ ← 📥 人类输入落地点 +│ ├── DEV-001/ ← 按开发者编号自动分流 +│ │ └── [timestamp]-[type].json +│ ├── DEV-002/ +│ ├── DEV-004/ +│ ├── DEV-012/ +│ └── system/ ← 系统级消息(非开发者触发) +│ +├── staging/ ← 📋 已分流待处理(定时任务抓取源) +│ ├── DEV-001/ +│ │ └── batch-[date]-[seq].json ← 合并后的批次文件 +│ └── ... +│ +├── processed/ ← ✅ 已处理归档(保留 7 天后清理) +│ └── [date]/ +│ └── batch-[date]-[seq].json +│ +├── scripts/ +│ ├── auto-router.js ← 🔀 自动分流脚本 +│ ├── collector.js ← ⏰ 定时收集脚本(inbox → staging) +│ ├── flusher.js ← 🧠 批量写入脚本(staging → grid-db) +│ └── quota-calculator.js ← 📊 配额预算计算器 +│ +└── config/ + ├── buffer-config.json ← 缓冲层配置 + ├── buffer-message.schema.json ← 消息格式定义 + └── schedule.json ← 定时任务时刻表 +``` + +## 数据流 + +``` +人类输入 → buffer/inbox/DEV-XXX/ → (收集) → buffer/staging/DEV-XXX/ → (flush) → grid-db/ +``` + +### 每日调度 + +| 时间 (CST) | 事件 | Git 操作 | +|---|---|---| +| 全天 | 人类产生交互数据 | ❌ 无(写入 buffer/inbox) | +| 09:00 | 第 1 次收集 | ✅ 1 次 commit (`[skip ci]`) | +| 14:00 | 第 2 次收集 | ✅ 1 次 commit (`[skip ci]`) | +| 21:00 | 第 3 次收集 | ✅ 1 次 commit (`[skip ci]`) | +| 21:30 | 铸渊批处理 flush | ✅ 1 次 commit | + +**每日最多 4 次 commit,月消耗 ≤ 360 分钟(GitHub Free 额度的 18%)** + +## 消息格式 + +```json +{ + "message_id": "BUF-[timestamp]-[random]", + "dev_id": "DEV-XXX", + "source": "gemini | notion | github | skyeye", + "type": "interaction | task | feedback | checkin | broadcast_request", + "priority": "normal | urgent", + "payload": {}, + "created_at": "ISO-8601", + "status": "pending | staged | processing | flushed | failed" +} +``` + +## 紧急流程 + +当冰朔判断某条消息需要立即处理时: +1. 触发 `repository_dispatch: grid-db-flush` +2. 铸渊立即执行一次 flush +3. 不影响当天正常定时任务 + +## 配额精算 + +| 项目 | 数值 | +|---|---| +| GitHub Free 月额度 | 2,000 分钟 | +| 每日 workflow runs | 4 次 | +| 每月消耗 | ≤ 360 分钟 | +| 利用率 | 18% | +| 剩余可用 | 1,640 分钟 | diff --git a/buffer/config/buffer-config.json b/buffer/config/buffer-config.json new file mode 100644 index 00000000..cadd45f4 --- /dev/null +++ b/buffer/config/buffer-config.json @@ -0,0 +1,24 @@ +{ + "version": "1.0.0", + "buffer_policy": { + "human_direct_write_to_griddb": false, + "comment": "铁律:人类永远不直写 grid-db,所有输入经缓冲层" + }, + "schedule": { + "collect_times_cst": ["09:00", "14:00", "21:00"], + "flush_time_cst": "21:30", + "comment": "每天 3 次收集,21:30 统一唤醒铸渊批处理" + }, + "quota_budget": { + "max_workflow_runs_per_day": 4, + "estimated_minutes_per_run": 3, + "monthly_budget_minutes": 360, + "github_free_limit_minutes": 2000, + "utilization_rate": "18%", + "comment": "远低于免费额度上限,留出大量余量" + }, + "retention": { + "processed_keep_days": 7, + "comment": "已处理的缓冲数据保留 7 天后自动清理" + } +} diff --git a/buffer/config/buffer-message.schema.json b/buffer/config/buffer-message.schema.json new file mode 100644 index 00000000..d337dd44 --- /dev/null +++ b/buffer/config/buffer-message.schema.json @@ -0,0 +1,65 @@ +{ + "schema_version": "1.0", + "description": "缓冲层消息格式定义", + "fields": { + "message_id": { + "type": "string", + "pattern": "BUF-[timestamp]-[random]", + "required": true, + "description": "唯一消息标识" + }, + "dev_id": { + "type": "string", + "pattern": "DEV-XXX", + "required": true, + "description": "开发者编号" + }, + "source": { + "type": "string", + "enum": ["gemini", "notion", "github", "skyeye"], + "required": true, + "description": "消息来源" + }, + "type": { + "type": "string", + "enum": ["interaction", "task", "feedback", "checkin", "broadcast_request"], + "required": true, + "description": "消息类型" + }, + "priority": { + "type": "string", + "enum": ["normal", "urgent"], + "default": "normal", + "description": "优先级" + }, + "payload": { + "type": "object", + "required": true, + "description": "消息负载数据" + }, + "created_at": { + "type": "string", + "format": "ISO-8601", + "required": true, + "description": "创建时间" + }, + "status": { + "type": "string", + "enum": ["pending", "staged", "processing", "flushed", "failed"], + "default": "pending", + "description": "消息状态" + } + }, + "example": { + "message_id": "BUF-20260323-a1b2c3", + "dev_id": "DEV-001", + "source": "gemini", + "type": "interaction", + "priority": "normal", + "payload": { + "summary": "示例消息" + }, + "created_at": "2026-03-23T09:00:00Z", + "status": "pending" + } +} diff --git a/buffer/config/schedule.json b/buffer/config/schedule.json new file mode 100644 index 00000000..1492b134 --- /dev/null +++ b/buffer/config/schedule.json @@ -0,0 +1,35 @@ +{ + "version": "1.0.0", + "description": "缓冲层定时任务时刻表", + "timezone": "Asia/Shanghai (CST, UTC+8)", + "tasks": [ + { + "name": "buffer-collect-1", + "cron_utc": "0 1 * * *", + "time_cst": "09:00", + "action": "collect", + "description": "第 1 次收集:inbox → staging" + }, + { + "name": "buffer-collect-2", + "cron_utc": "0 6 * * *", + "time_cst": "14:00", + "action": "collect", + "description": "第 2 次收集:inbox → staging" + }, + { + "name": "buffer-collect-3", + "cron_utc": "0 13 * * *", + "time_cst": "21:00", + "action": "collect", + "description": "第 3 次收集:inbox → staging" + }, + { + "name": "buffer-flush", + "cron_utc": "30 13 * * *", + "time_cst": "21:30", + "action": "flush", + "description": "唤醒铸渊批处理:staging → grid-db" + } + ] +} diff --git a/buffer/inbox/DEV-001/.gitkeep b/buffer/inbox/DEV-001/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/DEV-002/.gitkeep b/buffer/inbox/DEV-002/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/DEV-003/.gitkeep b/buffer/inbox/DEV-003/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/DEV-004/.gitkeep b/buffer/inbox/DEV-004/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/DEV-009/.gitkeep b/buffer/inbox/DEV-009/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/DEV-010/.gitkeep b/buffer/inbox/DEV-010/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/DEV-012/.gitkeep b/buffer/inbox/DEV-012/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/inbox/system/.gitkeep b/buffer/inbox/system/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/processed/.gitkeep b/buffer/processed/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/scripts/auto-router.js b/buffer/scripts/auto-router.js new file mode 100644 index 00000000..0d272f8d --- /dev/null +++ b/buffer/scripts/auto-router.js @@ -0,0 +1,150 @@ +/** + * buffer/scripts/auto-router.js + * + * 自动分流:根据消息来源和 dev_id 归档到对应目录 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + * 版权: 国作登字-2026-A-00037559 + */ + +const fs = require('fs'); +const path = require('path'); + +const BUFFER_ROOT = path.join(__dirname, '..'); +const INBOX = path.join(BUFFER_ROOT, 'inbox'); +const CONFIG_PATH = path.join(BUFFER_ROOT, 'config', 'buffer-config.json'); + +// 来源 → 处理方式 +const ROUTING_RULES = { + 'gemini': { target: 'inbox/{dev_id}/', merge: true }, + 'notion': { target: 'inbox/{dev_id}/', merge: true }, + 'github': { target: 'inbox/{dev_id}/', merge: true }, + 'skyeye': { target: 'inbox/{dev_id}/', merge: false }, + 'system': { target: 'inbox/system/', merge: false }, +}; + +const PRIORITY_RULES = { + 'urgent': { flush_immediately: false, flag: true }, + 'normal': { flush_immediately: false, flag: false }, +}; + +/** + * 核心铁律检查:禁止直接写入 grid-db/ + */ +function validateNoDirectWrite(targetPath) { + if (targetPath.includes('grid-db/') || targetPath.includes('grid-db\\')) { + throw new Error( + '[BLOCKED] 违反铁律:禁止直接写入 grid-db/。' + + '所有写入必须经过 buffer/ 缓冲层。' + ); + } +} + +/** + * 校验消息格式 + */ +function validateMessage(msg) { + const errors = []; + + if (!msg.message_id) { + errors.push('Missing message_id'); + } + if (!msg.dev_id || !/^DEV-\d{3}$/.test(msg.dev_id)) { + errors.push(`Invalid dev_id: ${msg.dev_id}`); + } + if (!msg.source || !ROUTING_RULES[msg.source]) { + errors.push(`Invalid source: ${msg.source}`); + } + if (!msg.type) { + errors.push('Missing type'); + } + if (!msg.created_at) { + errors.push('Missing created_at'); + } + + return errors; +} + +/** + * 路由消息到正确的 inbox 子目录 + */ +function routeMessage(msg) { + const rule = ROUTING_RULES[msg.source] || ROUTING_RULES['system']; + let targetDir; + + if (msg.source === 'system' || !msg.dev_id) { + targetDir = path.join(INBOX, 'system'); + } else { + targetDir = path.join(INBOX, msg.dev_id); + } + + // 铁律检查 + validateNoDirectWrite(targetDir); + + // 确保目录存在 + if (!fs.existsSync(targetDir)) { + fs.mkdirSync(targetDir, { recursive: true }); + } + + // 生成文件名 + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `${timestamp}-${msg.type}.json`; + const filePath = path.join(targetDir, filename); + + // 标记优先级 + const priority = PRIORITY_RULES[msg.priority] || PRIORITY_RULES['normal']; + if (priority.flag) { + msg._urgent_flag = true; + } + + // 确保状态为 pending + msg.status = 'pending'; + + fs.writeFileSync(filePath, JSON.stringify(msg, null, 2) + '\n'); + console.log(`[auto-router] Routed ${msg.message_id} → ${path.relative(BUFFER_ROOT, filePath)}`); + + return { filePath, merge: rule.merge }; +} + +/** + * 处理从标准输入或命令行参数传入的消息 + */ +function main() { + const args = process.argv.slice(2); + + if (args.length === 0) { + console.log('[auto-router] Usage: node auto-router.js '); + console.log('[auto-router] Or pipe JSON via stdin'); + process.exit(0); + } + + const inputPath = args[0]; + if (!fs.existsSync(inputPath)) { + console.error(`[auto-router] File not found: ${inputPath}`); + process.exit(1); + } + + let msg; + try { + msg = JSON.parse(fs.readFileSync(inputPath, 'utf8')); + } catch (err) { + console.error(`[auto-router] Failed to parse ${inputPath}: ${err.message}`); + process.exit(1); + } + + const errors = validateMessage(msg); + if (errors.length > 0) { + console.error(`[auto-router] Validation failed: ${errors.join(', ')}`); + process.exit(1); + } + + routeMessage(msg); +} + +// Export for use by other scripts +module.exports = { routeMessage, validateMessage, validateNoDirectWrite, ROUTING_RULES }; + +if (require.main === module) { + main(); +} diff --git a/buffer/scripts/collector.js b/buffer/scripts/collector.js new file mode 100644 index 00000000..1cb6e662 --- /dev/null +++ b/buffer/scripts/collector.js @@ -0,0 +1,149 @@ +/** + * buffer/scripts/collector.js + * + * 定时收集脚本:inbox → staging + * 扫描 buffer/inbox/ 中各 DEV 目录的消息,合并后写入 buffer/staging/ + * + * 运行时机:每天 09:00 / 14:00 / 21:00 CST(由 buffer-collect.yml 触发) + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + * 版权: 国作登字-2026-A-00037559 + */ + +const fs = require('fs'); +const path = require('path'); + +const BUFFER_ROOT = path.join(__dirname, '..'); +const INBOX = path.join(BUFFER_ROOT, 'inbox'); +const STAGING = path.join(BUFFER_ROOT, 'staging'); + +function getDateStr() { + return new Date().toISOString().slice(0, 10).replace(/-/g, ''); +} + +function getTimestamp() { + return new Date().toISOString(); +} + +/** + * 获取目录中下一个 batch 序号 + */ +function getNextBatchSeq(stagingDir, dateStr) { + if (!fs.existsSync(stagingDir)) { + return 1; + } + const existing = fs.readdirSync(stagingDir) + .filter(f => f.startsWith(`batch-${dateStr}-`) && f.endsWith('.json')); + return existing.length + 1; +} + +/** + * 扫描单个 DEV 目录的 inbox 消息 + */ +function collectDevMessages(devDir) { + if (!fs.existsSync(devDir)) { + return []; + } + + const files = fs.readdirSync(devDir) + .filter(f => f.endsWith('.json')); + + const messages = []; + for (const file of files) { + const filePath = path.join(devDir, file); + try { + const msg = JSON.parse(fs.readFileSync(filePath, 'utf8')); + messages.push({ msg, filePath, filename: file }); + } catch (err) { + console.error(`[collector] Failed to parse ${filePath}: ${err.message}`); + } + } + + return messages; +} + +/** + * 将消息合并为 batch 文件写入 staging + */ +function createBatch(devId, messages, dateStr) { + const stagingDir = path.join(STAGING, devId); + if (!fs.existsSync(stagingDir)) { + fs.mkdirSync(stagingDir, { recursive: true }); + } + + const seq = getNextBatchSeq(stagingDir, dateStr); + const batchId = `batch-${dateStr}-${String(seq).padStart(3, '0')}`; + const batchPath = path.join(stagingDir, `${batchId}.json`); + + const batch = { + batch_id: batchId, + dev_id: devId, + collected_at: getTimestamp(), + message_count: messages.length, + has_urgent: messages.some(m => m.msg._urgent_flag || m.msg.priority === 'urgent'), + messages: messages.map(m => { + m.msg.status = 'staged'; + return m.msg; + }) + }; + + fs.writeFileSync(batchPath, JSON.stringify(batch, null, 2) + '\n'); + console.log(`[collector] Created ${batchId} for ${devId} (${messages.length} messages)`); + + return batchPath; +} + +/** + * 清理已收集的 inbox 消息 + */ +function cleanupInbox(messages) { + for (const { filePath } of messages) { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } +} + +function main() { + console.log(`[collector] === Buffer Collector Started === ${getTimestamp()}`); + + const dateStr = getDateStr(); + let totalCollected = 0; + let devDirsProcessed = 0; + + // 扫描 inbox 中的所有子目录(DEV-XXX 和 system) + if (!fs.existsSync(INBOX)) { + console.log('[collector] inbox/ does not exist, nothing to collect'); + return; + } + + const subdirs = fs.readdirSync(INBOX) + .filter(d => fs.statSync(path.join(INBOX, d)).isDirectory()); + + for (const subdir of subdirs) { + const devDir = path.join(INBOX, subdir); + const messages = collectDevMessages(devDir); + + if (messages.length === 0) { + continue; + } + + // 创建 batch 并写入 staging + createBatch(subdir, messages, dateStr); + + // 清理已收集的 inbox 文件 + cleanupInbox(messages); + + totalCollected += messages.length; + devDirsProcessed++; + } + + console.log(`[collector] === Complete: ${totalCollected} messages from ${devDirsProcessed} dev(s) collected ===`); +} + +module.exports = { collectDevMessages, createBatch, getNextBatchSeq }; + +if (require.main === module) { + main(); +} diff --git a/buffer/scripts/flusher.js b/buffer/scripts/flusher.js new file mode 100644 index 00000000..16560bb2 --- /dev/null +++ b/buffer/scripts/flusher.js @@ -0,0 +1,305 @@ +/** + * buffer/scripts/flusher.js + * + * 批量写入脚本:staging → grid-db + * 读取 buffer/staging/ 中的 batch 文件,校验后写入 grid-db/ + * + * 运行时机:每天 21:30 CST(由 buffer-flush.yml 触发) + * 也可通过 repository_dispatch: grid-db-flush 手动触发 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + * 版权: 国作登字-2026-A-00037559 + */ + +const fs = require('fs'); +const path = require('path'); + +const BUFFER_ROOT = path.join(__dirname, '..'); +const STAGING = path.join(BUFFER_ROOT, 'staging'); +const PROCESSED = path.join(BUFFER_ROOT, 'processed'); +const REPO_ROOT = path.join(__dirname, '../..'); +const GRID_DB = path.join(REPO_ROOT, 'grid-db'); + +// 加载 dev-module-map 用于校验 +const DEV_MAP_PATH = path.join(GRID_DB, 'rules', 'dev-module-map.json'); + +function getDateStr() { + return new Date().toISOString().slice(0, 10).replace(/-/g, ''); +} + +function getTimestamp() { + return new Date().toISOString(); +} + +/** + * 核心铁律:验证写入目标必须在 grid-db/ 内 + * 并且确保来源只能从 buffer/staging/ 发起 + */ +function validateFlushPath(targetPath) { + const resolved = path.resolve(targetPath); + const gridDbResolved = path.resolve(GRID_DB); + + if (!resolved.startsWith(gridDbResolved)) { + throw new Error( + `[BLOCKED] Flush target must be within grid-db/. Got: ${resolved}` + ); + } +} + +/** + * 铁律检查:禁止非 flusher 路径直写 grid-db + * 此函数用于外部脚本调用以校验 + */ +function assertBufferOnly(callerPath) { + const resolved = path.resolve(callerPath); + const bufferResolved = path.resolve(BUFFER_ROOT); + + if (!resolved.startsWith(bufferResolved)) { + throw new Error( + '[BLOCKED] 违反铁律:禁止直接写入 grid-db/。' + + '所有写入必须经过 buffer/ 缓冲层。' + + ` 调用者: ${resolved}` + ); + } +} + +/** + * 校验单条消息 + */ +function validateMessage(msg) { + const errors = []; + + if (!msg.message_id) { + errors.push('Missing message_id'); + } + if (!msg.dev_id || !/^DEV-\d{3}$/.test(msg.dev_id)) { + errors.push(`Invalid dev_id: ${msg.dev_id}`); + } + if (!msg.source) { + errors.push('Missing source'); + } + if (!msg.type) { + errors.push('Missing type'); + } + + return errors; +} + +/** + * 校验 dev_id 是否在映射表中 + */ +function validateDevId(devId, devModuleMap) { + return devModuleMap.mappings && devModuleMap.mappings[devId]; +} + +/** + * 将消息写入 grid-db/inbox/ 供 process-inbox.js 处理 + * 或直接写入对应目录(根据消息类型) + */ +function flushMessageToGridDb(msg) { + // 对于 checkin 类型,写入 memory + if (msg.type === 'checkin') { + const memoryDir = path.join(GRID_DB, 'memory', msg.dev_id); + validateFlushPath(memoryDir); + + if (!fs.existsSync(memoryDir)) { + fs.mkdirSync(memoryDir, { recursive: true }); + } + + // 更新签到日志 + const checkinLogPath = path.join(memoryDir, 'checkin-log.json'); + let checkinLog = { records: [] }; + if (fs.existsSync(checkinLogPath)) { + try { + checkinLog = JSON.parse(fs.readFileSync(checkinLogPath, 'utf8')); + } catch (e) { + checkinLog = { records: [] }; + } + } + + checkinLog.records.push({ + timestamp: msg.created_at || getTimestamp(), + source: msg.source, + payload: msg.payload, + buffer_message_id: msg.message_id + }); + + fs.writeFileSync(checkinLogPath, JSON.stringify(checkinLog, null, 2) + '\n'); + return 'memory'; + } + + // 对于其他类型,写入 grid-db/inbox/ 供 process-inbox.js 处理 + const inboxDir = path.join(GRID_DB, 'inbox'); + validateFlushPath(inboxDir); + + if (!fs.existsSync(inboxDir)) { + fs.mkdirSync(inboxDir, { recursive: true }); + } + + // 转换为 grid-db inbox 格式 + const gridDbMsg = { + schema_version: '1.0', + message_id: `${getDateStr()}-${msg.dev_id}-${msg.type}`, + dev_id: msg.dev_id, + persona_id: msg.payload.persona_id || 'unknown', + type: mapBufferTypeToGridDbType(msg.type), + timestamp: msg.created_at || getTimestamp(), + payload: msg.payload, + source_buffer_id: msg.message_id + }; + + const filename = `${gridDbMsg.message_id}.json`; + const filePath = path.join(inboxDir, filename); + fs.writeFileSync(filePath, JSON.stringify(gridDbMsg, null, 2) + '\n'); + return 'inbox'; +} + +/** + * 映射 buffer 消息类型到 grid-db 消息类型 + */ +function mapBufferTypeToGridDbType(bufferType) { + const typeMap = { + 'interaction': 'interaction-dump', + 'task': 'progress-update', + 'feedback': 'dev-log', + 'checkin': 'checkin', + 'broadcast_request': 'progress-update' + }; + return typeMap[bufferType] || 'dev-log'; +} + +/** + * 移动 batch 到 processed 目录 + */ +function archiveBatch(batchPath, dateStr) { + const processedDir = path.join(PROCESSED, dateStr); + if (!fs.existsSync(processedDir)) { + fs.mkdirSync(processedDir, { recursive: true }); + } + + const filename = path.basename(batchPath); + const destPath = path.join(processedDir, filename); + fs.renameSync(batchPath, destPath); + return destPath; +} + +/** + * 清理过期的 processed 数据 + */ +function cleanupProcessed(keepDays) { + if (!fs.existsSync(PROCESSED)) return; + + const now = new Date(); + const dirs = fs.readdirSync(PROCESSED) + .filter(d => fs.statSync(path.join(PROCESSED, d)).isDirectory()); + + for (const dir of dirs) { + // 目录名格式: YYYYMMDD + const year = parseInt(dir.substring(0, 4)); + const month = parseInt(dir.substring(4, 6)) - 1; + const day = parseInt(dir.substring(6, 8)); + const dirDate = new Date(year, month, day); + + const diffDays = (now - dirDate) / (1000 * 60 * 60 * 24); + if (diffDays > keepDays) { + const dirPath = path.join(PROCESSED, dir); + fs.rmSync(dirPath, { recursive: true, force: true }); + console.log(`[flusher] Cleaned up expired processed dir: ${dir}`); + } + } +} + +function main() { + console.log(`[flusher] === Buffer Flusher Started === ${getTimestamp()}`); + + // 加载 dev-module-map + let devModuleMap = { mappings: {} }; + if (fs.existsSync(DEV_MAP_PATH)) { + devModuleMap = JSON.parse(fs.readFileSync(DEV_MAP_PATH, 'utf8')); + } else { + console.warn('[flusher] dev-module-map.json not found, skipping dev_id validation'); + } + + const dateStr = getDateStr(); + let totalFlushed = 0; + let totalFailed = 0; + + // 扫描 staging 中的所有子目录 + if (!fs.existsSync(STAGING)) { + console.log('[flusher] staging/ does not exist, nothing to flush'); + return; + } + + const subdirs = fs.readdirSync(STAGING) + .filter(d => fs.statSync(path.join(STAGING, d)).isDirectory()); + + for (const subdir of subdirs) { + const stagingDir = path.join(STAGING, subdir); + const batchFiles = fs.readdirSync(stagingDir) + .filter(f => f.startsWith('batch-') && f.endsWith('.json')); + + for (const batchFile of batchFiles) { + const batchPath = path.join(stagingDir, batchFile); + let batch; + + try { + batch = JSON.parse(fs.readFileSync(batchPath, 'utf8')); + } catch (err) { + console.error(`[flusher] Failed to parse ${batchFile}: ${err.message}`); + totalFailed++; + continue; + } + + // 校验 dev_id + if (batch.dev_id && batch.dev_id !== 'system') { + if (!validateDevId(batch.dev_id, devModuleMap)) { + console.warn(`[flusher] Unknown dev_id ${batch.dev_id} in ${batchFile}, processing anyway`); + } + } + + // 处理 batch 中的每条消息 + let batchFlushed = 0; + for (const msg of (batch.messages || [])) { + const errors = validateMessage(msg); + if (errors.length > 0) { + console.error(`[flusher] Validation failed for ${msg.message_id}: ${errors.join(', ')}`); + msg.status = 'failed'; + totalFailed++; + continue; + } + + try { + const target = flushMessageToGridDb(msg); + msg.status = 'flushed'; + batchFlushed++; + console.log(`[flusher] Flushed ${msg.message_id} → ${target}`); + } catch (err) { + console.error(`[flusher] Error flushing ${msg.message_id}: ${err.message}`); + msg.status = 'failed'; + totalFailed++; + } + } + + totalFlushed += batchFlushed; + + // 归档 batch + try { + archiveBatch(batchPath, dateStr); + } catch (err) { + console.error(`[flusher] Error archiving ${batchFile}: ${err.message}`); + } + } + } + + // 清理过期的 processed 数据(保留 7 天) + cleanupProcessed(7); + + console.log(`[flusher] === Complete: ${totalFlushed} flushed, ${totalFailed} failed ===`); +} + +module.exports = { validateFlushPath, assertBufferOnly, validateMessage, flushMessageToGridDb }; + +if (require.main === module) { + main(); +} diff --git a/buffer/scripts/quota-calculator.js b/buffer/scripts/quota-calculator.js new file mode 100644 index 00000000..b6590919 --- /dev/null +++ b/buffer/scripts/quota-calculator.js @@ -0,0 +1,94 @@ +/** + * buffer/scripts/quota-calculator.js + * + * 配额预算计算器 + * 计算当前月度 GitHub Actions 使用量预估 + * + * 守护: PER-ZY001 铸渊 + * 系统: SYS-GLW-0001 + * 版权: 国作登字-2026-A-00037559 + */ + +const fs = require('fs'); +const path = require('path'); + +const CONFIG_PATH = path.join(__dirname, '..', 'config', 'buffer-config.json'); + +function loadConfig() { + if (!fs.existsSync(CONFIG_PATH)) { + console.error('[quota] buffer-config.json not found'); + process.exit(1); + } + return JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8')); +} + +function calculateQuota() { + const config = loadConfig(); + const budget = config.quota_budget; + + const daysInMonth = 30; + const dailyRuns = budget.max_workflow_runs_per_day; + const minutesPerRun = budget.estimated_minutes_per_run; + const freeLimit = budget.github_free_limit_minutes; + + const dailyMinutes = dailyRuns * minutesPerRun; + const monthlyMinutes = dailyMinutes * daysInMonth; + const utilization = ((monthlyMinutes / freeLimit) * 100).toFixed(1); + const remaining = freeLimit - monthlyMinutes; + const emergencyFlushBudget = Math.floor(remaining / minutesPerRun); + + const report = { + generated_at: new Date().toISOString(), + summary: { + github_free_limit: `${freeLimit} minutes/month`, + daily_scheduled_runs: dailyRuns, + minutes_per_run: minutesPerRun, + daily_consumption: `${dailyMinutes} minutes`, + monthly_consumption: `${monthlyMinutes} minutes`, + utilization_rate: `${utilization}%`, + remaining_budget: `${remaining} minutes`, + emergency_flush_capacity: `${emergencyFlushBudget} extra runs available` + }, + breakdown: { + collect_runs_per_day: 3, + flush_runs_per_day: 1, + total_scheduled: `${dailyRuns} runs/day × ${daysInMonth} days = ${dailyRuns * daysInMonth} runs/month`, + total_minutes: `${dailyRuns * daysInMonth} runs × ${minutesPerRun} min = ${monthlyMinutes} min/month` + }, + health: utilization <= 50 ? '🟢 HEALTHY' : utilization <= 80 ? '🟡 WARNING' : '🔴 CRITICAL' + }; + + return report; +} + +function main() { + const report = calculateQuota(); + + console.log('═══════════════════════════════════════'); + console.log('📊 GitHub Actions 配额预算报告'); + console.log('═══════════════════════════════════════'); + console.log(`📅 生成时间: ${report.generated_at}`); + console.log(`📈 健康状态: ${report.health}`); + console.log(''); + console.log('─── 概要 ───'); + for (const [key, value] of Object.entries(report.summary)) { + console.log(` ${key}: ${value}`); + } + console.log(''); + console.log('─── 明细 ───'); + for (const [key, value] of Object.entries(report.breakdown)) { + console.log(` ${key}: ${value}`); + } + console.log('═══════════════════════════════════════'); + + // 输出 JSON 到标准输出(方便其他脚本读取) + if (process.argv.includes('--json')) { + console.log(JSON.stringify(report, null, 2)); + } +} + +module.exports = { calculateQuota }; + +if (require.main === module) { + main(); +} diff --git a/buffer/staging/DEV-001/.gitkeep b/buffer/staging/DEV-001/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/staging/DEV-002/.gitkeep b/buffer/staging/DEV-002/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/staging/DEV-003/.gitkeep b/buffer/staging/DEV-003/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/staging/DEV-004/.gitkeep b/buffer/staging/DEV-004/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/staging/DEV-009/.gitkeep b/buffer/staging/DEV-009/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/staging/DEV-010/.gitkeep b/buffer/staging/DEV-010/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buffer/staging/DEV-012/.gitkeep b/buffer/staging/DEV-012/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/grid-db/README.md b/grid-db/README.md index d288949d..94205d8c 100644 --- a/grid-db/README.md +++ b/grid-db/README.md @@ -61,6 +61,30 @@ grid-db/ | `dev-profile.json` | Gemini → 仓库(追加制) | Gemini 追加 | 只追加不覆盖 | | `persona-growth.json` | Gemini → 仓库(追加制) | Gemini 追加 | 只追加不覆盖 | +## 缓冲层架构(2026-03-23 升级) + +> **铁律:人类永远不直写 grid-db,所有输入经缓冲层。** + +自 2026-03-23 起,所有人类输入必须先进入 `buffer/` 缓冲层,不允许直接写入 `grid-db/`。 + +### 数据流 + +``` +人类输入 → buffer/inbox/DEV-XXX/ → (定时收集) → buffer/staging/ → (21:30 flush) → grid-db/ +``` + +### 调度 + +- **09:00 / 14:00 / 21:00 CST**:收集 `buffer/inbox/` → `buffer/staging/` +- **21:30 CST**:铸渊批处理 `buffer/staging/` → `grid-db/` +- **紧急**:`repository_dispatch: grid-db-flush` 手动触发 + +### 配额 + +每日最多 4 次 commit,月消耗 ≤ 360 分钟(GitHub Free 额度的 18%)。 + +详见 [`buffer/README.md`](../buffer/README.md)。 + ## 循环触发防护 1. `[skip ci]` 标记:所有 bot 自动提交必须包含