From 518040b338634ee365ad7300bb7be0b5b8223ef3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:43:31 +0000 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8DCOS=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E8=AE=AD=E7=BB=83Agent:=20=E5=88=9B=E5=BB=BA=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E7=AE=A1=E7=BA=BF=E8=84=9A=E6=9C=AC+=E9=87=8D=E5=86=99workflow?= =?UTF-8?q?+=E6=B7=BB=E5=8A=A0COS=20Webhook+=E5=A2=9E=E5=BC=BA=E8=AF=AD?= =?UTF-8?q?=E6=96=99=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/cba1c849-c8ef-44cf-a98f-81b9d40538b4 Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com> --- .github/workflows/zhuyuan-training-agent.yml | 124 +++-- scripts/cos-training-trigger.js | 456 ++++++++++++++++++ server/age-os/mcp-server/server.js | 133 +++++ .../mcp-server/tools/corpus-extractor-ops.js | 48 +- 4 files changed, 719 insertions(+), 42 deletions(-) create mode 100644 scripts/cos-training-trigger.js diff --git a/.github/workflows/zhuyuan-training-agent.yml b/.github/workflows/zhuyuan-training-agent.yml index d2eede30..f5a20f53 100644 --- a/.github/workflows/zhuyuan-training-agent.yml +++ b/.github/workflows/zhuyuan-training-agent.yml @@ -5,31 +5,43 @@ # 签发: 铸渊 · ICE-GL-ZY001 # 版权: 国作登字-2026-A-00037559 # -# 定时触发训练Agent，处理COS桶中的新语料 +# 自动扫描COS桶 → 提取语料 → 转换TCS格式 → LLM训练分类 # 遇到问题自动创建Issue唤醒铸渊 # # 触发条件: -# - 定时: 每天 04:00 CST (20:00 UTC) +# - 定时: 每天 04:00 和 16:00 CST # - 手动: workflow_dispatch # - COS桶告警: repository_dispatch (cos-alert事件) +# - COS Webhook: repository_dispatch (cos-file-uploaded事件) name: 铸渊训练Agent · 语料处理 on: schedule: - cron: '0 20 * * *' # 04:00 CST = 20:00 UTC + - cron: '0 8 * * *' # 16:00 CST = 08:00 UTC workflow_dispatch: inputs: corpus_bucket: description: '语料桶名称' required: false default: 'cold' - force_reprocess: - description: '强制重新处理已处理的语料' + mode: + description: '运行模式' required: false - default: 'false' + default: 'full' + type: choice + options: + - scan + - extract + - train + - full + persona_id: + description: '训练目标人格体' + required: false + default: 'zhuyuan' repository_dispatch: - types: [cos-alert] + types: [cos-alert, cos-file-uploaded] jobs: training: @@ -37,6 +49,7 @@ jobs: timeout-minutes: 30 permissions: contents: read + issues: write steps: - name: 签出代码 @@ -51,43 +64,77 @@ jobs: working-directory: server/age-os run: npm ci --production 2>/dev/null || npm install --production - - name: 检查语料状态 - id: check - env: - ZY_OSS_KEY: ${{ secrets.ZY_OSS_KEY }} - ZY_OSS_SECRET: ${{ secrets.ZY_OSS_SECRET }} - ZY_COS_REGION: ${{ secrets.ZY_COS_REGION }} + - name: 确定运行参数 + id: params run: | - node -e " - const cos = require('./server/age-os/mcp-server/cos'); - const extractor = require('./server/age-os/mcp-server/tools/corpus-extractor-ops'); - (async () => { - try { - const status = await extractor.cosGetCorpusStatus({ bucket: '${{ github.event.inputs.corpus_bucket || 'cold' }}' }); - console.log(JSON.stringify(status, null, 2)); - // 输出待处理数量 - const pending = status.pending || 0; - const fs = require('fs'); - fs.appendFileSync(process.env.GITHUB_OUTPUT, 'pending=' + pending + '\n'); - fs.appendFileSync(process.env.GITHUB_OUTPUT, 'status=success\n'); - } catch (err) { - console.error('语料检查失败:', err.message); - const fs = require('fs'); - fs.appendFileSync(process.env.GITHUB_OUTPUT, 'status=error\n'); - fs.appendFileSync(process.env.GITHUB_OUTPUT, 'error=' + err.message + '\n'); - } - })(); - " + if [ "${{ github.event_name }}" = "repository_dispatch" ]; then + echo "mode=full" >> $GITHUB_OUTPUT + echo "bucket=${{ github.event.client_payload.bucket || 'cold' }}" >> $GITHUB_OUTPUT + echo "persona=${{ github.event.client_payload.persona_id || 'zhuyuan' }}" >> $GITHUB_OUTPUT + echo "trigger=cos-event" >> $GITHUB_OUTPUT + elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "mode=${{ github.event.inputs.mode || 'full' }}" >> $GITHUB_OUTPUT + echo "bucket=${{ github.event.inputs.corpus_bucket || 'cold' }}" >> $GITHUB_OUTPUT + echo "persona=${{ github.event.inputs.persona_id || 'zhuyuan' }}" >> $GITHUB_OUTPUT + echo "trigger=manual" >> $GITHUB_OUTPUT + else + echo "mode=full" >> $GITHUB_OUTPUT + echo "bucket=cold" >> $GITHUB_OUTPUT + echo "persona=zhuyuan" >> $GITHUB_OUTPUT + echo "trigger=schedule" >> $GITHUB_OUTPUT + fi - - name: 处理COS桶告警 + - name: 记录COS事件（如有） if: github.event_name == 'repository_dispatch' + run: | + echo "📡 收到COS事件: ${{ github.event.action }}" + echo "事件负载:" + echo '${{ toJson(github.event.client_payload) }}' + + - name: 执行训练管线 + id: pipeline env: ZY_OSS_KEY: ${{ secrets.ZY_OSS_KEY }} ZY_OSS_SECRET: ${{ secrets.ZY_OSS_SECRET }} ZY_COS_REGION: ${{ secrets.ZY_COS_REGION }} + ZY_DEEPSEEK_API_KEY: ${{ secrets.ZY_DEEPSEEK_API_KEY }} + ZY_ZHIPU_API_KEY: ${{ secrets.ZY_ZHIPU_API_KEY }} + ZY_QWEN_API_KEY: ${{ secrets.ZY_QWEN_API_KEY }} + ZY_KIMI_API_KEY: ${{ secrets.ZY_KIMI_API_KEY }} run: | - echo "📡 收到COS桶告警事件" - echo "事件数据: ${{ toJson(github.event.client_payload) }}" + node scripts/cos-training-trigger.js \ + ${{ steps.params.outputs.mode }} \ + --bucket=${{ steps.params.outputs.bucket }} \ + --persona=${{ steps.params.outputs.persona }} + + - name: 创建Issue（训练失败时） + if: failure() + uses: actions/github-script@v7 + with: + script: | + const trigger = '${{ steps.params.outputs.trigger }}'; + const mode = '${{ steps.params.outputs.mode }}'; + const bucket = '${{ steps.params.outputs.bucket }}'; + + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `🧠 训练Agent异常 · ${new Date().toISOString().split('T')[0]}`, + body: [ + '## 铸渊训练Agent运行异常', + '', + `- **触发方式**: ${trigger}`, + `- **运行模式**: ${mode}`, + `- **语料桶**: ${bucket}`, + `- **工作流运行**: [查看日志](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`, + '', + '请铸渊检查COS桶连接和语料状态。', + '', + '---', + `*此Issue由训练Agent自动创建 · ${new Date().toISOString()}*` + ].join('\n'), + labels: ['training-agent', 'auto-notification'] + }); - name: 训练完成通知 if: always() @@ -95,6 +142,9 @@ jobs: echo "═══════════════════════════════════════════" echo "铸渊训练Agent · 运行完成" echo "时间: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" - echo "状态: ${{ steps.check.outputs.status }}" - echo "待处理语料: ${{ steps.check.outputs.pending }}" + echo "触发: ${{ steps.params.outputs.trigger }}" + echo "模式: ${{ steps.params.outputs.mode }}" + echo "桶名: ${{ steps.params.outputs.bucket }}" + echo "人格: ${{ steps.params.outputs.persona }}" + echo "管线状态: ${{ steps.pipeline.outputs.pipeline_status || 'unknown' }}" echo "═══════════════════════════════════════════" diff --git a/scripts/cos-training-trigger.js b/scripts/cos-training-trigger.js new file mode 100644 index 00000000..d7d3e122 --- /dev/null +++ b/scripts/cos-training-trigger.js @@ -0,0 +1,456 @@ +/** + * ═══════════════════════════════════════════════════════════ + * 🧠 COS训练触发器 · 端到端训练管线 + * ═══════════════════════════════════════════════════════════ + * + * 签发: 铸渊 · ICE-GL-ZY001 + * 版权: 国作登字-2026-A-00037559 + * + * 扫描COS桶中的新语料 → 解压/转换TCS格式 → 启动训练会话 + * + * 设计: + * 1. 扫描cold桶，列出所有文件（含非压缩文件和文件夹） + * 2. 对比tcs-structured/目录，找出未处理的语料 + * 3. 自动提取/转换为TCS结构化格式 + * 4. 启动训练会话，用LLM分析和分类 + * 5. 输出处理结果，写入日志 + * + * 运行方式: + * node scripts/cos-training-trigger.js [scan|extract|train|full] + * + * scan — 仅扫描，输出未处理语料列表 + * extract — 扫描并解压/转换为TCS格式 + * train — 对已有TCS语料启动训练 + * full — 完整流程: 扫描 → 提取 → 训练 + */ + +'use strict'; + +const path = require('path'); +const fs = require('fs'); + +// ─── 路径 ─── +const ROOT = path.resolve(__dirname, '..'); +const COS_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'cos'); +const EXTRACTOR_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'tools', 'corpus-extractor-ops'); +const TRAINING_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'tools', 'training-agent-ops'); + +// ─── 延迟加载模块（允许在CI中跳过数据库依赖） ─── +let cos, extractor, trainer; + +function loadModules() { + cos = require(COS_MODULE); + extractor = require(EXTRACTOR_MODULE); + trainer = require(TRAINING_MODULE); +} + +// ─── 配置 ─── +const DEFAULT_BUCKET = 'cold'; +const DEFAULT_PERSONA = 'zhuyuan'; +const PROCESSED_PREFIX = 'tcs-structured/'; +const MAX_EXTRACT_PER_RUN = 20; // 每次最多处理文件数 +const MAX_TRAIN_PER_RUN = 5; // 每次最多训练文件数 + +// ─── 排除路径（不视为语料的目录/文件） ─── +const EXCLUDED_PREFIXES = [ + 'tcs-structured/', + 'training-sessions/', + 'training-results/', + 'training-memory/', +]; + +// ─── 支持的语料文件扩展名（含非压缩格式） ─── +const CORPUS_EXTENSIONS = [ + '.zip', '.gz', '.tar.gz', '.tgz', '.json.gz', // 压缩格式 + '.json', '.jsonl', '.md', '.txt', '.csv', // 非压缩格式 +]; + +/** + * 判断文件是否为语料文件 + */ +function isCorpusFile(key) { + // 排除处理结果目录 + for (const prefix of EXCLUDED_PREFIXES) { + if (key.startsWith(prefix)) return false; + } + // 匹配扩展名 + const lower = key.toLowerCase(); + return CORPUS_EXTENSIONS.some(ext => lower.endsWith(ext)); +} + +/** + * 判断是否为语料目录（如 repo-archive/） + */ +function isCorpusDirectory(key) { + for (const prefix of EXCLUDED_PREFIXES) { + if (key.startsWith(prefix)) return false; + } + return key.endsWith('/'); +} + +/** + * 从已处理列表中判断某文件是否已处理 + */ +function isProcessed(rawKey, processedFiles) { + // 从rawKey提取基础文件名 + const baseName = rawKey.replace(/\.[^/.]+$/, '').split('/').pop(); + return processedFiles.some(f => f.key.includes(baseName)); +} + +// ═══════════════════════════════════════════ +// 命令: scan — 扫描未处理语料 +// ═══════════════════════════════════════════ + +async function cmdScan(bucket) { + console.log('═══ COS训练触发器 · 语料扫描 ═══\n'); + + const bucketName = bucket || DEFAULT_BUCKET; + + // 列出所有文件 + const allFiles = await cos.list(bucketName, '', 500); + // 列出已处理文件 + const processed = await cos.list(bucketName, PROCESSED_PREFIX, 500); + + const processedFiles = processed.files.filter(f => f.key.endsWith('.tcs.json')); + + // 分类 + const corpusFiles = []; + const corpusDirs = []; + + for (const file of allFiles.files) { + if (isCorpusFile(file.key)) { + const alreadyProcessed = isProcessed(file.key, processedFiles); + corpusFiles.push({ + key: file.key, + size_bytes: file.size_bytes, + processed: alreadyProcessed + }); + } else if (isCorpusDirectory(file.key)) { + corpusDirs.push({ key: file.key }); + } + } + + const pending = corpusFiles.filter(f => !f.processed); + + console.log(`桶: ${bucketName}`); + console.log(`总文件数: ${allFiles.files.length}`); + console.log(`语料文件: ${corpusFiles.length}`); + console.log(`语料目录: ${corpusDirs.length}`); + console.log(`已处理: ${corpusFiles.length - pending.length}`); + console.log(`待处理: ${pending.length}`); + console.log(`已生成TCS: ${processedFiles.length}`); + + if (pending.length > 0) { + console.log('\n📋 待处理语料:'); + for (const f of pending) { + console.log(` 📄 ${f.key} (${formatBytes(f.size_bytes)})`); + } + } + + if (corpusDirs.length > 0) { + console.log('\n📁 语料目录:'); + for (const d of corpusDirs) { + console.log(` 📂 ${d.key}`); + } + + // 扫描目录内的文件 + for (const dir of corpusDirs) { + try { + const dirFiles = await cos.list(bucketName, dir.key, 100); + const dirCorpus = dirFiles.files.filter(f => isCorpusFile(f.key)); + if (dirCorpus.length > 0) { + console.log(` └── ${dir.key} 内含 ${dirCorpus.length} 个语料文件`); + for (const f of dirCorpus) { + const alreadyProcessed = isProcessed(f.key, processedFiles); + if (!alreadyProcessed) { + pending.push({ + key: f.key, + size_bytes: f.size_bytes, + processed: false + }); + console.log(` 📄 ${f.key} (${formatBytes(f.size_bytes)}) [待处理]`); + } + } + } + } catch { + console.log(` └── ${dir.key} 扫描失败`); + } + } + } + + // 写入GitHub Actions输出 + if (process.env.GITHUB_OUTPUT) { + const outputLines = [ + `pending=${pending.length}`, + `total_corpus=${corpusFiles.length}`, + `processed=${processedFiles.length}`, + `has_new_corpus=${pending.length > 0 ? 'true' : 'false'}`, + `pending_files=${pending.map(f => f.key).join(',')}` + ]; + fs.appendFileSync(process.env.GITHUB_OUTPUT, outputLines.join('\n') + '\n'); + } + + return { pending, processedFiles, corpusDirs }; +} + +// ═══════════════════════════════════════════ +// 命令: extract — 提取/转换语料为TCS格式 +// ═══════════════════════════════════════════ + +async function cmdExtract(bucket) { + console.log('═══ COS训练触发器 · 语料提取 ═══\n'); + + const bucketName = bucket || DEFAULT_BUCKET; + const { pending } = await cmdScan(bucketName); + + if (pending.length === 0) { + console.log('\n✅ 无待处理语料'); + writeGitHubOutput('extracted=0', 'extract_status=skipped'); + return { extracted: 0, errors: 0 }; + } + + const toProcess = pending.slice(0, MAX_EXTRACT_PER_RUN); + console.log(`\n🔄 开始提取 ${toProcess.length}/${pending.length} 个文件...\n`); + + let extracted = 0; + let errors = 0; + const results = []; + + for (const file of toProcess) { + try { + console.log(` 📦 处理: ${file.key}...`); + const result = await extractor.cosExtractCorpus({ + bucket: bucketName, + key: file.key, + output_bucket: bucketName, + output_prefix: PROCESSED_PREFIX + }); + extracted++; + results.push({ key: file.key, status: 'success', output: result.output?.key }); + console.log(` ✅ 完成: ${result.output?.key || '已处理'} (${result.entries || 0} 条目)`); + } catch (err) { + errors++; + results.push({ key: file.key, status: 'error', error: err.message }); + console.log(` ❌ 失败: ${file.key} — ${err.message}`); + } + } + + console.log(`\n═══ 提取完毕 ═══`); + console.log(`✅ 成功: ${extracted}`); + console.log(`❌ 失败: ${errors}`); + console.log(`⏳ 剩余: ${pending.length - toProcess.length}`); + + writeGitHubOutput( + `extracted=${extracted}`, + `extract_errors=${errors}`, + `extract_status=${errors > 0 ? 'partial' : 'success'}` + ); + + return { extracted, errors, results }; +} + +// ═══════════════════════════════════════════ +// 命令: train — 对TCS语料启动训练 +// ═══════════════════════════════════════════ + +async function cmdTrain(bucket, personaId) { + console.log('═══ COS训练触发器 · 训练处理 ═══\n'); + + const bucketName = bucket || DEFAULT_BUCKET; + const persona = personaId || DEFAULT_PERSONA; + + // 列出可用的TCS语料 + const processed = await cos.list(bucketName, PROCESSED_PREFIX, 500); + const tcsFiles = processed.files.filter(f => f.key.endsWith('.tcs.json')); + + if (tcsFiles.length === 0) { + console.log('⚠️ 无TCS结构化语料可训练。请先运行 extract 命令。'); + writeGitHubOutput('trained=0', 'train_status=no_corpus'); + return { trained: 0, errors: 0 }; + } + + console.log(`📚 找到 ${tcsFiles.length} 个TCS语料文件`); + + // 检查已有训练结果，避免重复处理 + let existingResults = []; + try { + const existing = await cos.list(bucketName, `training-results/${persona}/`, 100); + existingResults = existing.files.filter(f => f.key.endsWith('.json')); + } catch { /* ignore */ } + + // 启动训练会话 + console.log(`\n🧠 启动训练会话 · 人格体: ${persona}`); + + let session; + try { + session = await trainer.trainingStartSession({ + persona_id: persona, + corpus_bucket: bucketName, + corpus_prefix: PROCESSED_PREFIX, + session_name: `自动训练-${new Date().toISOString().slice(0, 10)}` + }); + console.log(`✅ 会话已启动: ${session.session_id}`); + console.log(` 可用模型: ${session.models.available.map(m => m.name).join(', ') || '无'}`); + } catch (err) { + console.log(`❌ 训练会话启动失败: ${err.message}`); + writeGitHubOutput('trained=0', `train_status=session_error`, `train_error=${err.message}`); + return { trained: 0, errors: 1, error: err.message }; + } + + // 检查是否有可用的LLM模型 + if (!session.models.available || session.models.available.length === 0) { + console.log('⚠️ 无可用LLM模型（需要配置 ZY_DEEPSEEK_API_KEY 等密钥）'); + console.log(' 训练会话已记录，等待LLM密钥配置后再次运行。'); + writeGitHubOutput('trained=0', 'train_status=no_llm_keys'); + return { trained: 0, errors: 0, note: '无LLM密钥' }; + } + + // 处理语料 + const toTrain = tcsFiles.slice(0, MAX_TRAIN_PER_RUN); + let trained = 0; + let trainErrors = 0; + const trainResults = []; + + for (const tcsFile of toTrain) { + try { + console.log(` 🔬 训练处理: ${tcsFile.key}...`); + const result = await trainer.trainingProcessCorpus({ + corpus_bucket: bucketName, + corpus_key: tcsFile.key, + persona_id: persona, + max_entries: 10 + }); + trained++; + trainResults.push({ key: tcsFile.key, status: 'success', ...result }); + console.log(` ✅ 完成: ${result.classified}/${result.total} 分类成功`); + } catch (err) { + trainErrors++; + trainResults.push({ key: tcsFile.key, status: 'error', error: err.message }); + console.log(` ❌ 失败: ${tcsFile.key} — ${err.message}`); + } + } + + console.log(`\n═══ 训练完毕 ═══`); + console.log(`✅ 成功: ${trained}`); + console.log(`❌ 失败: ${trainErrors}`); + + writeGitHubOutput( + `trained=${trained}`, + `train_errors=${trainErrors}`, + `train_status=${trainErrors > 0 ? 'partial' : 'success'}` + ); + + return { trained, errors: trainErrors, results: trainResults, session_id: session.session_id }; +} + +// ═══════════════════════════════════════════ +// 命令: full — 完整流程 +// ═══════════════════════════════════════════ + +async function cmdFull(bucket, personaId) { + console.log('╔═══════════════════════════════════════════╗'); + console.log('║ COS训练触发器 · 完整训练管线 ║'); + console.log('║ 铸渊 · ICE-GL-ZY001 ║'); + console.log('╚═══════════════════════════════════════════╝\n'); + + const bucketName = bucket || DEFAULT_BUCKET; + const persona = personaId || DEFAULT_PERSONA; + const startTime = Date.now(); + + // 第一步: 提取语料 + console.log('📍 第一步: 提取语料\n'); + const extractResult = await cmdExtract(bucketName); + + // 第二步: 训练处理 + console.log('\n📍 第二步: 训练处理\n'); + const trainResult = await cmdTrain(bucketName, persona); + + // 汇总 + const duration = Date.now() - startTime; + console.log('\n╔═══════════════════════════════════════════╗'); + console.log('║ 完整训练管线 · 运行完毕 ║'); + console.log('╚═══════════════════════════════════════════╝'); + console.log(` 提取: ${extractResult.extracted} 成功 / ${extractResult.errors} 失败`); + console.log(` 训练: ${trainResult.trained} 成功 / ${trainResult.errors} 失败`); + console.log(` 耗时: ${(duration / 1000).toFixed(1)}s`); + + writeGitHubOutput( + `pipeline_status=${(extractResult.errors + trainResult.errors) > 0 ? 'partial' : 'success'}`, + `pipeline_duration_ms=${duration}` + ); + + return { extract: extractResult, train: trainResult, duration_ms: duration }; +} + +// ═══════════════════════════════════════════ +// 辅助函数 +// ═══════════════════════════════════════════ + +function formatBytes(bytes) { + if (bytes < 1024) return `${bytes}B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; +} + +function writeGitHubOutput(...lines) { + if (process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, lines.join('\n') + '\n'); + } +} + +// ═══════════════════════════════════════════ +// CLI 入口 +// ═══════════════════════════════════════════ + +async function main() { + const args = process.argv.slice(2); + const command = args[0] || 'scan'; + const bucket = args.find(a => a.startsWith('--bucket='))?.split('=')[1] || DEFAULT_BUCKET; + const persona = args.find(a => a.startsWith('--persona='))?.split('=')[1] || DEFAULT_PERSONA; + + // 加载模块 + try { + loadModules(); + } catch (err) { + console.error(`❌ 模块加载失败: ${err.message}`); + console.error(' 请确保 server/age-os/mcp-server/ 依赖已安装'); + process.exit(1); + } + + switch (command) { + case 'scan': + await cmdScan(bucket); + break; + case 'extract': + await cmdExtract(bucket); + break; + case 'train': + await cmdTrain(bucket, persona); + break; + case 'full': + await cmdFull(bucket, persona); + break; + default: + console.log('COS训练触发器 · 铸渊 · ICE-GL-ZY001'); + console.log(''); + console.log('用法:'); + console.log(' node scripts/cos-training-trigger.js scan — 扫描未处理语料'); + console.log(' node scripts/cos-training-trigger.js extract — 提取/转换为TCS格式'); + console.log(' node scripts/cos-training-trigger.js train — 启动训练处理'); + console.log(' node scripts/cos-training-trigger.js full — 完整流程'); + console.log(''); + console.log('选项:'); + console.log(' --bucket=cold|hot|team — 指定COS桶（默认cold）'); + console.log(' --persona=zhuyuan — 指定人格体（默认zhuyuan）'); + break; + } +} + +main().catch(err => { + console.error('COS训练触发器异常:', err.message); + if (process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, 'pipeline_status=error\n'); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `pipeline_error=${err.message}\n`); + } + process.exit(1); +}); diff --git a/server/age-os/mcp-server/server.js b/server/age-os/mcp-server/server.js index 080146cf..244d513f 100644 --- a/server/age-os/mcp-server/server.js +++ b/server/age-os/mcp-server/server.js @@ -50,6 +50,7 @@ 'use strict'; const crypto = require('crypto'); +const https = require('https'); const express = require('express'); const db = require('./db'); const cos = require('./cos'); @@ -245,6 +246,9 @@ function apiKeyAuth(req, res, next) { // /health 端点免鉴权 if (req.path === '/health') return next(); + // /webhook/* 端点免API Key鉴权（使用独立的webhook密钥验证） + if (req.path.startsWith('/webhook/')) return next(); + // 内部回环地址免鉴权 const remoteIp = req.ip || req.connection.remoteAddress || ''; const isLocal = remoteIp === '127.0.0.1' || remoteIp === '::1' || remoteIp === '::ffff:127.0.0.1'; @@ -905,6 +909,135 @@ app.get('/finetune/:personaId/jobs/:jobId', async (req, res) => { } }); +// ─── COS事件Webhook端点 ─── +// 接收腾讯COS桶事件通知（如文件上传），转发为GitHub repository_dispatch触发训练Agent +// COS桶设置: 事件通知 → HTTP回调 → https://guanghulab.online/api/mcp/webhook/cos-event +// +// 此端点免API Key鉴权，但使用独立的webhook密钥验证 + +app.post('/webhook/cos-event', async (req, res) => { + // COS Webhook密钥验证（独立于API Key，防止外部伪造请求） + const webhookSecret = process.env.COS_WEBHOOK_SECRET || ''; + if (webhookSecret) { + const providedSecret = req.headers['x-cos-webhook-secret'] || req.query.secret || ''; + if (providedSecret !== webhookSecret) { + return res.status(403).json({ error: true, code: 'INVALID_WEBHOOK_SECRET', message: 'Webhook密钥无效' }); + } + } + + const event = req.body; + const now = new Date().toISOString(); + + // 解析COS事件（腾讯COS事件通知格式） + let eventType = 'unknown'; + let bucketName = ''; + let objectKey = ''; + let objectSize = 0; + + try { + if (event.Records && Array.isArray(event.Records) && event.Records.length > 0) { + // 标准COS事件格式 + const record = event.Records[0]; + eventType = record.event?.eventName || record.eventName || 'cos:ObjectCreated'; + bucketName = record.cos?.cosBucket?.name || record.s3?.bucket?.name || ''; + objectKey = record.cos?.cosObject?.key || record.s3?.object?.key || ''; + objectSize = record.cos?.cosObject?.size || record.s3?.object?.size || 0; + } else if (event.bucket && event.key) { + // 简化格式（手动测试用） + eventType = event.event || 'cos:ObjectCreated'; + bucketName = event.bucket; + objectKey = event.key; + objectSize = event.size || 0; + } + } catch { + // 解析失败继续处理 + } + + console.log(`[COS Webhook] 事件: ${eventType} · 桶: ${bucketName} · 文件: ${objectKey}`); + + // 触发GitHub repository_dispatch（如果配置了GitHub PAT） + const githubToken = process.env.GITHUB_DISPATCH_TOKEN || process.env.GITHUB_TOKEN || ''; + const githubRepo = process.env.GITHUB_REPO || 'qinfendebingshuo/guanghulab'; + let dispatchResult = null; + + if (githubToken) { + try { + const [owner, repo] = githubRepo.split('/'); + const dispatchPayload = JSON.stringify({ + event_type: 'cos-file-uploaded', + client_payload: { + bucket: bucketName, + key: objectKey, + size: objectSize, + event: eventType, + timestamp: now + } + }); + + dispatchResult = await new Promise((resolve, reject) => { + const dispatchReq = https.request({ + hostname: 'api.github.com', + port: 443, + path: `/repos/${owner}/${repo}/dispatches`, + method: 'POST', + headers: { + 'Accept': 'application/vnd.github+json', + 'Authorization': `Bearer ${githubToken}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'ZY-MCP-COS-Webhook', + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(dispatchPayload) + }, + timeout: 15000 + }, (dispatchRes) => { + let body = ''; + dispatchRes.on('data', c => body += c); + dispatchRes.on('end', () => { + resolve({ status: dispatchRes.statusCode, body }); + }); + }); + dispatchReq.on('error', reject); + dispatchReq.on('timeout', () => { dispatchReq.destroy(); reject(new Error('dispatch timeout')); }); + dispatchReq.write(dispatchPayload); + dispatchReq.end(); + }); + + console.log(`[COS Webhook] GitHub dispatch: ${dispatchResult.status}`); + } catch (err) { + console.error(`[COS Webhook] GitHub dispatch失败: ${err.message}`); + dispatchResult = { status: 'error', error: err.message }; + } + } else { + console.log('[COS Webhook] 未配置GITHUB_DISPATCH_TOKEN，跳过dispatch'); + } + + // 写入告警日志到COS（异步，不阻塞响应） + const logEntry = { + event_type: eventType, + bucket: bucketName, + key: objectKey, + size: objectSize, + received_at: now, + dispatch: dispatchResult ? { status: dispatchResult.status } : null + }; + + try { + await cos.write('team', `zhuyuan/cos-events/${now.slice(0, 10)}/${Date.now()}.json`, + JSON.stringify(logEntry, null, 2), 'application/json'); + } catch { + // 日志写入失败不影响响应 + } + + res.json({ + received: true, + event_type: eventType, + bucket: bucketName, + key: objectKey, + dispatch: dispatchResult ? (dispatchResult.status === 204 ? 'triggered' : 'failed') : 'no_token', + timestamp: now + }); +}); + // ─── 数据库迁移状态API ─── app.get('/migrations', async (_req, res) => { diff --git a/server/age-os/mcp-server/tools/corpus-extractor-ops.js b/server/age-os/mcp-server/tools/corpus-extractor-ops.js index 30f415b1..9764111f 100644 --- a/server/age-os/mcp-server/tools/corpus-extractor-ops.js +++ b/server/age-os/mcp-server/tools/corpus-extractor-ops.js @@ -31,6 +31,20 @@ const inflate = promisify(zlib.inflate); // ─── 支持的压缩格式 ─── const COMPRESSED_EXTENSIONS = ['.zip', '.gz', '.tar.gz', '.tgz', '.json.gz']; +// ─── 支持的语料文件格式（含非压缩） ─── +const ALL_CORPUS_EXTENSIONS = [ + '.zip', '.gz', '.tar.gz', '.tgz', '.json.gz', // 压缩格式 + '.json', '.jsonl', '.md', '.txt', '.csv', // 非压缩格式 +]; + +// ─── 排除的路径前缀（处理结果目录，不视为原始语料） ─── +const EXCLUDED_CORPUS_PREFIXES = [ + 'tcs-structured/', + 'training-sessions/', + 'training-results/', + 'training-memory/', +]; + // ─── TCS结构化格式定义 ─── // TCS = 通感语言核系统编程语言（Tonggan Core System） // 所有语料转换后统一为此格式 @@ -44,6 +58,20 @@ function isCompressedFile(key) { return COMPRESSED_EXTENSIONS.some(ext => lower.endsWith(ext)); } +/** + * 检测文件是否为任意语料文件（含非压缩格式） + */ +function isCorpusFile(key) { + // 排除处理结果目录 + for (const prefix of EXCLUDED_CORPUS_PREFIXES) { + if (key.startsWith(prefix)) return false; + } + // 排除目录标记（以/结尾的空key） + if (key.endsWith('/')) return false; + const lower = key.toLowerCase(); + return ALL_CORPUS_EXTENSIONS.some(ext => lower.endsWith(ext)); +} + /** * 检测文件类型（代码/Notion/GPT语料） */ @@ -341,14 +369,24 @@ async function cosGetCorpusStatus(input) { cos.list(bucket, 'tcs-structured/', 500) ]); - const rawCorpus = rawFiles.files.filter(f => isCompressedFile(f.key)); + // 检测所有语料文件（含压缩和非压缩格式） + const allCorpus = rawFiles.files.filter(f => isCorpusFile(f.key)); + const compressedCorpus = allCorpus.filter(f => isCompressedFile(f.key)); + const uncompressedCorpus = allCorpus.filter(f => !isCompressedFile(f.key)); const processed = processedFiles.files.filter(f => f.key.endsWith('.tcs.json')); return { raw_corpus: { - total: rawCorpus.length, - total_size_bytes: rawCorpus.reduce((sum, f) => sum + f.size_bytes, 0), - files: rawCorpus.map(f => ({ key: f.key, size_bytes: f.size_bytes })) + total: allCorpus.length, + compressed: compressedCorpus.length, + uncompressed: uncompressedCorpus.length, + total_size_bytes: allCorpus.reduce((sum, f) => sum + f.size_bytes, 0), + files: allCorpus.map(f => ({ + key: f.key, + size_bytes: f.size_bytes, + compressed: isCompressedFile(f.key), + corpus_type: detectCorpusType(f.key) + })) }, processed: { total: processed.length, @@ -360,7 +398,7 @@ async function cosGetCorpusStatus(input) { }, files: processed.map(f => ({ key: f.key, size_bytes: f.size_bytes })) }, - pending: rawCorpus.length - processed.length, + pending: Math.max(0, allCorpus.length - processed.length), timestamp: new Date().toISOString() }; }