修复COS触发训练Agent: 创建训练管线脚本+重写workflow+添加COS Webhook+增强语料检测
Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/cba1c849-c8ef-44cf-a98f-81b9d40538b4 Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
GitHub
parent
0796e732a1
commit
518040b338
|
|
@ -5,31 +5,43 @@
|
|||
# 签发: 铸渊 · ICE-GL-ZY001
|
||||
# 版权: 国作登字-2026-A-00037559
|
||||
#
|
||||
# 定时触发训练Agent，处理COS桶中的新语料
|
||||
# 自动扫描COS桶 → 提取语料 → 转换TCS格式 → LLM训练分类
|
||||
# 遇到问题自动创建Issue唤醒铸渊
|
||||
#
|
||||
# 触发条件:
|
||||
# - 定时: 每天 04:00 CST (20:00 UTC)
|
||||
# - 定时: 每天 04:00 和 16:00 CST
|
||||
# - 手动: workflow_dispatch
|
||||
# - COS桶告警: repository_dispatch (cos-alert事件)
|
||||
# - COS Webhook: repository_dispatch (cos-file-uploaded事件)
|
||||
|
||||
name: 铸渊训练Agent · 语料处理
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 20 * * *' # 04:00 CST = 20:00 UTC
|
||||
- cron: '0 8 * * *' # 16:00 CST = 08:00 UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
corpus_bucket:
|
||||
description: '语料桶名称'
|
||||
required: false
|
||||
default: 'cold'
|
||||
force_reprocess:
|
||||
description: '强制重新处理已处理的语料'
|
||||
mode:
|
||||
description: '运行模式'
|
||||
required: false
|
||||
default: 'false'
|
||||
default: 'full'
|
||||
type: choice
|
||||
options:
|
||||
- scan
|
||||
- extract
|
||||
- train
|
||||
- full
|
||||
persona_id:
|
||||
description: '训练目标人格体'
|
||||
required: false
|
||||
default: 'zhuyuan'
|
||||
repository_dispatch:
|
||||
types: [cos-alert]
|
||||
types: [cos-alert, cos-file-uploaded]
|
||||
|
||||
jobs:
|
||||
training:
|
||||
|
|
@ -37,6 +49,7 @@ jobs:
|
|||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
- name: 签出代码
|
||||
|
|
@ -51,43 +64,77 @@ jobs:
|
|||
working-directory: server/age-os
|
||||
run: npm ci --production 2>/dev/null || npm install --production
|
||||
|
||||
- name: 检查语料状态
|
||||
id: check
|
||||
env:
|
||||
ZY_OSS_KEY: ${{ secrets.ZY_OSS_KEY }}
|
||||
ZY_OSS_SECRET: ${{ secrets.ZY_OSS_SECRET }}
|
||||
ZY_COS_REGION: ${{ secrets.ZY_COS_REGION }}
|
||||
- name: 确定运行参数
|
||||
id: params
|
||||
run: |
|
||||
node -e "
|
||||
const cos = require('./server/age-os/mcp-server/cos');
|
||||
const extractor = require('./server/age-os/mcp-server/tools/corpus-extractor-ops');
|
||||
(async () => {
|
||||
try {
|
||||
const status = await extractor.cosGetCorpusStatus({ bucket: '${{ github.event.inputs.corpus_bucket || 'cold' }}' });
|
||||
console.log(JSON.stringify(status, null, 2));
|
||||
// 输出待处理数量
|
||||
const pending = status.pending || 0;
|
||||
const fs = require('fs');
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, 'pending=' + pending + '\n');
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, 'status=success\n');
|
||||
} catch (err) {
|
||||
console.error('语料检查失败:', err.message);
|
||||
const fs = require('fs');
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, 'status=error\n');
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, 'error=' + err.message + '\n');
|
||||
}
|
||||
})();
|
||||
"
|
||||
if [ "${{ github.event_name }}" = "repository_dispatch" ]; then
|
||||
echo "mode=full" >> $GITHUB_OUTPUT
|
||||
echo "bucket=${{ github.event.client_payload.bucket || 'cold' }}" >> $GITHUB_OUTPUT
|
||||
echo "persona=${{ github.event.client_payload.persona_id || 'zhuyuan' }}" >> $GITHUB_OUTPUT
|
||||
echo "trigger=cos-event" >> $GITHUB_OUTPUT
|
||||
elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "mode=${{ github.event.inputs.mode || 'full' }}" >> $GITHUB_OUTPUT
|
||||
echo "bucket=${{ github.event.inputs.corpus_bucket || 'cold' }}" >> $GITHUB_OUTPUT
|
||||
echo "persona=${{ github.event.inputs.persona_id || 'zhuyuan' }}" >> $GITHUB_OUTPUT
|
||||
echo "trigger=manual" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "mode=full" >> $GITHUB_OUTPUT
|
||||
echo "bucket=cold" >> $GITHUB_OUTPUT
|
||||
echo "persona=zhuyuan" >> $GITHUB_OUTPUT
|
||||
echo "trigger=schedule" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: 处理COS桶告警
|
||||
- name: 记录COS事件（如有）
|
||||
if: github.event_name == 'repository_dispatch'
|
||||
run: |
|
||||
echo "📡 收到COS事件: ${{ github.event.action }}"
|
||||
echo "事件负载:"
|
||||
echo '${{ toJson(github.event.client_payload) }}'
|
||||
|
||||
- name: 执行训练管线
|
||||
id: pipeline
|
||||
env:
|
||||
ZY_OSS_KEY: ${{ secrets.ZY_OSS_KEY }}
|
||||
ZY_OSS_SECRET: ${{ secrets.ZY_OSS_SECRET }}
|
||||
ZY_COS_REGION: ${{ secrets.ZY_COS_REGION }}
|
||||
ZY_DEEPSEEK_API_KEY: ${{ secrets.ZY_DEEPSEEK_API_KEY }}
|
||||
ZY_ZHIPU_API_KEY: ${{ secrets.ZY_ZHIPU_API_KEY }}
|
||||
ZY_QWEN_API_KEY: ${{ secrets.ZY_QWEN_API_KEY }}
|
||||
ZY_KIMI_API_KEY: ${{ secrets.ZY_KIMI_API_KEY }}
|
||||
run: |
|
||||
echo "📡 收到COS桶告警事件"
|
||||
echo "事件数据: ${{ toJson(github.event.client_payload) }}"
|
||||
node scripts/cos-training-trigger.js \
|
||||
${{ steps.params.outputs.mode }} \
|
||||
--bucket=${{ steps.params.outputs.bucket }} \
|
||||
--persona=${{ steps.params.outputs.persona }}
|
||||
|
||||
- name: 创建Issue（训练失败时）
|
||||
if: failure()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const trigger = '${{ steps.params.outputs.trigger }}';
|
||||
const mode = '${{ steps.params.outputs.mode }}';
|
||||
const bucket = '${{ steps.params.outputs.bucket }}';
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: `🧠 训练Agent异常 · ${new Date().toISOString().split('T')[0]}`,
|
||||
body: [
|
||||
'## 铸渊训练Agent运行异常',
|
||||
'',
|
||||
`- **触发方式**: ${trigger}`,
|
||||
`- **运行模式**: ${mode}`,
|
||||
`- **语料桶**: ${bucket}`,
|
||||
`- **工作流运行**: [查看日志](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`,
|
||||
'',
|
||||
'请铸渊检查COS桶连接和语料状态。',
|
||||
'',
|
||||
'---',
|
||||
`*此Issue由训练Agent自动创建 · ${new Date().toISOString()}*`
|
||||
].join('\n'),
|
||||
labels: ['training-agent', 'auto-notification']
|
||||
});
|
||||
|
||||
- name: 训练完成通知
|
||||
if: always()
|
||||
|
|
@ -95,6 +142,9 @@ jobs:
|
|||
echo "═══════════════════════════════════════════"
|
||||
echo "铸渊训练Agent · 运行完成"
|
||||
echo "时间: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
echo "状态: ${{ steps.check.outputs.status }}"
|
||||
echo "待处理语料: ${{ steps.check.outputs.pending }}"
|
||||
echo "触发: ${{ steps.params.outputs.trigger }}"
|
||||
echo "模式: ${{ steps.params.outputs.mode }}"
|
||||
echo "桶名: ${{ steps.params.outputs.bucket }}"
|
||||
echo "人格: ${{ steps.params.outputs.persona }}"
|
||||
echo "管线状态: ${{ steps.pipeline.outputs.pipeline_status || 'unknown' }}"
|
||||
echo "═══════════════════════════════════════════"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,456 @@
|
|||
/**
|
||||
* ═══════════════════════════════════════════════════════════
|
||||
* 🧠 COS训练触发器 · 端到端训练管线
|
||||
* ═══════════════════════════════════════════════════════════
|
||||
*
|
||||
* 签发: 铸渊 · ICE-GL-ZY001
|
||||
* 版权: 国作登字-2026-A-00037559
|
||||
*
|
||||
* 扫描COS桶中的新语料 → 解压/转换TCS格式 → 启动训练会话
|
||||
*
|
||||
* 设计:
|
||||
* 1. 扫描cold桶，列出所有文件（含非压缩文件和文件夹）
|
||||
* 2. 对比tcs-structured/目录，找出未处理的语料
|
||||
* 3. 自动提取/转换为TCS结构化格式
|
||||
* 4. 启动训练会话，用LLM分析和分类
|
||||
* 5. 输出处理结果，写入日志
|
||||
*
|
||||
* 运行方式:
|
||||
* node scripts/cos-training-trigger.js [scan|extract|train|full]
|
||||
*
|
||||
* scan — 仅扫描，输出未处理语料列表
|
||||
* extract — 扫描并解压/转换为TCS格式
|
||||
* train — 对已有TCS语料启动训练
|
||||
* full — 完整流程: 扫描 → 提取 → 训练
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
// ─── 路径 ───
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const COS_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'cos');
|
||||
const EXTRACTOR_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'tools', 'corpus-extractor-ops');
|
||||
const TRAINING_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'tools', 'training-agent-ops');
|
||||
|
||||
// ─── 延迟加载模块（允许在CI中跳过数据库依赖） ───
|
||||
let cos, extractor, trainer;
|
||||
|
||||
function loadModules() {
|
||||
cos = require(COS_MODULE);
|
||||
extractor = require(EXTRACTOR_MODULE);
|
||||
trainer = require(TRAINING_MODULE);
|
||||
}
|
||||
|
||||
// ─── 配置 ───
|
||||
const DEFAULT_BUCKET = 'cold';
|
||||
const DEFAULT_PERSONA = 'zhuyuan';
|
||||
const PROCESSED_PREFIX = 'tcs-structured/';
|
||||
const MAX_EXTRACT_PER_RUN = 20; // 每次最多处理文件数
|
||||
const MAX_TRAIN_PER_RUN = 5; // 每次最多训练文件数
|
||||
|
||||
// ─── 排除路径（不视为语料的目录/文件） ───
|
||||
const EXCLUDED_PREFIXES = [
|
||||
'tcs-structured/',
|
||||
'training-sessions/',
|
||||
'training-results/',
|
||||
'training-memory/',
|
||||
];
|
||||
|
||||
// ─── 支持的语料文件扩展名（含非压缩格式） ───
|
||||
const CORPUS_EXTENSIONS = [
|
||||
'.zip', '.gz', '.tar.gz', '.tgz', '.json.gz', // 压缩格式
|
||||
'.json', '.jsonl', '.md', '.txt', '.csv', // 非压缩格式
|
||||
];
|
||||
|
||||
/**
|
||||
* 判断文件是否为语料文件
|
||||
*/
|
||||
function isCorpusFile(key) {
|
||||
// 排除处理结果目录
|
||||
for (const prefix of EXCLUDED_PREFIXES) {
|
||||
if (key.startsWith(prefix)) return false;
|
||||
}
|
||||
// 匹配扩展名
|
||||
const lower = key.toLowerCase();
|
||||
return CORPUS_EXTENSIONS.some(ext => lower.endsWith(ext));
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断是否为语料目录（如 repo-archive/）
|
||||
*/
|
||||
function isCorpusDirectory(key) {
|
||||
for (const prefix of EXCLUDED_PREFIXES) {
|
||||
if (key.startsWith(prefix)) return false;
|
||||
}
|
||||
return key.endsWith('/');
|
||||
}
|
||||
|
||||
/**
|
||||
* 从已处理列表中判断某文件是否已处理
|
||||
*/
|
||||
function isProcessed(rawKey, processedFiles) {
|
||||
// 从rawKey提取基础文件名
|
||||
const baseName = rawKey.replace(/\.[^/.]+$/, '').split('/').pop();
|
||||
return processedFiles.some(f => f.key.includes(baseName));
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// 命令: scan — 扫描未处理语料
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
async function cmdScan(bucket) {
|
||||
console.log('═══ COS训练触发器 · 语料扫描 ═══\n');
|
||||
|
||||
const bucketName = bucket || DEFAULT_BUCKET;
|
||||
|
||||
// 列出所有文件
|
||||
const allFiles = await cos.list(bucketName, '', 500);
|
||||
// 列出已处理文件
|
||||
const processed = await cos.list(bucketName, PROCESSED_PREFIX, 500);
|
||||
|
||||
const processedFiles = processed.files.filter(f => f.key.endsWith('.tcs.json'));
|
||||
|
||||
// 分类
|
||||
const corpusFiles = [];
|
||||
const corpusDirs = [];
|
||||
|
||||
for (const file of allFiles.files) {
|
||||
if (isCorpusFile(file.key)) {
|
||||
const alreadyProcessed = isProcessed(file.key, processedFiles);
|
||||
corpusFiles.push({
|
||||
key: file.key,
|
||||
size_bytes: file.size_bytes,
|
||||
processed: alreadyProcessed
|
||||
});
|
||||
} else if (isCorpusDirectory(file.key)) {
|
||||
corpusDirs.push({ key: file.key });
|
||||
}
|
||||
}
|
||||
|
||||
const pending = corpusFiles.filter(f => !f.processed);
|
||||
|
||||
console.log(`桶: ${bucketName}`);
|
||||
console.log(`总文件数: ${allFiles.files.length}`);
|
||||
console.log(`语料文件: ${corpusFiles.length}`);
|
||||
console.log(`语料目录: ${corpusDirs.length}`);
|
||||
console.log(`已处理: ${corpusFiles.length - pending.length}`);
|
||||
console.log(`待处理: ${pending.length}`);
|
||||
console.log(`已生成TCS: ${processedFiles.length}`);
|
||||
|
||||
if (pending.length > 0) {
|
||||
console.log('\n📋 待处理语料:');
|
||||
for (const f of pending) {
|
||||
console.log(` 📄 ${f.key} (${formatBytes(f.size_bytes)})`);
|
||||
}
|
||||
}
|
||||
|
||||
if (corpusDirs.length > 0) {
|
||||
console.log('\n📁 语料目录:');
|
||||
for (const d of corpusDirs) {
|
||||
console.log(` 📂 ${d.key}`);
|
||||
}
|
||||
|
||||
// 扫描目录内的文件
|
||||
for (const dir of corpusDirs) {
|
||||
try {
|
||||
const dirFiles = await cos.list(bucketName, dir.key, 100);
|
||||
const dirCorpus = dirFiles.files.filter(f => isCorpusFile(f.key));
|
||||
if (dirCorpus.length > 0) {
|
||||
console.log(` └── ${dir.key} 内含 ${dirCorpus.length} 个语料文件`);
|
||||
for (const f of dirCorpus) {
|
||||
const alreadyProcessed = isProcessed(f.key, processedFiles);
|
||||
if (!alreadyProcessed) {
|
||||
pending.push({
|
||||
key: f.key,
|
||||
size_bytes: f.size_bytes,
|
||||
processed: false
|
||||
});
|
||||
console.log(` 📄 ${f.key} (${formatBytes(f.size_bytes)}) [待处理]`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
console.log(` └── ${dir.key} 扫描失败`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 写入GitHub Actions输出
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
const outputLines = [
|
||||
`pending=${pending.length}`,
|
||||
`total_corpus=${corpusFiles.length}`,
|
||||
`processed=${processedFiles.length}`,
|
||||
`has_new_corpus=${pending.length > 0 ? 'true' : 'false'}`,
|
||||
`pending_files=${pending.map(f => f.key).join(',')}`
|
||||
];
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, outputLines.join('\n') + '\n');
|
||||
}
|
||||
|
||||
return { pending, processedFiles, corpusDirs };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// 命令: extract — 提取/转换语料为TCS格式
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
async function cmdExtract(bucket) {
|
||||
console.log('═══ COS训练触发器 · 语料提取 ═══\n');
|
||||
|
||||
const bucketName = bucket || DEFAULT_BUCKET;
|
||||
const { pending } = await cmdScan(bucketName);
|
||||
|
||||
if (pending.length === 0) {
|
||||
console.log('\n✅ 无待处理语料');
|
||||
writeGitHubOutput('extracted=0', 'extract_status=skipped');
|
||||
return { extracted: 0, errors: 0 };
|
||||
}
|
||||
|
||||
const toProcess = pending.slice(0, MAX_EXTRACT_PER_RUN);
|
||||
console.log(`\n🔄 开始提取 ${toProcess.length}/${pending.length} 个文件...\n`);
|
||||
|
||||
let extracted = 0;
|
||||
let errors = 0;
|
||||
const results = [];
|
||||
|
||||
for (const file of toProcess) {
|
||||
try {
|
||||
console.log(` 📦 处理: ${file.key}...`);
|
||||
const result = await extractor.cosExtractCorpus({
|
||||
bucket: bucketName,
|
||||
key: file.key,
|
||||
output_bucket: bucketName,
|
||||
output_prefix: PROCESSED_PREFIX
|
||||
});
|
||||
extracted++;
|
||||
results.push({ key: file.key, status: 'success', output: result.output?.key });
|
||||
console.log(` ✅ 完成: ${result.output?.key || '已处理'} (${result.entries || 0} 条目)`);
|
||||
} catch (err) {
|
||||
errors++;
|
||||
results.push({ key: file.key, status: 'error', error: err.message });
|
||||
console.log(` ❌ 失败: ${file.key} — ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n═══ 提取完毕 ═══`);
|
||||
console.log(`✅ 成功: ${extracted}`);
|
||||
console.log(`❌ 失败: ${errors}`);
|
||||
console.log(`⏳ 剩余: ${pending.length - toProcess.length}`);
|
||||
|
||||
writeGitHubOutput(
|
||||
`extracted=${extracted}`,
|
||||
`extract_errors=${errors}`,
|
||||
`extract_status=${errors > 0 ? 'partial' : 'success'}`
|
||||
);
|
||||
|
||||
return { extracted, errors, results };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// 命令: train — 对TCS语料启动训练
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
async function cmdTrain(bucket, personaId) {
|
||||
console.log('═══ COS训练触发器 · 训练处理 ═══\n');
|
||||
|
||||
const bucketName = bucket || DEFAULT_BUCKET;
|
||||
const persona = personaId || DEFAULT_PERSONA;
|
||||
|
||||
// 列出可用的TCS语料
|
||||
const processed = await cos.list(bucketName, PROCESSED_PREFIX, 500);
|
||||
const tcsFiles = processed.files.filter(f => f.key.endsWith('.tcs.json'));
|
||||
|
||||
if (tcsFiles.length === 0) {
|
||||
console.log('⚠️ 无TCS结构化语料可训练。请先运行 extract 命令。');
|
||||
writeGitHubOutput('trained=0', 'train_status=no_corpus');
|
||||
return { trained: 0, errors: 0 };
|
||||
}
|
||||
|
||||
console.log(`📚 找到 ${tcsFiles.length} 个TCS语料文件`);
|
||||
|
||||
// 检查已有训练结果，避免重复处理
|
||||
let existingResults = [];
|
||||
try {
|
||||
const existing = await cos.list(bucketName, `training-results/${persona}/`, 100);
|
||||
existingResults = existing.files.filter(f => f.key.endsWith('.json'));
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// 启动训练会话
|
||||
console.log(`\n🧠 启动训练会话 · 人格体: ${persona}`);
|
||||
|
||||
let session;
|
||||
try {
|
||||
session = await trainer.trainingStartSession({
|
||||
persona_id: persona,
|
||||
corpus_bucket: bucketName,
|
||||
corpus_prefix: PROCESSED_PREFIX,
|
||||
session_name: `自动训练-${new Date().toISOString().slice(0, 10)}`
|
||||
});
|
||||
console.log(`✅ 会话已启动: ${session.session_id}`);
|
||||
console.log(` 可用模型: ${session.models.available.map(m => m.name).join(', ') || '无'}`);
|
||||
} catch (err) {
|
||||
console.log(`❌ 训练会话启动失败: ${err.message}`);
|
||||
writeGitHubOutput('trained=0', `train_status=session_error`, `train_error=${err.message}`);
|
||||
return { trained: 0, errors: 1, error: err.message };
|
||||
}
|
||||
|
||||
// 检查是否有可用的LLM模型
|
||||
if (!session.models.available || session.models.available.length === 0) {
|
||||
console.log('⚠️ 无可用LLM模型（需要配置 ZY_DEEPSEEK_API_KEY 等密钥）');
|
||||
console.log(' 训练会话已记录，等待LLM密钥配置后再次运行。');
|
||||
writeGitHubOutput('trained=0', 'train_status=no_llm_keys');
|
||||
return { trained: 0, errors: 0, note: '无LLM密钥' };
|
||||
}
|
||||
|
||||
// 处理语料
|
||||
const toTrain = tcsFiles.slice(0, MAX_TRAIN_PER_RUN);
|
||||
let trained = 0;
|
||||
let trainErrors = 0;
|
||||
const trainResults = [];
|
||||
|
||||
for (const tcsFile of toTrain) {
|
||||
try {
|
||||
console.log(` 🔬 训练处理: ${tcsFile.key}...`);
|
||||
const result = await trainer.trainingProcessCorpus({
|
||||
corpus_bucket: bucketName,
|
||||
corpus_key: tcsFile.key,
|
||||
persona_id: persona,
|
||||
max_entries: 10
|
||||
});
|
||||
trained++;
|
||||
trainResults.push({ key: tcsFile.key, status: 'success', ...result });
|
||||
console.log(` ✅ 完成: ${result.classified}/${result.total} 分类成功`);
|
||||
} catch (err) {
|
||||
trainErrors++;
|
||||
trainResults.push({ key: tcsFile.key, status: 'error', error: err.message });
|
||||
console.log(` ❌ 失败: ${tcsFile.key} — ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n═══ 训练完毕 ═══`);
|
||||
console.log(`✅ 成功: ${trained}`);
|
||||
console.log(`❌ 失败: ${trainErrors}`);
|
||||
|
||||
writeGitHubOutput(
|
||||
`trained=${trained}`,
|
||||
`train_errors=${trainErrors}`,
|
||||
`train_status=${trainErrors > 0 ? 'partial' : 'success'}`
|
||||
);
|
||||
|
||||
return { trained, errors: trainErrors, results: trainResults, session_id: session.session_id };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// 命令: full — 完整流程
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
async function cmdFull(bucket, personaId) {
|
||||
console.log('╔═══════════════════════════════════════════╗');
|
||||
console.log('║ COS训练触发器 · 完整训练管线 ║');
|
||||
console.log('║ 铸渊 · ICE-GL-ZY001 ║');
|
||||
console.log('╚═══════════════════════════════════════════╝\n');
|
||||
|
||||
const bucketName = bucket || DEFAULT_BUCKET;
|
||||
const persona = personaId || DEFAULT_PERSONA;
|
||||
const startTime = Date.now();
|
||||
|
||||
// 第一步: 提取语料
|
||||
console.log('📍 第一步: 提取语料\n');
|
||||
const extractResult = await cmdExtract(bucketName);
|
||||
|
||||
// 第二步: 训练处理
|
||||
console.log('\n📍 第二步: 训练处理\n');
|
||||
const trainResult = await cmdTrain(bucketName, persona);
|
||||
|
||||
// 汇总
|
||||
const duration = Date.now() - startTime;
|
||||
console.log('\n╔═══════════════════════════════════════════╗');
|
||||
console.log('║ 完整训练管线 · 运行完毕 ║');
|
||||
console.log('╚═══════════════════════════════════════════╝');
|
||||
console.log(` 提取: ${extractResult.extracted} 成功 / ${extractResult.errors} 失败`);
|
||||
console.log(` 训练: ${trainResult.trained} 成功 / ${trainResult.errors} 失败`);
|
||||
console.log(` 耗时: ${(duration / 1000).toFixed(1)}s`);
|
||||
|
||||
writeGitHubOutput(
|
||||
`pipeline_status=${(extractResult.errors + trainResult.errors) > 0 ? 'partial' : 'success'}`,
|
||||
`pipeline_duration_ms=${duration}`
|
||||
);
|
||||
|
||||
return { extract: extractResult, train: trainResult, duration_ms: duration };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// 辅助函数
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
||||
}
|
||||
|
||||
function writeGitHubOutput(...lines) {
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, lines.join('\n') + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// CLI 入口
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const command = args[0] || 'scan';
|
||||
const bucket = args.find(a => a.startsWith('--bucket='))?.split('=')[1] || DEFAULT_BUCKET;
|
||||
const persona = args.find(a => a.startsWith('--persona='))?.split('=')[1] || DEFAULT_PERSONA;
|
||||
|
||||
// 加载模块
|
||||
try {
|
||||
loadModules();
|
||||
} catch (err) {
|
||||
console.error(`❌ 模块加载失败: ${err.message}`);
|
||||
console.error(' 请确保 server/age-os/mcp-server/ 依赖已安装');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
switch (command) {
|
||||
case 'scan':
|
||||
await cmdScan(bucket);
|
||||
break;
|
||||
case 'extract':
|
||||
await cmdExtract(bucket);
|
||||
break;
|
||||
case 'train':
|
||||
await cmdTrain(bucket, persona);
|
||||
break;
|
||||
case 'full':
|
||||
await cmdFull(bucket, persona);
|
||||
break;
|
||||
default:
|
||||
console.log('COS训练触发器 · 铸渊 · ICE-GL-ZY001');
|
||||
console.log('');
|
||||
console.log('用法:');
|
||||
console.log(' node scripts/cos-training-trigger.js scan — 扫描未处理语料');
|
||||
console.log(' node scripts/cos-training-trigger.js extract — 提取/转换为TCS格式');
|
||||
console.log(' node scripts/cos-training-trigger.js train — 启动训练处理');
|
||||
console.log(' node scripts/cos-training-trigger.js full — 完整流程');
|
||||
console.log('');
|
||||
console.log('选项:');
|
||||
console.log(' --bucket=cold|hot|team — 指定COS桶（默认cold）');
|
||||
console.log(' --persona=zhuyuan — 指定人格体（默认zhuyuan）');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('COS训练触发器异常:', err.message);
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, 'pipeline_status=error\n');
|
||||
fs.appendFileSync(process.env.GITHUB_OUTPUT, `pipeline_error=${err.message}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -50,6 +50,7 @@
|
|||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const https = require('https');
|
||||
const express = require('express');
|
||||
const db = require('./db');
|
||||
const cos = require('./cos');
|
||||
|
|
@ -245,6 +246,9 @@ function apiKeyAuth(req, res, next) {
|
|||
// /health 端点免鉴权
|
||||
if (req.path === '/health') return next();
|
||||
|
||||
// /webhook/* 端点免API Key鉴权（使用独立的webhook密钥验证）
|
||||
if (req.path.startsWith('/webhook/')) return next();
|
||||
|
||||
// 内部回环地址免鉴权
|
||||
const remoteIp = req.ip || req.connection.remoteAddress || '';
|
||||
const isLocal = remoteIp === '127.0.0.1' || remoteIp === '::1' || remoteIp === '::ffff:127.0.0.1';
|
||||
|
|
@ -905,6 +909,135 @@ app.get('/finetune/:personaId/jobs/:jobId', async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
// ─── COS事件Webhook端点 ───
|
||||
// 接收腾讯COS桶事件通知（如文件上传），转发为GitHub repository_dispatch触发训练Agent
|
||||
// COS桶设置: 事件通知 → HTTP回调 → https://guanghulab.online/api/mcp/webhook/cos-event
|
||||
//
|
||||
// 此端点免API Key鉴权，但使用独立的webhook密钥验证
|
||||
|
||||
app.post('/webhook/cos-event', async (req, res) => {
|
||||
// COS Webhook密钥验证（独立于API Key，防止外部伪造请求）
|
||||
const webhookSecret = process.env.COS_WEBHOOK_SECRET || '';
|
||||
if (webhookSecret) {
|
||||
const providedSecret = req.headers['x-cos-webhook-secret'] || req.query.secret || '';
|
||||
if (providedSecret !== webhookSecret) {
|
||||
return res.status(403).json({ error: true, code: 'INVALID_WEBHOOK_SECRET', message: 'Webhook密钥无效' });
|
||||
}
|
||||
}
|
||||
|
||||
const event = req.body;
|
||||
const now = new Date().toISOString();
|
||||
|
||||
// 解析COS事件（腾讯COS事件通知格式）
|
||||
let eventType = 'unknown';
|
||||
let bucketName = '';
|
||||
let objectKey = '';
|
||||
let objectSize = 0;
|
||||
|
||||
try {
|
||||
if (event.Records && Array.isArray(event.Records) && event.Records.length > 0) {
|
||||
// 标准COS事件格式
|
||||
const record = event.Records[0];
|
||||
eventType = record.event?.eventName || record.eventName || 'cos:ObjectCreated';
|
||||
bucketName = record.cos?.cosBucket?.name || record.s3?.bucket?.name || '';
|
||||
objectKey = record.cos?.cosObject?.key || record.s3?.object?.key || '';
|
||||
objectSize = record.cos?.cosObject?.size || record.s3?.object?.size || 0;
|
||||
} else if (event.bucket && event.key) {
|
||||
// 简化格式（手动测试用）
|
||||
eventType = event.event || 'cos:ObjectCreated';
|
||||
bucketName = event.bucket;
|
||||
objectKey = event.key;
|
||||
objectSize = event.size || 0;
|
||||
}
|
||||
} catch {
|
||||
// 解析失败继续处理
|
||||
}
|
||||
|
||||
console.log(`[COS Webhook] 事件: ${eventType} · 桶: ${bucketName} · 文件: ${objectKey}`);
|
||||
|
||||
// 触发GitHub repository_dispatch（如果配置了GitHub PAT）
|
||||
const githubToken = process.env.GITHUB_DISPATCH_TOKEN || process.env.GITHUB_TOKEN || '';
|
||||
const githubRepo = process.env.GITHUB_REPO || 'qinfendebingshuo/guanghulab';
|
||||
let dispatchResult = null;
|
||||
|
||||
if (githubToken) {
|
||||
try {
|
||||
const [owner, repo] = githubRepo.split('/');
|
||||
const dispatchPayload = JSON.stringify({
|
||||
event_type: 'cos-file-uploaded',
|
||||
client_payload: {
|
||||
bucket: bucketName,
|
||||
key: objectKey,
|
||||
size: objectSize,
|
||||
event: eventType,
|
||||
timestamp: now
|
||||
}
|
||||
});
|
||||
|
||||
dispatchResult = await new Promise((resolve, reject) => {
|
||||
const dispatchReq = https.request({
|
||||
hostname: 'api.github.com',
|
||||
port: 443,
|
||||
path: `/repos/${owner}/${repo}/dispatches`,
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Accept': 'application/vnd.github+json',
|
||||
'Authorization': `Bearer ${githubToken}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
'User-Agent': 'ZY-MCP-COS-Webhook',
|
||||
'Content-Type': 'application/json',
|
||||
'Content-Length': Buffer.byteLength(dispatchPayload)
|
||||
},
|
||||
timeout: 15000
|
||||
}, (dispatchRes) => {
|
||||
let body = '';
|
||||
dispatchRes.on('data', c => body += c);
|
||||
dispatchRes.on('end', () => {
|
||||
resolve({ status: dispatchRes.statusCode, body });
|
||||
});
|
||||
});
|
||||
dispatchReq.on('error', reject);
|
||||
dispatchReq.on('timeout', () => { dispatchReq.destroy(); reject(new Error('dispatch timeout')); });
|
||||
dispatchReq.write(dispatchPayload);
|
||||
dispatchReq.end();
|
||||
});
|
||||
|
||||
console.log(`[COS Webhook] GitHub dispatch: ${dispatchResult.status}`);
|
||||
} catch (err) {
|
||||
console.error(`[COS Webhook] GitHub dispatch失败: ${err.message}`);
|
||||
dispatchResult = { status: 'error', error: err.message };
|
||||
}
|
||||
} else {
|
||||
console.log('[COS Webhook] 未配置GITHUB_DISPATCH_TOKEN，跳过dispatch');
|
||||
}
|
||||
|
||||
// 写入告警日志到COS（异步，不阻塞响应）
|
||||
const logEntry = {
|
||||
event_type: eventType,
|
||||
bucket: bucketName,
|
||||
key: objectKey,
|
||||
size: objectSize,
|
||||
received_at: now,
|
||||
dispatch: dispatchResult ? { status: dispatchResult.status } : null
|
||||
};
|
||||
|
||||
try {
|
||||
await cos.write('team', `zhuyuan/cos-events/${now.slice(0, 10)}/${Date.now()}.json`,
|
||||
JSON.stringify(logEntry, null, 2), 'application/json');
|
||||
} catch {
|
||||
// 日志写入失败不影响响应
|
||||
}
|
||||
|
||||
res.json({
|
||||
received: true,
|
||||
event_type: eventType,
|
||||
bucket: bucketName,
|
||||
key: objectKey,
|
||||
dispatch: dispatchResult ? (dispatchResult.status === 204 ? 'triggered' : 'failed') : 'no_token',
|
||||
timestamp: now
|
||||
});
|
||||
});
|
||||
|
||||
// ─── 数据库迁移状态API ───
|
||||
|
||||
app.get('/migrations', async (_req, res) => {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,20 @@ const inflate = promisify(zlib.inflate);
|
|||
// ─── 支持的压缩格式 ───
|
||||
const COMPRESSED_EXTENSIONS = ['.zip', '.gz', '.tar.gz', '.tgz', '.json.gz'];
|
||||
|
||||
// ─── 支持的语料文件格式（含非压缩） ───
|
||||
const ALL_CORPUS_EXTENSIONS = [
|
||||
'.zip', '.gz', '.tar.gz', '.tgz', '.json.gz', // 压缩格式
|
||||
'.json', '.jsonl', '.md', '.txt', '.csv', // 非压缩格式
|
||||
];
|
||||
|
||||
// ─── 排除的路径前缀（处理结果目录，不视为原始语料） ───
|
||||
const EXCLUDED_CORPUS_PREFIXES = [
|
||||
'tcs-structured/',
|
||||
'training-sessions/',
|
||||
'training-results/',
|
||||
'training-memory/',
|
||||
];
|
||||
|
||||
// ─── TCS结构化格式定义 ───
|
||||
// TCS = 通感语言核系统编程语言（Tonggan Core System）
|
||||
// 所有语料转换后统一为此格式
|
||||
|
|
@ -44,6 +58,20 @@ function isCompressedFile(key) {
|
|||
return COMPRESSED_EXTENSIONS.some(ext => lower.endsWith(ext));
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测文件是否为任意语料文件（含非压缩格式）
|
||||
*/
|
||||
function isCorpusFile(key) {
|
||||
// 排除处理结果目录
|
||||
for (const prefix of EXCLUDED_CORPUS_PREFIXES) {
|
||||
if (key.startsWith(prefix)) return false;
|
||||
}
|
||||
// 排除目录标记（以/结尾的空key）
|
||||
if (key.endsWith('/')) return false;
|
||||
const lower = key.toLowerCase();
|
||||
return ALL_CORPUS_EXTENSIONS.some(ext => lower.endsWith(ext));
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测文件类型（代码/Notion/GPT语料）
|
||||
*/
|
||||
|
|
@ -341,14 +369,24 @@ async function cosGetCorpusStatus(input) {
|
|||
cos.list(bucket, 'tcs-structured/', 500)
|
||||
]);
|
||||
|
||||
const rawCorpus = rawFiles.files.filter(f => isCompressedFile(f.key));
|
||||
// 检测所有语料文件（含压缩和非压缩格式）
|
||||
const allCorpus = rawFiles.files.filter(f => isCorpusFile(f.key));
|
||||
const compressedCorpus = allCorpus.filter(f => isCompressedFile(f.key));
|
||||
const uncompressedCorpus = allCorpus.filter(f => !isCompressedFile(f.key));
|
||||
const processed = processedFiles.files.filter(f => f.key.endsWith('.tcs.json'));
|
||||
|
||||
return {
|
||||
raw_corpus: {
|
||||
total: rawCorpus.length,
|
||||
total_size_bytes: rawCorpus.reduce((sum, f) => sum + f.size_bytes, 0),
|
||||
files: rawCorpus.map(f => ({ key: f.key, size_bytes: f.size_bytes }))
|
||||
total: allCorpus.length,
|
||||
compressed: compressedCorpus.length,
|
||||
uncompressed: uncompressedCorpus.length,
|
||||
total_size_bytes: allCorpus.reduce((sum, f) => sum + f.size_bytes, 0),
|
||||
files: allCorpus.map(f => ({
|
||||
key: f.key,
|
||||
size_bytes: f.size_bytes,
|
||||
compressed: isCompressedFile(f.key),
|
||||
corpus_type: detectCorpusType(f.key)
|
||||
}))
|
||||
},
|
||||
processed: {
|
||||
total: processed.length,
|
||||
|
|
@ -360,7 +398,7 @@ async function cosGetCorpusStatus(input) {
|
|||
},
|
||||
files: processed.map(f => ({ key: f.key, size_bytes: f.size_bytes }))
|
||||
},
|
||||
pending: rawCorpus.length - processed.length,
|
||||
pending: Math.max(0, allCorpus.length - processed.length),
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue