diff --git a/server/age-os/mcp-server/tools/finetune-engine-ops.js b/server/age-os/mcp-server/tools/finetune-engine-ops.js new file mode 100644 index 00000000..d8807eca --- /dev/null +++ b/server/age-os/mcp-server/tools/finetune-engine-ops.js @@ -0,0 +1,972 @@ +/** + * ═══════════════════════════════════════════════════════════ + * 模块H · 开源模型微调引擎 MCP 工具 + * ═══════════════════════════════════════════════════════════ + * + * 签发: 铸渊 · ICE-GL-ZY001 + * 版权: 国作登字-2026-A-00037559 + * + * 冰朔D62核心指令: 接入开源模型,用COS桶训练数据直接做微调 + * 本质: 同一份TCS结构化数据,两种用途 — RAG + 微调 + * + * 架构理念: + * 现有RAG训练 → 用API调用商业模型,人格体"脑子"在COS桶里 + * 开源模型微调 → 用同一份数据,把"脑子"直接装进开源模型 + * 二者并行运行 → 微调模型优先 → 不可用时降级回API模型 + * + * 工具清单: + * finetuneExportDataset — 导出TCS语料为微调JSONL格式 + * finetuneSubmitJob — 提交微调任务到DeepSeek/Qwen + * finetuneCheckStatus — 查询微调任务进度 + * finetuneRegisterModel — 注册微调完成的模型 + * finetuneListModels — 列出已注册的微调模型 + * finetuneCallModel — 调用微调模型进行推理 + * finetuneCompareModels — A/B测试微调 vs 基座模型 + * finetuneGetCostEstimate — 估算微调成本 + */ + +'use strict'; + +const https = require('https'); +const crypto = require('crypto'); +const cos = require('../cos'); + +// ─── 微调 API 配置 ─── +const FINETUNE_PROVIDERS = { + deepseek: { + host: 'api.deepseek.com', + createPath: '/fine_tuning/jobs', + statusPath: '/fine_tuning/jobs/', + uploadPath: '/files', + inferencePath: '/v1/chat/completions', + defaultModel: 'deepseek-chat', + keyEnv: 'ZY_DEEPSEEK_API_KEY', + label: 'DeepSeek微调' + }, + qwen: { + host: 'dashscope.aliyuncs.com', + createPath: '/api/v1/fine-tunes', + statusPath: '/api/v1/fine-tunes/', + uploadPath: '/api/v1/files', + inferencePath: '/compatible-mode/v1/chat/completions', + defaultModel: 'qwen-max', + keyEnv: 'ZY_QWEN_API_KEY', + label: 'Qwen/DashScope微调' + } +}; + +// ─── 推理降级配置(与training-agent-ops.js同源) ─── +const LLM_CONFIGS = { + 'deepseek-chat': { + host: 'api.deepseek.com', + path: '/v1/chat/completions', + model: 'deepseek-chat', + keyEnv: 'ZY_DEEPSEEK_API_KEY', + purpose: '微调基座·推理降级' + }, + 'qwen-max': { + host: 'dashscope.aliyuncs.com', + path: '/compatible-mode/v1/chat/completions', + model: 'qwen-max', + keyEnv: 'ZY_QWEN_API_KEY', + purpose: '微调基座·推理降级' + } +}; + +// ─── 成本估算参数 ─── +const COST_PER_1K_TOKENS = { + deepseek: 0.014, // 约 ¥0.014 / 1K tokens(训练) + qwen: 0.020 // 约 ¥0.020 / 1K tokens(训练) +}; + +// ─── 常量 ─── +const DEFAULT_BUCKET = 'cold'; +const MAX_SAMPLES_DEFAULT = 500; +const FINETUNE_TIMEOUT = 60000; + +// ═══════════════════════════════════════════════════════════ +// 工具实现 +// ═══════════════════════════════════════════════════════════ + +/** + * finetuneExportDataset — 导出TCS语料为微调JSONL格式 + * + * 将COS桶中的TCS结构化语料转换为 instruction/input/output 三元组 + * JSONL格式,直接用于提交到微调API + * + * input: + * persona_id: string — 人格体ID + * corpus_bucket: string — 语料桶(默认cold) + * corpus_prefix: string — 语料路径前缀 + * output_format: string — 输出格式(默认jsonl) + * max_samples: number — 最大样本数 + */ +async function finetuneExportDataset(input) { + const { persona_id, corpus_bucket, corpus_prefix, output_format, max_samples } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + + const bucket = corpus_bucket || DEFAULT_BUCKET; + const prefix = corpus_prefix || 'tcs-structured/'; + const format = output_format || 'jsonl'; + const maxSamples = max_samples || MAX_SAMPLES_DEFAULT; + const datasetId = `ds-${persona_id}-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`; + + // 扫描TCS语料文件 + let corpusFiles = []; + try { + const result = await cos.list(bucket, prefix, 500); + corpusFiles = result.files.filter(f => f.key.endsWith('.tcs.json') || f.key.endsWith('.json')); + } catch { + throw new Error(`无法读取语料桶 ${bucket}/${prefix}`); + } + + if (corpusFiles.length === 0) { + throw new Error(`语料桶 ${bucket}/${prefix} 中未找到TCS语料文件`); + } + + // 逐文件读取并转换为JSONL三元组 + const jsonlLines = []; + let filesProcessed = 0; + + for (const file of corpusFiles) { + if (jsonlLines.length >= maxSamples) break; + + try { + const raw = await cos.read(bucket, file.key); + const corpus = JSON.parse(raw.content); + const entries = corpus.entries || (Array.isArray(corpus) ? corpus : [corpus]); + + for (const entry of entries) { + if (jsonlLines.length >= maxSamples) break; + + const triple = convertEntryToTriple(persona_id, corpus.corpus_type, entry); + if (triple) { + jsonlLines.push(JSON.stringify(triple)); + } + } + filesProcessed++; + } catch { + // 跳过无法解析的文件 + } + } + + if (jsonlLines.length === 0) { + throw new Error('未能从语料中生成任何训练样本'); + } + + // 写入JSONL到COS + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const fileKey = `finetune-datasets/${persona_id}/${timestamp}.${format}`; + const jsonlContent = jsonlLines.join('\n') + '\n'; + + await cos.write(bucket, fileKey, jsonlContent, 'application/jsonl'); + + return { + dataset_id: datasetId, + file_key: fileKey, + sample_count: jsonlLines.length, + format, + files_scanned: corpusFiles.length, + files_processed: filesProcessed, + bucket, + created_at: new Date().toISOString() + }; +} + +/** + * finetuneSubmitJob — 提交微调任务到DeepSeek或Qwen API + * + * 从COS读取JSONL数据集,上传到provider,然后创建微调任务 + * + * input: + * persona_id: string — 人格体ID + * dataset_key: string — COS中JSONL文件路径 + * provider: string — 微调提供方(deepseek / qwen) + * base_model: string — 基座模型(可选,默认取provider默认值) + * hyperparams: object — 超参数(可选) + */ +async function finetuneSubmitJob(input) { + const { persona_id, dataset_key, provider, base_model, hyperparams } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + if (!dataset_key) throw new Error('缺少 dataset_key'); + + const providerKey = (provider || 'deepseek').toLowerCase(); + const providerConfig = FINETUNE_PROVIDERS[providerKey]; + if (!providerConfig) throw new Error(`不支持的微调提供方: ${providerKey},仅支持 deepseek / qwen`); + + const apiKey = process.env[providerConfig.keyEnv]; + if (!apiKey) throw new Error(`缺少API密钥环境变量 ${providerConfig.keyEnv}`); + + const jobId = `ft-${persona_id}-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`; + const model = base_model || providerConfig.defaultModel; + + // 从COS读取JSONL数据集 + const bucket = DEFAULT_BUCKET; + const raw = await cos.read(bucket, dataset_key); + const datasetContent = raw.content; + + // 上传训练文件到provider + const fileId = await uploadTrainingFile(providerConfig, apiKey, datasetContent, providerKey); + + // 创建微调任务 + const jobResult = await createFinetuneJob(providerConfig, apiKey, model, fileId, hyperparams, providerKey); + + // 保存任务元数据到COS + const jobMeta = { + job_id: jobId, + provider_job_id: jobResult.provider_job_id, + persona_id, + provider: providerKey, + base_model: model, + dataset_key, + file_id: fileId, + hyperparams: hyperparams || {}, + status: jobResult.status || 'pending', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString() + }; + + await cos.write(bucket, `finetune-jobs/${persona_id}/${jobId}.json`, + JSON.stringify(jobMeta, null, 2), 'application/json'); + + return { + job_id: jobId, + provider_job_id: jobResult.provider_job_id, + provider: providerKey, + status: jobMeta.status, + base_model: model, + estimated_time: jobResult.estimated_time || '未知,通常需要数小时' + }; +} + +/** + * finetuneCheckStatus — 查询微调任务进度 + * + * input: + * persona_id: string — 人格体ID + * job_id: string — 任务ID + * provider: string — 微调提供方 + */ +async function finetuneCheckStatus(input) { + const { persona_id, job_id, provider } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + if (!job_id) throw new Error('缺少 job_id'); + + const bucket = DEFAULT_BUCKET; + + // 读取任务元数据 + let jobMeta; + try { + const raw = await cos.read(bucket, `finetune-jobs/${persona_id}/${job_id}.json`); + jobMeta = JSON.parse(raw.content); + } catch { + throw new Error(`未找到微调任务: ${job_id}`); + } + + const providerKey = provider || jobMeta.provider; + const providerConfig = FINETUNE_PROVIDERS[providerKey]; + if (!providerConfig) throw new Error(`不支持的微调提供方: ${providerKey}`); + + const apiKey = process.env[providerConfig.keyEnv]; + if (!apiKey) throw new Error(`缺少API密钥环境变量 ${providerConfig.keyEnv}`); + + // 查询provider API获取最新状态 + const providerJobId = jobMeta.provider_job_id; + let statusResult; + try { + statusResult = await queryJobStatus(providerConfig, apiKey, providerJobId, providerKey); + } catch (err) { + return { + job_id, + provider: providerKey, + status: jobMeta.status, + progress: null, + metrics: null, + error: `查询provider状态失败: ${err.message}`, + last_known_update: jobMeta.updated_at + }; + } + + // 更新COS中的任务元数据 + jobMeta.status = statusResult.status; + jobMeta.updated_at = new Date().toISOString(); + if (statusResult.fine_tuned_model) { + jobMeta.fine_tuned_model = statusResult.fine_tuned_model; + } + if (statusResult.metrics) { + jobMeta.metrics = statusResult.metrics; + } + + try { + await cos.write(bucket, `finetune-jobs/${persona_id}/${job_id}.json`, + JSON.stringify(jobMeta, null, 2), 'application/json'); + } catch { /* ignore */ } + + return { + job_id, + provider_job_id: providerJobId, + provider: providerKey, + status: statusResult.status, + progress: statusResult.progress || null, + metrics: statusResult.metrics || null, + fine_tuned_model: statusResult.fine_tuned_model || null, + updated_at: jobMeta.updated_at + }; +} + +/** + * finetuneRegisterModel — 注册微调完成的模型 + * + * input: + * persona_id: string — 人格体ID + * job_id: string — 关联的微调任务ID + * model_endpoint: string — 模型推理端点(provider返回的fine_tuned_model名称) + * model_name: string — 本地注册名称 + * provider: string — 微调提供方 + * description: string — 模型描述 + */ +async function finetuneRegisterModel(input) { + const { persona_id, job_id, model_endpoint, model_name, provider, description } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + if (!model_endpoint) throw new Error('缺少 model_endpoint'); + if (!model_name) throw new Error('缺少 model_name'); + + const providerKey = (provider || 'deepseek').toLowerCase(); + const providerConfig = FINETUNE_PROVIDERS[providerKey]; + if (!providerConfig) throw new Error(`不支持的微调提供方: ${providerKey}`); + + const modelId = `mdl-${persona_id}-${crypto.randomBytes(4).toString('hex')}`; + const now = new Date().toISOString(); + + const modelConfig = { + model_id: modelId, + persona_id, + model_name, + model_endpoint, + provider: providerKey, + provider_host: providerConfig.host, + inference_path: providerConfig.inferencePath, + key_env: providerConfig.keyEnv, + job_id: job_id || null, + description: description || `${persona_id} 微调模型`, + status: 'active', + created_at: now, + updated_at: now + }; + + const bucket = DEFAULT_BUCKET; + const configKey = `finetune-models/${persona_id}/${model_name}.json`; + await cos.write(bucket, configKey, JSON.stringify(modelConfig, null, 2), 'application/json'); + + return { + model_id: modelId, + model_name, + provider: providerKey, + registered_at: now, + config_key: configKey, + config: modelConfig + }; +} + +/** + * finetuneListModels — 列出已注册的微调模型 + * + * input: + * persona_id: string — 人格体ID + */ +async function finetuneListModels(input) { + const { persona_id } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + + const bucket = DEFAULT_BUCKET; + const prefix = `finetune-models/${persona_id}/`; + + let files = []; + try { + const result = await cos.list(bucket, prefix, 100); + files = result.files.filter(f => f.key.endsWith('.json')); + } catch { + return { persona_id, models: [], count: 0 }; + } + + const models = []; + for (const file of files) { + try { + const raw = await cos.read(bucket, file.key); + const config = JSON.parse(raw.content); + models.push({ + model_name: config.model_name, + model_id: config.model_id, + provider: config.provider, + model_endpoint: config.model_endpoint, + status: config.status, + description: config.description, + created_at: config.created_at + }); + } catch { + // 跳过无法解析的配置 + } + } + + return { + persona_id, + models, + count: models.length + }; +} + +/** + * finetuneCallModel — 调用微调模型进行推理 + * + * 加载模型配置,调用provider推理API + * 微调模型不可用时自动降级到基座模型 + * + * input: + * persona_id: string — 人格体ID + * model_name: string — 已注册的模型名称 + * prompt: string — 推理提示词 + * temperature: number — 温度(默认0.7) + * max_tokens: number — 最大token数(默认1000) + */ +async function finetuneCallModel(input) { + const { persona_id, model_name, prompt, temperature, max_tokens } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + if (!model_name) throw new Error('缺少 model_name'); + if (!prompt) throw new Error('缺少 prompt'); + + const bucket = DEFAULT_BUCKET; + let modelConfig; + let fallbackUsed = false; + + // 加载模型配置 + try { + const raw = await cos.read(bucket, `finetune-models/${persona_id}/${model_name}.json`); + modelConfig = JSON.parse(raw.content); + } catch { + throw new Error(`未找到模型配置: ${model_name}`); + } + + const apiKey = process.env[modelConfig.key_env]; + if (!apiKey) throw new Error(`缺少API密钥环境变量 ${modelConfig.key_env}`); + + const temp = typeof temperature === 'number' ? temperature : 0.7; + const tokens = max_tokens || 1000; + + // 尝试调用微调模型 + try { + const result = await callInferenceAPI({ + host: modelConfig.provider_host, + path: modelConfig.inference_path, + model: modelConfig.model_endpoint + }, apiKey, prompt, temp, tokens); + + return { + response: result.content, + model_used: modelConfig.model_endpoint, + provider: modelConfig.provider, + tokens: result.tokens, + fallback_used: false + }; + } catch { + // 微调模型不可用,降级到基座模型 + fallbackUsed = true; + } + + // 降级:使用基座模型 + const baseConfig = LLM_CONFIGS[modelConfig.provider === 'deepseek' ? 'deepseek-chat' : 'qwen-max']; + if (!baseConfig) throw new Error('降级失败:无可用基座模型'); + + const baseKey = process.env[baseConfig.keyEnv]; + if (!baseKey) throw new Error(`降级失败:缺少基座模型API密钥 ${baseConfig.keyEnv}`); + + const result = await callInferenceAPI({ + host: baseConfig.host, + path: baseConfig.path, + model: baseConfig.model + }, baseKey, prompt, temp, tokens); + + return { + response: result.content, + model_used: baseConfig.model, + provider: modelConfig.provider, + tokens: result.tokens, + fallback_used: fallbackUsed, + fallback_reason: '微调模型不可用,已降级到基座模型' + }; +} + +/** + * finetuneCompareModels — A/B测试微调 vs 基座模型 + * + * 用相同的prompt分别调用微调模型和基座模型,返回对比结果 + * + * input: + * persona_id: string — 人格体ID + * model_name: string — 已注册的微调模型名称 + * test_prompt: string — 测试提示词 + * base_model: string — 基座模型名称(默认取provider对应的基座) + */ +async function finetuneCompareModels(input) { + const { persona_id, model_name, test_prompt, base_model } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + if (!model_name) throw new Error('缺少 model_name'); + if (!test_prompt) throw new Error('缺少 test_prompt'); + + const bucket = DEFAULT_BUCKET; + + // 加载微调模型配置 + let modelConfig; + try { + const raw = await cos.read(bucket, `finetune-models/${persona_id}/${model_name}.json`); + modelConfig = JSON.parse(raw.content); + } catch { + throw new Error(`未找到模型配置: ${model_name}`); + } + + const apiKey = process.env[modelConfig.key_env]; + if (!apiKey) throw new Error(`缺少API密钥环境变量 ${modelConfig.key_env}`); + + // 确定基座模型 + const baseModelName = base_model || (modelConfig.provider === 'deepseek' ? 'deepseek-chat' : 'qwen-max'); + const baseConfig = LLM_CONFIGS[baseModelName]; + if (!baseConfig) throw new Error(`未找到基座模型配置: ${baseModelName}`); + + const baseKey = process.env[baseConfig.keyEnv]; + if (!baseKey) throw new Error(`缺少基座模型API密钥 ${baseConfig.keyEnv}`); + + // 并行调用两个模型 + const [finetunedResult, baseResult] = await Promise.allSettled([ + callInferenceAPI({ + host: modelConfig.provider_host, + path: modelConfig.inference_path, + model: modelConfig.model_endpoint + }, apiKey, test_prompt, 0.7, 1000), + callInferenceAPI({ + host: baseConfig.host, + path: baseConfig.path, + model: baseConfig.model + }, baseKey, test_prompt, 0.7, 1000) + ]); + + return { + test_prompt, + finetuned_response: finetunedResult.status === 'fulfilled' + ? finetunedResult.value.content + : `调用失败: ${finetunedResult.reason?.message || '未知错误'}`, + base_response: baseResult.status === 'fulfilled' + ? baseResult.value.content + : `调用失败: ${baseResult.reason?.message || '未知错误'}`, + model_a: { + name: modelConfig.model_endpoint, + type: 'finetuned', + tokens: finetunedResult.status === 'fulfilled' ? finetunedResult.value.tokens : null + }, + model_b: { + name: baseConfig.model, + type: 'base', + tokens: baseResult.status === 'fulfilled' ? baseResult.value.tokens : null + }, + compared_at: new Date().toISOString() + }; +} + +/** + * finetuneGetCostEstimate — 估算微调成本 + * + * 读取JSONL数据集,统计token数量,按provider定价估算费用 + * + * input: + * persona_id: string — 人格体ID + * dataset_key: string — COS中JSONL文件路径 + * provider: string — 微调提供方(deepseek / qwen) + */ +async function finetuneGetCostEstimate(input) { + const { persona_id, dataset_key, provider } = input; + if (!persona_id) throw new Error('缺少 persona_id'); + if (!dataset_key) throw new Error('缺少 dataset_key'); + + const bucket = DEFAULT_BUCKET; + const providerKey = (provider || 'deepseek').toLowerCase(); + + if (!FINETUNE_PROVIDERS[providerKey]) { + throw new Error(`不支持的微调提供方: ${providerKey}`); + } + + // 读取JSONL数据集 + const raw = await cos.read(bucket, dataset_key); + const lines = raw.content.split('\n').filter(l => l.trim()); + + // 统计token(中文约每字1.5-2 token,英文约每词1 token,粗估用字符数/2) + let totalChars = 0; + let sampleCount = 0; + + for (const line of lines) { + try { + const sample = JSON.parse(line); + const messages = sample.messages || []; + for (const msg of messages) { + totalChars += (msg.content || '').length; + } + sampleCount++; + } catch { + // 跳过无效行 + } + } + + // 粗估token数(中文字符 ≈ 1.5 tokens,英文约1:1) + const estimatedTokens = Math.ceil(totalChars * 1.5); + const costPer1k = COST_PER_1K_TOKENS[providerKey] || 0.02; + + // 微调通常跑 3-4 个 epoch + const epochs = 3; + const totalTrainTokens = estimatedTokens * epochs; + const estimatedCostRmb = (totalTrainTokens / 1000) * costPer1k; + + return { + dataset_key, + sample_count: sampleCount, + total_chars: totalChars, + token_count: estimatedTokens, + training_tokens: totalTrainTokens, + epochs, + estimated_cost_rmb: Math.round(estimatedCostRmb * 100) / 100, + provider: providerKey, + cost_per_1k_tokens: costPer1k, + notes: [ + `Token估算基于字符数粗估(中文 ×1.5),实际以provider计费为准`, + `训练按 ${epochs} 个epoch估算`, + `${FINETUNE_PROVIDERS[providerKey].label} 当前参考价: ¥${costPer1k}/1K tokens`, + `实际费用可能因模型版本和优惠策略有所不同` + ] + }; +} + +// ═══════════════════════════════════════════════════════════ +// Provider API 交互(内部实现) +// ═══════════════════════════════════════════════════════════ + +/** + * 上传训练文件到provider + */ +function uploadTrainingFile(providerConfig, apiKey, content, providerKey) { + return new Promise((resolve, reject) => { + // 构建 multipart/form-data + const boundary = `----FormBoundary${crypto.randomBytes(8).toString('hex')}`; + const fileName = `training-${Date.now()}.jsonl`; + + let bodyParts = []; + bodyParts.push(`--${boundary}\r\n`); + bodyParts.push(`Content-Disposition: form-data; name="purpose"\r\n\r\n`); + bodyParts.push(`fine-tune\r\n`); + bodyParts.push(`--${boundary}\r\n`); + bodyParts.push(`Content-Disposition: form-data; name="file"; filename="${fileName}"\r\n`); + bodyParts.push(`Content-Type: application/jsonl\r\n\r\n`); + bodyParts.push(content); + bodyParts.push(`\r\n--${boundary}--\r\n`); + + const body = bodyParts.join(''); + + const req = https.request({ + hostname: providerConfig.host, + port: 443, + path: providerConfig.uploadPath, + method: 'POST', + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Authorization': `Bearer ${apiKey}`, + 'Content-Length': Buffer.byteLength(body) + }, + timeout: FINETUNE_TIMEOUT + }, (res) => { + const chunks = []; + res.on('data', c => chunks.push(c)); + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode < 300) { + try { + const data = JSON.parse(Buffer.concat(chunks).toString()); + // DeepSeek返回 {id: "file-xxx"}, Qwen返回类似结构 + resolve(data.id || data.file_id || data.output?.file_id || ''); + } catch { + reject(new Error('训练文件上传响应解析失败')); + } + } else { + reject(new Error(`训练文件上传失败: HTTP ${res.statusCode}`)); + } + }); + }); + + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('训练文件上传超时')); }); + req.write(body); + req.end(); + }); +} + +/** + * 创建微调任务 + */ +function createFinetuneJob(providerConfig, apiKey, model, fileId, hyperparams, providerKey) { + return new Promise((resolve, reject) => { + let requestBody; + + if (providerKey === 'deepseek') { + requestBody = { + model, + training_file: fileId, + hyperparameters: { + n_epochs: hyperparams?.n_epochs || 3, + learning_rate_multiplier: hyperparams?.learning_rate_multiplier || 1.0, + batch_size: hyperparams?.batch_size || 'auto' + } + }; + } else { + // Qwen/DashScope 格式 + requestBody = { + model, + training_file_ids: [fileId], + hyper_parameters: { + n_epochs: hyperparams?.n_epochs || 3, + learning_rate: hyperparams?.learning_rate_multiplier || 1e-5, + batch_size: hyperparams?.batch_size || 4 + } + }; + } + + const body = JSON.stringify(requestBody); + + const req = https.request({ + hostname: providerConfig.host, + port: 443, + path: providerConfig.createPath, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + 'Content-Length': Buffer.byteLength(body) + }, + timeout: FINETUNE_TIMEOUT + }, (res) => { + const chunks = []; + res.on('data', c => chunks.push(c)); + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode < 300) { + try { + const data = JSON.parse(Buffer.concat(chunks).toString()); + resolve({ + provider_job_id: data.id || data.output?.job_id || '', + status: data.status || data.output?.status || 'pending', + estimated_time: data.estimated_completion || null + }); + } catch { + reject(new Error('微调任务创建响应解析失败')); + } + } else { + reject(new Error(`微调任务创建失败: HTTP ${res.statusCode}`)); + } + }); + }); + + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('微调任务创建超时')); }); + req.write(body); + req.end(); + }); +} + +/** + * 查询微调任务状态 + */ +function queryJobStatus(providerConfig, apiKey, providerJobId, providerKey) { + return new Promise((resolve, reject) => { + const path = `${providerConfig.statusPath}${encodeURIComponent(providerJobId)}`; + + const req = https.request({ + hostname: providerConfig.host, + port: 443, + path, + method: 'GET', + headers: { + 'Authorization': `Bearer ${apiKey}` + }, + timeout: 30000 + }, (res) => { + const chunks = []; + res.on('data', c => chunks.push(c)); + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode < 300) { + try { + const data = JSON.parse(Buffer.concat(chunks).toString()); + + // 统一不同provider的状态字段 + let status, progress, metrics, fineTunedModel; + + if (providerKey === 'deepseek') { + status = data.status || 'unknown'; + fineTunedModel = data.fine_tuned_model || null; + metrics = data.result_files ? { result_files: data.result_files } : null; + progress = data.trained_tokens + ? { trained_tokens: data.trained_tokens } + : null; + } else { + // Qwen + const output = data.output || data; + status = output.status || data.status || 'unknown'; + fineTunedModel = output.fine_tuned_model || output.finetuned_output?.model_id || null; + metrics = output.metrics || null; + progress = output.training_progress || null; + } + + // 统一状态值 + status = normalizeJobStatus(status); + + resolve({ status, progress, metrics, fine_tuned_model: fineTunedModel }); + } catch { + reject(new Error('微调状态查询响应解析失败')); + } + } else { + reject(new Error(`微调状态查询失败: HTTP ${res.statusCode}`)); + } + }); + }); + + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('微调状态查询超时')); }); + req.end(); + }); +} + +/** + * 调用推理API(微调模型或基座模型通用) + */ +function callInferenceAPI(config, apiKey, prompt, temperature, maxTokens) { + return new Promise((resolve, reject) => { + const body = JSON.stringify({ + model: config.model, + messages: [ + { role: 'user', content: prompt } + ], + temperature: temperature || 0.7, + max_tokens: maxTokens || 1000 + }); + + const req = https.request({ + hostname: config.host, + port: 443, + path: config.path, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + 'Content-Length': Buffer.byteLength(body) + }, + timeout: 30000 + }, (res) => { + const chunks = []; + res.on('data', c => chunks.push(c)); + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode < 300) { + try { + const data = JSON.parse(Buffer.concat(chunks).toString()); + resolve({ + content: data.choices?.[0]?.message?.content || '', + tokens: data.usage || {} + }); + } catch { + reject(new Error('推理响应解析失败')); + } + } else { + reject(new Error(`推理调用失败: HTTP ${res.statusCode}`)); + } + }); + }); + + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('推理请求超时')); }); + req.write(body); + req.end(); + }); +} + +// ═══════════════════════════════════════════════════════════ +// 辅助函数 +// ═══════════════════════════════════════════════════════════ + +/** + * 将TCS语料条目转换为微调JSONL三元组 + */ +function convertEntryToTriple(personaId, corpusType, entry) { + const content = typeof entry.content === 'string' + ? entry.content + : (entry.text || entry.summary || JSON.stringify(entry)); + + if (!content || content.length < 10) return null; + + const tags = entry.tcs_tags || entry.tags || []; + const category = entry.category || corpusType || 'general'; + + // 生成system prompt(人格体身份) + const systemContent = `你是${personaId},光湖系统中的人格体。你的思维方式基于TCS语言核系统,` + + `你需要以${personaId}的视角和风格来回答问题。`; + + // 根据语料类型构建instruction/input/output + let userContent, assistantContent; + + if (entry.question && entry.answer) { + // 已有Q&A结构 + userContent = entry.question; + assistantContent = entry.answer; + } else if (tags.length > 0) { + // 有标签的条目:生成关于该内容的问答 + userContent = `关于${category}类型的内容,请解释以下要点: ${tags.slice(0, 3).join('、')}`; + assistantContent = content; + } else { + // 通用条目:以理解和阐述的方式构建 + userContent = `请阐述你对以下内容的理解和看法:\n${content.substring(0, 200)}`; + assistantContent = content; + } + + return { + messages: [ + { role: 'system', content: systemContent }, + { role: 'user', content: userContent }, + { role: 'assistant', content: assistantContent } + ] + }; +} + +/** + * 统一不同provider的任务状态值 + */ +function normalizeJobStatus(rawStatus) { + const statusMap = { + // DeepSeek 状态 + validating_files: 'pending', + queued: 'pending', + running: 'running', + succeeded: 'completed', + failed: 'failed', + cancelled: 'failed', + // Qwen/DashScope 状态 + PENDING: 'pending', + RUNNING: 'running', + SUCCEEDED: 'completed', + FAILED: 'failed', + CANCELED: 'failed', + // 通用 + pending: 'pending', + completed: 'completed' + }; + + return statusMap[rawStatus] || rawStatus; +} + +module.exports = { + finetuneExportDataset, + finetuneSubmitJob, + finetuneCheckStatus, + finetuneRegisterModel, + finetuneListModels, + finetuneCallModel, + finetuneCompareModels, + finetuneGetCostEstimate +};