diff --git a/.github/workflows/deploy-to-server.yml b/.github/workflows/deploy-to-server.yml index d9504e19..30f6fdf7 100644 --- a/.github/workflows/deploy-to-server.yml +++ b/.github/workflows/deploy-to-server.yml @@ -279,14 +279,15 @@ jobs: echo '⚠️ WebSocket pm2 启动失败' fi - # 部署 src HLI 中间层 + # 部署 src HLI 中间层(端口 3001,对应 Nginx /api/v1/ 路由) + # 注意:guanghulab-backend 已占用端口 3000,HLI 使用 3001 避免冲突 if [ -f '${{ secrets.DEPLOY_PATH }}/src/index.js' ]; then echo '📦 安装 src 依赖...' cd '${{ secrets.DEPLOY_PATH }}' npm install --production 2>/dev/null || echo '⚠️ src npm install 失败' - echo '🔄 重启 HLI 服务...' - pm2 restart guanghulab 2>/dev/null || \ - pm2 start src/index.js --name guanghulab 2>/dev/null || \ + echo '🔄 重启 HLI 服务(端口 3001)...' + pm2 delete guanghulab 2>/dev/null || true + PORT=3001 pm2 start src/index.js --name guanghulab --update-env 2>/dev/null || \ echo '⚠️ src pm2 启动失败' fi diff --git a/.github/workflows/pm2-server-diagnose.yml b/.github/workflows/pm2-server-diagnose.yml new file mode 100644 index 00000000..642a594d --- /dev/null +++ b/.github/workflows/pm2-server-diagnose.yml @@ -0,0 +1,316 @@ +name: "🔧 铸渊 · PM2 服务诊断与健康检查" + +# ━━━ PM2 服务诊断 + 全系统健康检查 + Notion 报告 ━━━ +# 手动触发:对服务器 PM2 进程进行全面排查, +# 自动处理 errored 进程,并将诊断报告写入 Notion 工单队列。 +# +# 必需 GitHub Secrets: +# DEPLOY_HOST — 服务器 IP +# DEPLOY_USER — SSH 用户名 +# DEPLOY_KEY — SSH 私钥(完整 PEM 内容) +# DEPLOY_PATH — 网站根目录(如 /var/www/guanghulab) +# +# 可选 Secrets(有则自动写 Notion 工单): +# NOTION_API_TOKEN — Notion 集成 token +# NOTION_TICKET_DB_ID — 工单队列数据库 ID(默认 84edd0640bf146a9a5a7840107013e8c) + +on: + workflow_dispatch: + inputs: + auto_cleanup: + description: '是否自动清理 errored 进程(删除启动路径不存在的进程)' + required: false + default: 'true' + type: choice + options: + - 'true' + - 'false' + +permissions: + contents: read + +jobs: + diagnose: + name: "🔍 PM2 诊断 + 系统健康检查" + runs-on: ubuntu-latest + + steps: + - name: 📥 检出代码 + uses: actions/checkout@v4 + + - name: 🟢 配置 Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: 🔑 配置 SSH 连接 + run: | + mkdir -p ~/.ssh + echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + ssh-keyscan -H "${{ secrets.DEPLOY_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null || { + echo "⚠️ ssh-keyscan 失败,使用 StrictHostKeyChecking=no 后备" + } + + # ━━━ 阶段一:PM2 服务诊断 ━━━ + - name: "🔧 阶段一 · PM2 errored 服务排查" + id: pm2_diagnose + run: | + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " 阶段一 · PM2 errored 服务排查" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + PM2_REPORT=$(ssh -i ~/.ssh/deploy_key \ + -o StrictHostKeyChecking=no \ + "${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" ' + echo "══════════════════════════════════════" + echo " PM2 进程状态概览" + echo "══════════════════════════════════════" + pm2 status 2>/dev/null || echo "❌ pm2 未安装或不可用" + + echo "" + echo "══════════════════════════════════════" + echo " errored 进程详细日志" + echo "══════════════════════════════════════" + + # 遍历所有 errored 进程 + ERRORED_LIST=$(pm2 jlist 2>/dev/null | python3 -c " + import sys, json + try: + procs = json.load(sys.stdin) + for p in procs: + if p.get(\"pm2_env\", {}).get(\"status\") == \"errored\": + name = p.get(\"name\", \"unknown\") + pm_id = p.get(\"pm_id\", \"?\") + script = p.get(\"pm2_env\", {}).get(\"pm_exec_path\", \"unknown\") + restarts = p.get(\"pm2_env\", {}).get(\"restart_time\", 0) + print(f\"{pm_id}|{name}|{script}|{restarts}\") + except: + pass + " 2>/dev/null) + + if [ -z "$ERRORED_LIST" ]; then + echo "✅ 没有 errored 进程" + else + echo "$ERRORED_LIST" | while IFS="|" read -r pm_id name script restarts; do + echo "" + echo "━━━ 进程: $name (PM2 ID: $pm_id) ━━━" + echo " 启动脚本: $script" + echo " 重启次数: $restarts" + + # 检查脚本文件是否存在 + if [ -f "$script" ]; then + echo " 脚本文件: ✅ 存在" + else + echo " 脚本文件: ❌ 不存在" + fi + + # 查看最近日志 + echo " 最近错误日志 (最后 20 行):" + pm2 logs "$name" --lines 20 --nostream --err 2>/dev/null || echo " (无法读取日志)" + echo "" + done + fi + + echo "" + echo "══════════════════════════════════════" + echo " 所有进程详细信息" + echo "══════════════════════════════════════" + pm2 jlist 2>/dev/null | python3 -c " + import sys, json + try: + procs = json.load(sys.stdin) + for p in procs: + env = p.get(\"pm2_env\", {}) + print(f\" [{p.get(\"pm_id\")}] {p.get(\"name\")} | status={env.get(\"status\")} | script={env.get(\"pm_exec_path\",\"?\")} | restarts={env.get(\"restart_time\",0)} | memory={p.get(\"monit\",{}).get(\"memory\",0)}\") + except Exception as e: + print(f\" 解析失败: {e}\") + " 2>/dev/null + + echo "" + echo "══════════════════════════════════════" + echo " 端口占用情况" + echo "══════════════════════════════════════" + netstat -tlnp 2>/dev/null | grep -E "3000|3001|3002|3721|8080" || \ + ss -tlnp 2>/dev/null | grep -E "3000|3001|3002|3721|8080" || \ + echo " (netstat/ss 不可用)" + ') + + echo "$PM2_REPORT" + + # 保存到文件供后续步骤使用 + echo "$PM2_REPORT" > /tmp/pm2-report.txt + + # 设置输出 + { + echo "pm2_report<> "$GITHUB_OUTPUT" + + # ━━━ 阶段一 · 自动清理 ━━━ + - name: "🧹 阶段一 · 自动清理 errored 进程" + if: ${{ inputs.auto_cleanup == 'true' }} + id: pm2_cleanup + run: | + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " 自动清理 errored 进程" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + CLEANUP_REPORT=$(ssh -i ~/.ssh/deploy_key \ + -o StrictHostKeyChecking=no \ + "${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" ' + ERRORED_LIST=$(pm2 jlist 2>/dev/null | python3 -c " + import sys, json + try: + procs = json.load(sys.stdin) + for p in procs: + if p.get(\"pm2_env\", {}).get(\"status\") == \"errored\": + name = p.get(\"name\", \"unknown\") + script = p.get(\"pm2_env\", {}).get(\"pm_exec_path\", \"unknown\") + print(f\"{name}|{script}\") + except: + pass + " 2>/dev/null) + + CLEANED="" + SKIPPED="" + + if [ -z "$ERRORED_LIST" ]; then + echo "✅ 没有需要清理的 errored 进程" + else + echo "$ERRORED_LIST" | while IFS="|" read -r name script; do + if [ ! -f "$script" ]; then + echo "🗑️ 删除 $name(脚本 $script 不存在)" + pm2 delete "$name" 2>/dev/null && echo " ✅ 已删除" || echo " ❌ 删除失败" + else + echo "⚠️ 保留 $name(脚本 $script 存在,可能是运行时错误)" + echo " 尝试重启..." + pm2 restart "$name" 2>/dev/null && echo " ✅ 重启成功" || echo " ❌ 重启失败" + fi + done + + echo "" + echo "📊 清理后 PM2 状态:" + pm2 status 2>/dev/null + pm2 save 2>/dev/null || echo "⚠️ pm2 save 失败" + fi + ') + + echo "$CLEANUP_REPORT" + echo "$CLEANUP_REPORT" > /tmp/cleanup-report.txt + + { + echo "cleanup_report<> "$GITHUB_OUTPUT" + + # ━━━ 阶段二:全系统健康检查 ━━━ + - name: "🏥 阶段二 · 全系统健康检查" + id: health_check + run: | + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " 阶段二 · 全系统健康检查" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + HEALTH_REPORT=$(ssh -i ~/.ssh/deploy_key \ + -o StrictHostKeyChecking=no \ + "${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" ' + echo "══════════════════════════════════════" + echo " Nginx 状态" + echo "══════════════════════════════════════" + nginx -t 2>&1 || echo "❌ Nginx 配置检测失败" + systemctl status nginx --no-pager -l 2>/dev/null | head -15 || echo "⚠️ systemctl 不可用" + + echo "" + echo "══════════════════════════════════════" + echo " SSL 证书有效期" + echo "══════════════════════════════════════" + if [ -f "/etc/letsencrypt/live/guanghulab.com/fullchain.pem" ]; then + openssl x509 -in /etc/letsencrypt/live/guanghulab.com/fullchain.pem -noout -dates 2>/dev/null || echo "⚠️ 无法读取 SSL 证书" + else + echo "⚠️ SSL 证书文件不存在于默认路径" + # 尝试查找其他证书路径 + find /etc/letsencrypt/live/ -name "fullchain.pem" 2>/dev/null | head -3 || echo " 未找到 Let'\''s Encrypt 证书" + find /etc/nginx/ -name "*.pem" -o -name "*.crt" 2>/dev/null | head -3 || echo " 未找到 Nginx SSL 文件" + fi + + echo "" + echo "══════════════════════════════════════" + echo " 磁盘空间" + echo "══════════════════════════════════════" + df -h 2>/dev/null || echo "❌ df 命令不可用" + + echo "" + echo "══════════════════════════════════════" + echo " 内存使用" + echo "══════════════════════════════════════" + free -h 2>/dev/null || echo "❌ free 命令不可用" + + echo "" + echo "══════════════════════════════════════" + echo " PM2 最终状态" + echo "══════════════════════════════════════" + pm2 status 2>/dev/null || echo "❌ pm2 不可用" + + echo "" + echo "══════════════════════════════════════" + echo " Node.js 版本" + echo "══════════════════════════════════════" + node -v 2>/dev/null || echo "❌ Node.js 未安装" + npm -v 2>/dev/null || echo "❌ npm 未安装" + + echo "" + echo "══════════════════════════════════════" + echo " 网站可访问性检查" + echo "══════════════════════════════════════" + curl -sI https://guanghulab.com 2>/dev/null | head -5 || echo "⚠️ HTTPS 访问失败" + echo "---" + curl -s http://127.0.0.1:3000/health 2>/dev/null | head -c 200 || echo "⚠️ 端口 3000 无响应" + echo "" + echo "---" + curl -s http://127.0.0.1:3001/api/v1/ 2>/dev/null | head -c 200 || echo "⚠️ 端口 3001 无响应" + echo "" + echo "---" + curl -s http://127.0.0.1:3002/api/ps/ 2>/dev/null | head -c 200 || echo "⚠️ 端口 3002 无响应" + echo "" + + echo "" + echo "══════════════════════════════════════" + echo " 系统运行时间" + echo "══════════════════════════════════════" + uptime 2>/dev/null || echo "❌ uptime 不可用" + ') + + echo "$HEALTH_REPORT" + echo "$HEALTH_REPORT" > /tmp/health-report.txt + + { + echo "health_report<> "$GITHUB_OUTPUT" + + # ━━━ 阶段三:生成 Notion 诊断报告 ━━━ + - name: "📋 阶段三 · 写入 Notion 工单" + if: ${{ secrets.NOTION_API_TOKEN != '' }} + env: + NOTION_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + NOTION_TICKET_DB_ID: ${{ secrets.NOTION_TICKET_DB_ID || '84edd0640bf146a9a5a7840107013e8c' }} + PM2_REPORT: ${{ steps.pm2_diagnose.outputs.pm2_report }} + CLEANUP_REPORT: ${{ steps.pm2_cleanup.outputs.cleanup_report }} + HEALTH_REPORT: ${{ steps.health_check.outputs.health_report }} + run: node scripts/server-diagnose-report.js + + - name: "📋 阶段三 · 写入 Notion 工单(环境检查)" + if: ${{ secrets.NOTION_API_TOKEN == '' }} + run: | + echo "⚠️ Notion credentials 未配置,跳过工单创建" + echo "诊断报告仅输出到 Actions 日志" + + - name: "✅ 诊断完成" + run: | + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " ✅ PM2 诊断完毕 · $(date '+%Y-%m-%d %H:%M:%S')" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/ecosystem.config.js b/ecosystem.config.js index e1b55c1d..a5359737 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -1,32 +1,32 @@ // ecosystem.config.js // PM2 进程管理配置 +// +// 注意:进程名必须与 deploy-to-server.yml 中的 --name 参数保持一致。 +// 当前活跃服务(2026-03 更新): +// guanghulab → HLI 中间层 → 端口 3001(/api/v1/) +// guanghulab-proxy → AI Chat API 代理 → 端口 3721 +// guanghulab-backend → Express 后端 API → 端口 3000(/webhook/feishu) +// guanghulab-ws → Status Board WS → 端口 8080 +// persona-studio → Persona Studio API → 端口 3002(/api/ps/) module.exports = { apps: [ { name: 'guanghulab', script: 'src/index.js', - instances: 'max', - exec_mode: 'cluster', + instances: 1, + exec_mode: 'fork', watch: false, env: { NODE_ENV: 'production', - PORT: 3000, - }, - env_development: { - NODE_ENV: 'development', - PORT: 3000, - }, - env_test: { - NODE_ENV: 'test', PORT: 3001, }, log_date_format: 'YYYY-MM-DD HH:mm:ss', - error_file: 'logs/error.log', - out_file: 'logs/out.log', + error_file: 'logs/hli-error.log', + out_file: 'logs/hli-out.log', }, { - name: 'api-proxy', + name: 'guanghulab-proxy', script: 'backend-integration/api-proxy.js', instances: 1, exec_mode: 'fork', @@ -39,6 +39,34 @@ module.exports = { error_file: 'logs/api-proxy-error.log', out_file: 'logs/api-proxy-out.log', }, + { + name: 'guanghulab-backend', + script: 'backend/server.js', + instances: 1, + exec_mode: 'fork', + watch: false, + env: { + NODE_ENV: 'production', + PORT: 3000, + }, + log_date_format: 'YYYY-MM-DD HH:mm:ss', + error_file: 'logs/backend-error.log', + out_file: 'logs/backend-out.log', + }, + { + name: 'guanghulab-ws', + script: 'status-board/mock-ws-server.js', + instances: 1, + exec_mode: 'fork', + watch: false, + env: { + NODE_ENV: 'production', + PORT: 8080, + }, + log_date_format: 'YYYY-MM-DD HH:mm:ss', + error_file: 'logs/ws-error.log', + out_file: 'logs/ws-out.log', + }, { name: 'persona-studio', script: 'persona-studio/backend/server.js', diff --git a/scripts/server-diagnose-report.js b/scripts/server-diagnose-report.js new file mode 100644 index 00000000..abe380ab --- /dev/null +++ b/scripts/server-diagnose-report.js @@ -0,0 +1,201 @@ +// scripts/server-diagnose-report.js +// 铸渊 · PM2 服务诊断报告 → Notion 工单 +// +// 将 PM2 诊断和系统健康检查结果写入 Notion 工单队列。 +// 由 pm2-server-diagnose.yml 工作流调用。 +// +// 环境变量: +// NOTION_TOKEN Notion API token +// NOTION_TICKET_DB_ID 工单队列数据库 ID +// PM2_REPORT PM2 诊断输出 +// CLEANUP_REPORT 清理操作输出(可选) +// HEALTH_REPORT 健康检查输出 + +'use strict'; + +const https = require('https'); + +const NOTION_TOKEN = process.env.NOTION_TOKEN || ''; +const NOTION_TICKET_DB_ID = process.env.NOTION_TICKET_DB_ID || ''; +const PM2_REPORT = process.env.PM2_REPORT || ''; +const CLEANUP_REPORT = process.env.CLEANUP_REPORT || ''; +const HEALTH_REPORT = process.env.HEALTH_REPORT || ''; + +const NOTION_VERSION = '2022-06-28'; +const NOTION_API_HOSTNAME = 'api.notion.com'; +const NOTION_RICH_TEXT_MAX = 2000; +const MAX_PM2_REPORT_LENGTH = 3000; +const MAX_CLEANUP_REPORT_LENGTH = 2000; +const MAX_HEALTH_REPORT_LENGTH = 3000; + +// ══════════════════════════════════════════════════════════ +// Notion API 工具 +// ══════════════════════════════════════════════════════════ + +function notionPost(endpoint, body) { + return new Promise(function (resolve, reject) { + var payload = JSON.stringify(body); + var opts = { + hostname: NOTION_API_HOSTNAME, + port: 443, + path: endpoint, + method: 'POST', + headers: { + 'Authorization': 'Bearer ' + NOTION_TOKEN, + 'Content-Type': 'application/json', + 'Notion-Version': NOTION_VERSION, + 'Content-Length': Buffer.byteLength(payload), + }, + }; + var req = https.request(opts, function (res) { + var data = ''; + res.on('data', function (chunk) { data += chunk; }); + res.on('end', function () { + try { + var parsed = JSON.parse(data); + if (res.statusCode >= 200 && res.statusCode < 300) { + resolve(parsed); + } else { + reject(new Error('Notion API ' + res.statusCode + ': ' + (parsed.message || data))); + } + } catch (e) { + reject(new Error('Notion API parse error: ' + data)); + } + }); + }); + req.on('error', reject); + req.write(payload); + req.end(); + }); +} + +function richTextChunks(content) { + var str = String(content || ''); + var chunks = []; + for (var i = 0; i < str.length; i += NOTION_RICH_TEXT_MAX) { + chunks.push({ type: 'text', text: { content: str.slice(i, i + NOTION_RICH_TEXT_MAX) } }); + } + if (chunks.length === 0) { + chunks.push({ type: 'text', text: { content: '' } }); + } + return chunks; +} + +// ══════════════════════════════════════════════════════════ +// 构建报告内容 +// ══════════════════════════════════════════════════════════ + +function buildReportContent() { + var now = new Date().toISOString(); + var dateStr = now.slice(0, 10); + + var sections = []; + + sections.push('## 📋 PM2 服务排查报告 · ' + dateStr); + sections.push(''); + sections.push('| 字段 | 值 |'); + sections.push('|------|-----|'); + sections.push('| 执行者 | 铸渊Agent |'); + sections.push('| 执行时间 | ' + now + ' |'); + sections.push('| 来源 | GitHub Actions · pm2-server-diagnose |'); + sections.push(''); + + if (PM2_REPORT) { + sections.push('## 阶段一 · PM2 诊断结果'); + sections.push(''); + sections.push(PM2_REPORT.slice(0, MAX_PM2_REPORT_LENGTH)); + sections.push(''); + } + + if (CLEANUP_REPORT) { + sections.push('## 清理操作结果'); + sections.push(''); + sections.push(CLEANUP_REPORT.slice(0, MAX_CLEANUP_REPORT_LENGTH)); + sections.push(''); + } + + if (HEALTH_REPORT) { + sections.push('## 阶段二 · 全系统健康检查'); + sections.push(''); + sections.push(HEALTH_REPORT.slice(0, MAX_HEALTH_REPORT_LENGTH)); + sections.push(''); + } + + return sections.join('\n'); +} + +// ══════════════════════════════════════════════════════════ +// 主函数 — 创建 Notion 诊断工单 +// ══════════════════════════════════════════════════════════ + +async function createDiagnoseTicket() { + if (!NOTION_TOKEN || !NOTION_TICKET_DB_ID) { + console.log('⚠️ Notion credentials not configured, skipping ticket creation'); + console.log(' NOTION_TOKEN: ' + (NOTION_TOKEN ? '✅' : '❌')); + console.log(' NOTION_TICKET_DB_ID: ' + (NOTION_TICKET_DB_ID ? '✅' : '❌')); + process.exit(0); + } + + var now = new Date(); + var dateStr = now.toISOString().slice(0, 10); + var title = 'PM2服务排查报告 · ' + dateStr; + var contentText = buildReportContent(); + + // 将内容拆分成多个段落块(Notion 单个 rich_text 限制 2000 字符) + var contentBlocks = []; + var remaining = contentText; + + while (remaining.length > 0) { + var chunk = remaining.slice(0, NOTION_RICH_TEXT_MAX); + remaining = remaining.slice(NOTION_RICH_TEXT_MAX); + contentBlocks.push({ + object: 'block', + type: 'paragraph', + paragraph: { + rich_text: [{ type: 'text', text: { content: chunk } }], + }, + }); + } + + // 确保至少有一个内容块 + if (contentBlocks.length === 0) { + contentBlocks.push({ + object: 'block', + type: 'paragraph', + paragraph: { + rich_text: [{ type: 'text', text: { content: '诊断报告(无详细输出)' } }], + }, + }); + } + + var body = { + parent: { database_id: NOTION_TICKET_DB_ID }, + properties: { + '标题': { title: [{ type: 'text', text: { content: title } }] }, + '操作类型': { select: { name: '其他' } }, + '提交者': { rich_text: [{ type: 'text', text: { content: '铸渊Agent' } }] }, + '状态': { select: { name: '已完成' } }, + '优先级': { select: { name: 'P1' } }, + }, + children: contentBlocks, + }; + + console.log('📋 创建 PM2 诊断工单...'); + console.log(' 标题: ' + title); + console.log(' 数据库: ' + NOTION_TICKET_DB_ID); + console.log(' 内容长度: ' + contentText.length + ' 字符'); + console.log(' 内容块数: ' + contentBlocks.length); + + try { + var result = await notionPost('/v1/pages', body); + console.log('✅ Notion 诊断工单已创建: ' + result.id); + console.log(' URL: ' + (result.url || 'N/A')); + } catch (err) { + console.error('❌ 工单创建失败: ' + err.message); + // 不以失败退出,诊断本身已成功 + console.log('⚠️ 诊断结果已输出到 Actions 日志,Notion 写入失败不影响诊断'); + process.exit(0); + } +} + +createDiagnoseTicket();