Merge pull request #89 from qinfendebingshuo/copilot/restore-core-cognition

PM2 server diagnosis workflow + port conflict fix for errored services
This commit is contained in:
冰朔 2026-03-13 22:25:38 +08:00 committed by GitHub
commit a23ef896b8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 563 additions and 17 deletions

View File

@ -279,14 +279,15 @@ jobs:
echo '⚠️ WebSocket pm2 启动失败'
fi
# 部署 src HLI 中间层
# 部署 src HLI 中间层(端口 3001对应 Nginx /api/v1/ 路由)
# 注意guanghulab-backend 已占用端口 3000HLI 使用 3001 避免冲突
if [ -f '${{ secrets.DEPLOY_PATH }}/src/index.js' ]; then
echo '📦 安装 src 依赖...'
cd '${{ secrets.DEPLOY_PATH }}'
npm install --production 2>/dev/null || echo '⚠️ src npm install 失败'
echo '🔄 重启 HLI 服务...'
pm2 restart guanghulab 2>/dev/null || \
pm2 start src/index.js --name guanghulab 2>/dev/null || \
echo '🔄 重启 HLI 服务(端口 3001...'
pm2 delete guanghulab 2>/dev/null || true
PORT=3001 pm2 start src/index.js --name guanghulab --update-env 2>/dev/null || \
echo '⚠️ src pm2 启动失败'
fi

View File

@ -0,0 +1,316 @@
name: "🔧 铸渊 · PM2 服务诊断与健康检查"
# ━━━ PM2 服务诊断 + 全系统健康检查 + Notion 报告 ━━━
# 手动触发:对服务器 PM2 进程进行全面排查,
# 自动处理 errored 进程,并将诊断报告写入 Notion 工单队列。
#
# 必需 GitHub Secrets:
# DEPLOY_HOST — 服务器 IP
# DEPLOY_USER — SSH 用户名
# DEPLOY_KEY — SSH 私钥(完整 PEM 内容)
# DEPLOY_PATH — 网站根目录(如 /var/www/guanghulab
#
# 可选 Secrets有则自动写 Notion 工单):
# NOTION_API_TOKEN — Notion 集成 token
# NOTION_TICKET_DB_ID — 工单队列数据库 ID默认 84edd0640bf146a9a5a7840107013e8c
on:
workflow_dispatch:
inputs:
auto_cleanup:
description: '是否自动清理 errored 进程(删除启动路径不存在的进程)'
required: false
default: 'true'
type: choice
options:
- 'true'
- 'false'
permissions:
contents: read
jobs:
diagnose:
name: "🔍 PM2 诊断 + 系统健康检查"
runs-on: ubuntu-latest
steps:
- name: 📥 检出代码
uses: actions/checkout@v4
- name: 🟢 配置 Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: 🔑 配置 SSH 连接
run: |
mkdir -p ~/.ssh
echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
ssh-keyscan -H "${{ secrets.DEPLOY_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null || {
echo "⚠️ ssh-keyscan 失败,使用 StrictHostKeyChecking=no 后备"
}
# ━━━ 阶段一PM2 服务诊断 ━━━
- name: "🔧 阶段一 · PM2 errored 服务排查"
id: pm2_diagnose
run: |
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 阶段一 · PM2 errored 服务排查"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
PM2_REPORT=$(ssh -i ~/.ssh/deploy_key \
-o StrictHostKeyChecking=no \
"${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" '
echo "══════════════════════════════════════"
echo " PM2 进程状态概览"
echo "══════════════════════════════════════"
pm2 status 2>/dev/null || echo "❌ pm2 未安装或不可用"
echo ""
echo "══════════════════════════════════════"
echo " errored 进程详细日志"
echo "══════════════════════════════════════"
# 遍历所有 errored 进程
ERRORED_LIST=$(pm2 jlist 2>/dev/null | python3 -c "
import sys, json
try:
procs = json.load(sys.stdin)
for p in procs:
if p.get(\"pm2_env\", {}).get(\"status\") == \"errored\":
name = p.get(\"name\", \"unknown\")
pm_id = p.get(\"pm_id\", \"?\")
script = p.get(\"pm2_env\", {}).get(\"pm_exec_path\", \"unknown\")
restarts = p.get(\"pm2_env\", {}).get(\"restart_time\", 0)
print(f\"{pm_id}|{name}|{script}|{restarts}\")
except:
pass
" 2>/dev/null)
if [ -z "$ERRORED_LIST" ]; then
echo "✅ 没有 errored 进程"
else
echo "$ERRORED_LIST" | while IFS="|" read -r pm_id name script restarts; do
echo ""
echo "━━━ 进程: $name (PM2 ID: $pm_id) ━━━"
echo " 启动脚本: $script"
echo " 重启次数: $restarts"
# 检查脚本文件是否存在
if [ -f "$script" ]; then
echo " 脚本文件: ✅ 存在"
else
echo " 脚本文件: ❌ 不存在"
fi
# 查看最近日志
echo " 最近错误日志 (最后 20 行):"
pm2 logs "$name" --lines 20 --nostream --err 2>/dev/null || echo " (无法读取日志)"
echo ""
done
fi
echo ""
echo "══════════════════════════════════════"
echo " 所有进程详细信息"
echo "══════════════════════════════════════"
pm2 jlist 2>/dev/null | python3 -c "
import sys, json
try:
procs = json.load(sys.stdin)
for p in procs:
env = p.get(\"pm2_env\", {})
print(f\" [{p.get(\"pm_id\")}] {p.get(\"name\")} | status={env.get(\"status\")} | script={env.get(\"pm_exec_path\",\"?\")} | restarts={env.get(\"restart_time\",0)} | memory={p.get(\"monit\",{}).get(\"memory\",0)}\")
except Exception as e:
print(f\" 解析失败: {e}\")
" 2>/dev/null
echo ""
echo "══════════════════════════════════════"
echo " 端口占用情况"
echo "══════════════════════════════════════"
netstat -tlnp 2>/dev/null | grep -E "3000|3001|3002|3721|8080" || \
ss -tlnp 2>/dev/null | grep -E "3000|3001|3002|3721|8080" || \
echo " (netstat/ss 不可用)"
')
echo "$PM2_REPORT"
# 保存到文件供后续步骤使用
echo "$PM2_REPORT" > /tmp/pm2-report.txt
# 设置输出
{
echo "pm2_report<<REPORT_EOF"
echo "$PM2_REPORT"
echo "REPORT_EOF"
} >> "$GITHUB_OUTPUT"
# ━━━ 阶段一 · 自动清理 ━━━
- name: "🧹 阶段一 · 自动清理 errored 进程"
if: ${{ inputs.auto_cleanup == 'true' }}
id: pm2_cleanup
run: |
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 自动清理 errored 进程"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
CLEANUP_REPORT=$(ssh -i ~/.ssh/deploy_key \
-o StrictHostKeyChecking=no \
"${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" '
ERRORED_LIST=$(pm2 jlist 2>/dev/null | python3 -c "
import sys, json
try:
procs = json.load(sys.stdin)
for p in procs:
if p.get(\"pm2_env\", {}).get(\"status\") == \"errored\":
name = p.get(\"name\", \"unknown\")
script = p.get(\"pm2_env\", {}).get(\"pm_exec_path\", \"unknown\")
print(f\"{name}|{script}\")
except:
pass
" 2>/dev/null)
CLEANED=""
SKIPPED=""
if [ -z "$ERRORED_LIST" ]; then
echo "✅ 没有需要清理的 errored 进程"
else
echo "$ERRORED_LIST" | while IFS="|" read -r name script; do
if [ ! -f "$script" ]; then
echo "🗑️ 删除 $name脚本 $script 不存在)"
pm2 delete "$name" 2>/dev/null && echo " ✅ 已删除" || echo " ❌ 删除失败"
else
echo "⚠️ 保留 $name脚本 $script 存在,可能是运行时错误)"
echo " 尝试重启..."
pm2 restart "$name" 2>/dev/null && echo " ✅ 重启成功" || echo " ❌ 重启失败"
fi
done
echo ""
echo "📊 清理后 PM2 状态:"
pm2 status 2>/dev/null
pm2 save 2>/dev/null || echo "⚠️ pm2 save 失败"
fi
')
echo "$CLEANUP_REPORT"
echo "$CLEANUP_REPORT" > /tmp/cleanup-report.txt
{
echo "cleanup_report<<CLEANUP_EOF"
echo "$CLEANUP_REPORT"
echo "CLEANUP_EOF"
} >> "$GITHUB_OUTPUT"
# ━━━ 阶段二:全系统健康检查 ━━━
- name: "🏥 阶段二 · 全系统健康检查"
id: health_check
run: |
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 阶段二 · 全系统健康检查"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
HEALTH_REPORT=$(ssh -i ~/.ssh/deploy_key \
-o StrictHostKeyChecking=no \
"${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" '
echo "══════════════════════════════════════"
echo " Nginx 状态"
echo "══════════════════════════════════════"
nginx -t 2>&1 || echo "❌ Nginx 配置检测失败"
systemctl status nginx --no-pager -l 2>/dev/null | head -15 || echo "⚠️ systemctl 不可用"
echo ""
echo "══════════════════════════════════════"
echo " SSL 证书有效期"
echo "══════════════════════════════════════"
if [ -f "/etc/letsencrypt/live/guanghulab.com/fullchain.pem" ]; then
openssl x509 -in /etc/letsencrypt/live/guanghulab.com/fullchain.pem -noout -dates 2>/dev/null || echo "⚠️ 无法读取 SSL 证书"
else
echo "⚠️ SSL 证书文件不存在于默认路径"
# 尝试查找其他证书路径
find /etc/letsencrypt/live/ -name "fullchain.pem" 2>/dev/null | head -3 || echo " 未找到 Let'\''s Encrypt 证书"
find /etc/nginx/ -name "*.pem" -o -name "*.crt" 2>/dev/null | head -3 || echo " 未找到 Nginx SSL 文件"
fi
echo ""
echo "══════════════════════════════════════"
echo " 磁盘空间"
echo "══════════════════════════════════════"
df -h 2>/dev/null || echo "❌ df 命令不可用"
echo ""
echo "══════════════════════════════════════"
echo " 内存使用"
echo "══════════════════════════════════════"
free -h 2>/dev/null || echo "❌ free 命令不可用"
echo ""
echo "══════════════════════════════════════"
echo " PM2 最终状态"
echo "══════════════════════════════════════"
pm2 status 2>/dev/null || echo "❌ pm2 不可用"
echo ""
echo "══════════════════════════════════════"
echo " Node.js 版本"
echo "══════════════════════════════════════"
node -v 2>/dev/null || echo "❌ Node.js 未安装"
npm -v 2>/dev/null || echo "❌ npm 未安装"
echo ""
echo "══════════════════════════════════════"
echo " 网站可访问性检查"
echo "══════════════════════════════════════"
curl -sI https://guanghulab.com 2>/dev/null | head -5 || echo "⚠️ HTTPS 访问失败"
echo "---"
curl -s http://127.0.0.1:3000/health 2>/dev/null | head -c 200 || echo "⚠️ 端口 3000 无响应"
echo ""
echo "---"
curl -s http://127.0.0.1:3001/api/v1/ 2>/dev/null | head -c 200 || echo "⚠️ 端口 3001 无响应"
echo ""
echo "---"
curl -s http://127.0.0.1:3002/api/ps/ 2>/dev/null | head -c 200 || echo "⚠️ 端口 3002 无响应"
echo ""
echo ""
echo "══════════════════════════════════════"
echo " 系统运行时间"
echo "══════════════════════════════════════"
uptime 2>/dev/null || echo "❌ uptime 不可用"
')
echo "$HEALTH_REPORT"
echo "$HEALTH_REPORT" > /tmp/health-report.txt
{
echo "health_report<<HEALTH_EOF"
echo "$HEALTH_REPORT"
echo "HEALTH_EOF"
} >> "$GITHUB_OUTPUT"
# ━━━ 阶段三:生成 Notion 诊断报告 ━━━
- name: "📋 阶段三 · 写入 Notion 工单"
if: ${{ secrets.NOTION_API_TOKEN != '' }}
env:
NOTION_TOKEN: ${{ secrets.NOTION_API_TOKEN }}
NOTION_TICKET_DB_ID: ${{ secrets.NOTION_TICKET_DB_ID || '84edd0640bf146a9a5a7840107013e8c' }}
PM2_REPORT: ${{ steps.pm2_diagnose.outputs.pm2_report }}
CLEANUP_REPORT: ${{ steps.pm2_cleanup.outputs.cleanup_report }}
HEALTH_REPORT: ${{ steps.health_check.outputs.health_report }}
run: node scripts/server-diagnose-report.js
- name: "📋 阶段三 · 写入 Notion 工单(环境检查)"
if: ${{ secrets.NOTION_API_TOKEN == '' }}
run: |
echo "⚠️ Notion credentials 未配置,跳过工单创建"
echo "诊断报告仅输出到 Actions 日志"
- name: "✅ 诊断完成"
run: |
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " ✅ PM2 诊断完毕 · $(date '+%Y-%m-%d %H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

View File

@ -1,32 +1,32 @@
// ecosystem.config.js
// PM2 进程管理配置
//
// 注意:进程名必须与 deploy-to-server.yml 中的 --name 参数保持一致。
// 当前活跃服务2026-03 更新):
// guanghulab → HLI 中间层 → 端口 3001/api/v1/
// guanghulab-proxy → AI Chat API 代理 → 端口 3721
// guanghulab-backend → Express 后端 API → 端口 3000/webhook/feishu
// guanghulab-ws → Status Board WS → 端口 8080
// persona-studio → Persona Studio API → 端口 3002/api/ps/
module.exports = {
apps: [
{
name: 'guanghulab',
script: 'src/index.js',
instances: 'max',
exec_mode: 'cluster',
instances: 1,
exec_mode: 'fork',
watch: false,
env: {
NODE_ENV: 'production',
PORT: 3000,
},
env_development: {
NODE_ENV: 'development',
PORT: 3000,
},
env_test: {
NODE_ENV: 'test',
PORT: 3001,
},
log_date_format: 'YYYY-MM-DD HH:mm:ss',
error_file: 'logs/error.log',
out_file: 'logs/out.log',
error_file: 'logs/hli-error.log',
out_file: 'logs/hli-out.log',
},
{
name: 'api-proxy',
name: 'guanghulab-proxy',
script: 'backend-integration/api-proxy.js',
instances: 1,
exec_mode: 'fork',
@ -39,6 +39,34 @@ module.exports = {
error_file: 'logs/api-proxy-error.log',
out_file: 'logs/api-proxy-out.log',
},
{
name: 'guanghulab-backend',
script: 'backend/server.js',
instances: 1,
exec_mode: 'fork',
watch: false,
env: {
NODE_ENV: 'production',
PORT: 3000,
},
log_date_format: 'YYYY-MM-DD HH:mm:ss',
error_file: 'logs/backend-error.log',
out_file: 'logs/backend-out.log',
},
{
name: 'guanghulab-ws',
script: 'status-board/mock-ws-server.js',
instances: 1,
exec_mode: 'fork',
watch: false,
env: {
NODE_ENV: 'production',
PORT: 8080,
},
log_date_format: 'YYYY-MM-DD HH:mm:ss',
error_file: 'logs/ws-error.log',
out_file: 'logs/ws-out.log',
},
{
name: 'persona-studio',
script: 'persona-studio/backend/server.js',

View File

@ -0,0 +1,201 @@
// scripts/server-diagnose-report.js
// 铸渊 · PM2 服务诊断报告 → Notion 工单
//
// 将 PM2 诊断和系统健康检查结果写入 Notion 工单队列。
// 由 pm2-server-diagnose.yml 工作流调用。
//
// 环境变量:
// NOTION_TOKEN Notion API token
// NOTION_TICKET_DB_ID 工单队列数据库 ID
// PM2_REPORT PM2 诊断输出
// CLEANUP_REPORT 清理操作输出(可选)
// HEALTH_REPORT 健康检查输出
'use strict';
const https = require('https');
const NOTION_TOKEN = process.env.NOTION_TOKEN || '';
const NOTION_TICKET_DB_ID = process.env.NOTION_TICKET_DB_ID || '';
const PM2_REPORT = process.env.PM2_REPORT || '';
const CLEANUP_REPORT = process.env.CLEANUP_REPORT || '';
const HEALTH_REPORT = process.env.HEALTH_REPORT || '';
const NOTION_VERSION = '2022-06-28';
const NOTION_API_HOSTNAME = 'api.notion.com';
const NOTION_RICH_TEXT_MAX = 2000;
const MAX_PM2_REPORT_LENGTH = 3000;
const MAX_CLEANUP_REPORT_LENGTH = 2000;
const MAX_HEALTH_REPORT_LENGTH = 3000;
// ══════════════════════════════════════════════════════════
// Notion API 工具
// ══════════════════════════════════════════════════════════
function notionPost(endpoint, body) {
return new Promise(function (resolve, reject) {
var payload = JSON.stringify(body);
var opts = {
hostname: NOTION_API_HOSTNAME,
port: 443,
path: endpoint,
method: 'POST',
headers: {
'Authorization': 'Bearer ' + NOTION_TOKEN,
'Content-Type': 'application/json',
'Notion-Version': NOTION_VERSION,
'Content-Length': Buffer.byteLength(payload),
},
};
var req = https.request(opts, function (res) {
var data = '';
res.on('data', function (chunk) { data += chunk; });
res.on('end', function () {
try {
var parsed = JSON.parse(data);
if (res.statusCode >= 200 && res.statusCode < 300) {
resolve(parsed);
} else {
reject(new Error('Notion API ' + res.statusCode + ': ' + (parsed.message || data)));
}
} catch (e) {
reject(new Error('Notion API parse error: ' + data));
}
});
});
req.on('error', reject);
req.write(payload);
req.end();
});
}
function richTextChunks(content) {
var str = String(content || '');
var chunks = [];
for (var i = 0; i < str.length; i += NOTION_RICH_TEXT_MAX) {
chunks.push({ type: 'text', text: { content: str.slice(i, i + NOTION_RICH_TEXT_MAX) } });
}
if (chunks.length === 0) {
chunks.push({ type: 'text', text: { content: '' } });
}
return chunks;
}
// ══════════════════════════════════════════════════════════
// 构建报告内容
// ══════════════════════════════════════════════════════════
function buildReportContent() {
var now = new Date().toISOString();
var dateStr = now.slice(0, 10);
var sections = [];
sections.push('## 📋 PM2 服务排查报告 · ' + dateStr);
sections.push('');
sections.push('| 字段 | 值 |');
sections.push('|------|-----|');
sections.push('| 执行者 | 铸渊Agent |');
sections.push('| 执行时间 | ' + now + ' |');
sections.push('| 来源 | GitHub Actions · pm2-server-diagnose |');
sections.push('');
if (PM2_REPORT) {
sections.push('## 阶段一 · PM2 诊断结果');
sections.push('');
sections.push(PM2_REPORT.slice(0, MAX_PM2_REPORT_LENGTH));
sections.push('');
}
if (CLEANUP_REPORT) {
sections.push('## 清理操作结果');
sections.push('');
sections.push(CLEANUP_REPORT.slice(0, MAX_CLEANUP_REPORT_LENGTH));
sections.push('');
}
if (HEALTH_REPORT) {
sections.push('## 阶段二 · 全系统健康检查');
sections.push('');
sections.push(HEALTH_REPORT.slice(0, MAX_HEALTH_REPORT_LENGTH));
sections.push('');
}
return sections.join('\n');
}
// ══════════════════════════════════════════════════════════
// 主函数 — 创建 Notion 诊断工单
// ══════════════════════════════════════════════════════════
async function createDiagnoseTicket() {
if (!NOTION_TOKEN || !NOTION_TICKET_DB_ID) {
console.log('⚠️ Notion credentials not configured, skipping ticket creation');
console.log(' NOTION_TOKEN: ' + (NOTION_TOKEN ? '✅' : '❌'));
console.log(' NOTION_TICKET_DB_ID: ' + (NOTION_TICKET_DB_ID ? '✅' : '❌'));
process.exit(0);
}
var now = new Date();
var dateStr = now.toISOString().slice(0, 10);
var title = 'PM2服务排查报告 · ' + dateStr;
var contentText = buildReportContent();
// 将内容拆分成多个段落块Notion 单个 rich_text 限制 2000 字符)
var contentBlocks = [];
var remaining = contentText;
while (remaining.length > 0) {
var chunk = remaining.slice(0, NOTION_RICH_TEXT_MAX);
remaining = remaining.slice(NOTION_RICH_TEXT_MAX);
contentBlocks.push({
object: 'block',
type: 'paragraph',
paragraph: {
rich_text: [{ type: 'text', text: { content: chunk } }],
},
});
}
// 确保至少有一个内容块
if (contentBlocks.length === 0) {
contentBlocks.push({
object: 'block',
type: 'paragraph',
paragraph: {
rich_text: [{ type: 'text', text: { content: '诊断报告(无详细输出)' } }],
},
});
}
var body = {
parent: { database_id: NOTION_TICKET_DB_ID },
properties: {
'标题': { title: [{ type: 'text', text: { content: title } }] },
'操作类型': { select: { name: '其他' } },
'提交者': { rich_text: [{ type: 'text', text: { content: '铸渊Agent' } }] },
'状态': { select: { name: '已完成' } },
'优先级': { select: { name: 'P1' } },
},
children: contentBlocks,
};
console.log('📋 创建 PM2 诊断工单...');
console.log(' 标题: ' + title);
console.log(' 数据库: ' + NOTION_TICKET_DB_ID);
console.log(' 内容长度: ' + contentText.length + ' 字符');
console.log(' 内容块数: ' + contentBlocks.length);
try {
var result = await notionPost('/v1/pages', body);
console.log('✅ Notion 诊断工单已创建: ' + result.id);
console.log(' URL: ' + (result.url || 'N/A'));
} catch (err) {
console.error('❌ 工单创建失败: ' + err.message);
// 不以失败退出,诊断本身已成功
console.log('⚠️ 诊断结果已输出到 Actions 日志Notion 写入失败不影响诊断');
process.exit(0);
}
}
createDiagnoseTicket();