修复工单observer-23906843495三个Bug:告警阈值/IP子域名误判/邮件次数写死
Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/fd18e928-cba0-469e-b2c5-ce77d8efc2b1 Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com>
This commit is contained in:
parent
78738907ad
commit
816357b477
|
|
@ -114,6 +114,7 @@ jobs:
|
||||||
outputs:
|
outputs:
|
||||||
repair_success: ${{ steps.repair.outputs.repair_success }}
|
repair_success: ${{ steps.repair.outputs.repair_success }}
|
||||||
repair_attempt: ${{ steps.repair.outputs.repair_attempt }}
|
repair_attempt: ${{ steps.repair.outputs.repair_attempt }}
|
||||||
|
needs_human: ${{ steps.repair.outputs.needs_human }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: '📥 检出代码'
|
- name: '📥 检出代码'
|
||||||
|
|
@ -212,7 +213,7 @@ jobs:
|
||||||
if: >-
|
if: >-
|
||||||
always() &&
|
always() &&
|
||||||
needs.analyze-and-repair.result != 'skipped' &&
|
needs.analyze-and-repair.result != 'skipped' &&
|
||||||
needs.analyze-and-repair.outputs.repair_success != 'true'
|
needs.analyze-and-repair.outputs.needs_human == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
|
@ -269,7 +270,8 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
if [ -n "$ZY_SMTP_USER" ] && [ -n "$ZY_SMTP_PASS" ]; then
|
if [ -n "$ZY_SMTP_USER" ] && [ -n "$ZY_SMTP_PASS" ]; then
|
||||||
node scripts/staging-ops-agent.js alert \
|
node scripts/staging-ops-agent.js alert \
|
||||||
--order-id "observer-${{ needs.collect-logs.outputs.run_id }}" || true
|
--order-id "observer-${{ needs.collect-logs.outputs.run_id }}" \
|
||||||
|
--attempt "${{ needs.analyze-and-repair.outputs.repair_attempt }}" || true
|
||||||
else
|
else
|
||||||
echo "ℹ️ SMTP未配置 · 跳过邮件告警 · 已创建GitHub Issue"
|
echo "ℹ️ SMTP未配置 · 跳过邮件告警 · 已创建GitHub Issue"
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -116,7 +116,7 @@
|
||||||
{
|
{
|
||||||
"id": "WO-20260402-1457",
|
"id": "WO-20260402-1457",
|
||||||
"title": "Merge pull request #258 from qinfendebingshuo/copilot/update-homepage-three-column-layout",
|
"title": "Merge pull request #258 from qinfendebingshuo/copilot/update-homepage-three-column-layout",
|
||||||
"status": "failed",
|
"status": "human-intervened",
|
||||||
"commit_sha": "b3f38e263f6f6aa6e03a516445cbea8b63bd9c09",
|
"commit_sha": "b3f38e263f6f6aa6e03a516445cbea8b63bd9c09",
|
||||||
"branch": "main",
|
"branch": "main",
|
||||||
"created_by": "qinfendebingshuo",
|
"created_by": "qinfendebingshuo",
|
||||||
|
|
@ -141,6 +141,12 @@
|
||||||
"status": "failed",
|
"status": "failed",
|
||||||
"actor": "Agent",
|
"actor": "Agent",
|
||||||
"message": "部署失败"
|
"message": "部署失败"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"timestamp": "2026-04-02T16:33:46.728Z",
|
||||||
|
"status": "human-intervened",
|
||||||
|
"actor": "铸渊",
|
||||||
|
"message": "铸渊人工干预 · 已定位根因:健康检查IP子域名误判Bug + 告警阈值Bug · 正在修复"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"deploy_logs": [
|
"deploy_logs": [
|
||||||
|
|
@ -150,7 +156,7 @@
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"created_at": "2026-04-02T14:57:46.729Z",
|
"created_at": "2026-04-02T14:57:46.729Z",
|
||||||
"updated_at": "2026-04-02T14:58:27.523Z"
|
"updated_at": "2026-04-02T16:33:46.728Z"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -284,6 +284,7 @@ async function repair() {
|
||||||
console.log(`❌ 已达最大修复次数(${MAX_REPAIR_ATTEMPTS}) · 放弃修复 · 通知人类`);
|
console.log(`❌ 已达最大修复次数(${MAX_REPAIR_ATTEMPTS}) · 放弃修复 · 通知人类`);
|
||||||
setOutput('repair_success', 'false');
|
setOutput('repair_success', 'false');
|
||||||
setOutput('repair_attempt', String(attemptNumber - 1));
|
setOutput('repair_attempt', String(attemptNumber - 1));
|
||||||
|
setOutput('needs_human', 'true');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -460,16 +461,18 @@ async function repair() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// §7 设置输出
|
// §7 设置输出
|
||||||
|
const needsHuman = !repairSuccess && attemptNumber >= MAX_REPAIR_ATTEMPTS;
|
||||||
setOutput('repair_success', repairSuccess ? 'true' : 'false');
|
setOutput('repair_success', repairSuccess ? 'true' : 'false');
|
||||||
setOutput('repair_attempt', String(attemptNumber));
|
setOutput('repair_attempt', String(attemptNumber));
|
||||||
|
setOutput('needs_human', needsHuman ? 'true' : 'false');
|
||||||
|
|
||||||
console.log('═'.repeat(60));
|
console.log('═'.repeat(60));
|
||||||
if (repairSuccess) {
|
if (repairSuccess) {
|
||||||
console.log(`✅ 修复成功 · 第${attemptNumber}次尝试`);
|
console.log(`✅ 修复成功 · 第${attemptNumber}次尝试`);
|
||||||
} else if (attemptNumber >= MAX_REPAIR_ATTEMPTS) {
|
} else if (needsHuman) {
|
||||||
console.log(`❌ ${MAX_REPAIR_ATTEMPTS}次修复均失败 · 需要人工干预`);
|
console.log(`❌ ${MAX_REPAIR_ATTEMPTS}次修复均失败 · 需要人工干预`);
|
||||||
} else {
|
} else {
|
||||||
console.log(`❌ 第${attemptNumber}次修复失败 · 还有${MAX_REPAIR_ATTEMPTS - attemptNumber}次尝试机会`);
|
console.log(`❌ 第${attemptNumber}次修复失败 · 还有${MAX_REPAIR_ATTEMPTS - attemptNumber}次尝试机会 · 等待下次重试`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -100,14 +100,22 @@ async function runHealthCheck(host) {
|
||||||
console.log(`🔍 健康检查 · 目标: ${host}`);
|
console.log(`🔍 健康检查 · 目标: ${host}`);
|
||||||
console.log('─'.repeat(50));
|
console.log('─'.repeat(50));
|
||||||
|
|
||||||
|
// 如果host是IP地址,带host_prefix的检查无效(IP不支持子域名)→ 降级为non-critical
|
||||||
|
const isIpAddress = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
||||||
|
if (isIpAddress) {
|
||||||
|
console.log(' ℹ️ 目标为IP地址 · 子域名检查降级为非关键');
|
||||||
|
}
|
||||||
|
|
||||||
const results = [];
|
const results = [];
|
||||||
let criticalFail = false;
|
let criticalFail = false;
|
||||||
|
|
||||||
for (const endpoint of HEALTH_ENDPOINTS) {
|
for (const endpoint of HEALTH_ENDPOINTS) {
|
||||||
const checkHost = endpoint.host_prefix ? `${endpoint.host_prefix}.${host}` : host;
|
const checkHost = endpoint.host_prefix ? `${endpoint.host_prefix}.${host}` : host;
|
||||||
|
// IP地址不支持子域名前缀,降级为non-critical避免误报
|
||||||
|
const isCritical = endpoint.critical && !(isIpAddress && endpoint.host_prefix);
|
||||||
const result = await httpCheck(checkHost, endpoint.path, endpoint.port);
|
const result = await httpCheck(checkHost, endpoint.path, endpoint.port);
|
||||||
|
|
||||||
const icon = result.ok ? '✅' : (endpoint.critical ? '❌' : '⚠️');
|
const icon = result.ok ? '✅' : (isCritical ? '❌' : '⚠️');
|
||||||
console.log(` ${icon} ${endpoint.name}: ${result.ok ? 'OK' : result.error || `HTTP ${result.status}`}`);
|
console.log(` ${icon} ${endpoint.name}: ${result.ok ? 'OK' : result.error || `HTTP ${result.status}`}`);
|
||||||
|
|
||||||
results.push({
|
results.push({
|
||||||
|
|
@ -116,7 +124,7 @@ async function runHealthCheck(host) {
|
||||||
checked_at: new Date().toISOString()
|
checked_at: new Date().toISOString()
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!result.ok && endpoint.critical) {
|
if (!result.ok && isCritical) {
|
||||||
criticalFail = true;
|
criticalFail = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -302,16 +310,17 @@ async function analyzeLog(logContent, orderId) {
|
||||||
return deepResult;
|
return deepResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 邮件告警 (3次重试失败) ──────────────────
|
// ── 邮件告警 (最大重试失败) ──────────────────
|
||||||
async function sendAlert(orderId, email) {
|
async function sendAlert(orderId, email, options = {}) {
|
||||||
const smtpUser = process.env.ZY_SMTP_USER;
|
const smtpUser = process.env.ZY_SMTP_USER;
|
||||||
const smtpPass = process.env.ZY_SMTP_PASS;
|
const smtpPass = process.env.ZY_SMTP_PASS;
|
||||||
|
const attemptCount = options.attempt || process.env.REPAIR_ATTEMPT || '?';
|
||||||
|
|
||||||
if (!smtpUser || !smtpPass) {
|
if (!smtpUser || !smtpPass) {
|
||||||
console.error('❌ SMTP未配置 (需要ZY_SMTP_USER和ZY_SMTP_PASS)');
|
console.error('❌ SMTP未配置 (需要ZY_SMTP_USER和ZY_SMTP_PASS)');
|
||||||
console.log('📧 告警内容 (控制台输出):');
|
console.log('📧 告警内容 (控制台输出):');
|
||||||
console.log(` 工单: ${orderId}`);
|
console.log(` 工单: ${orderId}`);
|
||||||
console.log(` 状态: 3次自动修复未能解决,需要人工干预`);
|
console.log(` 状态: ${attemptCount}次自动修复未能解决,需要人工干预`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -345,7 +354,7 @@ async function sendAlert(orderId, email) {
|
||||||
<p style="color: #666; margin-top: 0;">ZY-Staging-Ops-Agent · 自动告警</p>
|
<p style="color: #666; margin-top: 0;">ZY-Staging-Ops-Agent · 自动告警</p>
|
||||||
|
|
||||||
<div style="background: #fef5f5; border: 1px solid #f5c6cb; border-radius: 8px; padding: 15px; margin: 20px 0;">
|
<div style="background: #fef5f5; border: 1px solid #f5c6cb; border-radius: 8px; padding: 15px; margin: 20px 0;">
|
||||||
<strong style="color: #c0392b;">⚠️ 自动修复已达最大重试次数(3次),需要冰朔人工干预</strong>
|
<strong style="color: #c0392b;">⚠️ 自动修复已达最大重试次数(${attemptCount}次),需要冰朔人工干预</strong>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<table style="width: 100%; border-collapse: collapse;">
|
<table style="width: 100%; border-collapse: collapse;">
|
||||||
|
|
@ -473,7 +482,7 @@ async function main() {
|
||||||
console.error('❌ 需要 --order-id 参数');
|
console.error('❌ 需要 --order-id 参数');
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
await sendAlert(orderId, args.email);
|
await sendAlert(orderId, args.email, { attempt: args.attempt });
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue