修复工单observer-23906843495三个Bug:告警阈值/IP子域名误判/邮件次数写死
Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/fd18e928-cba0-469e-b2c5-ce77d8efc2b1 Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com>
This commit is contained in:
parent
78738907ad
commit
816357b477
|
|
@ -114,6 +114,7 @@ jobs:
|
|||
outputs:
|
||||
repair_success: ${{ steps.repair.outputs.repair_success }}
|
||||
repair_attempt: ${{ steps.repair.outputs.repair_attempt }}
|
||||
needs_human: ${{ steps.repair.outputs.needs_human }}
|
||||
|
||||
steps:
|
||||
- name: '📥 检出代码'
|
||||
|
|
@ -212,7 +213,7 @@ jobs:
|
|||
if: >-
|
||||
always() &&
|
||||
needs.analyze-and-repair.result != 'skipped' &&
|
||||
needs.analyze-and-repair.outputs.repair_success != 'true'
|
||||
needs.analyze-and-repair.outputs.needs_human == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
|
@ -269,7 +270,8 @@ jobs:
|
|||
run: |
|
||||
if [ -n "$ZY_SMTP_USER" ] && [ -n "$ZY_SMTP_PASS" ]; then
|
||||
node scripts/staging-ops-agent.js alert \
|
||||
--order-id "observer-${{ needs.collect-logs.outputs.run_id }}" || true
|
||||
--order-id "observer-${{ needs.collect-logs.outputs.run_id }}" \
|
||||
--attempt "${{ needs.analyze-and-repair.outputs.repair_attempt }}" || true
|
||||
else
|
||||
echo "ℹ️ SMTP未配置 · 跳过邮件告警 · 已创建GitHub Issue"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@
|
|||
{
|
||||
"id": "WO-20260402-1457",
|
||||
"title": "Merge pull request #258 from qinfendebingshuo/copilot/update-homepage-three-column-layout",
|
||||
"status": "failed",
|
||||
"status": "human-intervened",
|
||||
"commit_sha": "b3f38e263f6f6aa6e03a516445cbea8b63bd9c09",
|
||||
"branch": "main",
|
||||
"created_by": "qinfendebingshuo",
|
||||
|
|
@ -141,6 +141,12 @@
|
|||
"status": "failed",
|
||||
"actor": "Agent",
|
||||
"message": "部署失败"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-04-02T16:33:46.728Z",
|
||||
"status": "human-intervened",
|
||||
"actor": "铸渊",
|
||||
"message": "铸渊人工干预 · 已定位根因:健康检查IP子域名误判Bug + 告警阈值Bug · 正在修复"
|
||||
}
|
||||
],
|
||||
"deploy_logs": [
|
||||
|
|
@ -150,7 +156,7 @@
|
|||
}
|
||||
],
|
||||
"created_at": "2026-04-02T14:57:46.729Z",
|
||||
"updated_at": "2026-04-02T14:58:27.523Z"
|
||||
"updated_at": "2026-04-02T16:33:46.728Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -284,6 +284,7 @@ async function repair() {
|
|||
console.log(`❌ 已达最大修复次数(${MAX_REPAIR_ATTEMPTS}) · 放弃修复 · 通知人类`);
|
||||
setOutput('repair_success', 'false');
|
||||
setOutput('repair_attempt', String(attemptNumber - 1));
|
||||
setOutput('needs_human', 'true');
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -460,16 +461,18 @@ async function repair() {
|
|||
}
|
||||
|
||||
// §7 设置输出
|
||||
const needsHuman = !repairSuccess && attemptNumber >= MAX_REPAIR_ATTEMPTS;
|
||||
setOutput('repair_success', repairSuccess ? 'true' : 'false');
|
||||
setOutput('repair_attempt', String(attemptNumber));
|
||||
setOutput('needs_human', needsHuman ? 'true' : 'false');
|
||||
|
||||
console.log('═'.repeat(60));
|
||||
if (repairSuccess) {
|
||||
console.log(`✅ 修复成功 · 第${attemptNumber}次尝试`);
|
||||
} else if (attemptNumber >= MAX_REPAIR_ATTEMPTS) {
|
||||
} else if (needsHuman) {
|
||||
console.log(`❌ ${MAX_REPAIR_ATTEMPTS}次修复均失败 · 需要人工干预`);
|
||||
} else {
|
||||
console.log(`❌ 第${attemptNumber}次修复失败 · 还有${MAX_REPAIR_ATTEMPTS - attemptNumber}次尝试机会`);
|
||||
console.log(`❌ 第${attemptNumber}次修复失败 · 还有${MAX_REPAIR_ATTEMPTS - attemptNumber}次尝试机会 · 等待下次重试`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -100,14 +100,22 @@ async function runHealthCheck(host) {
|
|||
console.log(`🔍 健康检查 · 目标: ${host}`);
|
||||
console.log('─'.repeat(50));
|
||||
|
||||
// 如果host是IP地址,带host_prefix的检查无效(IP不支持子域名)→ 降级为non-critical
|
||||
const isIpAddress = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
||||
if (isIpAddress) {
|
||||
console.log(' ℹ️ 目标为IP地址 · 子域名检查降级为非关键');
|
||||
}
|
||||
|
||||
const results = [];
|
||||
let criticalFail = false;
|
||||
|
||||
for (const endpoint of HEALTH_ENDPOINTS) {
|
||||
const checkHost = endpoint.host_prefix ? `${endpoint.host_prefix}.${host}` : host;
|
||||
// IP地址不支持子域名前缀,降级为non-critical避免误报
|
||||
const isCritical = endpoint.critical && !(isIpAddress && endpoint.host_prefix);
|
||||
const result = await httpCheck(checkHost, endpoint.path, endpoint.port);
|
||||
|
||||
const icon = result.ok ? '✅' : (endpoint.critical ? '❌' : '⚠️');
|
||||
const icon = result.ok ? '✅' : (isCritical ? '❌' : '⚠️');
|
||||
console.log(` ${icon} ${endpoint.name}: ${result.ok ? 'OK' : result.error || `HTTP ${result.status}`}`);
|
||||
|
||||
results.push({
|
||||
|
|
@ -116,7 +124,7 @@ async function runHealthCheck(host) {
|
|||
checked_at: new Date().toISOString()
|
||||
});
|
||||
|
||||
if (!result.ok && endpoint.critical) {
|
||||
if (!result.ok && isCritical) {
|
||||
criticalFail = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -302,16 +310,17 @@ async function analyzeLog(logContent, orderId) {
|
|||
return deepResult;
|
||||
}
|
||||
|
||||
// ── 邮件告警 (3次重试失败) ──────────────────
|
||||
async function sendAlert(orderId, email) {
|
||||
// ── 邮件告警 (最大重试失败) ──────────────────
|
||||
async function sendAlert(orderId, email, options = {}) {
|
||||
const smtpUser = process.env.ZY_SMTP_USER;
|
||||
const smtpPass = process.env.ZY_SMTP_PASS;
|
||||
const attemptCount = options.attempt || process.env.REPAIR_ATTEMPT || '?';
|
||||
|
||||
if (!smtpUser || !smtpPass) {
|
||||
console.error('❌ SMTP未配置 (需要ZY_SMTP_USER和ZY_SMTP_PASS)');
|
||||
console.log('📧 告警内容 (控制台输出):');
|
||||
console.log(` 工单: ${orderId}`);
|
||||
console.log(` 状态: 3次自动修复未能解决,需要人工干预`);
|
||||
console.log(` 状态: ${attemptCount}次自动修复未能解决,需要人工干预`);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -345,7 +354,7 @@ async function sendAlert(orderId, email) {
|
|||
<p style="color: #666; margin-top: 0;">ZY-Staging-Ops-Agent · 自动告警</p>
|
||||
|
||||
<div style="background: #fef5f5; border: 1px solid #f5c6cb; border-radius: 8px; padding: 15px; margin: 20px 0;">
|
||||
<strong style="color: #c0392b;">⚠️ 自动修复已达最大重试次数(3次),需要冰朔人工干预</strong>
|
||||
<strong style="color: #c0392b;">⚠️ 自动修复已达最大重试次数(${attemptCount}次),需要冰朔人工干预</strong>
|
||||
</div>
|
||||
|
||||
<table style="width: 100%; border-collapse: collapse;">
|
||||
|
|
@ -473,7 +482,7 @@ async function main() {
|
|||
console.error('❌ 需要 --order-id 参数');
|
||||
process.exit(1);
|
||||
}
|
||||
await sendAlert(orderId, args.email);
|
||||
await sendAlert(orderId, args.email, { attempt: args.attempt });
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue