fix: 修复content-range解析NaN问题和HEAD预检静默失败
- 提取parseTotalSize()工具函数,正确处理畸形content-range头 - HEAD预检失败时输出警告日志而非完全静默 Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/f10f1b02-ed4e-4ee8-ac27-0580c1cf8a7e Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com>
This commit is contained in:
parent
ba4c233509
commit
2e8badf100
|
|
@ -251,6 +251,23 @@ async function head(bucket, key) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从响应头解析文件总大小(content-range 或 content-length)
|
||||||
|
*/
|
||||||
|
function parseTotalSize(headers) {
|
||||||
|
// content-range 格式: bytes 0-1023/665000000
|
||||||
|
const range = headers['content-range'];
|
||||||
|
if (range) {
|
||||||
|
const parts = range.split('/');
|
||||||
|
if (parts.length === 2) {
|
||||||
|
const size = parseInt(parts[1], 10);
|
||||||
|
if (!isNaN(size)) return size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const cl = parseInt(headers['content-length'] || '0', 10);
|
||||||
|
return isNaN(cl) ? 0 : cl;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 分块读取大文件 — 仅读取前 N 字节用于预览/采样
|
* 分块读取大文件 — 仅读取前 N 字节用于预览/采样
|
||||||
* 适用于超大语料文件的类型检测和内容预览
|
* 适用于超大语料文件的类型检测和内容预览
|
||||||
|
|
@ -286,8 +303,7 @@ async function readPartial(bucket, key, maxBytes) {
|
||||||
res.on('end', () => {
|
res.on('end', () => {
|
||||||
try {
|
try {
|
||||||
const buffer = Buffer.concat(chunks);
|
const buffer = Buffer.concat(chunks);
|
||||||
const totalSize = parseInt(res.headers['content-range']?.split('/')[1] || '0', 10)
|
const totalSize = parseTotalSize(res.headers);
|
||||||
|| parseInt(res.headers['content-length'] || '0', 10);
|
|
||||||
resolve({
|
resolve({
|
||||||
content: buffer.toString('utf8'),
|
content: buffer.toString('utf8'),
|
||||||
size_bytes: buffer.length,
|
size_bytes: buffer.length,
|
||||||
|
|
@ -303,8 +319,7 @@ async function readPartial(bucket, key, maxBytes) {
|
||||||
if (chunks.length > 0) {
|
if (chunks.length > 0) {
|
||||||
try {
|
try {
|
||||||
const buffer = Buffer.concat(chunks);
|
const buffer = Buffer.concat(chunks);
|
||||||
const totalSize = parseInt(res.headers['content-range']?.split('/')[1] || '0', 10)
|
const totalSize = parseTotalSize(res.headers);
|
||||||
|| parseInt(res.headers['content-length'] || '0', 10);
|
|
||||||
resolve({
|
resolve({
|
||||||
content: buffer.toString('utf8'),
|
content: buffer.toString('utf8'),
|
||||||
size_bytes: buffer.length,
|
size_bytes: buffer.length,
|
||||||
|
|
|
||||||
|
|
@ -172,8 +172,12 @@ async function cosExtractCorpus(input) {
|
||||||
try {
|
try {
|
||||||
const meta = await cos.head(bucket, key);
|
const meta = await cos.head(bucket, key);
|
||||||
fileSize = meta.size_bytes;
|
fileSize = meta.size_bytes;
|
||||||
} catch {
|
} catch (headErr) {
|
||||||
// HEAD 不支持时跳过,由后续读取处理
|
// HEAD失败不阻断流程(部分COS配置不支持HEAD),用0继续
|
||||||
|
// 但如果是认证/桶不存在等严重错误,后续read也会失败
|
||||||
|
if (headErr.message && !headErr.message.includes('405') && !headErr.message.includes('403')) {
|
||||||
|
console.log(` ⚠️ HEAD预检失败: ${headErr.message},继续尝试读取`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ZIP文件始终返回提示(无论大小),因为需要二进制解析
|
// ZIP文件始终返回提示(无论大小),因为需要二进制解析
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue