fix: 修复content-range解析NaN问题和HEAD预检静默失败
- 提取parseTotalSize()工具函数,正确处理畸形content-range头 - HEAD预检失败时输出警告日志而非完全静默 Agent-Logs-Url: https://github.com/qinfendebingshuo/guanghulab/sessions/f10f1b02-ed4e-4ee8-ac27-0580c1cf8a7e Co-authored-by: qinfendebingshuo <207279273+qinfendebingshuo@users.noreply.github.com>
This commit is contained in:
parent
ba4c233509
commit
2e8badf100
|
|
@ -251,6 +251,23 @@ async function head(bucket, key) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 从响应头解析文件总大小(content-range 或 content-length)
|
||||
*/
|
||||
function parseTotalSize(headers) {
|
||||
// content-range 格式: bytes 0-1023/665000000
|
||||
const range = headers['content-range'];
|
||||
if (range) {
|
||||
const parts = range.split('/');
|
||||
if (parts.length === 2) {
|
||||
const size = parseInt(parts[1], 10);
|
||||
if (!isNaN(size)) return size;
|
||||
}
|
||||
}
|
||||
const cl = parseInt(headers['content-length'] || '0', 10);
|
||||
return isNaN(cl) ? 0 : cl;
|
||||
}
|
||||
|
||||
/**
|
||||
* 分块读取大文件 — 仅读取前 N 字节用于预览/采样
|
||||
* 适用于超大语料文件的类型检测和内容预览
|
||||
|
|
@ -286,8 +303,7 @@ async function readPartial(bucket, key, maxBytes) {
|
|||
res.on('end', () => {
|
||||
try {
|
||||
const buffer = Buffer.concat(chunks);
|
||||
const totalSize = parseInt(res.headers['content-range']?.split('/')[1] || '0', 10)
|
||||
|| parseInt(res.headers['content-length'] || '0', 10);
|
||||
const totalSize = parseTotalSize(res.headers);
|
||||
resolve({
|
||||
content: buffer.toString('utf8'),
|
||||
size_bytes: buffer.length,
|
||||
|
|
@ -303,8 +319,7 @@ async function readPartial(bucket, key, maxBytes) {
|
|||
if (chunks.length > 0) {
|
||||
try {
|
||||
const buffer = Buffer.concat(chunks);
|
||||
const totalSize = parseInt(res.headers['content-range']?.split('/')[1] || '0', 10)
|
||||
|| parseInt(res.headers['content-length'] || '0', 10);
|
||||
const totalSize = parseTotalSize(res.headers);
|
||||
resolve({
|
||||
content: buffer.toString('utf8'),
|
||||
size_bytes: buffer.length,
|
||||
|
|
|
|||
|
|
@ -172,8 +172,12 @@ async function cosExtractCorpus(input) {
|
|||
try {
|
||||
const meta = await cos.head(bucket, key);
|
||||
fileSize = meta.size_bytes;
|
||||
} catch {
|
||||
// HEAD 不支持时跳过,由后续读取处理
|
||||
} catch (headErr) {
|
||||
// HEAD失败不阻断流程(部分COS配置不支持HEAD),用0继续
|
||||
// 但如果是认证/桶不存在等严重错误,后续read也会失败
|
||||
if (headErr.message && !headErr.message.includes('405') && !headErr.message.includes('403')) {
|
||||
console.log(` ⚠️ HEAD预检失败: ${headErr.message},继续尝试读取`);
|
||||
}
|
||||
}
|
||||
|
||||
// ZIP文件始终返回提示(无论大小),因为需要二进制解析
|
||||
|
|
|
|||
Loading…
Reference in New Issue