feat: test262 supervise 心跳 COMPLETED 语义与 FAIL 旁路透传
This commit is contained in:
@@ -17,7 +17,8 @@ mod report;
|
|||||||
mod test262_log;
|
mod test262_log;
|
||||||
use oxide_log::{Level, LogConfig, Output, SUBSYSTEM_COUNT};
|
use oxide_log::{Level, LogConfig, Output, SUBSYSTEM_COUNT};
|
||||||
use report::{
|
use report::{
|
||||||
first_line, format_fail_categories, format_fail_list, FailRecord, MAX_FAIL_MSG_CHARS, MAX_FAIL_RECORD_BYTES,
|
append_fail_log, first_line, format_fail_categories, format_fail_list, parse_fail_log, FailRecord,
|
||||||
|
MAX_FAIL_MSG_CHARS, MAX_FAIL_RECORD_BYTES,
|
||||||
};
|
};
|
||||||
|
|
||||||
// 记录当前正在执行的测试路径(thread-local);每个测试执行前写入,
|
// 记录当前正在执行的测试路径(thread-local);每个测试执行前写入,
|
||||||
@@ -137,6 +138,13 @@ struct RunStats {
|
|||||||
categories: Vec<String>, // 类别 id 表(FailRecord.category_id 索引)
|
categories: Vec<String>, // 类别 id 表(FailRecord.category_id 索引)
|
||||||
fail_records: Vec<FailRecord>,
|
fail_records: Vec<FailRecord>,
|
||||||
fail_record_bytes: usize, // 累计 message 字节(OOM cap)
|
fail_record_bytes: usize, // 累计 message 字节(OOM cap)
|
||||||
|
// supervise 模式异常计数:spawn 失败 / try_wait 错误 / 心跳写失败(子进程侧
|
||||||
|
// 累计后经心跳第 6 字段回传)。
|
||||||
|
spawn_errors: usize,
|
||||||
|
wait_errors: usize,
|
||||||
|
hb_write_errors: usize,
|
||||||
|
/// 超时/崩溃清单:(index, elapsed_ms);elapsed_ms=0 表示非超时崩溃。
|
||||||
|
timeout_crashes: Vec<(usize, u64)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RunStats {
|
impl RunStats {
|
||||||
@@ -156,6 +164,10 @@ impl RunStats {
|
|||||||
self.fail_records.push(FailRecord { category_id: id, ..rec });
|
self.fail_records.push(FailRecord { category_id: id, ..rec });
|
||||||
}
|
}
|
||||||
self.fail_record_bytes += other.fail_record_bytes;
|
self.fail_record_bytes += other.fail_record_bytes;
|
||||||
|
self.spawn_errors += other.spawn_errors;
|
||||||
|
self.wait_errors += other.wait_errors;
|
||||||
|
self.hb_write_errors += other.hb_write_errors;
|
||||||
|
self.timeout_crashes.extend(other.timeout_crashes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 把单个测试结果记入运行累计;失败同时追加逐路径失败记录(含类别)。
|
/// 把单个测试结果记入运行累计;失败同时追加逐路径失败记录(含类别)。
|
||||||
@@ -949,11 +961,10 @@ fn run_chunked(args: &[String], skip_until: usize, end_index: usize, chunk_size:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 监督模式下子进程写入、父进程轮询的一条心跳记录。
|
/// 监督模式下子进程写入、父进程轮询的一条心跳记录。
|
||||||
/// `index` 是子进程即将运行的全局测试下标(`START`)或其完成窗口的
|
/// `COMPLETED`:每个测试完成后写,`index` 为刚完成的全局测试下标,计数覆盖
|
||||||
/// 结束下标(`DONE`);计数始终覆盖 `index` *之前* 已完成的测试,
|
/// ≤ index 的全部测试;`DONE`:窗口尾,`index` 为窗口结束下标。
|
||||||
/// 因此进行中的测试不会被计入。`categories` 为子进程失败分类计数快照
|
/// `categories` 为子进程失败分类计数快照(首行之后按 `类别\t计数` 逐行写出);
|
||||||
/// (首行之后按 `类别\t计数` 逐行写出),使父进程在 supervise 下也能按
|
/// `hb_write_errors` 为子进程侧心跳/旁路写失败累计,经第 6 字段回传父进程。
|
||||||
/// 根因拆分失败。
|
|
||||||
struct Heartbeat {
|
struct Heartbeat {
|
||||||
phase: String,
|
phase: String,
|
||||||
index: usize,
|
index: usize,
|
||||||
@@ -961,35 +972,57 @@ struct Heartbeat {
|
|||||||
fail: usize,
|
fail: usize,
|
||||||
skip: usize,
|
skip: usize,
|
||||||
categories: HashMap<String, usize>,
|
categories: HashMap<String, usize>,
|
||||||
|
hb_write_errors: usize, // 子进程侧写失败累计(第 6 字段)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 用单行心跳头 + 失败类别行覆写心跳文件。错误被忽略:漏写心跳只是把停滞
|
/// 用单行心跳头(phase index pass fail skip hb_write_errors)+ 失败类别行覆写心跳文件。
|
||||||
/// 检测推迟一个轮询间隔。假定单 worker(监督器强制 `OXIDE_TEST262_WORKERS=1`);
|
///
|
||||||
/// 多 worker 时运行下标有歧义且文件存在竞争。
|
/// 经 `{path}.tmp` 临时文件再 rename 原子落盘(同目录 POSIX 原子替换;不 fsync,
|
||||||
|
/// SIGKILL 下 page cache 幸存,ponytail)。写失败返回 Err 并清理临时文件,由调用方
|
||||||
|
/// 自增 hb_write_errors 随下一心跳回传父进程。
|
||||||
|
///
|
||||||
|
/// # 边界与前提
|
||||||
|
/// - 假定单 worker(监督器强制 `OXIDE_TEST262_WORKERS=1`);多 worker 时运行下标
|
||||||
|
/// 有歧义且对同一路径的覆写存在竞争。
|
||||||
|
///
|
||||||
|
/// # 副作用
|
||||||
|
/// - 覆写 `path`;失败时可能残留 `path.tmp`(调用方或 supervise 清理兜底)。
|
||||||
|
#[expect(clippy::too_many_arguments)]
|
||||||
fn write_heartbeat(
|
fn write_heartbeat(
|
||||||
path: &Path, phase: &str, index: usize, pass: usize, fail: usize, skip: usize, categories: &HashMap<String, usize>,
|
path: &Path, phase: &str, index: usize, pass: usize, fail: usize, skip: usize, categories: &HashMap<String, usize>,
|
||||||
) {
|
hb_write_errors: usize,
|
||||||
let mut content = format!("{phase} {index} {pass} {fail} {skip}\n");
|
) -> std::io::Result<()> {
|
||||||
|
let mut content = format!("{phase} {index} {pass} {fail} {skip} {hb_write_errors}\n");
|
||||||
for (cat, count) in categories {
|
for (cat, count) in categories {
|
||||||
// 类别文本内的制表符/换行会破坏行格式,写盘前压平。
|
// 类别文本内的制表符/换行会破坏行格式,写盘前压平。
|
||||||
let cat = cat.replace(['\t', '\n', '\r'], " ");
|
let cat = cat.replace(['\t', '\n', '\r'], " ");
|
||||||
content.push_str(&format!("{cat}\t{count}\n"));
|
content.push_str(&format!("{cat}\t{count}\n"));
|
||||||
}
|
}
|
||||||
let _ = std::fs::write(path, content);
|
let tmp = format!("{}.tmp", path.display());
|
||||||
|
if let Err(e) = std::fs::write(&tmp, content) {
|
||||||
|
let _ = std::fs::remove_file(&tmp);
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
std::fs::rename(&tmp, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 读取最新心跳(含失败类别行)。任何缺失/残缺/畸形内容均返回 `None`,
|
/// 读取最新心跳(含失败类别行与第 6 字段写失败计数)。任何缺失/残缺/畸形内容
|
||||||
/// 使轮询循环可直接在下一拍重试。
|
/// 均返回 `None`,使轮询循环可直接在下一拍重试。
|
||||||
|
///
|
||||||
|
/// # 边界与前提
|
||||||
|
/// - 旧 5 字段 `START` 格式宽容映射为 `COMPLETED(index - 1)`:旧 START(j) 计数
|
||||||
|
/// 覆盖 < j 的测试,与 COMPLETED(j-1) 语义等价;index=0 时 saturating_sub 防下溢。
|
||||||
fn read_heartbeat(path: &Path) -> Option<Heartbeat> {
|
fn read_heartbeat(path: &Path) -> Option<Heartbeat> {
|
||||||
let content = std::fs::read_to_string(path).ok()?;
|
let content = std::fs::read_to_string(path).ok()?;
|
||||||
let mut lines = content.lines();
|
let mut lines = content.lines();
|
||||||
let line = lines.next()?;
|
let line = lines.next()?;
|
||||||
let mut parts = line.split_whitespace();
|
let mut parts = line.split_whitespace();
|
||||||
let phase = parts.next()?.to_string();
|
let phase = parts.next()?.to_string();
|
||||||
let index = parts.next()?.parse().ok()?;
|
let index: usize = parts.next()?.parse().ok()?;
|
||||||
let pass = parts.next()?.parse().ok()?;
|
let pass = parts.next()?.parse().ok()?;
|
||||||
let fail = parts.next()?.parse().ok()?;
|
let fail = parts.next()?.parse().ok()?;
|
||||||
let skip = parts.next()?.parse().ok()?;
|
let skip = parts.next()?.parse().ok()?;
|
||||||
|
let hb_write_errors = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||||
let mut categories = HashMap::new();
|
let mut categories = HashMap::new();
|
||||||
for l in lines {
|
for l in lines {
|
||||||
if let Some((cat, count)) = l.split_once('\t') {
|
if let Some((cat, count)) = l.split_once('\t') {
|
||||||
@@ -998,6 +1031,11 @@ fn read_heartbeat(path: &Path) -> Option<Heartbeat> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let (phase, index) = if phase == "START" {
|
||||||
|
("COMPLETED".into(), index.saturating_sub(1))
|
||||||
|
} else {
|
||||||
|
(phase, index)
|
||||||
|
};
|
||||||
Some(Heartbeat {
|
Some(Heartbeat {
|
||||||
phase,
|
phase,
|
||||||
index,
|
index,
|
||||||
@@ -1005,37 +1043,62 @@ fn read_heartbeat(path: &Path) -> Option<Heartbeat> {
|
|||||||
fail,
|
fail,
|
||||||
skip,
|
skip,
|
||||||
categories,
|
categories,
|
||||||
|
hb_write_errors,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 把心跳快照并入累计统计(含失败类别)。
|
/// 把心跳快照并入累计统计(含失败类别与子进程侧写失败计数)。
|
||||||
fn merge_heartbeat(stats: &mut RunStats, hb: &Heartbeat) {
|
fn merge_heartbeat(stats: &mut RunStats, hb: &Heartbeat) {
|
||||||
stats.pass += hb.pass;
|
stats.pass += hb.pass;
|
||||||
stats.fail += hb.fail;
|
stats.fail += hb.fail;
|
||||||
stats.skip += hb.skip;
|
stats.skip += hb.skip;
|
||||||
|
stats.hb_write_errors += hb.hb_write_errors;
|
||||||
for (cat, count) in &hb.categories {
|
for (cat, count) in &hb.categories {
|
||||||
*stats.fail_categories.entry(cat.clone()).or_insert(0) += count;
|
*stats.fail_categories.entry(cat.clone()).or_insert(0) += count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 把 supervise 旁路失败行文件并入统计:逐行重建 FailRecord(类别经本表 id
|
||||||
|
/// 重映射 + push_fail_record 的 cap 逻辑)。
|
||||||
|
///
|
||||||
|
/// # 注意事项
|
||||||
|
/// - 失败计数(fail_categories)已由对应心跳合并覆盖,此处只补 fail_records,
|
||||||
|
/// 不得再累计计数,避免双计。worker 循环先写心跳后追加旁路,故"无心跳退出"
|
||||||
|
/// 分支旁路必为空(防御性 no-op)。
|
||||||
|
fn merge_fail_log(stats: &mut RunStats, hb_path: &Path) {
|
||||||
|
let content = match std::fs::read_to_string(hb_path.with_extension("fails")) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(_) => return,
|
||||||
|
};
|
||||||
|
for (index, cat, subkey, message) in parse_fail_log(&content) {
|
||||||
|
stats.push_fail_record(index, cat, subkey, message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// 记一笔超时/崩溃结果:默认计入 skip,`--no-skip` 下计入 fail(归入
|
/// 记一笔超时/崩溃结果:默认计入 skip,`--no-skip` 下计入 fail(归入
|
||||||
/// `timeout/crash` 类别,父进程无法进一步拆分根因)。
|
/// `timeout/crash` 类别,父进程无法进一步拆分根因)。
|
||||||
fn record_timeout_or_crash(stats: &mut RunStats, no_skip: bool) {
|
///
|
||||||
|
/// # 副作用
|
||||||
|
/// - 把 `(index, elapsed_ms)` 追加进 timeout_crashes 清单(elapsed_ms=0 表示
|
||||||
|
/// 非超时崩溃:spawn 失败 / 中途退出 / waiterr / 无心跳),供收尾逐路径报告。
|
||||||
|
fn record_timeout_or_crash(stats: &mut RunStats, no_skip: bool, index: usize, elapsed_ms: u64) {
|
||||||
if no_skip {
|
if no_skip {
|
||||||
stats.fail += 1;
|
stats.fail += 1;
|
||||||
*stats.fail_categories.entry("timeout/crash".into()).or_insert(0) += 1;
|
*stats.fail_categories.entry("timeout/crash".into()).or_insert(0) += 1;
|
||||||
} else {
|
} else {
|
||||||
stats.skip += 1;
|
stats.skip += 1;
|
||||||
}
|
}
|
||||||
|
stats.timeout_crashes.push((index, elapsed_ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 在监督下运行一个窗口 `[wstart, wend)`,返回经过多次子进程重启
|
/// 在监督下运行一个窗口 `[wstart, wend)`,返回经过多次子进程重启
|
||||||
/// 聚合的 `RunStats`(含失败类别)。
|
/// 聚合的 `RunStats`(含失败类别)。
|
||||||
///
|
///
|
||||||
/// 单 worker 子进程运行常规 in-process 路径(预热 kernel + harness 前缀缓存)
|
/// 单 worker 子进程运行常规 in-process 路径(预热 kernel + harness 前缀缓存)
|
||||||
/// 并在每个测试前发出心跳。若运行下标停滞超过 `timeout`,子进程被杀死、
|
/// 并在每个测试完成后发出心跳(COMPLETED)。若运行下标停滞超过 `timeout`,
|
||||||
/// 按路径报告肇事者,并由全新子进程从 `culprit + 1` 续跑。子进程在测试中途
|
/// 子进程被杀死、按路径报告肇事者,并由全新子进程从肇事者之后续跑;子进程
|
||||||
/// 崩溃也经同路径恢复。超时/崩溃默认计为 skip,`--no-skip` 下计为失败。
|
/// 在测试中途崩溃也经同路径恢复,恢复基于最后完成下标,无漏项。子进程死亡后
|
||||||
|
/// 读取旁路失败行并入 fail_records。超时/崩溃默认计为 skip,`--no-skip` 下计为失败。
|
||||||
#[expect(clippy::too_many_arguments)]
|
#[expect(clippy::too_many_arguments)]
|
||||||
fn supervise_window(
|
fn supervise_window(
|
||||||
exe: &Path, args: &[String], no_skip: bool, wstart: usize, wend: usize, timeout: Duration, startup_grace: Duration,
|
exe: &Path, args: &[String], no_skip: bool, wstart: usize, wend: usize, timeout: Duration, startup_grace: Duration,
|
||||||
@@ -1054,6 +1117,8 @@ fn supervise_window(
|
|||||||
|
|
||||||
while cur < wend {
|
while cur < wend {
|
||||||
let _ = std::fs::remove_file(&hb_path);
|
let _ = std::fs::remove_file(&hb_path);
|
||||||
|
let _ = std::fs::remove_file(format!("{}.tmp", hb_path.display()));
|
||||||
|
let _ = std::fs::remove_file(hb_path.with_extension("fails"));
|
||||||
let max_tests = wend - cur;
|
let max_tests = wend - cur;
|
||||||
|
|
||||||
let mut child = match Command::new(exe)
|
let mut child = match Command::new(exe)
|
||||||
@@ -1072,7 +1137,8 @@ fn supervise_window(
|
|||||||
Ok(child) => child,
|
Ok(child) => child,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
eprintln!(" window {window_id}: failed to spawn child at index {cur}: {err}");
|
eprintln!(" window {window_id}: failed to spawn child at index {cur}: {err}");
|
||||||
record_timeout_or_crash(&mut stats, no_skip);
|
stats.spawn_errors += 1;
|
||||||
|
record_timeout_or_crash(&mut stats, no_skip, cur, 0);
|
||||||
cur += 1;
|
cur += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1088,23 +1154,27 @@ fn supervise_window(
|
|||||||
match read_heartbeat(&hb_path) {
|
match read_heartbeat(&hb_path) {
|
||||||
Some(hb) if hb.phase == "DONE" => {
|
Some(hb) if hb.phase == "DONE" => {
|
||||||
merge_heartbeat(&mut stats, &hb);
|
merge_heartbeat(&mut stats, &hb);
|
||||||
|
merge_fail_log(&mut stats, &hb_path);
|
||||||
cur = wend;
|
cur = wend;
|
||||||
}
|
}
|
||||||
Some(hb) => {
|
Some(hb) => {
|
||||||
merge_heartbeat(&mut stats, &hb);
|
merge_heartbeat(&mut stats, &hb);
|
||||||
|
let culprit = hb.index + 1;
|
||||||
|
merge_fail_log(&mut stats, &hb_path);
|
||||||
eprintln!(
|
eprintln!(
|
||||||
" [warn] window {window_id}: child exited ({status}) mid-test #{}: {}",
|
" [warn] window {window_id}: child exited ({status}) mid-test #{}: {}",
|
||||||
hb.index,
|
culprit,
|
||||||
describe(hb.index)
|
describe(culprit)
|
||||||
);
|
);
|
||||||
record_timeout_or_crash(&mut stats, no_skip);
|
record_timeout_or_crash(&mut stats, no_skip, culprit, 0);
|
||||||
cur = hb.index + 1;
|
cur = culprit;
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
" [warn] window {window_id}: child exited ({status}) with no heartbeat at index {cur}; skipping one"
|
" [warn] window {window_id}: child exited ({status}) with no heartbeat at index {cur}; skipping one"
|
||||||
);
|
);
|
||||||
record_timeout_or_crash(&mut stats, no_skip);
|
merge_fail_log(&mut stats, &hb_path);
|
||||||
|
record_timeout_or_crash(&mut stats, no_skip, cur, 0);
|
||||||
cur += 1;
|
cur += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1113,8 +1183,11 @@ fn supervise_window(
|
|||||||
Ok(None) => {}
|
Ok(None) => {}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
eprintln!(" window {window_id}: try_wait error: {err}");
|
eprintln!(" window {window_id}: try_wait error: {err}");
|
||||||
|
stats.wait_errors += 1;
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
let _ = child.wait();
|
let _ = child.wait();
|
||||||
|
merge_fail_log(&mut stats, &hb_path);
|
||||||
|
record_timeout_or_crash(&mut stats, no_skip, cur, 0);
|
||||||
cur += 1;
|
cur += 1;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1134,19 +1207,21 @@ fn supervise_window(
|
|||||||
};
|
};
|
||||||
|
|
||||||
if elapsed > deadline {
|
if elapsed > deadline {
|
||||||
|
// 先杀再读:心跳与旁路失败行都须在子进程死亡后取最终状态。
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
let hb = read_heartbeat(&hb_path);
|
let hb = read_heartbeat(&hb_path);
|
||||||
let culprit = hb.as_ref().map(|h| h.index).unwrap_or(cur);
|
let culprit = hb.as_ref().map(|h| h.index + 1).unwrap_or(cur);
|
||||||
if let Some(h) = &hb {
|
if let Some(h) = &hb {
|
||||||
merge_heartbeat(&mut stats, h);
|
merge_heartbeat(&mut stats, h);
|
||||||
}
|
}
|
||||||
|
merge_fail_log(&mut stats, &hb_path);
|
||||||
eprintln!(
|
eprintln!(
|
||||||
" [timeout] window {window_id}: TIMEOUT ({}s) on test #{culprit}: {}",
|
" [timeout] window {window_id}: TIMEOUT ({}s) on test #{culprit}: {}",
|
||||||
deadline.as_secs(),
|
deadline.as_secs(),
|
||||||
describe(culprit)
|
describe(culprit)
|
||||||
);
|
);
|
||||||
let _ = child.kill();
|
record_timeout_or_crash(&mut stats, no_skip, culprit, elapsed.as_millis() as u64);
|
||||||
let _ = child.wait();
|
|
||||||
record_timeout_or_crash(&mut stats, no_skip);
|
|
||||||
cur = culprit + 1;
|
cur = culprit + 1;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1156,6 +1231,8 @@ fn supervise_window(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let _ = std::fs::remove_file(&hb_path);
|
let _ = std::fs::remove_file(&hb_path);
|
||||||
|
let _ = std::fs::remove_file(format!("{}.tmp", hb_path.display()));
|
||||||
|
let _ = std::fs::remove_file(hb_path.with_extension("fails"));
|
||||||
stats
|
stats
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1275,6 +1352,29 @@ fn run_supervised(args: &[String], skip_until: usize, end_index: usize, no_skip:
|
|||||||
if !fail_list.is_empty() {
|
if !fail_list.is_empty() {
|
||||||
print!("{fail_list}");
|
print!("{fail_list}");
|
||||||
}
|
}
|
||||||
|
let anomalies = stats.spawn_errors + stats.wait_errors + stats.hb_write_errors;
|
||||||
|
if anomalies > 0 {
|
||||||
|
println!(" --- supervise anomalies ---");
|
||||||
|
if stats.spawn_errors > 0 {
|
||||||
|
println!(" spawn errors : {}", stats.spawn_errors);
|
||||||
|
}
|
||||||
|
if stats.wait_errors > 0 {
|
||||||
|
println!(" wait errors : {}", stats.wait_errors);
|
||||||
|
}
|
||||||
|
if stats.hb_write_errors > 0 {
|
||||||
|
println!(" hb write errors: {}", stats.hb_write_errors);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !stats.timeout_crashes.is_empty() {
|
||||||
|
println!(" --- TIMEOUT/CRASH list ({}) ---", stats.timeout_crashes.len());
|
||||||
|
for (idx, elapsed_ms) in &stats.timeout_crashes {
|
||||||
|
let path = paths
|
||||||
|
.get(*idx)
|
||||||
|
.map(|p| p.display().to_string())
|
||||||
|
.unwrap_or_else(|| format!("#{idx}"));
|
||||||
|
println!(" {idx} {elapsed_ms}ms {path}");
|
||||||
|
}
|
||||||
|
}
|
||||||
println!("═══════════════════════════════════════");
|
println!("═══════════════════════════════════════");
|
||||||
|
|
||||||
stats.fail == 0
|
stats.fail == 0
|
||||||
@@ -1598,17 +1698,6 @@ fn run_tests() -> bool {
|
|||||||
test262_debug!("running: {}", path_str);
|
test262_debug!("running: {}", path_str);
|
||||||
eprintln!(" [{tid:?}] running: {path_str}");
|
eprintln!(" [{tid:?}] running: {path_str}");
|
||||||
}
|
}
|
||||||
if let Some(hb) = heartbeat_ref {
|
|
||||||
write_heartbeat(
|
|
||||||
hb,
|
|
||||||
"START",
|
|
||||||
i,
|
|
||||||
stats.pass,
|
|
||||||
stats.fail,
|
|
||||||
stats.skip,
|
|
||||||
&stats.fail_categories,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let result = process_path(
|
let result = process_path(
|
||||||
&paths_ref[i],
|
&paths_ref[i],
|
||||||
@@ -1620,6 +1709,30 @@ fn run_tests() -> bool {
|
|||||||
&harness_cache,
|
&harness_cache,
|
||||||
);
|
);
|
||||||
stats.record(i, &result);
|
stats.record(i, &result);
|
||||||
|
// COMPLETED 心跳:先写心跳再追加旁路失败行(崩溃恢复
|
||||||
|
// 无漏项、无双计的写序契约)。
|
||||||
|
if let Some(hb) = heartbeat_ref {
|
||||||
|
if let Err(e) = write_heartbeat(
|
||||||
|
hb,
|
||||||
|
"COMPLETED",
|
||||||
|
i,
|
||||||
|
stats.pass,
|
||||||
|
stats.fail,
|
||||||
|
stats.skip,
|
||||||
|
&stats.fail_categories,
|
||||||
|
stats.hb_write_errors,
|
||||||
|
) {
|
||||||
|
stats.hb_write_errors += 1;
|
||||||
|
eprintln!(" [warn] heartbeat write failed at #{i}: {e}");
|
||||||
|
}
|
||||||
|
if let TestOutcome::Fail(msg) = &result.outcome {
|
||||||
|
let cat = categorize_fail(msg);
|
||||||
|
if let Err(e) = append_fail_log(&hb.with_extension("fails"), i, &cat, "", msg) {
|
||||||
|
stats.hb_write_errors += 1;
|
||||||
|
eprintln!(" [warn] fail log append failed at #{i}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if verbose {
|
if verbose {
|
||||||
match &result.outcome {
|
match &result.outcome {
|
||||||
TestOutcome::Pass(_) => println!("PASS {}", paths_ref[i].display()),
|
TestOutcome::Pass(_) => println!("PASS {}", paths_ref[i].display()),
|
||||||
@@ -1665,7 +1778,19 @@ fn run_tests() -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(hb) = &heartbeat_path {
|
if let Some(hb) = &heartbeat_path {
|
||||||
write_heartbeat(hb, "DONE", end_index, stats.pass, stats.fail, stats.skip, &stats.fail_categories);
|
if let Err(e) = write_heartbeat(
|
||||||
|
hb,
|
||||||
|
"DONE",
|
||||||
|
end_index,
|
||||||
|
stats.pass,
|
||||||
|
stats.fail,
|
||||||
|
stats.skip,
|
||||||
|
&stats.fail_categories,
|
||||||
|
stats.hb_write_errors,
|
||||||
|
) {
|
||||||
|
stats.hb_write_errors += 1;
|
||||||
|
eprintln!(" [warn] final DONE heartbeat write failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
eprintln!();
|
eprintln!();
|
||||||
@@ -1838,7 +1963,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 心跳写读往返:类别行随心跳头一起持久化并完整还原(含制表符/换行压平)。
|
/// 心跳写读往返:类别行随心跳头一起持久化并完整还原(含制表符/换行压平),
|
||||||
|
/// 第 6 字段 hb_write_errors 同步往返。
|
||||||
#[test]
|
#[test]
|
||||||
fn heartbeat_round_trips_categories() {
|
fn heartbeat_round_trips_categories() {
|
||||||
let dir = std::env::temp_dir();
|
let dir = std::env::temp_dir();
|
||||||
@@ -1846,13 +1972,14 @@ mod tests {
|
|||||||
let mut categories = HashMap::new();
|
let mut categories = HashMap::new();
|
||||||
categories.insert("vm: not defined".to_string(), 3);
|
categories.insert("vm: not defined".to_string(), 3);
|
||||||
categories.insert("compile: unsupported".to_string(), 1);
|
categories.insert("compile: unsupported".to_string(), 1);
|
||||||
write_heartbeat(&path, "DONE", 42, 30, 4, 8, &categories);
|
write_heartbeat(&path, "DONE", 42, 30, 4, 8, &categories, 3).expect("心跳写失败");
|
||||||
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
||||||
assert_eq!(hb.phase, "DONE");
|
assert_eq!(hb.phase, "DONE");
|
||||||
assert_eq!(hb.index, 42);
|
assert_eq!(hb.index, 42);
|
||||||
assert_eq!(hb.pass, 30);
|
assert_eq!(hb.pass, 30);
|
||||||
assert_eq!(hb.fail, 4);
|
assert_eq!(hb.fail, 4);
|
||||||
assert_eq!(hb.skip, 8);
|
assert_eq!(hb.skip, 8);
|
||||||
|
assert_eq!(hb.hb_write_errors, 3);
|
||||||
assert_eq!(hb.categories.get("vm: not defined"), Some(&3));
|
assert_eq!(hb.categories.get("vm: not defined"), Some(&3));
|
||||||
assert_eq!(hb.categories.get("compile: unsupported"), Some(&1));
|
assert_eq!(hb.categories.get("compile: unsupported"), Some(&1));
|
||||||
let _ = std::fs::remove_file(&path);
|
let _ = std::fs::remove_file(&path);
|
||||||
@@ -1865,7 +1992,7 @@ mod tests {
|
|||||||
let path = dir.join(format!("oxide_t262_hb_test2_{}.txt", std::process::id()));
|
let path = dir.join(format!("oxide_t262_hb_test2_{}.txt", std::process::id()));
|
||||||
let mut categories = HashMap::new();
|
let mut categories = HashMap::new();
|
||||||
categories.insert("vm: other (multi\nline\tmessage)".to_string(), 2);
|
categories.insert("vm: other (multi\nline\tmessage)".to_string(), 2);
|
||||||
write_heartbeat(&path, "START", 7, 1, 2, 3, &categories);
|
write_heartbeat(&path, "COMPLETED", 7, 1, 2, 3, &categories, 0).expect("心跳写失败");
|
||||||
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
||||||
assert_eq!(hb.fail, 2);
|
assert_eq!(hb.fail, 2);
|
||||||
assert_eq!(hb.categories.len(), 1);
|
assert_eq!(hb.categories.len(), 1);
|
||||||
@@ -1960,4 +2087,128 @@ mod tests {
|
|||||||
let payload: Box<dyn std::any::Any + Send> = Box::new(42u32);
|
let payload: Box<dyn std::any::Any + Send> = Box::new(42u32);
|
||||||
assert_eq!(panic_payload_str(&payload), "engine panic (non-string payload)");
|
assert_eq!(panic_payload_str(&payload), "engine panic (non-string payload)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 心跳第 6 字段(hb_write_errors)写读往返。
|
||||||
|
#[test]
|
||||||
|
fn heartbeat_round_trips_write_errors_field() {
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join(format!("oxide_t262_hb_test_wr_{}.txt", std::process::id()));
|
||||||
|
let categories = HashMap::new();
|
||||||
|
write_heartbeat(&path, "COMPLETED", 9, 5, 1, 2, &categories, 5).expect("心跳写失败");
|
||||||
|
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
||||||
|
assert_eq!(hb.phase, "COMPLETED");
|
||||||
|
assert_eq!(hb.index, 9);
|
||||||
|
assert_eq!(hb.hb_write_errors, 5);
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 旧 5 字段 START 心跳宽容映射:START(j) → COMPLETED(j-1);j=0 不溢出。
|
||||||
|
#[test]
|
||||||
|
fn read_heartbeat_maps_legacy_start_to_completed() {
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join(format!("oxide_t262_hb_test_legacy_{}.txt", std::process::id()));
|
||||||
|
std::fs::write(&path, "START 7 1 2 3\n").expect("写原始心跳行失败");
|
||||||
|
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
||||||
|
assert_eq!(hb.phase, "COMPLETED");
|
||||||
|
assert_eq!(hb.index, 6);
|
||||||
|
std::fs::write(&path, "START 0 0 0 0\n").expect("写原始心跳行失败");
|
||||||
|
let hb = read_heartbeat(&path).expect("心跳应可读回");
|
||||||
|
assert_eq!(hb.index, 0, "saturating_sub 防下溢");
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 原子写:写后目标文件内容完整、无 `.tmp` 残留。
|
||||||
|
#[test]
|
||||||
|
fn write_heartbeat_atomic_no_tmp_leftover() {
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join(format!("oxide_t262_hb_test_atomic_{}.txt", std::process::id()));
|
||||||
|
let categories = HashMap::new();
|
||||||
|
write_heartbeat(&path, "DONE", 10, 3, 2, 1, &categories, 0).expect("心跳写失败");
|
||||||
|
let content = std::fs::read_to_string(&path).expect("心跳应可读回");
|
||||||
|
assert!(content.starts_with("DONE 10 3 2 1 0\n"), "内容应为 6 字段头,实际:\n{content}");
|
||||||
|
let tmp = format!("{}.tmp", path.display());
|
||||||
|
assert!(!Path::new(&tmp).exists(), "tmp 文件不应残留");
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 超时/崩溃记账:计数(no_skip 转 fail)与 timeout_crashes 入列。
|
||||||
|
#[test]
|
||||||
|
fn record_timeout_or_crash_records_index_elapsed() {
|
||||||
|
let mut stats = RunStats::default();
|
||||||
|
record_timeout_or_crash(&mut stats, true, 3, 2500);
|
||||||
|
assert_eq!(stats.fail, 1);
|
||||||
|
assert_eq!(stats.fail_categories.get("timeout/crash"), Some(&1));
|
||||||
|
assert_eq!(stats.timeout_crashes, vec![(3, 2500)]);
|
||||||
|
record_timeout_or_crash(&mut stats, false, 4, 0);
|
||||||
|
assert_eq!(stats.skip, 1);
|
||||||
|
assert_eq!(stats.timeout_crashes, vec![(3, 2500), (4, 0)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// merge 合并异常计数器:spawn/wait/hb 求和、timeout_crashes 拼接。
|
||||||
|
#[test]
|
||||||
|
fn runstats_merge_sums_error_counters() {
|
||||||
|
let mut a = RunStats {
|
||||||
|
spawn_errors: 1,
|
||||||
|
wait_errors: 2,
|
||||||
|
hb_write_errors: 3,
|
||||||
|
timeout_crashes: vec![(0, 100)],
|
||||||
|
..RunStats::default()
|
||||||
|
};
|
||||||
|
let b = RunStats {
|
||||||
|
spawn_errors: 4,
|
||||||
|
wait_errors: 5,
|
||||||
|
hb_write_errors: 6,
|
||||||
|
timeout_crashes: vec![(1, 200)],
|
||||||
|
..RunStats::default()
|
||||||
|
};
|
||||||
|
a.merge(b);
|
||||||
|
assert_eq!(a.spawn_errors, 5);
|
||||||
|
assert_eq!(a.wait_errors, 7);
|
||||||
|
assert_eq!(a.hb_write_errors, 9);
|
||||||
|
assert_eq!(a.timeout_crashes, vec![(0, 100), (1, 200)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 旁路失败行往返:append 后 parse 恢复全部字段;残缺/畸形尾行静默跳过。
|
||||||
|
#[test]
|
||||||
|
fn fail_log_round_trips_and_skips_truncated() {
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join(format!("oxide_t262_fails_test_{}.log", std::process::id()));
|
||||||
|
append_fail_log(&path, 3, "vm: not callable", "", "x is not callable").expect("追加失败");
|
||||||
|
append_fail_log(&path, 4, "compile: unsupported", "foo", "a\tb\nc").expect("追加失败");
|
||||||
|
let mut content = std::fs::read_to_string(&path).expect("读取失败");
|
||||||
|
content.push_str("5\tvm: x\n"); // 残缺:仅 2 字段(SIGKILL 半写形态)
|
||||||
|
content.push_str("x\tb\tc\td\n"); // 畸形:index 非数字
|
||||||
|
std::fs::write(&path, content).expect("写回失败");
|
||||||
|
let content = std::fs::read_to_string(&path).expect("读取失败");
|
||||||
|
let rows = parse_fail_log(&content);
|
||||||
|
assert_eq!(rows.len(), 2, "残缺/畸形行应被跳过,实际 {rows:?}");
|
||||||
|
assert_eq!(
|
||||||
|
rows[0],
|
||||||
|
(3, "vm: not callable".to_string(), "".to_string(), "x is not callable".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(rows[1], (4, "compile: unsupported".to_string(), "foo".to_string(), "a b c".to_string()));
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// merge_fail_log 重建 fail_records:类别经 id 表重映射、消息 cap;
|
||||||
|
/// fail_categories 保持空(计数不双计契约)。
|
||||||
|
#[test]
|
||||||
|
fn merge_fail_log_reconstructs_fail_records() {
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let hb_path = dir.join(format!("oxide_t262_hb_test_mrg_{}.txt", std::process::id()));
|
||||||
|
let sidecar = hb_path.with_extension("fails");
|
||||||
|
append_fail_log(&sidecar, 2, "vm: not callable", "", "x is not callable").expect("追加失败");
|
||||||
|
append_fail_log(&sidecar, 7, "compile: unsupported", "", "y").expect("追加失败");
|
||||||
|
let mut stats = RunStats::default();
|
||||||
|
merge_fail_log(&mut stats, &hb_path);
|
||||||
|
assert_eq!(stats.fail_records.len(), 2);
|
||||||
|
assert_eq!(stats.fail_records[0].index, 2);
|
||||||
|
assert_eq!(stats.categories[stats.fail_records[0].category_id as usize], "vm: not callable");
|
||||||
|
assert_eq!(stats.fail_records[0].message, "x is not callable");
|
||||||
|
assert_eq!(stats.fail_records[1].index, 7);
|
||||||
|
assert_eq!(stats.categories[stats.fail_records[1].category_id as usize], "compile: unsupported");
|
||||||
|
assert!(stats.fail_categories.is_empty(), "计数不双计:fail_categories 必须保持空");
|
||||||
|
let _ = std::fs::remove_file(&sidecar);
|
||||||
|
let _ = std::fs::remove_file(&hb_path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
//! 全部为纯函数/String builder(可单测),不持有全局状态。
|
//! 全部为纯函数/String builder(可单测),不持有全局状态。
|
||||||
|
|
||||||
use crate::RunStats;
|
use crate::RunStats;
|
||||||
use std::path::PathBuf;
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// 单条失败记录:路径用全局测试数组下标(8 字节),类别用 RunStats 内 id 表下标。
|
/// 单条失败记录:路径用全局测试数组下标(8 字节),类别用 RunStats 内 id 表下标。
|
||||||
/// 消息完整保留(单条 2 KiB cap + 全局 64 MiB cap),不截断到类别桶。
|
/// 消息完整保留(单条 2 KiB cap + 全局 64 MiB cap),不截断到类别桶。
|
||||||
@@ -29,8 +29,7 @@ pub fn first_line(s: &str) -> &str {
|
|||||||
s.lines().next().unwrap_or("")
|
s.lines().next().unwrap_or("")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 压平 \t \n \r 为空格(后续 fail-log / 心跳旁路行的可解析保证)。
|
/// 压平 \t \n \r 为空格(fail-log / 心跳旁路行 / 类别行的可解析保证)。
|
||||||
#[cfg_attr(not(test), expect(dead_code))] // 心跳旁路行接入后使用
|
|
||||||
pub fn escape_log_field(s: &str) -> String {
|
pub fn escape_log_field(s: &str) -> String {
|
||||||
s.replace(['\t', '\n', '\r'], " ")
|
s.replace(['\t', '\n', '\r'], " ")
|
||||||
}
|
}
|
||||||
@@ -97,6 +96,42 @@ pub fn format_fail_list(stats: &RunStats, paths: &[PathBuf]) -> String {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 把一条失败记录追加到 supervise 旁路文件(子进程 → 父进程通道)。
|
||||||
|
/// 行格式 `index\tcategory\tsubkey\tmessage`,字段经 escape_log_field 压平。
|
||||||
|
///
|
||||||
|
/// # 注意事项
|
||||||
|
/// - 追加写不做原子性:父进程只在子进程死亡后才读取,半写尾行由
|
||||||
|
/// [`parse_fail_log`] 防御性跳过。
|
||||||
|
pub fn append_fail_log(path: &Path, index: usize, category: &str, subkey: &str, message: &str) -> std::io::Result<()> {
|
||||||
|
use std::io::Write;
|
||||||
|
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(path)?;
|
||||||
|
writeln!(
|
||||||
|
f,
|
||||||
|
"{}\t{}\t{}\t{}",
|
||||||
|
index,
|
||||||
|
escape_log_field(category),
|
||||||
|
escape_log_field(subkey),
|
||||||
|
escape_log_field(message)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 解析旁路失败行文件为 `(index, category, subkey, message)` 列表。
|
||||||
|
///
|
||||||
|
/// # 边界与前提
|
||||||
|
/// - 残缺尾行(子进程被杀时半写)与畸形行(字段数不足 / index 非数字)静默跳过。
|
||||||
|
pub fn parse_fail_log(content: &str) -> Vec<(usize, String, String, String)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for line in content.lines() {
|
||||||
|
let mut it = line.splitn(4, '\t');
|
||||||
|
let (Some(index), Some(cat), Some(subkey), Some(message)) = (it.next(), it.next(), it.next(), it.next()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Ok(index) = index.parse::<usize>() else { continue };
|
||||||
|
out.push((index, cat.to_string(), subkey.to_string(), message.to_string()));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
Reference in New Issue
Block a user