feat: 添加基准测试基础设施

This commit is contained in:
爱特每羊队
2026-06-24 10:46:59 +08:00
parent 9d43314242
commit f602ae5f00
32 changed files with 1215 additions and 10 deletions
+2
View File
@@ -22,3 +22,5 @@ oxide_api = { path = "../oxide_api" }
clap.workspace = true
rustyline.workspace = true
ansi_term.workspace = true
serde_json.workspace = true
serde = { workspace = true, features = ["derive"] }
+94
View File
@@ -0,0 +1,94 @@
use crate::bench::metrics::MetricCollection;
pub struct Baseline {
pub entries: Vec<MetricCollection>,
}
impl Baseline {
pub fn empty() -> Self {
Self { entries: Vec::new() }
}
}
pub struct Regression {
pub metric: String,
pub test_name: String,
pub baseline: f64,
pub current: f64,
pub ratio: f64,
pub tolerance: f64,
}
pub fn load_baseline() -> Result<Baseline, String> {
let json_path = "benchmark_baseline.json";
let data = match std::fs::read_to_string(json_path) {
Ok(d) => d,
Err(_) => {
eprintln!("No baseline found at {} — run --update-baseline first", json_path);
return Ok(Baseline::empty());
}
};
let entries: Vec<MetricCollection> =
serde_json::from_str(&data).map_err(|e| format!("Failed to parse baseline: {}", e))?;
Ok(Baseline { entries })
}
pub fn save_baseline(results: &[MetricCollection]) -> Result<(), String> {
let json = serde_json::to_string_pretty(results).map_err(|e| format!("Failed to serialize baseline: {}", e))?;
std::fs::write("benchmark_baseline.json", &json)
.map_err(|e| format!("Failed to write benchmark_baseline.json: {}", e))?;
let md = crate::bench::output::format_text_table(results);
std::fs::write("BENCHMARK_BASELINE.md", md).map_err(|e| format!("Failed to write BENCHMARK_BASELINE.md: {}", e))?;
Ok(())
}
pub fn compare_baseline(current: &[MetricCollection]) -> Vec<Regression> {
let baseline = match load_baseline() {
Ok(b) => b,
Err(_) => return Vec::new(),
};
if baseline.entries.is_empty() {
eprintln!("No baseline data — skipping regression check");
return Vec::new();
}
let mut regressions = Vec::new();
for cur in current {
for base in &baseline.entries {
if base.test_name != cur.test_name {
continue;
}
for (name, cur_val) in cur.iter_metrics() {
if cur_val == 0.0 {
continue;
}
let base_val = base
.iter_metrics()
.iter()
.find(|(n, _)| *n == name)
.map(|(_, v)| *v)
.unwrap_or(cur_val);
if base_val == 0.0 {
continue;
}
let tolerance = match name {
"wall_time_us" => 0.10,
"gc_trigger_count" | "gc_bytes_freed" | "gc_objects_scanned" | "gc_collection_us" => 0.50,
"session_bytes" => 0.20,
_ => 0.20,
};
let ratio = cur_val / base_val;
if ratio > 1.0 + tolerance {
regressions.push(Regression {
metric: name.to_string(),
test_name: cur.test_name.clone(),
baseline: base_val,
current: cur_val,
ratio,
tolerance,
});
}
}
}
}
regressions
}
+208
View File
@@ -0,0 +1,208 @@
use std::fs;
use std::process::ExitCode;
use std::sync::Arc;
use std::time::Instant;
use oxide_compiler::compiler::{compiled_module_hash, Compiler};
use oxide_kernel::kernel::KernelCore;
use oxide_parser::Allocator;
use oxide_vm::vm_pool::VmPool;
use crate::bench::baseline::{compare_baseline, save_baseline};
use crate::bench::metrics::MetricCollection;
use crate::bench::output::{format_json, format_text_table};
use crate::bench::BenchConfig;
pub fn run_js_stress_bench(config: &BenchConfig, kernel: &Arc<KernelCore>, pool: &Arc<VmPool>) -> ExitCode {
let mut results: Vec<MetricCollection> = Vec::new();
let compiler = Compiler::new();
let test_dir = "tests/stress";
let mut files: Vec<_> = match fs::read_dir(test_dir) {
Ok(entries) => entries
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().is_some_and(|ext| ext == "js")
&& config
.filter
.as_ref()
.map_or(true, |f| e.file_name().to_string_lossy().contains(f.as_str()))
})
.map(|e| e.path())
.collect(),
Err(e) => {
eprintln!("Could not read {}: {}", test_dir, e);
return ExitCode::FAILURE;
}
};
files.sort();
if files.is_empty() {
eprintln!("No matching stress tests found in {}", test_dir);
return ExitCode::FAILURE;
}
for path in &files {
let test_name = path.file_stem().unwrap_or_default().to_string_lossy().to_string();
let mut js = match fs::read_to_string(path) {
Ok(s) => s,
Err(e) => {
eprintln!("Failed to read {:?}: {}", path, e);
continue;
}
};
if config.iterations > 0 {
js = override_iterations(&js, config.iterations);
}
let allocator = Allocator::default();
let program = match oxide_parser::parse(&allocator, &js) {
Ok(p) => p,
Err(errors) => {
eprintln!("Parse error in {}: {:?}", test_name, errors.first());
continue;
}
};
let hash = compiled_module_hash(&program);
let module = match kernel.code_forge().get_or_insert_with(hash, || compiler.compile(&program)) {
Ok(m) => m,
Err(e) => {
eprintln!("Compile error in {}: {}", test_name, e);
continue;
}
};
for _ in 0..config.warmup {
let mut guard = pool.spawn();
let _ = guard.vm_mut().run(&module);
}
let mut metrics = Vec::new();
for _ in 0..config.iterations {
let compile_start = Instant::now();
let mut guard = pool.spawn();
let vm = guard.vm_mut();
let compile_time = compile_start.elapsed();
let pre_session = vm.session_object_count();
let pre_epoch = vm.epoch_object_count();
let exec_start = Instant::now();
let _result = vm.run(&module);
let exec_time = exec_start.elapsed();
let gc_stats = vm.session_gc_stats();
metrics.push(MetricCollection {
test_name: test_name.clone(),
wall_time_us: (compile_time + exec_time).as_micros() as u64,
session_objects: vm.session_object_count().saturating_sub(pre_session),
session_bytes: vm.session_bytes_allocated(),
epoch_objects: vm.epoch_object_count().saturating_sub(pre_epoch),
epoch_bytes: 0,
gc_trigger_count: gc_stats.total_collections,
gc_bytes_freed: gc_stats.last_collection_bytes_freed,
gc_objects_scanned: gc_stats.last_collection_objects_scanned,
gc_collection_us: gc_stats.last_collection_duration_us,
instruction_count: vm.instruction_count(),
compile_time_us: compile_time.as_micros() as u64,
exec_time_us: exec_time.as_micros() as u64,
ic_hit_rate: vm.ic_hit_rate(),
ic_hits: vm.ic_hit_count(),
ic_misses: vm.ic_miss_count(),
});
}
let avg = average_metrics(&metrics);
results.push(avg);
}
let json = format_json(&results);
let table = format_text_table(&results);
eprintln!("{}", table);
println!("{}", json);
if config.update_baseline {
if let Err(e) = save_baseline(&results) {
eprintln!("Failed to save baseline: {}", e);
return ExitCode::FAILURE;
}
eprintln!("Baseline saved to BENCHMARK_BASELINE.md + benchmark_baseline.json");
} else {
let regressions = compare_baseline(&results);
if !regressions.is_empty() {
eprintln!("\nRegression detected:");
for r in &regressions {
eprintln!(
" {}: {} baseline={} current={} ratio={:.2}% tolerance={:.0}%",
r.test_name,
r.metric,
r.baseline,
r.current,
(r.ratio - 1.0) * 100.0,
r.tolerance * 100.0,
);
}
return ExitCode::FAILURE;
}
}
ExitCode::SUCCESS
}
fn average_metrics(metrics: &[MetricCollection]) -> MetricCollection {
let n = metrics.len() as f64;
if n == 0.0 || metrics.is_empty() {
return metrics.first().cloned().unwrap_or(MetricCollection {
test_name: String::new(),
wall_time_us: 0,
session_objects: 0,
session_bytes: 0,
epoch_objects: 0,
epoch_bytes: 0,
gc_trigger_count: 0,
gc_bytes_freed: 0,
gc_objects_scanned: 0,
gc_collection_us: 0,
instruction_count: 0,
compile_time_us: 0,
exec_time_us: 0,
ic_hit_rate: 0.0,
ic_hits: 0,
ic_misses: 0,
});
}
MetricCollection {
test_name: metrics[0].test_name.clone(),
wall_time_us: (metrics.iter().map(|m| m.wall_time_us as f64).sum::<f64>() / n) as u64,
session_objects: (metrics.iter().map(|m| m.session_objects as f64).sum::<f64>() / n) as usize,
session_bytes: (metrics.iter().map(|m| m.session_bytes as f64).sum::<f64>() / n) as usize,
epoch_objects: (metrics.iter().map(|m| m.epoch_objects as f64).sum::<f64>() / n) as usize,
epoch_bytes: (metrics.iter().map(|m| m.epoch_bytes as f64).sum::<f64>() / n) as u64,
gc_trigger_count: (metrics.iter().map(|m| m.gc_trigger_count as f64).sum::<f64>() / n) as u64,
gc_bytes_freed: (metrics.iter().map(|m| m.gc_bytes_freed as f64).sum::<f64>() / n) as u64,
gc_objects_scanned: (metrics.iter().map(|m| m.gc_objects_scanned as f64).sum::<f64>() / n) as u64,
gc_collection_us: (metrics.iter().map(|m| m.gc_collection_us as f64).sum::<f64>() / n) as u64,
instruction_count: (metrics.iter().map(|m| m.instruction_count as f64).sum::<f64>() / n) as u64,
compile_time_us: (metrics.iter().map(|m| m.compile_time_us as f64).sum::<f64>() / n) as u64,
exec_time_us: (metrics.iter().map(|m| m.exec_time_us as f64).sum::<f64>() / n) as u64,
ic_hit_rate: metrics.iter().map(|m| m.ic_hit_rate).sum::<f64>() / n,
ic_hits: (metrics.iter().map(|m| m.ic_hits as f64).sum::<f64>() / n) as u64,
ic_misses: (metrics.iter().map(|m| m.ic_misses as f64).sum::<f64>() / n) as u64,
}
}
fn override_iterations(js: &str, iters: u32) -> String {
js.lines()
.map(|line| {
if line.trim_start().starts_with("var ITERATIONS ") {
format!("var ITERATIONS = {};", iters)
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
}
+141
View File
@@ -0,0 +1,141 @@
use std::collections::VecDeque;
use std::process::ExitCode;
use std::sync::Arc;
use std::time::Instant;
use oxide_compiler::compiler::{compiled_module_hash, Compiler};
use oxide_kernel::kernel::KernelCore;
use oxide_parser::Allocator;
use oxide_vm::vm_pool::VmPool;
use crate::bench::metrics::MetricCollection;
use crate::bench::output::format_json;
use crate::bench::BenchConfig;
pub struct LeakSampler {
window: VecDeque<(usize, f64)>,
window_size: usize,
}
impl LeakSampler {
pub fn new(window_size: usize) -> Self {
Self {
window: VecDeque::new(),
window_size,
}
}
pub fn add_sample(&mut self, iteration: usize, value: f64) -> Option<LeakVerdict> {
self.window.push_back((iteration, value));
if self.window.len() >= self.window_size {
self.window.pop_front();
}
if self.window.len() < 10 {
return None;
}
let n = self.window.len() as f64;
let sum_x: f64 = self.window.iter().map(|(i, _)| *i as f64).sum();
let sum_y: f64 = self.window.iter().map(|(_, v)| v).sum();
let sum_xy: f64 = self.window.iter().map(|(i, v)| *i as f64 * v).sum();
let sum_x2: f64 = self.window.iter().map(|(i, _)| (*i as f64).powi(2)).sum();
let slope = (n * sum_xy - sum_x * sum_y) / (n * sum_x2 - sum_x * sum_x);
let intercept = (sum_y - slope * sum_x) / n;
let y_mean = sum_y / n;
let ss_res: f64 = self
.window
.iter()
.map(|(i, v)| (v - (slope * *i as f64 + intercept)).powi(2))
.sum();
let ss_tot: f64 = self.window.iter().map(|(_, v)| (v - y_mean).powi(2)).sum();
let r2 = if ss_tot == 0.0 { 1.0 } else { 1.0 - ss_res / ss_tot };
if r2 > 0.9 && slope > 0.0 {
Some(LeakVerdict { slope, r2 })
} else {
None
}
}
}
pub struct LeakVerdict {
pub slope: f64,
pub r2: f64,
}
pub fn run_leak_detect(config: &BenchConfig, kernel: &Arc<KernelCore>, pool: &Arc<VmPool>) -> ExitCode {
let js =
"var ITERATIONS = 1000; var obj = {}; for (var i = 0; i < ITERATIONS; i++) { obj['key' + i] = i; } obj['key0']";
let allocator = Allocator::default();
let program = match oxide_parser::parse(&allocator, js) {
Ok(p) => p,
Err(_) => return ExitCode::FAILURE,
};
let compiler = Compiler::new();
let hash = compiled_module_hash(&program);
let module = match kernel.code_forge().get_or_insert_with(hash, || compiler.compile(&program)) {
Ok(m) => m,
Err(_) => return ExitCode::FAILURE,
};
let mut samplers: Vec<(&str, LeakSampler)> = vec![
("session_objects", LeakSampler::new(20)),
("session_bytes", LeakSampler::new(20)),
("code_forge_entries", LeakSampler::new(20)),
("symbol_registry", LeakSampler::new(20)),
];
let mut results: Vec<MetricCollection> = Vec::new();
for i in 0..config.iterations as usize {
let start = Instant::now();
let mut guard = pool.spawn();
let vm = guard.vm_mut();
let _ = vm.run(&module);
let elapsed = start.elapsed();
for (name, sampler) in &mut samplers {
let value: f64 = match *name {
"session_objects" => vm.session_object_count() as f64,
"session_bytes" => vm.session_bytes_allocated() as f64,
"code_forge_entries" => kernel.code_forge().len() as f64,
"symbol_registry" => vm.symbol_registry_len() as f64,
_ => continue,
};
if let Some(verdict) = sampler.add_sample(i, value) {
eprintln!(
"[LEAK] {}: slope={:.6} R²={:.4} over {} samples",
name,
verdict.slope,
verdict.r2,
sampler.window.len(),
);
}
}
if i == config.iterations as usize - 1 {
let guard = pool.spawn();
let vm = guard.vm();
results.push(MetricCollection {
test_name: "leak_detect".to_string(),
wall_time_us: elapsed.as_micros() as u64,
session_objects: vm.session_object_count(),
session_bytes: vm.session_bytes_allocated(),
epoch_objects: vm.epoch_object_count(),
epoch_bytes: 0,
gc_trigger_count: vm.session_gc_stats().total_collections,
gc_bytes_freed: vm.session_gc_stats().last_collection_bytes_freed,
gc_objects_scanned: vm.session_gc_stats().last_collection_objects_scanned,
gc_collection_us: vm.session_gc_stats().last_collection_duration_us,
instruction_count: vm.instruction_count(),
compile_time_us: 0,
exec_time_us: 0,
ic_hit_rate: vm.ic_hit_rate(),
ic_hits: vm.ic_hit_count(),
ic_misses: vm.ic_miss_count(),
});
}
}
let json = format_json(&results);
println!("{}", json);
ExitCode::SUCCESS
}
+43
View File
@@ -0,0 +1,43 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MetricCollection {
pub test_name: String,
pub wall_time_us: u64,
pub session_objects: usize,
pub session_bytes: usize,
pub epoch_objects: usize,
pub epoch_bytes: u64,
pub gc_trigger_count: u64,
pub gc_bytes_freed: u64,
pub gc_objects_scanned: u64,
pub gc_collection_us: u64,
pub instruction_count: u64,
pub compile_time_us: u64,
pub exec_time_us: u64,
pub ic_hit_rate: f64,
pub ic_hits: u64,
pub ic_misses: u64,
}
impl MetricCollection {
pub fn iter_metrics(&self) -> Vec<(&'static str, f64)> {
vec![
("wall_time_us", self.wall_time_us as f64),
("session_objects", self.session_objects as f64),
("session_bytes", self.session_bytes as f64),
("epoch_objects", self.epoch_objects as f64),
("epoch_bytes", self.epoch_bytes as f64),
("gc_trigger_count", self.gc_trigger_count as f64),
("gc_bytes_freed", self.gc_bytes_freed as f64),
("gc_objects_scanned", self.gc_objects_scanned as f64),
("gc_collection_us", self.gc_collection_us as f64),
("instruction_count", self.instruction_count as f64),
("compile_time_us", self.compile_time_us as f64),
("exec_time_us", self.exec_time_us as f64),
("ic_hit_rate", self.ic_hit_rate),
("ic_hits", self.ic_hits as f64),
("ic_misses", self.ic_misses as f64),
]
}
}
+34
View File
@@ -0,0 +1,34 @@
pub mod baseline;
pub mod js_stress;
pub mod leak_detect;
pub mod metrics;
pub mod output;
pub mod rust_bench;
use std::process::ExitCode;
use std::sync::Arc;
use oxide_kernel::kernel::KernelCore;
use oxide_vm::vm_pool::VmPool;
pub struct BenchConfig {
pub mode: String,
pub filter: Option<String>,
pub warmup: u32,
pub iterations: u32,
pub process: bool,
pub update_baseline: bool,
pub leak_check_interval: usize,
}
pub fn run_benchmarks(config: BenchConfig, kernel: Arc<KernelCore>, pool: Arc<VmPool>) -> ExitCode {
match config.mode.as_str() {
"js" => js_stress::run_js_stress_bench(&config, &kernel, &pool),
"rust" => rust_bench::run_rust_bench(config.filter.as_deref()),
"leak" => leak_detect::run_leak_detect(&config, &kernel, &pool),
_ => {
eprintln!("Unknown bench mode: {}. Use: js, rust, or leak", config.mode);
ExitCode::FAILURE
}
}
}
+40
View File
@@ -0,0 +1,40 @@
use crate::bench::metrics::MetricCollection;
pub fn format_json(results: &[MetricCollection]) -> String {
serde_json::to_string_pretty(results).unwrap_or_else(|e| format!("{{\"error\": \"{}\"}}", e))
}
pub fn format_text_table(results: &[MetricCollection]) -> String {
let mut out = String::new();
out.push_str(&format!(
"{:<35} {:>10} {:>10} {:>10} {:>12} {:>8} {:>8} {:>10} {:>10}\n",
"Test Name", "Wall(ms)", "Comp(ms)", "Exec(ms)", "Instrs", "GC Ct", "IC Rate", "Sess Objs", "Sess B",
));
out.push_str(&"-".repeat(120));
out.push('\n');
for m in results {
out.push_str(&format!(
"{:<35} {:>10.2} {:>10.3} {:>10.3} {:>12} {:>8} {:>7.1}% {:>10} {:>10}",
truncate(&m.test_name, 35),
m.wall_time_us as f64 / 1000.0,
m.compile_time_us as f64 / 1000.0,
m.exec_time_us as f64 / 1000.0,
m.instruction_count,
m.gc_trigger_count,
m.ic_hit_rate * 100.0,
m.session_objects,
m.session_bytes,
));
out.push('\n');
}
out
}
fn truncate(s: &str, max: usize) -> &str {
if s.len() <= max {
s
} else {
&s[..max]
}
}
+20
View File
@@ -0,0 +1,20 @@
pub fn run_rust_bench(filter: Option<&str>) -> std::process::ExitCode {
let mut cmd = std::process::Command::new("cargo");
cmd.args(["bench", "-p", "oxide_vm"]);
if let Some(name) = filter {
cmd.args(["--bench", name]);
}
match cmd.status() {
Ok(status) => {
if status.success() {
std::process::ExitCode::SUCCESS
} else {
std::process::ExitCode::from(1)
}
}
Err(e) => {
eprintln!("Failed to run cargo bench: {}", e);
std::process::ExitCode::from(1)
}
}
}
+39 -2
View File
@@ -1,4 +1,5 @@
#![allow(clippy::arc_with_non_send_sync)]
#![allow(dead_code)]
use std::fs;
use std::process::ExitCode;
@@ -15,6 +16,8 @@ use oxide_types::object::JsObject;
use oxide_vm::vm_pool::VmPool;
use oxide_vm::JsValue;
mod bench;
#[derive(Parser)]
#[command(name = "oxide", version, about = "OxideJS - Rust JavaScript engine")]
struct Cli {
@@ -42,7 +45,20 @@ enum Commands {
file: Option<String>,
},
Bench {
suite: Option<String>,
#[arg(default_value = "js")]
mode: Option<String>,
#[arg(short, long)]
filter: Option<String>,
#[arg(long, default_value = "2")]
warmup: u32,
#[arg(long, default_value = "10")]
iterations: u32,
#[arg(long)]
process: bool,
#[arg(long)]
update_baseline: bool,
#[arg(long, default_value = "1000")]
leak_check_interval: usize,
},
Test {
suite: Option<String>,
@@ -64,7 +80,28 @@ fn main() -> ExitCode {
run(&file, &kernel, &pool)
}
Some(Commands::Compile { expr, file }) => compile(expr, file),
Some(Commands::Bench { .. }) => not_implemented("bench"),
Some(Commands::Bench {
mode,
filter,
warmup,
iterations,
process,
update_baseline,
leak_check_interval,
}) => {
let kernel = make_kernel();
let pool = make_pool(&kernel);
let config = bench::BenchConfig {
mode: mode.unwrap_or_else(|| "js".to_string()),
filter,
warmup,
iterations,
process,
update_baseline,
leak_check_interval,
};
bench::run_benchmarks(config, kernel, pool)
}
Some(Commands::Test { .. }) => not_implemented("test"),
None => repl(),
}
+5 -5
View File
@@ -38,12 +38,12 @@ fn run_file() {
}
#[test]
fn bench_not_implemented() {
fn bench_help() {
let output = Command::new("cargo")
.args(["run", "--", "bench"])
.args(["run", "--", "bench", "--help"])
.output()
.expect("failed to run oxide bench");
.expect("failed to run oxide bench --help");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("not yet implemented"), "bench should print not-implemented message");
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(stdout.contains("--iterations"), "bench --help should list --iterations flag");
}