Compare commits

..
Author SHA1 Message Date
David Peter 16f53eb8d6 Format CSV measurements through FormatQuantity 2026-10-05 13:41:43 +02:00
David Peter 4aa36c8051 Merge pull request #970 from sharkdp/codex/truncate-export-rewrites
Truncate export files before rewriting results
2026-10-05 12:57:52 +02:00
David Peter e689520fb5 Merge pull request #969 from sharkdp/codex/result-measurements
Store typed Measurements in BenchmarkResult
2026-10-05 12:51:22 +02:00
David Peter 8cf4e010e9 Truncate export files before rewriting results 2026-10-05 12:48:52 +02:00
David Peter 30a2b61d4c Merge pull request #807 from willcl-ark/expose-run-var
expose HYPERFINE_ITERATION to intermediate commands
2026-10-05 12:45:29 +02:00
David Peter 3558138cea Further simplifications 2026-10-05 12:40:33 +02:00
David Peter 1eafcf4739 Combine iteration environment variable integration tests 2026-10-05 12:38:53 +02:00
willcl-ark 44c0cc231b expose HYPERFINE_ITERATION to intermediate commands
Fixes: #781

Expose the `$HYPERFINE_ITERATION` variable to intermediate commands.
This will allow iteration-specific prepare/conclude, such as:

```bash
hyperfine --runs 5 --prepare 'echo touch file${HYPERFINE_ITERATION}.dat' \
    'echo testing file${HYPERFINE_ITERATION}.dat' \
    --conclude 'echo rm file${HYPERFINE_ITERATION}.dat' --show-output
```
2026-10-05 12:24:43 +02:00
5 changed files with 99 additions and 31 deletions

No files matched your search

+1
View File
@@ -15,6 +15,7 @@ use super::measurement::{Measurement, Measurements};
use anyhow::{bail, Context, Result};
#[derive(Clone, Copy)]
pub enum BenchmarkIteration {
NonBenchmarkRun,
Warmup(u64),
+38 -18
View File
@@ -57,11 +57,12 @@ impl<'a> Benchmark<'a> {
command: &Command<'_>,
error_output: &'static str,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.executor
.run_command_and_measure(
command,
executor::BenchmarkIteration::NonBenchmarkRun,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
@@ -84,7 +85,14 @@ impl<'a> Benchmark<'a> {
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
.map(|cmd| {
self.run_intermediate_command(
&cmd,
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
})
.transpose()?
.unwrap_or_default())
}
@@ -105,7 +113,14 @@ impl<'a> Benchmark<'a> {
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
.map(|cmd| {
self.run_intermediate_command(
&cmd,
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
})
.transpose()?
.unwrap_or_default())
}
@@ -115,11 +130,12 @@ impl<'a> Benchmark<'a> {
&self,
command: &Command<'_>,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy)
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
/// Run the command specified by `--conclude`.
@@ -127,11 +143,12 @@ impl<'a> Benchmark<'a> {
&self,
command: &Command<'_>,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy)
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
/// Run the benchmark for a single command
@@ -164,10 +181,10 @@ impl<'a> Benchmark<'a> {
)
});
let run_preparation_command = || {
let run_preparation_command = |iteration: executor::BenchmarkIteration| {
preparation_command
.as_ref()
.map(|cmd| self.run_preparation_command(cmd, output_policy))
.map(|cmd| self.run_preparation_command(cmd, output_policy, iteration))
.transpose()
};
@@ -183,10 +200,10 @@ impl<'a> Benchmark<'a> {
self.command.get_parameters().iter().cloned(),
)
});
let run_conclusion_command = || {
let run_conclusion_command = |iteration: executor::BenchmarkIteration| {
conclusion_command
.as_ref()
.map(|cmd| self.run_conclusion_command(cmd, output_policy))
.map(|cmd| self.run_conclusion_command(cmd, output_policy, iteration))
.transpose()
};
@@ -205,14 +222,15 @@ impl<'a> Benchmark<'a> {
};
for i in 0..self.options.warmup_count {
let _ = run_preparation_command()?;
let warmup_iteration = BenchmarkIteration::Warmup(i);
let _ = run_preparation_command(warmup_iteration)?;
let _ = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Warmup(i),
warmup_iteration,
None,
output_policy,
)?;
let _ = run_conclusion_command()?;
let _ = run_conclusion_command(warmup_iteration)?;
if let Some(bar) = progress_bar.as_ref() {
bar.inc(1)
}
@@ -233,7 +251,8 @@ impl<'a> Benchmark<'a> {
None
};
let preparation_result = run_preparation_command()?;
let benchmark_iteration = BenchmarkIteration::Benchmark(0);
let preparation_result = run_preparation_command(benchmark_iteration)?;
let preparation_overhead = preparation_result.map_or(Time::zero(), |res| {
res.time_wall_clock + self.executor.time_overhead()
});
@@ -241,13 +260,13 @@ impl<'a> Benchmark<'a> {
// Initial timing run
let res = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(0),
benchmark_iteration,
None,
output_policy,
)?;
let success = res.exit_status.success();
let conclusion_result = run_conclusion_command()?;
let conclusion_result = run_conclusion_command(benchmark_iteration)?;
let conclusion_overhead = conclusion_result.map_or(Time::zero(), |res| {
res.time_wall_clock + self.executor.time_overhead()
});
@@ -288,7 +307,8 @@ impl<'a> Benchmark<'a> {
// Gather statistics (perform the actual benchmark)
for i in 0..count_remaining {
run_preparation_command()?;
let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
run_preparation_command(benchmark_iteration)?;
let msg = {
let t_wall_clock_mean = measurements.time_wall_clock_mean();
@@ -306,7 +326,7 @@ impl<'a> Benchmark<'a> {
let res = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(i + 1),
benchmark_iteration,
None,
output_policy,
)?;
@@ -320,7 +340,7 @@ impl<'a> Benchmark<'a> {
bar.inc(1)
}
run_conclusion_command()?;
run_conclusion_command(benchmark_iteration)?;
}
if let Some(bar) = progress_bar.as_ref() {
+13 -12
View File
@@ -5,7 +5,7 @@ use csv::WriterBuilder;
use super::Exporter;
use crate::benchmark::benchmark_result::BenchmarkResult;
use crate::options::SortOrder;
use crate::quantity::{second, TimeUnit};
use crate::quantity::{FormatQuantity, TimeUnit};
use anyhow::Result;
@@ -19,6 +19,7 @@ impl Exporter for CsvExporter {
_unit: Option<TimeUnit>,
_sort_order: SortOrder,
) -> Result<Vec<u8>> {
const CSV_UNIT: TimeUnit = TimeUnit::Second;
const CSV_PRECISION: usize = 6;
let mut writer = WriterBuilder::new().from_writer(vec![]);
@@ -42,18 +43,18 @@ impl Exporter for CsvExporter {
for res in results {
let mut fields = vec![Cow::Borrowed(res.command.as_bytes())];
for f in &[
res.mean_wall_clock_time().get::<second>(),
res.measurements
.stddev()
.unwrap_or_default()
.get::<second>(),
res.measurements.median().get::<second>(),
res.measurements.time_user_mean().get::<second>(),
res.measurements.time_system_mean().get::<second>(),
res.measurements.min().get::<second>(),
res.measurements.max().get::<second>(),
res.mean_wall_clock_time(),
res.measurements.stddev().unwrap_or_default(),
res.measurements.median(),
res.measurements.time_user_mean(),
res.measurements.time_system_mean(),
res.measurements.min(),
res.measurements.max(),
] {
fields.push(Cow::Owned(format!("{f:.CSV_PRECISION$}").into_bytes()))
fields.push(Cow::Owned(
f.format_with_precision(CSV_UNIT, CSV_PRECISION)
.into_bytes(),
))
}
for v in res.parameters.values() {
fields.push(Cow::Borrowed(v.value.as_bytes()))
+5 -1
View File
@@ -158,7 +158,11 @@ impl ExportManager {
/// Write the given content to a file with the specified name
fn write_to_file(filename: &str, content: &[u8]) -> Result<()> {
let mut file = OpenOptions::new().write(true).open(filename)?;
let mut file = OpenOptions::new()
.write(true)
// Avoid leaving trailing bytes when the new export is shorter.
.truncate(true)
.open(filename)?;
file.write_all(content)
.with_context(|| format!("Failed to export results to '{filename}'"))
}
+42
View File
@@ -1037,3 +1037,45 @@ fn windows_quote_before_quote_args() {
.assert()
.success();
}
#[test]
fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() {
let tempdir = tempfile::tempdir().unwrap();
let output_path = tempdir.path().join("iteration output.txt");
let iteration = if cfg!(windows) {
"%HYPERFINE_ITERATION%"
} else {
"$HYPERFINE_ITERATION"
};
let command = |phase| format!(r#"echo {phase}:{iteration} >> "iteration output.txt""#);
hyperfine()
.current_dir(tempdir.path())
.arg("--runs=2")
.arg("--warmup=1")
.arg("--prepare")
.arg(command("prepare"))
.arg(command("benchmark"))
.arg("--conclude")
.arg(command("conclude"))
.assert()
.success();
let contents = std::fs::read_to_string(output_path).unwrap();
// cmd.exe's echo includes the space before the redirection operator.
let lines: Vec<&str> = contents.lines().map(str::trim_end).collect();
assert_eq!(
lines,
[
"prepare:warmup-0",
"benchmark:warmup-0",
"conclude:warmup-0",
"prepare:0",
"benchmark:0",
"conclude:0",
"prepare:1",
"benchmark:1",
"conclude:1",
]
);
}