Compare commits

..
Author SHA1 Message Date
David Peter 0fe93f9d0b Keep the nonempty-sample assertion in median 2026-10-05 14:02:52 +02:00
David Peter 8084a9eb7f Adopt the remaining statistics changes from #790 2026-10-05 13:57:31 +02:00
David Peter efd57b3a89 Merge pull request #975 from sharkdp/codex/unified-windows-timer
Unify the Windows CPU timer stop interface
2026-10-05 13:55:36 +02:00
David Peter 580b500052 Merge pull request #973 from sharkdp/dependabot/github_actions/softprops/action-gh-release-3
Bump softprops/action-gh-release from 2 to 3
2026-10-05 13:54:02 +02:00
David Peter ee96552dae Merge pull request #974 from sharkdp/dependabot/github_actions/actions/checkout-7
Bump actions/checkout from 6 to 7
2026-10-05 13:53:45 +02:00
David Peter 6af23bb885 Unify the Windows CPU timer stop interface 2026-10-05 13:51:06 +02:00
David Peter 683885a6c8 Merge pull request #972 from sharkdp/codex/csv-quantity-formatting
Format CSV measurements through FormatQuantity
2026-10-05 13:45:52 +02:00
David Peter 33aafdca07 Merge pull request #971 from sharkdp/codex/typed-relative-speed
Use typed quantities for relative-speed calculations
2026-10-05 13:44:35 +02:00
dependabot[bot] 0e2acb67e5 Bump actions/checkout from 6 to 7
Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v6...v7)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-05 11:42:47 +00:00
dependabot[bot] de0afb81c5 Bump softprops/action-gh-release from 2 to 3
Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 2 to 3.
- [Release notes](https://github.com/softprops/action-gh-release/releases)
- [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md)
- [Commits](https://github.com/softprops/action-gh-release/compare/v2...v3)

---
updated-dependencies:
- dependency-name: softprops/action-gh-release
  dependency-version: '3'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-05 11:42:43 +00:00
David Peter 16f53eb8d6 Format CSV measurements through FormatQuantity 2026-10-05 13:41:43 +02:00
David Peter df49a04a66 Use typed quantities for relative-speed calculations 2026-10-05 13:41:20 +02:00
David Peter 4aa36c8051 Merge pull request #970 from sharkdp/codex/truncate-export-rewrites
Truncate export files before rewriting results
2026-10-05 12:57:52 +02:00
David Peter e689520fb5 Merge pull request #969 from sharkdp/codex/result-measurements
Store typed Measurements in BenchmarkResult
2026-10-05 12:51:22 +02:00
David Peter 8cf4e010e9 Truncate export files before rewriting results 2026-10-05 12:48:52 +02:00
David Peter 30a2b61d4c Merge pull request #807 from willcl-ark/expose-run-var
expose HYPERFINE_ITERATION to intermediate commands
2026-10-05 12:45:29 +02:00
David Peter 3558138cea Further simplifications 2026-10-05 12:40:33 +02:00
David Peter 1eafcf4739 Combine iteration environment variable integration tests 2026-10-05 12:38:53 +02:00
willcl-ark 44c0cc231b expose HYPERFINE_ITERATION to intermediate commands
Fixes: #781

Expose the `$HYPERFINE_ITERATION` variable to intermediate commands.
This will allow iteration-specific prepare/conclude, such as:

```bash
hyperfine --runs 5 --prepare 'echo touch file${HYPERFINE_ITERATION}.dat' \
    'echo testing file${HYPERFINE_ITERATION}.dat' \
    --conclude 'echo rm file${HYPERFINE_ITERATION}.dat' --show-output
```
2026-10-05 12:24:43 +02:00
11 changed files with 169 additions and 105 deletions

No files matched your search

+5 -5
View File
@@ -18,7 +18,7 @@ jobs:
name: Extract crate metadata
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- name: Extract crate information
id: crate_metadata
run: |
@@ -49,7 +49,7 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- run: cargo fmt -- --check
min_version:
@@ -58,7 +58,7 @@ jobs:
needs: crate_metadata
steps:
- name: Checkout source code
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
uses: dtolnay/rust-toolchain@master
@@ -96,7 +96,7 @@ jobs:
BUILD_CMD: cargo
steps:
- name: Checkout source code
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Install prerequisites
shell: bash
@@ -337,7 +337,7 @@ jobs:
echo "IS_RELEASE=${IS_RELEASE}" >> $GITHUB_OUTPUT
- name: Publish archives and packages
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@v3
if: steps.is-release.outputs.IS_RELEASE
with:
files: |
+1
View File
@@ -15,6 +15,7 @@ use super::measurement::{Measurement, Measurements};
use anyhow::{bail, Context, Result};
#[derive(Clone, Copy)]
pub enum BenchmarkIteration {
NonBenchmarkRun,
Warmup(u64),
+5 -4
View File
@@ -54,13 +54,14 @@ impl Measurements {
mean(self.wall_clock_times())
}
/// The standard deviation of wall clock times, if at least two runs were measured.
/// The standard deviation of all wall clock times. Not available if only one run has been performed
pub fn stddev(&self) -> Option<Time> {
if self.len() < 2 {
let times: Vec<_> = self.wall_clock_times().collect(); // TODO: Avoid collecting
if times.len() < 2 {
None
} else {
let times: Vec<_> = self.wall_clock_times().collect();
Some(standard_deviation(&times, self.time_wall_clock_mean()))
Some(standard_deviation(times))
}
}
+38 -18
View File
@@ -57,11 +57,12 @@ impl<'a> Benchmark<'a> {
command: &Command<'_>,
error_output: &'static str,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.executor
.run_command_and_measure(
command,
executor::BenchmarkIteration::NonBenchmarkRun,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
@@ -84,7 +85,14 @@ impl<'a> Benchmark<'a> {
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
.map(|cmd| {
self.run_intermediate_command(
&cmd,
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
})
.transpose()?
.unwrap_or_default())
}
@@ -105,7 +113,14 @@ impl<'a> Benchmark<'a> {
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
.map(|cmd| {
self.run_intermediate_command(
&cmd,
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
})
.transpose()?
.unwrap_or_default())
}
@@ -115,11 +130,12 @@ impl<'a> Benchmark<'a> {
&self,
command: &Command<'_>,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy)
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
/// Run the command specified by `--conclude`.
@@ -127,11 +143,12 @@ impl<'a> Benchmark<'a> {
&self,
command: &Command<'_>,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy)
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
/// Run the benchmark for a single command
@@ -164,10 +181,10 @@ impl<'a> Benchmark<'a> {
)
});
let run_preparation_command = || {
let run_preparation_command = |iteration: executor::BenchmarkIteration| {
preparation_command
.as_ref()
.map(|cmd| self.run_preparation_command(cmd, output_policy))
.map(|cmd| self.run_preparation_command(cmd, output_policy, iteration))
.transpose()
};
@@ -183,10 +200,10 @@ impl<'a> Benchmark<'a> {
self.command.get_parameters().iter().cloned(),
)
});
let run_conclusion_command = || {
let run_conclusion_command = |iteration: executor::BenchmarkIteration| {
conclusion_command
.as_ref()
.map(|cmd| self.run_conclusion_command(cmd, output_policy))
.map(|cmd| self.run_conclusion_command(cmd, output_policy, iteration))
.transpose()
};
@@ -205,14 +222,15 @@ impl<'a> Benchmark<'a> {
};
for i in 0..self.options.warmup_count {
let _ = run_preparation_command()?;
let warmup_iteration = BenchmarkIteration::Warmup(i);
let _ = run_preparation_command(warmup_iteration)?;
let _ = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Warmup(i),
warmup_iteration,
None,
output_policy,
)?;
let _ = run_conclusion_command()?;
let _ = run_conclusion_command(warmup_iteration)?;
if let Some(bar) = progress_bar.as_ref() {
bar.inc(1)
}
@@ -233,7 +251,8 @@ impl<'a> Benchmark<'a> {
None
};
let preparation_result = run_preparation_command()?;
let benchmark_iteration = BenchmarkIteration::Benchmark(0);
let preparation_result = run_preparation_command(benchmark_iteration)?;
let preparation_overhead = preparation_result.map_or(Time::zero(), |res| {
res.time_wall_clock + self.executor.time_overhead()
});
@@ -241,13 +260,13 @@ impl<'a> Benchmark<'a> {
// Initial timing run
let res = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(0),
benchmark_iteration,
None,
output_policy,
)?;
let success = res.exit_status.success();
let conclusion_result = run_conclusion_command()?;
let conclusion_result = run_conclusion_command(benchmark_iteration)?;
let conclusion_overhead = conclusion_result.map_or(Time::zero(), |res| {
res.time_wall_clock + self.executor.time_overhead()
});
@@ -288,7 +307,8 @@ impl<'a> Benchmark<'a> {
// Gather statistics (perform the actual benchmark)
for i in 0..count_remaining {
run_preparation_command()?;
let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
run_preparation_command(benchmark_iteration)?;
let msg = {
let t_wall_clock_mean = measurements.time_wall_clock_mean();
@@ -306,7 +326,7 @@ impl<'a> Benchmark<'a> {
let res = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(i + 1),
benchmark_iteration,
None,
output_policy,
)?;
@@ -320,7 +340,7 @@ impl<'a> Benchmark<'a> {
bar.inc(1)
}
run_conclusion_command()?;
run_conclusion_command(benchmark_iteration)?;
}
if let Some(bar) = progress_bar.as_ref() {
+20 -28
View File
@@ -1,8 +1,10 @@
use std::cmp::Ordering;
use super::benchmark_result::BenchmarkResult;
use crate::options::SortOrder;
use crate::quantity::second;
use crate::{
options::SortOrder,
quantity::{self, Ratio, Time, Zero},
};
#[derive(Debug)]
pub struct BenchmarkResultWithRelativeSpeed<'a> {
@@ -16,8 +18,7 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
l.mean_wall_clock_time()
.get::<second>()
.partial_cmp(&r.mean_wall_clock_time().get::<second>())
.partial_cmp(&r.mean_wall_clock_time())
.unwrap_or(Ordering::Equal)
}
@@ -39,7 +40,7 @@ fn compute_relative_speeds<'a>(
let is_reference = result == reference;
let relative_ordering = compare_mean_time(result, reference);
if result.mean_wall_clock_time().get::<second>() == 0.0 {
if result.mean_wall_clock_time() == Time::zero() {
return BenchmarkResultWithRelativeSpeed {
result,
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -50,34 +51,25 @@ fn compute_relative_speeds<'a>(
}
let ratio = match relative_ordering {
Ordering::Less => {
reference.mean_wall_clock_time().get::<second>()
/ result.mean_wall_clock_time().get::<second>()
}
Ordering::Equal => 1.0,
Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(),
Ordering::Equal => Ratio::new::<quantity::ratio>(1.0),
Ordering::Greater => {
result.mean_wall_clock_time().get::<second>()
/ reference.mean_wall_clock_time().get::<second>()
result.mean_wall_clock_time() / reference.mean_wall_clock_time()
}
};
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance assumed to be 0, i.e. variables are assumed to be independent
let ratio_stddev = match (
result
.measurements
.stddev()
.map(|time| time.get::<second>()),
reference
.measurements
.stddev()
.map(|time| time.get::<second>()),
result.measurements.stddev(),
reference.measurements.stddev(),
) {
(Some(result_stddev), Some(fastest_stddev)) => Some(
ratio
* ((result_stddev / result.mean_wall_clock_time().get::<second>()).powi(2)
+ (fastest_stddev / reference.mean_wall_clock_time().get::<second>())
.powi(2))
* ((result_stddev / result.mean_wall_clock_time())
.powi(uom::typenum::P2::new())
+ (fastest_stddev / reference.mean_wall_clock_time())
.powi(uom::typenum::P2::new()))
.sqrt(),
),
_ => None,
@@ -85,8 +77,8 @@ fn compute_relative_speeds<'a>(
BenchmarkResultWithRelativeSpeed {
result,
relative_speed: ratio,
relative_speed_stddev: ratio_stddev,
relative_speed: ratio.get::<quantity::ratio>(),
relative_speed_stddev: ratio_stddev.map(|r| r.get::<quantity::ratio>()),
is_reference,
relative_ordering,
}
@@ -108,8 +100,8 @@ pub fn compute_with_check_from_reference<'a>(
reference: &'a BenchmarkResult,
sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
if fastest_of(results).mean_wall_clock_time().get::<second>() == 0.0
|| reference.mean_wall_clock_time().get::<second>() == 0.0
if fastest_of(results).mean_wall_clock_time() == Time::zero()
|| reference.mean_wall_clock_time() == Time::zero()
{
return None;
}
@@ -123,7 +115,7 @@ pub fn compute_with_check(
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
let fastest = fastest_of(results);
if fastest.mean_wall_clock_time().get::<second>() == 0.0 {
if fastest.mean_wall_clock_time() == Time::zero() {
return None;
}
+13 -12
View File
@@ -5,7 +5,7 @@ use csv::WriterBuilder;
use super::Exporter;
use crate::benchmark::benchmark_result::BenchmarkResult;
use crate::options::SortOrder;
use crate::quantity::{second, TimeUnit};
use crate::quantity::{FormatQuantity, TimeUnit};
use anyhow::Result;
@@ -19,6 +19,7 @@ impl Exporter for CsvExporter {
_unit: Option<TimeUnit>,
_sort_order: SortOrder,
) -> Result<Vec<u8>> {
const CSV_UNIT: TimeUnit = TimeUnit::Second;
const CSV_PRECISION: usize = 6;
let mut writer = WriterBuilder::new().from_writer(vec![]);
@@ -42,18 +43,18 @@ impl Exporter for CsvExporter {
for res in results {
let mut fields = vec![Cow::Borrowed(res.command.as_bytes())];
for f in &[
res.mean_wall_clock_time().get::<second>(),
res.measurements
.stddev()
.unwrap_or_default()
.get::<second>(),
res.measurements.median().get::<second>(),
res.measurements.time_user_mean().get::<second>(),
res.measurements.time_system_mean().get::<second>(),
res.measurements.min().get::<second>(),
res.measurements.max().get::<second>(),
res.mean_wall_clock_time(),
res.measurements.stddev().unwrap_or_default(),
res.measurements.median(),
res.measurements.time_user_mean(),
res.measurements.time_system_mean(),
res.measurements.min(),
res.measurements.max(),
] {
fields.push(Cow::Owned(format!("{f:.CSV_PRECISION$}").into_bytes()))
fields.push(Cow::Owned(
f.format_with_precision(CSV_UNIT, CSV_PRECISION)
.into_bytes(),
))
}
for v in res.parameters.values() {
fields.push(Cow::Borrowed(v.value.as_bytes()))
+5 -1
View File
@@ -158,7 +158,11 @@ impl ExportManager {
/// Write the given content to a file with the specified name
fn write_to_file(filename: &str, content: &[u8]) -> Result<()> {
let mut file = OpenOptions::new().write(true).open(filename)?;
let mut file = OpenOptions::new()
.write(true)
// Avoid leaving trailing bytes when the new export is shorter.
.truncate(true)
.open(filename)?;
file.write_all(content)
.with_context(|| format!("Failed to export results to '{filename}'"))
}
+30 -25
View File
@@ -1,8 +1,12 @@
//! Statistics for non-empty samples of finite measurements.
use std::ops::Add;
use std::ops::AddAssign;
use std::ops::Div;
use std::ops::{Add, AddAssign, Div};
use uom::num_traits;
use uom::si::f64::Ratio;
use uom::si::ratio::ratio;
use crate::quantity::{byte, ratio, second, Information, Ratio, Time, Zero};
use super::{byte, second, Information, Time};
/// A min function that assumes no NaNs and at least one element
pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
@@ -20,14 +24,14 @@ pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
.expect("'max' requires at least one element")
}
/// Arithmetic mean, summing measurements in their original order.
/// A mean function that assumes at least one element
pub fn mean<Q, P>(values: impl IntoIterator<Item = Q>) -> Q
where
Q: AddAssign + Zero + Div<Ratio, Output = P>,
Q: AddAssign + num_traits::Zero + Div<Ratio, Output = P>,
P: Into<Q>,
{
let mut sum = Q::zero();
let mut count = 0usize;
let mut count = 0;
for value in values {
sum += value;
count += 1;
@@ -37,7 +41,6 @@ where
(sum / count).into()
}
/// Median without modifying the input sample.
pub fn median<Q, P>(values: impl IntoIterator<Item = Q>) -> Q
where
Q: Copy + PartialOrd + Add<Output = Q> + Div<Ratio, Output = P>,
@@ -48,14 +51,14 @@ where
!values.is_empty(),
"median requires at least one measurement"
);
values.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
values.sort_by(|a, b| a.partial_cmp(b).expect("No NaN values"));
let len = values.len();
if len % 2 == 0 {
let mid = len / 2;
let a = &values[mid - 1];
let b = &values[mid];
((*b + *a) / Ratio::new::<ratio>(2.)).into()
((*a + *b) / Ratio::new::<ratio>(2.)).into()
} else {
values[len / 2]
}
@@ -88,22 +91,24 @@ impl UnsafeRawValue for Information {
}
}
/// Sample standard deviation, using an already computed mean.
pub fn standard_deviation<Q: UnsafeRawValue>(values: &[Q], mean: Q) -> Q {
let mean = mean.unsafe_raw_value();
assert!(
values.len() > 1,
"standard deviation requires at least two measurements"
);
let sum = values
.iter()
.map(|value| (value.unsafe_raw_value() - mean) * (value.unsafe_raw_value() - mean))
.fold(0.0, |sum, deviation| sum + deviation);
assert!(sum >= 0.0, "invalid sum of squared deviations");
fn standard_deviation_f64(values: &[f64]) -> f64 {
let mean_value = mean(values.iter().copied());
// Preserve the operation order of statistical 1.0: dividing by n - 1
// before taking the square root also preserves exported floating-point values.
Q::unsafe_from_raw_value((sum / (values.len() as f64 - 1.0)).sqrt())
let mut squared_deviations = 0.;
let mut n = 0;
for value in values {
let deviation = value - mean_value;
squared_deviations += deviation * deviation;
n += 1;
}
(1. / ((n - 1) as f64) * squared_deviations).sqrt()
}
pub fn standard_deviation<Q: UnsafeRawValue>(values: impl IntoIterator<Item = Q> + Clone) -> Q {
let values: Vec<_> = values.into_iter().map(|q| q.unsafe_raw_value()).collect();
let result = standard_deviation_f64(&values);
Q::unsafe_from_raw_value(result)
}
/// Compute modified Z-scores for a given sample. A (unmodified) Z-score is defined by
@@ -219,7 +224,7 @@ mod tests {
Time::new::<millisecond>(200.0),
Time::new::<microsecond>(300_000.0),
];
let result = standard_deviation(&values, mean(values));
let result = standard_deviation(values);
assert_relative_eq!(result.get::<millisecond>(), 100.0);
}
}
-8
View File
@@ -82,17 +82,9 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
discard(output);
}
#[cfg(not(windows))]
let (time_user, time_system, peak_memory_usage, exit_status) = cpu_timer.stop(child)?;
#[cfg(windows)]
let exit_status = child.wait()?;
let time_wall_clock = wallclock_timer.stop();
#[cfg(windows)]
let (time_user, time_system, peak_memory_usage) = cpu_timer.stop();
Ok(Measurement {
time_wall_clock,
time_user,
+10 -4
View File
@@ -1,7 +1,10 @@
#![cfg(windows)]
#![warn(unsafe_op_in_unsafe_fn)]
use std::{mem, os::windows::io::AsRawHandle, process, ptr};
use std::process::{self, Child, ExitStatus};
use std::{mem, os::windows::io::AsRawHandle, ptr};
use anyhow::Result;
use windows_sys::Win32::{
Foundation::{CloseHandle, HANDLE},
@@ -84,7 +87,9 @@ impl CPUTimer {
Self { job_object }
}
pub fn stop(&self) -> (Time, Time, Information) {
pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Information, ExitStatus)> {
let status = child.wait()?;
let mut job_object_info =
mem::MaybeUninit::<JOBOBJECT_BASIC_ACCOUNTING_INFORMATION>::uninit();
@@ -113,9 +118,10 @@ impl CPUTimer {
// processes no longer associated with the job, in 100-nanosecond ticks."
let system_time =
Time::new::<nanosecond>((job_object_info.TotalKernelTime as f64) * 100.0);
(user_time, system_time, Information::zero())
Ok((user_time, system_time, Information::zero(), status))
} else {
(Time::zero(), Time::zero(), Information::zero())
Ok((Time::zero(), Time::zero(), Information::zero(), status))
}
}
}
+42
View File
@@ -1037,3 +1037,45 @@ fn windows_quote_before_quote_args() {
.assert()
.success();
}
#[test]
fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() {
let tempdir = tempfile::tempdir().unwrap();
let output_path = tempdir.path().join("iteration output.txt");
let iteration = if cfg!(windows) {
"%HYPERFINE_ITERATION%"
} else {
"$HYPERFINE_ITERATION"
};
let command = |phase| format!(r#"echo {phase}:{iteration} >> "iteration output.txt""#);
hyperfine()
.current_dir(tempdir.path())
.arg("--runs=2")
.arg("--warmup=1")
.arg("--prepare")
.arg(command("prepare"))
.arg(command("benchmark"))
.arg("--conclude")
.arg(command("conclude"))
.assert()
.success();
let contents = std::fs::read_to_string(output_path).unwrap();
// cmd.exe's echo includes the space before the redirection operator.
let lines: Vec<&str> = contents.lines().map(str::trim_end).collect();
assert_eq!(
lines,
[
"prepare:warmup-0",
"benchmark:warmup-0",
"conclude:warmup-0",
"prepare:0",
"benchmark:0",
"conclude:0",
"prepare:1",
"benchmark:1",
"conclude:1",
]
);
}