Merge master and resolve dependency lockfile conflict

Collect benchmark runs in Measurements

Cargo.lock

@@ -382,6 +382,7 @@ dependencies = [
  "csv",
  "indicatif",
  "insta",
+ "insta-cmd",
  "libc",
  "nix",
  "once_cell",
@@ -427,11 +428,24 @@ checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
 dependencies = [
  "console 0.16.6",
  "once_cell",
+ "regex",
  "serde",
  "similar",
+ "strip-ansi-escapes",
  "tempfile",
 ]
 
+[[package]]
+name = "insta-cmd"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bffdf4af1db390cf0401535d7c1303cd079a074d28d8473b026fdb6559c41403"
+dependencies = [
+ "insta",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.13"
@@ -802,6 +816,15 @@ version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
 
+[[package]]
+name = "strip-ansi-escapes"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
+dependencies = [
+ "vte",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -933,6 +956,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "vte"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "wait-timeout"
 version = "0.2.1"

Cargo.toml

@@ -63,7 +63,8 @@ features = [
 [dev-dependencies]
 approx = "0.5"
 assert_cmd = "2.2"
-insta = { version = "1.48", features = ["yaml"] }
+insta = { version = "1.48", features = ["yaml", "filters"] }
+insta-cmd = "0.7"
 predicates = "3.1"
 tempfile = "3.27"
 

src/benchmark/executor.rs

@@ -9,10 +9,9 @@ use crate::options::{
 use crate::output::progress_bar::get_progress_bar;
 use crate::timer::execute_and_measure;
 use crate::util::randomized_environment_offset;
-use crate::util::statistics::mean;
 use crate::util::units::Second;
 
-use super::measurement::Measurement;
+use super::measurement::{Measurement, Measurements};
 
 use anyhow::{bail, Context, Result};
 
@@ -79,7 +78,7 @@ fn run_command_and_measure_common(
     let result = execute_and_measure(command)
         .with_context(|| format!("Failed to run command '{command_name}'"))?;
 
-    if !result.status.success() {
+    if !result.exit_status.success() {
         use crate::util::exit_code::extract_exit_code;
 
         let should_fail = match command_failure_action {
@@ -87,7 +86,7 @@ fn run_command_and_measure_common(
             CmdFailureAction::IgnoreAllFailures => false,
             CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
                 // Only fail if the exit code is not in the list of codes to ignore
-                if let Some(exit_code) = extract_exit_code(result.status) {
+                if let Some(exit_code) = extract_exit_code(result.exit_status) {
                     !codes.contains(&exit_code)
                 } else {
                     // If we can't extract an exit code, treat it as a failure
@@ -107,7 +106,7 @@ fn run_command_and_measure_common(
             bail!(
                 "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
                 Alternatively, use the '--show-output' option to debug what went wrong.",
-                cause=result.status.code().map_or(
+                cause=result.exit_status.code().map_or(
                     "The process has been terminated by a signal".into(),
                     |c| format!("Command terminated with non-zero exit code {c}")
 
@@ -203,7 +202,8 @@ impl Executor for ShellExecutor<'_> {
 
         // Subtract shell spawning time
         if let Some(spawning_time) = self.shell_spawning_time {
-            result.time_real = (result.time_real - spawning_time.time_real).max(0.0);
+            result.time_wall_clock =
+                (result.time_wall_clock - spawning_time.time_wall_clock).max(0.0);
             result.time_user = (result.time_user - spawning_time.time_user).max(0.0);
             result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
         }
@@ -224,9 +224,7 @@ impl Executor for ShellExecutor<'_> {
             None
         };
 
-        let mut times_real: Vec<Second> = vec![];
-        let mut times_user: Vec<Second> = vec![];
-        let mut times_system: Vec<Second> = vec![];
+        let mut measurements = Measurements::default();
 
         for _ in 0..COUNT {
             // Just run the shell without any command
@@ -251,9 +249,7 @@ impl Executor for ShellExecutor<'_> {
                     );
                 }
                 Ok(r) => {
-                    times_real.push(r.time_real);
-                    times_user.push(r.time_user);
-                    times_system.push(r.time_system);
+                    measurements.push(r);
                 }
             }
 
@@ -267,18 +263,18 @@ impl Executor for ShellExecutor<'_> {
         }
 
         self.shell_spawning_time = Some(Measurement {
-            time_real: mean(&times_real),
-            time_user: mean(&times_user),
-            time_system: mean(&times_system),
-            memory_usage_byte: 0,
-            status: ExitStatus::default(),
+            time_wall_clock: measurements.time_wall_clock_mean(),
+            time_user: measurements.time_user_mean(),
+            time_system: measurements.time_system_mean(),
+            peak_memory_usage: 0,
+            exit_status: ExitStatus::default(),
         });
 
         Ok(())
     }
 
     fn time_overhead(&self) -> Second {
-        self.shell_spawning_time.unwrap().time_real
+        self.shell_spawning_time.unwrap().time_wall_clock
     }
 }
 
@@ -311,23 +307,23 @@ impl Executor for MockExecutor {
         _output_policy: &CommandOutputPolicy,
     ) -> Result<Measurement> {
         #[cfg(unix)]
-        let status = {
+        let exit_status = {
             use std::os::unix::process::ExitStatusExt;
             ExitStatus::from_raw(0)
         };
 
         #[cfg(windows)]
-        let status = {
+        let exit_status = {
             use std::os::windows::process::ExitStatusExt;
             ExitStatus::from_raw(0)
         };
 
         Ok(Measurement {
-            time_real: Self::extract_time(command.get_command_line()),
+            time_wall_clock: Self::extract_time(command.get_command_line()),
             time_user: 0.0,
             time_system: 0.0,
-            memory_usage_byte: 0,
-            status,
+            peak_memory_usage: 0,
+            exit_status,
         })
     }
 

src/benchmark/measurement.rs

@@ -1,12 +1,14 @@
 use std::process::ExitStatus;
 
+use crate::outlier_detection::modified_zscores;
+use crate::util::statistics::{max, mean, median, min, standard_deviation};
 use crate::util::units::Second;
 
 /// Performance measurements and exit status from running a single command
 #[derive(Debug, Default, Copy, Clone)]
 pub struct Measurement {
     /// Wall clock time
-    pub time_real: Second,
+    pub time_wall_clock: Second,
 
     /// Time spent in user mode
     pub time_user: Second,
@@ -15,8 +17,77 @@ pub struct Measurement {
     pub time_system: Second,
 
     /// Maximum amount of memory used, in bytes
-    pub memory_usage_byte: u64,
+    pub peak_memory_usage: u64,
 
     /// The exit status of the process
-    pub status: ExitStatus,
+    pub exit_status: ExitStatus,
+}
+
+/// Performance measurements and exit statuses for all runs of a command.
+#[derive(Debug, Default, Clone)]
+pub struct Measurements {
+    pub measurements: Vec<Measurement>,
+}
+
+impl Measurements {
+    pub fn len(&self) -> usize {
+        self.measurements.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.measurements.is_empty()
+    }
+
+    pub fn push(&mut self, measurement: Measurement) {
+        self.measurements.push(measurement);
+    }
+
+    pub fn wall_clock_times(&self) -> impl Iterator<Item = Second> + '_ {
+        self.measurements.iter().map(|m| m.time_wall_clock)
+    }
+
+    /// The average wall clock time.
+    pub fn time_wall_clock_mean(&self) -> Second {
+        mean(self.wall_clock_times())
+    }
+
+    /// The standard deviation of wall clock times, if at least two runs were measured.
+    pub fn stddev(&self) -> Option<Second> {
+        if self.len() < 2 {
+            None
+        } else {
+            let times: Vec<_> = self.wall_clock_times().collect();
+            Some(standard_deviation(&times, self.time_wall_clock_mean()))
+        }
+    }
+
+    /// The median wall clock time.
+    pub fn median(&self) -> Second {
+        median(&self.wall_clock_times().collect::<Vec<_>>())
+    }
+
+    /// The minimum wall clock time.
+    pub fn min(&self) -> Second {
+        min(self.wall_clock_times())
+    }
+
+    /// The maximum wall clock time.
+    pub fn max(&self) -> Second {
+        max(self.wall_clock_times())
+    }
+
+    /// Compute modified Z-scores for the wall clock times.
+    pub fn modified_zscores(&self) -> Vec<f64> {
+        modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
+    }
+
+    /// The average user time.
+    pub fn time_user_mean(&self) -> Second {
+        mean(self.measurements.iter().map(|m| m.time_user))
+    }
+
+    /// The average system time.
+    pub fn time_system_mean(&self) -> Second {
+        mean(self.measurements.iter().map(|m| m.time_system))
+    }
 }

src/benchmark/mod.rs

@@ -12,17 +12,16 @@ use crate::command::Command;
 use crate::options::{
     CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
 };
-use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
+use crate::outlier_detection::OUTLIER_THRESHOLD;
 use crate::output::console_writeln;
 use crate::output::format::{format_duration, format_duration_unit};
 use crate::output::progress_bar::get_progress_bar;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
 use crate::util::exit_code::extract_exit_code;
-use crate::util::statistics::{max, mean, median, min, standard_deviation};
 use crate::util::units::Second;
 use benchmark_result::BenchmarkResult;
-use measurement::Measurement;
+use measurement::{Measurement, Measurements};
 
 use anyhow::{anyhow, Result};
 use colored::*;
@@ -149,11 +148,7 @@ impl<'a> Benchmark<'a> {
             )?;
         }
 
-        let mut times_real: Vec<Second> = vec![];
-        let mut times_user: Vec<Second> = vec![];
-        let mut times_system: Vec<Second> = vec![];
-        let mut memory_usage_byte: Vec<u64> = vec![];
-        let mut exit_codes: Vec<Option<i32>> = vec![];
+        let mut measurements = Measurements::default();
         let mut all_succeeded = true;
 
         let output_policy = &self.options.command_output_policies[self.number];
@@ -241,8 +236,9 @@ impl<'a> Benchmark<'a> {
         };
 
         let preparation_result = run_preparation_command()?;
-        let preparation_overhead =
-            preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
+        let preparation_overhead = preparation_result.map_or(0.0, |res| {
+            res.time_wall_clock + self.executor.time_overhead()
+        });
 
         // Initial timing run
         let res = self.executor.run_command_and_measure(
@@ -251,15 +247,16 @@ impl<'a> Benchmark<'a> {
             None,
             output_policy,
         )?;
-        let success = res.status.success();
+        let success = res.exit_status.success();
 
         let conclusion_result = run_conclusion_command()?;
-        let conclusion_overhead =
-            conclusion_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
+        let conclusion_overhead = conclusion_result.map_or(0.0, |res| {
+            res.time_wall_clock + self.executor.time_overhead()
+        });
 
         // Determine number of benchmark runs
         let runs_in_min_time = (self.options.min_benchmarking_time
-            / (res.time_real
+            / (res.time_wall_clock
                 + self.executor.time_overhead()
                 + preparation_overhead
                 + conclusion_overhead)) as u64;
@@ -278,11 +275,7 @@ impl<'a> Benchmark<'a> {
         let count_remaining = count - 1;
 
         // Save the first result
-        times_real.push(res.time_real);
-        times_user.push(res.time_user);
-        times_system.push(res.time_system);
-        memory_usage_byte.push(res.memory_usage_byte);
-        exit_codes.push(extract_exit_code(res.status));
+        measurements.push(res);
 
         all_succeeded = all_succeeded && success;
 
@@ -299,7 +292,8 @@ impl<'a> Benchmark<'a> {
             run_preparation_command()?;
 
             let msg = {
-                let mean = format_duration(mean(&times_real), self.options.time_unit);
+                let mean =
+                    format_duration(measurements.time_wall_clock_mean(), self.options.time_unit);
                 format!("Current estimate: {}", mean.to_string().green())
             };
 
@@ -313,13 +307,9 @@ impl<'a> Benchmark<'a> {
                 None,
                 output_policy,
             )?;
-            let success = res.status.success();
+            let success = res.exit_status.success();
 
-            times_real.push(res.time_real);
-            times_user.push(res.time_user);
-            times_system.push(res.time_system);
-            memory_usage_byte.push(res.memory_usage_byte);
-            exit_codes.push(extract_exit_code(res.status));
+            measurements.push(res);
 
             all_succeeded = all_succeeded && success;
 
@@ -335,19 +325,15 @@ impl<'a> Benchmark<'a> {
         }
 
         // Compute statistical quantities
-        let t_num = times_real.len();
-        let t_mean = mean(&times_real);
-        let t_stddev = if times_real.len() > 1 {
-            Some(standard_deviation(&times_real, t_mean))
-        } else {
-            None
-        };
-        let t_median = median(&times_real);
-        let t_min = min(times_real.iter().copied());
-        let t_max = max(times_real.iter().copied());
+        let t_num = measurements.len();
+        let t_mean = measurements.time_wall_clock_mean();
+        let t_stddev = measurements.stddev();
+        let t_median = measurements.median();
+        let t_min = measurements.min();
+        let t_max = measurements.max();
 
-        let user_mean = mean(&times_user);
-        let system_mean = mean(&times_system);
+        let user_mean = measurements.time_user_mean();
+        let system_mean = measurements.time_system_mean();
 
         // Formatting and console output
         let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);
@@ -360,7 +346,7 @@ impl<'a> Benchmark<'a> {
 
         if self.options.output_style != OutputStyleOption::Disabled {
             let mut stdout = io::stdout().lock();
-            if times_real.len() == 1 {
+            if measurements.len() == 1 {
                 console_writeln!(
                     stdout,
                     "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
@@ -401,7 +387,9 @@ impl<'a> Benchmark<'a> {
 
         // Check execution time
         if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
-            && times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
+            && measurements
+                .wall_clock_times()
+                .any(|t| t < MIN_EXECUTION_TIME)
         {
             warnings.push(Warnings::FastExecutionTime);
         }
@@ -412,7 +400,7 @@ impl<'a> Benchmark<'a> {
         }
 
         // Run outlier detection
-        let scores = modified_zscores(&times_real);
+        let scores = measurements.modified_zscores();
 
         let outlier_warning_options = OutlierWarningOptions {
             warmup_in_use: self.options.warmup_count > 0,
@@ -427,7 +415,7 @@ impl<'a> Benchmark<'a> {
 
         if scores[0] > OUTLIER_THRESHOLD {
             warnings.push(Warnings::SlowInitialRun(
-                times_real[0],
+                measurements.wall_clock_times().next().unwrap(),
                 outlier_warning_options,
             ));
         } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
@@ -459,9 +447,19 @@ impl<'a> Benchmark<'a> {
             system: system_mean,
             min: t_min,
             max: t_max,
-            times: Some(times_real),
-            memory_usage_byte: Some(memory_usage_byte),
-            exit_codes,
+            times: Some(measurements.wall_clock_times().collect()),
+            memory_usage_byte: Some(
+                measurements
+                    .measurements
+                    .iter()
+                    .map(|m| m.peak_memory_usage)
+                    .collect(),
+            ),
+            exit_codes: measurements
+                .measurements
+                .iter()
+                .map(|m| extract_exit_code(m.exit_status))
+                .collect(),
             parameters: self
                 .command
                 .get_parameters()

src/benchmark/relative_speed.rs

@@ -52,7 +52,7 @@ fn compute_relative_speeds<'a>(
             };
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
-            // Covariance asssumed to be 0, i.e. variables are assumed to be independent
+            // Covariance assumed to be 0, i.e. variables are assumed to be independent
             let ratio_stddev = match (result.stddev, reference.stddev) {
                 (Some(result_stddev), Some(fastest_stddev)) => Some(
                     ratio

src/cli.rs

@@ -167,7 +167,7 @@ fn build_command() -> Command {
                      string '{VAR}' in each command by the current parameter value.\n\n  \
                      Example:  hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\
                      This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
-                     To have the value increase following different patterns, use shell arithmetics.\n\n  \
+                     To have the value increase following different patterns, use shell arithmetic.\n\n  \
                      Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\
                      This performs benchmarks with power of 2 increases: 'sleep 1', 'sleep 2', 'sleep 4', …\n\
                      The exact syntax may vary depending on your shell and OS."

src/timer/mod.rs

@@ -95,16 +95,16 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
         discard(output);
     }
 
-    let status = child.wait()?;
+    let exit_status = child.wait()?;
 
-    let time_real = wallclock_timer.stop();
-    let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
+    let time_wall_clock = wallclock_timer.stop();
+    let (time_user, time_system, peak_memory_usage) = cpu_timer.stop();
 
     Ok(Measurement {
-        time_real,
+        time_wall_clock,
         time_user,
         time_system,
-        memory_usage_byte,
-        status,
+        peak_memory_usage,
+        exit_status,
     })
 }

src/util/statistics.rs

@@ -17,8 +17,14 @@ pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
 }
 
 /// Arithmetic mean, summing measurements in their original order.
-pub fn mean(values: &[f64]) -> f64 {
-    values.iter().fold(0.0, |sum, value| sum + value) / values.len() as f64
+pub fn mean(values: impl IntoIterator<Item = f64>) -> f64 {
+    let mut sum = 0.0;
+    let mut count = 0usize;
+    for value in values {
+        sum += value;
+        count += 1;
+    }
+    sum / count as f64
 }
 
 /// Median without modifying the input sample.
@@ -86,11 +92,11 @@ mod tests {
 
     #[test]
     fn test_mean() {
-        assert_eq!(1.0, mean(&[1.0]));
-        assert_relative_eq!(2.0, mean(&[1.0, 3.0]));
+        assert_eq!(1.0, mean([1.0]));
+        assert_relative_eq!(2.0, mean([1.0, 3.0]));
 
         let values = [0.1, 0.2, 0.6];
-        assert_relative_eq!(mean(&values), 0.3);
+        assert_relative_eq!(mean(values), 0.3);
     }
 
     #[test]
@@ -108,6 +114,6 @@ mod tests {
     #[test]
     fn test_standard_deviation() {
         let values = [0.1, 0.2, 0.3];
-        assert_relative_eq!(standard_deviation(&values, mean(&values)), 0.1);
+        assert_relative_eq!(standard_deviation(&values, mean(values)), 0.1);
     }
 }

tests/integration_tests.rs

@@ -1,6 +1,8 @@
 mod common;
 use common::{hyperfine, hyperfine_raw_command};
 
+use assert_cmd::assert::OutputAssertExt;
+use insta_cmd::assert_cmd_snapshot;
 use predicates::prelude::*;
 
 /// Platform-specific I/O utility.
@@ -9,12 +11,19 @@ use predicates::prelude::*;
 ///   See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
 const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
 
-pub fn hyperfine_debug() -> assert_cmd::Command {
-    let mut cmd = hyperfine();
+pub fn hyperfine_debug() -> std::process::Command {
+    let mut cmd = hyperfine_raw_command();
     cmd.arg("--debug-mode");
     cmd
 }
 
+fn snapshot_settings() -> insta::Settings {
+    let mut settings = insta::Settings::clone_current();
+    // Ignore trailing report padding while preserving indentation and internal spacing.
+    settings.add_filter(r"(?m)[ \t]+$", "");
+    settings
+}
+
 #[test]
 fn runs_successfully() {
     hyperfine()
@@ -594,86 +603,187 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
 
 #[test]
 fn shows_benchmark_comparison_with_relative_times() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("sleep 1.0")
         .arg("sleep 2.0")
-        .arg("sleep 3.0")
-        .assert()
-        .success()
-        .stdout(
-            predicate::str::contains("2.00 ± 0.00 times faster")
-                .and(predicate::str::contains("3.00 ± 0.00 times faster")),
-        );
+        .arg("sleep 3.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: sleep 1.0
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Benchmark 2: sleep 2.0
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Benchmark 3: sleep 3.0
+      Time (mean ± σ):      3.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    3.000 s …  3.000 s    10 runs
+
+    Summary
+      sleep 1.0 ran
+        2.00 ± 0.00 times faster than sleep 2.0
+        3.00 ± 0.00 times faster than sleep 3.0
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
 fn shows_benchmark_comparison_with_same_time() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("--command-name=A")
         .arg("--command-name=B")
         .arg("sleep 1.0")
         .arg("sleep 1.0")
         .arg("sleep 2.0")
-        .arg("sleep 1000.0")
-        .assert()
-        .success()
-        .stdout(
-            predicate::str::contains("As fast (1.00 ± 0.00) as")
-                .and(predicate::str::contains("2.00 ± 0.00 times faster"))
-                .and(predicate::str::contains("1000.00 ± 0.00 times faster")),
-        );
+        .arg("sleep 1000.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: A
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Benchmark 2: B
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Benchmark 3: sleep 2.0
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Benchmark 4: sleep 1000.0
+      Time (mean ± σ):     1000.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   1000.000 s … 1000.000 s    10 runs
+
+    Summary
+      A ran
+        As fast (1.00 ± 0.00) as B
+        2.00 ± 0.00 times faster than sleep 2.0
+     1000.00 ± 0.00 times faster than sleep 1000.0
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
 fn shows_benchmark_comparison_relative_to_reference() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("--reference=sleep 2.0")
         .arg("sleep 1.0")
-        .arg("sleep 3.0")
-        .assert()
-        .success()
-        .stdout(
-            predicate::str::contains("2.00 ± 0.00 times slower")
-                .and(predicate::str::contains("1.50 ± 0.00 times faster")),
-        );
+        .arg("sleep 3.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: sleep 2.0
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Benchmark 2: sleep 1.0
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Benchmark 3: sleep 3.0
+      Time (mean ± σ):      3.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    3.000 s …  3.000 s    10 runs
+
+    Summary
+      sleep 2.0 ran
+        2.00 ± 0.00 times slower than sleep 1.0
+        1.50 ± 0.00 times faster than sleep 3.0
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
 fn shows_reference_name() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("--reference=sleep 2.0")
         .arg("--reference-name=refabc123")
         .arg("sleep 1.0")
-        .arg("sleep 3.0")
-        .assert()
-        .success()
-        .stdout(predicate::str::contains("Benchmark 1: refabc123"));
+        .arg("sleep 3.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: refabc123
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Benchmark 2: sleep 1.0
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Benchmark 3: sleep 3.0
+      Time (mean ± σ):      3.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    3.000 s …  3.000 s    10 runs
+
+    Summary
+      refabc123 ran
+        2.00 ± 0.00 times slower than sleep 1.0
+        1.50 ± 0.00 times faster than sleep 3.0
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
 fn performs_all_benchmarks_in_parameter_scan() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("--parameter-scan")
         .arg("time")
         .arg("30")
         .arg("45")
         .arg("--parameter-step-size")
         .arg("5")
-        .arg("sleep {time}")
-        .assert()
-        .success()
-        .stdout(
-            predicate::str::contains("Benchmark 1: sleep 30")
-                .and(predicate::str::contains("Benchmark 2: sleep 35"))
-                .and(predicate::str::contains("Benchmark 3: sleep 40"))
-                .and(predicate::str::contains("Benchmark 4: sleep 45"))
-                .and(predicate::str::contains("Benchmark 5: sleep 50").not()),
-        );
+        .arg("sleep {time}"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: sleep 30
+      Time (mean ± σ):     30.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   30.000 s … 30.000 s    10 runs
+
+    Benchmark 2: sleep 35
+      Time (mean ± σ):     35.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   35.000 s … 35.000 s    10 runs
+
+    Benchmark 3: sleep 40
+      Time (mean ± σ):     40.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   40.000 s … 40.000 s    10 runs
+
+    Benchmark 4: sleep 45
+      Time (mean ± σ):     45.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   45.000 s … 45.000 s    10 runs
+
+    Summary
+      sleep 30 ran
+        1.17 ± 0.00 times faster than sleep 35
+        1.33 ± 0.00 times faster than sleep 40
+        1.50 ± 0.00 times faster than sleep 45
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
 fn performs_reference_and_all_benchmarks_in_parameter_scan() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("--reference=sleep 25")
         .arg("--parameter-scan")
         .arg("time")
@@ -681,33 +791,62 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
         .arg("45")
         .arg("--parameter-step-size")
         .arg("5")
-        .arg("sleep {time}")
-        .assert()
-        .success()
-        .stdout(
-            predicate::str::contains("Benchmark 1: sleep 25")
-                .and(predicate::str::contains("Benchmark 2: sleep 30"))
-                .and(predicate::str::contains("Benchmark 3: sleep 35"))
-                .and(predicate::str::contains("Benchmark 4: sleep 40"))
-                .and(predicate::str::contains("Benchmark 5: sleep 45"))
-                .and(predicate::str::contains("Benchmark 6: sleep 50").not()),
-        );
+        .arg("sleep {time}"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: sleep 25
+      Time (mean ± σ):     25.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   25.000 s … 25.000 s    10 runs
+
+    Benchmark 2: sleep 30
+      Time (mean ± σ):     30.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   30.000 s … 30.000 s    10 runs
+
+    Benchmark 3: sleep 35
+      Time (mean ± σ):     35.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   35.000 s … 35.000 s    10 runs
+
+    Benchmark 4: sleep 40
+      Time (mean ± σ):     40.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   40.000 s … 40.000 s    10 runs
+
+    Benchmark 5: sleep 45
+      Time (mean ± σ):     45.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):   45.000 s … 45.000 s    10 runs
+
+    Summary
+      sleep 25 ran
+        1.20 ± 0.00 times faster than sleep 30
+        1.40 ± 0.00 times faster than sleep 35
+        1.60 ± 0.00 times faster than sleep 40
+        1.80 ± 0.00 times faster than sleep 45
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
 fn intermediate_results_are_not_exported_to_stdout() {
-    hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=none") // To only see the Markdown export on stdout
         .arg("--export-markdown")
         .arg("-")
         .arg("sleep 1")
-        .arg("sleep 2")
-        .assert()
-        .success()
-        .stdout(
-            (predicate::str::contains("sleep 1").count(1))
-                .and(predicate::str::contains("sleep 2").count(1)),
-        );
+        .arg("sleep 2"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+
+    | Command | Mean [s] | Min [s] | Max [s] | Relative |
+    |:---|---:|---:|---:|---:|
+    | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
+    | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
+
+
+    ----- stderr -----
+    ");
 }
 
 #[test]
@@ -749,27 +888,56 @@ fn unused_parameters_are_shown_in_benchmark_name() {
 
 #[test]
 fn speed_comparison_sort_order() {
-    for sort_order in ["auto", "mean-time"] {
-        hyperfine_debug()
-            .arg("sleep 2")
-            .arg("sleep 1")
-            .arg(format!("--sort={sort_order}"))
-            .assert()
-            .success()
-            .stdout(predicate::str::contains(
-                "sleep 1 ran\n    2.00 ± 0.00 times faster than sleep 2",
-            ));
+    let _settings = snapshot_settings().bind_to_scope();
+    insta::allow_duplicates! {
+        for sort_order in ["auto", "mean-time"] {
+            assert_cmd_snapshot!(hyperfine_debug()
+                .arg("--style=basic")
+                .arg("sleep 2")
+                .arg("sleep 1")
+                .arg(format!("--sort={sort_order}")), @r"
+            success: true
+            exit_code: 0
+            ----- stdout -----
+            Benchmark 1: sleep 2
+              Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+              Range (min … max):    2.000 s …  2.000 s    10 runs
+
+            Benchmark 2: sleep 1
+              Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+              Range (min … max):    1.000 s …  1.000 s    10 runs
+
+            Summary
+              sleep 1 ran
+                2.00 ± 0.00 times faster than sleep 2
+
+            ----- stderr -----
+            ");
+        }
     }
 
-    hyperfine_debug()
+    assert_cmd_snapshot!(hyperfine_debug()
+        .arg("--style=basic")
         .arg("sleep 2")
         .arg("sleep 1")
-        .arg("--sort=command")
-        .assert()
-        .success()
-        .stdout(predicate::str::contains(
-            "2.00 ±  0.00  sleep 2\n        1.00          sleep 1",
-        ));
+        .arg("--sort=command"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: sleep 2
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Benchmark 2: sleep 1
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Relative speed comparison
+            2.00 ±  0.00  sleep 2
+            1.00          sleep 1
+
+    ----- stderr -----
+    ");
 }
 
 #[cfg(windows)]