Unify timer and executor measurement results

* ci: add native Windows ARM64 releases

* ci: keep existing macOS release runners

.github/workflows/CICD.yml

@@ -89,6 +89,7 @@ jobs:
           - { target: aarch64-apple-darwin        , os: macos-15                      }
           # - { target: x86_64-pc-windows-gnu       , os: windows-2022                  }
           - { target: x86_64-pc-windows-msvc      , os: windows-2022                  }
+          - { target: aarch64-pc-windows-msvc     , os: windows-11-arm                }
           - { target: x86_64-unknown-linux-gnu    , os: ubuntu-24.04, use-cross: true }
           - { target: x86_64-unknown-linux-musl   , os: ubuntu-24.04, use-cross: true }
     env:
@@ -160,7 +161,7 @@ jobs:
       run: |
         # test only library unit tests and binary for arm-type targets
         unset CARGO_TEST_OPTIONS
-        unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ steps.bin.outputs.BIN_NAME }}" ;; esac;
+        unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ needs.crate_metadata.outputs.bin-name }}" ;; esac;
         echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT
 
     - name: Run tests

CHANGELOG.md

@@ -5,7 +5,7 @@
 - Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
 - The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
 - Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
-- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
+- Python scripts: Add new `plot_benchmark_comparison.py` script for plotting collections of benchmarks, see #806 (@marxin)
 
 ## Bugfixes
 
@@ -99,7 +99,7 @@
 ## Other
 
 - Thanks to @berombau for working on dependency upgrades, see #584
-- Fixed installationm on Windows, see #595 and #596 (@AntoniosBarotsis)
+- Fixed installation on Windows, see #595 and #596 (@AntoniosBarotsis)
 
 
 # v1.15.0
@@ -244,10 +244,10 @@ Package authors: note that Hyperfine now comes with a set of shell completion fi
 - The new `--parameter-list <VAR> <VALUES>` option can be used to run
   a parametrized benchmark on a user-specified list of values.
   This is similar to `--parameter-scan <VAR> <MIN> <MAX>`, but doesn't
-  necessarily required numeric arguments.
+  necessarily require numeric arguments.
 
   ``` bash
-  hyperfine --parameter-list compiler "gcc,clang" \
+  hyperfine --parameter-list compiler "g++,clang++" \
       "{compiler} -O2 main.cpp"
   ```
 
@@ -415,7 +415,7 @@ for reviewing pull requests and for giving very valuable feedback.
 ## Other
 
 - [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2).
-- Ubuntu/Debian packages are now are available.
+- Ubuntu/Debian packages are now available.
 
 # v0.2.0
 

README.md

@@ -33,8 +33,9 @@ hyperfine 'sleep 0.3'
 ```
 
 Hyperfine will automatically determine the number of runs to perform for each command. By default,
-it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
-you can use the `-r`/`--runs` option:
+it will perform *at least* 10 benchmarking runs and estimate a run count targeting roughly
+3 seconds, including shell overhead and preparation and conclusion commands. To set an exact
+number of runs, you can use the `-r`/`--runs` option:
 ```sh
 hyperfine --runs 5 'sleep 0.3'
 ```
@@ -56,8 +57,8 @@ hyperfine --warmup 3 'grep -R TODO *'
 ```
 
 Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`
-option to run a special command before *each* timing run. For example, to clear harddisk caches
-on Linux, you can run
+option to run a special command before *each* timing run. For example, to clear Linux filesystem caches,
+you can run
 ```sh
 sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
 ```
@@ -84,12 +85,12 @@ This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
 For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
 option:
 ```
-hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
+hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'
 ```
 
 ### Intermediate shell
 
-By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
+By default, commands are executed using `sh` on Unix (resolved through `PATH`) or `cmd.exe` on Windows.
 If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
 ```sh
 hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
@@ -104,7 +105,7 @@ If you want to run a benchmark *without an intermediate shell*, you can use the
 option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
 produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case.
 ```
-hyperfine -N 'grep TODO /home/user'
+hyperfine -N 'grep -R TODO /home/user'
 ```
 
 
@@ -212,13 +213,6 @@ cave resolve -x repository/rust
 cave resolve -x hyperfine
 ```
 
-### On Funtoo Linux
-
-On Funtoo Linux, hyperfine can be installed [from core-kit](https://github.com/funtoo/core-kit/tree/1.4-release/app-benchmarks/hyperfine):
-```
-emerge app-benchmarks/hyperfine
-```
-
 ### On NixOS
 
 On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine):
@@ -334,7 +328,7 @@ The name *hyperfine* was chosen in reference to the hyperfine levels of caesium
 
 ## Citing hyperfine
 
-Thank you for considering to cite hyperfine in your research work. Please see the information
+Thank you for considering citing hyperfine in your research work. Please see the information
 in the sidebar on how to properly cite hyperfine.
 
 ## License

doc/hyperfine.1

@@ -3,7 +3,8 @@
 hyperfine \- command\-line benchmarking tool
 .SH SYNOPSIS
 .B hyperfine
-.RB [ \-ihVN ]
+.RB [ \-hVN ]
+.RB [ \-\-ignore\-failure[=MODE] ]
 .RB [ \-\-warmup
 .IR NUM ]
 .RB [ \-\-min\-runs
@@ -14,6 +15,10 @@ hyperfine \- command\-line benchmarking tool
 .IR NUM ]
 .RB [ \-\-setup
 .IR CMD ]
+.RB [ \-\-reference
+.IR CMD ]
+.RB [ \-\-reference\-name
+.IR NAME ]
 .RB [ \-\-prepare
 .IR CMD ]
 .RB [ \-\-conclude
@@ -105,10 +110,20 @@ Perform exactly \fINUM\fP runs for each command. If this option is not specified
 \fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP
 .IP
 Execute \fICMD\fP once before each set of timing runs. This is useful
-for compiling your software or with the provided parameters, or to do any
+for compiling your software with the provided parameters, or to do any
 other work that should happen once before a series of benchmark runs,
 not every time as would happen with the \fB\-\-prepare\fR option.
 .HP
+\fB\-\-reference\fR \fICMD\fP
+.IP
+Benchmark \fICMD\fP as the reference for the relative comparison of results.
+If this option is not specified, the fastest command is used as the reference.
+.HP
+\fB\-\-reference\-name\fR \fINAME\fP
+.IP
+Give a meaningful \fINAME\fP to the reference command.
+Requires \fB\-\-reference\fR.
+.HP
 \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP
 .IP
 Execute \fICMD\fP before each timing run. This is useful for clearing disk caches,
@@ -117,7 +132,7 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
 once for each command. In the latter case, each preparation command will be
 run prior to the corresponding benchmark command.
 .HP
-\fB\-\-conclude\fR \fICMD...\fP
+\fB\-C\fR, \fB\-\-conclude\fR \fICMD...\fP
 .IP
 Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
 for example.
@@ -129,8 +144,8 @@ run after the corresponding benchmark command.
 .IP
 Execute \fICMD\fP after the completion of all benchmarking runs for each individual
 command to be benchmarked. This is useful if the commands to be benchmarked
-produce artifacts that need to be cleaned up. It only runs once a series of
-benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
+produce artifacts that need to be cleaned up. It only runs once per series of
+benchmark runs, as opposed to the \fB\-\-conclude\fR option which runs after
 every run.
 .HP
 \fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
@@ -141,14 +156,14 @@ string '{\fIVAR\fP}' in each command by the current parameter value.
 .RS
 Example:
 .RS
-\fBhyperfine\fR \fB\-P\fR threads 1 8 'make \-j {threads}'
+\fBhyperfine\fR \fB\-\-prepare\fR 'make clean' \fB\-P\fR threads 1 8 'make \-j {threads}'
 .RE
 .RE
 .IP
 This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'.
 .IP
 To have the value increase following different patterns, use shell
-arithmetics.
+arithmetic.
 .IP
 .RS
 Example:
@@ -184,11 +199,11 @@ Replaces the string '{\fIVAR\fP}' in each command by the current parameter value
 .RS
 Example:
 .RS
-\fBhyperfine\fR \fB\-L\fR compiler gcc,clang '{compiler} \-O2 main.cpp'
+\fBhyperfine\fR \fB\-L\fR compiler g++,clang++ '{compiler} \-O2 main.cpp'
 .RE
 .RE
 .IP
-This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'.
+This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'.
 .IP
 The option can be specified multiple times to run benchmarks for all
 possible parameter combinations.
@@ -207,9 +222,13 @@ things like "sleep 0.1; sleep 0.2" are not possible without a shell.
 .IP
 An alias for '\-\-shell=none'.
 .HP
-\fB\-i\fR, \fB\-\-ignore\-failure\fR
+\fB\-i\fR, \fB\-\-ignore\-failure\fR[=\fIMODE\fP]
 .IP
-Ignore non\-zero exit codes of the benchmarked programs.
+Ignore failures of the benchmarked programs. Without a value or with
+\fBall\-non\-zero\fR, all non\-zero exit codes are ignored.
+Alternatively, provide a comma\-separated list of exit codes to ignore,
+for example \fB\-\-ignore\-failure=1,2\fR.
+An explicit value must be attached to the option with an equals sign.
 .HP
 \fB\-\-style\fR \fITYPE\fP
 .IP
@@ -309,9 +328,9 @@ Read from \fI\,/dev/null\/\fP (the default).
 Read the input from the given file.
 .RE
 .HP
-\fB\-n\fR, \fB\-\-command\-name\fR \fiNAME\fP
+\fB\-n\fR, \fB\-\-command\-name\fR \fINAME\fP
 .IP
-Give a meaningful \fiNAME\fP to a command. This can be specified multiple times
+Give a meaningful \fINAME\fP to a command. This can be specified multiple times
 if several commands are benchmarked.
 .HP
 \fB\-h\fR, \fB\-\-help\fR
@@ -363,7 +382,6 @@ Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fB\-\-conclude\fR,
 	'echo command n={n}'
 .fi
 .RE
-.RE
 .SH AUTHOR
 .LP
 David Peter <mail@david-peter.de>

scripts/README.md

@@ -1,26 +1,17 @@
-This folder contains scripts that can be used in combination with hyperfines `--export-json` option.
+This folder contains scripts that can be used in combination with hyperfine's `--export-json` option.
 
 ### Example:
 
 ```bash
 hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
-./plot_whisker.py sleep.json
-```
-
-### Pre-requisites
-
-To make these scripts work, you will need `numpy`, `matplotlib` and `scipy`.
-
-If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/)
-inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx),
-you can directly run the scripts using
-
-```bash
 uv run plot_whisker.py sleep.json
 ```
 
-Otherwise, install the dependencies via your system package manager or using `pip`:
+### Prerequisites
 
-```bash
-pip install numpy matplotlib scipy  # pip3, if you are using python3
-```
+Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
+commands above from this directory.
+
+The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
+inline script metadata. `uv run` automatically installs these dependencies in an
+isolated environment before running the script.

scripts/welch_ttest.py

@@ -7,8 +7,7 @@
 # ///
 
 """This script performs Welch's t-test on a JSON export file with two
-benchmark results to test whether or not the two distributions are
-the same."""
+benchmark results to test for a difference in their population mean runtimes."""
 
 import argparse
 import json
@@ -40,6 +39,10 @@ print(f"t = {t:.3}, p = {p:.3}")
 print()
 
 if dispose:
-    print(f"There is a difference between the two benchmarks (p < {th}).")
+    print(
+        f"A statistically significant difference in mean runtime was detected (p < {th})."
+    )
 else:
-    print(f"The two benchmarks are almost the same (p >= {th}).")
+    print(
+        f"No statistically significant difference in mean runtime was detected (p >= {th})."
+    )

src/benchmark/executor.rs

@@ -7,11 +7,11 @@ use crate::options::{
     CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
 };
 use crate::output::progress_bar::get_progress_bar;
-use crate::timer::{execute_and_measure, TimerResult};
+use crate::timer::execute_and_measure;
 use crate::util::randomized_environment_offset;
 use crate::util::units::Second;
 
-use super::timing_result::TimingResult;
+use super::measurement::Measurement;
 
 use anyhow::{bail, Context, Result};
 use statistical::mean;
@@ -40,7 +40,7 @@ pub trait Executor {
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)>;
+    ) -> Result<Measurement>;
 
     /// Perform a calibration of this executor. For example,
     /// when running commands through a shell, we need to
@@ -62,7 +62,7 @@ fn run_command_and_measure_common(
     command_input_policy: &CommandInputPolicy,
     command_output_policy: &CommandOutputPolicy,
     command_name: &str,
-) -> Result<TimerResult> {
+) -> Result<Measurement> {
     let stdin = command_input_policy.get_stdin()?;
     let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
     command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -136,25 +136,15 @@ impl Executor for RawExecutor<'_> {
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)> {
-        let result = run_command_and_measure_common(
+    ) -> Result<Measurement> {
+        run_command_and_measure_common(
             command.get_command()?,
             iteration,
             command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
             &self.options.command_input_policy,
             output_policy,
             &command.get_command_line(),
-        )?;
-
-        Ok((
-            TimingResult {
-                time_real: result.time_real,
-                time_user: result.time_user,
-                time_system: result.time_system,
-                memory_usage_byte: result.memory_usage_byte,
-            },
-            result.status,
-        ))
+        )
     }
 
     fn calibrate(&mut self) -> Result<()> {
@@ -169,7 +159,7 @@ impl Executor for RawExecutor<'_> {
 pub struct ShellExecutor<'a> {
     options: &'a Options,
     shell: &'a Shell,
-    shell_spawning_time: Option<TimingResult>,
+    shell_spawning_time: Option<Measurement>,
 }
 
 impl<'a> ShellExecutor<'a> {
@@ -189,7 +179,7 @@ impl Executor for ShellExecutor<'_> {
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)> {
+    ) -> Result<Measurement> {
         let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
         let mut command_builder = self.shell.command();
         command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
@@ -218,15 +208,7 @@ impl Executor for ShellExecutor<'_> {
             result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
         }
 
-        Ok((
-            TimingResult {
-                time_real: result.time_real,
-                time_user: result.time_user,
-                time_system: result.time_system,
-                memory_usage_byte: result.memory_usage_byte,
-            },
-            result.status,
-        ))
+        Ok(result)
     }
 
     /// Measure the average shell spawning time
@@ -268,7 +250,7 @@ impl Executor for ShellExecutor<'_> {
                         shell_cmd
                     );
                 }
-                Ok((r, _)) => {
+                Ok(r) => {
                     times_real.push(r.time_real);
                     times_user.push(r.time_user);
                     times_system.push(r.time_system);
@@ -284,11 +266,12 @@ impl Executor for ShellExecutor<'_> {
             bar.finish_and_clear()
         }
 
-        self.shell_spawning_time = Some(TimingResult {
+        self.shell_spawning_time = Some(Measurement {
             time_real: mean(&times_real),
             time_user: mean(&times_user),
             time_system: mean(&times_system),
             memory_usage_byte: 0,
+            status: ExitStatus::default(),
         });
 
         Ok(())
@@ -326,7 +309,7 @@ impl Executor for MockExecutor {
         _iteration: BenchmarkIteration,
         _command_failure_action: Option<CmdFailureAction>,
         _output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)> {
+    ) -> Result<Measurement> {
         #[cfg(unix)]
         let status = {
             use std::os::unix::process::ExitStatusExt;
@@ -339,15 +322,13 @@ impl Executor for MockExecutor {
             ExitStatus::from_raw(0)
         };
 
-        Ok((
-            TimingResult {
-                time_real: Self::extract_time(command.get_command_line()),
-                time_user: 0.0,
-                time_system: 0.0,
-                memory_usage_byte: 0,
-            },
+        Ok(Measurement {
+            time_real: Self::extract_time(command.get_command_line()),
+            time_user: 0.0,
+            time_system: 0.0,
+            memory_usage_byte: 0,
             status,
-        ))
+        })
     }
 
     fn calibrate(&mut self) -> Result<()> {

src/benchmark/measurement.rs

@@ -1,8 +1,10 @@
+use std::process::ExitStatus;
+
 use crate::util::units::Second;
 
-/// Results from timing a single command
+/// Performance measurements and exit status from running a single command
 #[derive(Debug, Default, Copy, Clone)]
-pub struct TimingResult {
+pub struct Measurement {
     /// Wall clock time
     pub time_real: Second,
 
@@ -14,4 +16,7 @@ pub struct TimingResult {
 
     /// Maximum amount of memory used, in bytes
     pub memory_usage_byte: u64,
+
+    /// The exit status of the process
+    pub status: ExitStatus,
 }

src/benchmark/mod.rs

@@ -1,8 +1,8 @@
 pub mod benchmark_result;
 pub mod executor;
+pub mod measurement;
 pub mod relative_speed;
 pub mod scheduler;
-pub mod timing_result;
 
 use std::cmp;
 use std::io::{self, Write};
@@ -22,7 +22,7 @@ use crate::util::exit_code::extract_exit_code;
 use crate::util::min_max::{max, min};
 use crate::util::units::Second;
 use benchmark_result::BenchmarkResult;
-use timing_result::TimingResult;
+use measurement::Measurement;
 
 use anyhow::{anyhow, Result};
 use colored::*;
@@ -61,7 +61,7 @@ impl<'a> Benchmark<'a> {
         command: &Command<'_>,
         error_output: &'static str,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         self.executor
             .run_command_and_measure(
                 command,
@@ -69,7 +69,6 @@ impl<'a> Benchmark<'a> {
                 Some(CmdFailureAction::RaiseError),
                 output_policy,
             )
-            .map(|r| r.0)
             .map_err(|_| anyhow!(error_output))
     }
 
@@ -78,7 +77,7 @@ impl<'a> Benchmark<'a> {
         &self,
         parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let command = self
             .options
             .setup_command
@@ -99,7 +98,7 @@ impl<'a> Benchmark<'a> {
         &self,
         parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let command = self
             .options
             .cleanup_command
@@ -120,7 +119,7 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let error_output = "The preparation command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
@@ -132,7 +131,7 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let error_output = "The conclusion command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
@@ -247,13 +246,13 @@ impl<'a> Benchmark<'a> {
             preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
 
         // Initial timing run
-        let (res, status) = self.executor.run_command_and_measure(
+        let res = self.executor.run_command_and_measure(
             self.command,
             BenchmarkIteration::Benchmark(0),
             None,
             output_policy,
         )?;
-        let success = status.success();
+        let success = res.status.success();
 
         let conclusion_result = run_conclusion_command()?;
         let conclusion_overhead =
@@ -284,7 +283,7 @@ impl<'a> Benchmark<'a> {
         times_user.push(res.time_user);
         times_system.push(res.time_system);
         memory_usage_byte.push(res.memory_usage_byte);
-        exit_codes.push(extract_exit_code(status));
+        exit_codes.push(extract_exit_code(res.status));
 
         all_succeeded = all_succeeded && success;
 
@@ -309,19 +308,19 @@ impl<'a> Benchmark<'a> {
                 bar.set_message(msg.to_owned())
             }
 
-            let (res, status) = self.executor.run_command_and_measure(
+            let res = self.executor.run_command_and_measure(
                 self.command,
                 BenchmarkIteration::Benchmark(i + 1),
                 None,
                 output_policy,
             )?;
-            let success = status.success();
+            let success = res.status.success();
 
             times_real.push(res.time_real);
             times_user.push(res.time_user);
             times_system.push(res.time_system);
             memory_usage_byte.push(res.memory_usage_byte);
-            exit_codes.push(extract_exit_code(status));
+            exit_codes.push(extract_exit_code(res.status));
 
             all_succeeded = all_succeeded && success;
 

src/cli.rs

@@ -41,6 +41,7 @@ fn build_command() -> Command {
                 .short('w')
                 .value_name("NUM")
                 .action(ArgAction::Set)
+                .value_parser(clap::value_parser!(u64))
                 .help(
                     "Perform NUM warmup runs before the actual benchmark. This can be used \
                      to fill (disk) caches for I/O-heavy programs.",
@@ -52,6 +53,7 @@ fn build_command() -> Command {
                 .short('m')
                 .action(ArgAction::Set)
                 .value_name("NUM")
+                .value_parser(clap::value_parser!(u64).range(1..))
                 .help("Perform at least NUM runs for each command (default: 10)."),
         )
         .arg(
@@ -60,6 +62,7 @@ fn build_command() -> Command {
                 .short('M')
                 .action(ArgAction::Set)
                 .value_name("NUM")
+                .value_parser(clap::value_parser!(u64).range(1..))
                 .help("Perform at most NUM runs for each command. By default, there is no limit."),
         )
         .arg(
@@ -69,6 +72,7 @@ fn build_command() -> Command {
                 .short('r')
                 .action(ArgAction::Set)
                 .value_name("NUM")
+                .value_parser(clap::value_parser!(u64).range(1..))
                 .help("Perform exactly NUM runs for each command. If this option is not specified, \
                        hyperfine automatically determines the number of runs."),
         )
@@ -161,7 +165,7 @@ fn build_command() -> Command {
                 .help(
                     "Perform benchmark runs for each value in the range MIN..MAX. Replaces the \
                      string '{VAR}' in each command by the current parameter value.\n\n  \
-                     Example:  hyperfine -P threads 1 8 'make -j {threads}'\n\n\
+                     Example:  hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\
                      This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
                      To have the value increase following different patterns, use shell arithmetics.\n\n  \
                      Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\
@@ -194,8 +198,8 @@ fn build_command() -> Command {
                 .help(
                     "Perform benchmark runs for each value in the comma-separated list VALUES. \
                      Replaces the string '{VAR}' in each command by the current parameter value\
-                     .\n\nExample:  hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\n\n\
-                     This performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp'.\n\n\
+                     .\n\nExample:  hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'\n\n\
+                     This performs benchmarks for 'g++ -O2 main.cpp' and 'clang++ -O2 main.cpp'.\n\n\
                      The option can be specified multiple times to run benchmarks for all \
                      possible parameter combinations.\n"
                 ),

src/options.rs

@@ -275,22 +275,15 @@ impl Default for Options {
 impl Options {
     pub fn from_cli_arguments<'a>(matches: &ArgMatches) -> Result<Self, OptionsError<'a>> {
         let mut options = Self::default();
-        let param_to_u64 = |param| {
-            matches
-                .get_one::<String>(param)
-                .map(|n| {
-                    n.parse::<u64>()
-                        .map_err(|e| OptionsError::IntParsingError(param, e))
-                })
-                .transpose()
-        };
+        options.warmup_count = matches
+            .get_one::<u64>("warmup")
+            .copied()
+            .unwrap_or(options.warmup_count);
 
-        options.warmup_count = param_to_u64("warmup")?.unwrap_or(options.warmup_count);
+        let mut min_runs = matches.get_one::<u64>("min-runs").copied();
+        let mut max_runs = matches.get_one::<u64>("max-runs").copied();
 
-        let mut min_runs = param_to_u64("min-runs")?;
-        let mut max_runs = param_to_u64("max-runs")?;
-
-        if let Some(runs) = param_to_u64("runs")? {
+        if let Some(&runs) = matches.get_one::<u64>("runs") {
             min_runs = Some(runs);
             max_runs = Some(runs);
         }

src/timer/mod.rs

@@ -16,11 +16,11 @@ use std::os::fd::AsFd;
 #[cfg(target_os = "windows")]
 use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
 
-use crate::util::units::Second;
+use crate::benchmark::measurement::Measurement;
 use wall_clock_timer::WallClockTimer;
 
 use std::io::Read;
-use std::process::{ChildStdout, Command, ExitStatus};
+use std::process::{ChildStdout, Command};
 
 use anyhow::Result;
 
@@ -37,17 +37,6 @@ struct CPUTimes {
     pub memory_usage_byte: u64,
 }
 
-/// Used to indicate the result of running a command
-#[derive(Debug, Copy, Clone)]
-pub struct TimerResult {
-    pub time_real: Second,
-    pub time_user: Second,
-    pub time_system: Second,
-    pub memory_usage_byte: u64,
-    /// The exit status of the process
-    pub status: ExitStatus,
-}
-
 /// Discard the output of a child process.
 fn discard(output: ChildStdout) {
     const CHUNK_SIZE: usize = 64 << 10;
@@ -80,7 +69,7 @@ fn discard(output: ChildStdout) {
 }
 
 /// Execute the given command and return a timing summary
-pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
+pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
     #[cfg(not(windows))]
     let cpu_timer = self::unix_timer::CPUTimer::start();
 
@@ -111,7 +100,7 @@ pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
     let time_real = wallclock_timer.stop();
     let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
 
-    Ok(TimerResult {
+    Ok(Measurement {
         time_real,
         time_user,
         time_system,

tests/integration_tests.rs

@@ -74,6 +74,32 @@ fn exits_quietly_when_stdout_is_closed() {
     }
 }
 
+#[test]
+fn fails_with_zero_runs() {
+    for option in ["--runs", "--min-runs", "--max-runs"] {
+        hyperfine()
+            .arg(option)
+            .arg("0")
+            .arg("echo dummy benchmark")
+            .assert()
+            .code(2)
+            .stderr(predicate::str::contains(format!(
+                "invalid value '0' for '{option} <NUM>'"
+            )));
+    }
+}
+
+#[test]
+fn min_runs_of_one_still_performs_one_run() {
+    hyperfine_debug()
+        .arg("--min-runs=1")
+        .arg("--warmup=0")
+        .arg("sleep 4")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Time (abs ≡)"));
+}
+
 #[test]
 fn can_run_commands_without_a_shell() {
     hyperfine()