Compare commits

..
13 changed files with 128 additions and 141 deletions

No files matched your search

+1 -2
View File
@@ -89,7 +89,6 @@ jobs:
- { target: aarch64-apple-darwin , os: macos-15 } - { target: aarch64-apple-darwin , os: macos-15 }
# - { target: x86_64-pc-windows-gnu , os: windows-2022 } # - { target: x86_64-pc-windows-gnu , os: windows-2022 }
- { target: x86_64-pc-windows-msvc , os: windows-2022 } - { target: x86_64-pc-windows-msvc , os: windows-2022 }
- { target: aarch64-pc-windows-msvc , os: windows-11-arm }
- { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } - { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
- { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } - { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
env: env:
@@ -161,7 +160,7 @@ jobs:
run: | run: |
# test only library unit tests and binary for arm-type targets # test only library unit tests and binary for arm-type targets
unset CARGO_TEST_OPTIONS unset CARGO_TEST_OPTIONS
unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ needs.crate_metadata.outputs.bin-name }}" ;; esac; unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ steps.bin.outputs.BIN_NAME }}" ;; esac;
echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT
- name: Run tests - name: Run tests
+5 -5
View File
@@ -5,7 +5,7 @@
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally) - Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp) - The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp) - Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
- Python scripts: Add new `plot_benchmark_comparison.py` script for plotting collections of benchmarks, see #806 (@marxin) - Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
## Bugfixes ## Bugfixes
@@ -99,7 +99,7 @@
## Other ## Other
- Thanks to @berombau for working on dependency upgrades, see #584 - Thanks to @berombau for working on dependency upgrades, see #584
- Fixed installation on Windows, see #595 and #596 (@AntoniosBarotsis) - Fixed installationm on Windows, see #595 and #596 (@AntoniosBarotsis)
# v1.15.0 # v1.15.0
@@ -244,10 +244,10 @@ Package authors: note that Hyperfine now comes with a set of shell completion fi
- The new `--parameter-list <VAR> <VALUES>` option can be used to run - The new `--parameter-list <VAR> <VALUES>` option can be used to run
a parametrized benchmark on a user-specified list of values. a parametrized benchmark on a user-specified list of values.
This is similar to `--parameter-scan <VAR> <MIN> <MAX>`, but doesn't This is similar to `--parameter-scan <VAR> <MIN> <MAX>`, but doesn't
necessarily require numeric arguments. necessarily required numeric arguments.
``` bash ``` bash
hyperfine --parameter-list compiler "g++,clang++" \ hyperfine --parameter-list compiler "gcc,clang" \
"{compiler} -O2 main.cpp" "{compiler} -O2 main.cpp"
``` ```
@@ -415,7 +415,7 @@ for reviewing pull requests and for giving very valuable feedback.
## Other ## Other
- [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2). - [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2).
- Ubuntu/Debian packages are now available. - Ubuntu/Debian packages are now are available.
# v0.2.0 # v0.2.0
+15 -9
View File
@@ -33,9 +33,8 @@ hyperfine 'sleep 0.3'
``` ```
Hyperfine will automatically determine the number of runs to perform for each command. By default, Hyperfine will automatically determine the number of runs to perform for each command. By default,
it will perform *at least* 10 benchmarking runs and estimate a run count targeting roughly it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
3 seconds, including shell overhead and preparation and conclusion commands. To set an exact you can use the `-r`/`--runs` option:
number of runs, you can use the `-r`/`--runs` option:
```sh ```sh
hyperfine --runs 5 'sleep 0.3' hyperfine --runs 5 'sleep 0.3'
``` ```
@@ -57,8 +56,8 @@ hyperfine --warmup 3 'grep -R TODO *'
``` ```
Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare` Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`
option to run a special command before *each* timing run. For example, to clear Linux filesystem caches, option to run a special command before *each* timing run. For example, to clear harddisk caches
you can run on Linux, you can run
```sh ```sh
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
``` ```
@@ -85,12 +84,12 @@ This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list` For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
option: option:
``` ```
hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp' hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
``` ```
### Intermediate shell ### Intermediate shell
By default, commands are executed using `sh` on Unix (resolved through `PATH`) or `cmd.exe` on Windows. By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option: If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
```sh ```sh
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done' hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
@@ -105,7 +104,7 @@ If you want to run a benchmark *without an intermediate shell*, you can use the
option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case. produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case.
``` ```
hyperfine -N 'grep -R TODO /home/user' hyperfine -N 'grep TODO /home/user'
``` ```
@@ -213,6 +212,13 @@ cave resolve -x repository/rust
cave resolve -x hyperfine cave resolve -x hyperfine
``` ```
### On Funtoo Linux
On Funtoo Linux, hyperfine can be installed [from core-kit](https://github.com/funtoo/core-kit/tree/1.4-release/app-benchmarks/hyperfine):
```
emerge app-benchmarks/hyperfine
```
### On NixOS ### On NixOS
On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine): On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine):
@@ -328,7 +334,7 @@ The name *hyperfine* was chosen in reference to the hyperfine levels of caesium
## Citing hyperfine ## Citing hyperfine
Thank you for considering citing hyperfine in your research work. Please see the information Thank you for considering to cite hyperfine in your research work. Please see the information
in the sidebar on how to properly cite hyperfine. in the sidebar on how to properly cite hyperfine.
## License ## License
+14 -32
View File
@@ -3,8 +3,7 @@
hyperfine \- command\-line benchmarking tool hyperfine \- command\-line benchmarking tool
.SH SYNOPSIS .SH SYNOPSIS
.B hyperfine .B hyperfine
.RB [ \-hVN ] .RB [ \-ihVN ]
.RB [ \-\-ignore\-failure[=MODE] ]
.RB [ \-\-warmup .RB [ \-\-warmup
.IR NUM ] .IR NUM ]
.RB [ \-\-min\-runs .RB [ \-\-min\-runs
@@ -15,10 +14,6 @@ hyperfine \- command\-line benchmarking tool
.IR NUM ] .IR NUM ]
.RB [ \-\-setup .RB [ \-\-setup
.IR CMD ] .IR CMD ]
.RB [ \-\-reference
.IR CMD ]
.RB [ \-\-reference\-name
.IR NAME ]
.RB [ \-\-prepare .RB [ \-\-prepare
.IR CMD ] .IR CMD ]
.RB [ \-\-conclude .RB [ \-\-conclude
@@ -110,20 +105,10 @@ Perform exactly \fINUM\fP runs for each command. If this option is not specified
\fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP \fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP
.IP .IP
Execute \fICMD\fP once before each set of timing runs. This is useful Execute \fICMD\fP once before each set of timing runs. This is useful
for compiling your software with the provided parameters, or to do any for compiling your software or with the provided parameters, or to do any
other work that should happen once before a series of benchmark runs, other work that should happen once before a series of benchmark runs,
not every time as would happen with the \fB\-\-prepare\fR option. not every time as would happen with the \fB\-\-prepare\fR option.
.HP .HP
\fB\-\-reference\fR \fICMD\fP
.IP
Benchmark \fICMD\fP as the reference for the relative comparison of results.
If this option is not specified, the fastest command is used as the reference.
.HP
\fB\-\-reference\-name\fR \fINAME\fP
.IP
Give a meaningful \fINAME\fP to the reference command.
Requires \fB\-\-reference\fR.
.HP
\fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP
.IP .IP
Execute \fICMD\fP before each timing run. This is useful for clearing disk caches, Execute \fICMD\fP before each timing run. This is useful for clearing disk caches,
@@ -132,7 +117,7 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
once for each command. In the latter case, each preparation command will be once for each command. In the latter case, each preparation command will be
run prior to the corresponding benchmark command. run prior to the corresponding benchmark command.
.HP .HP
\fB\-C\fR, \fB\-\-conclude\fR \fICMD...\fP \fB\-\-conclude\fR \fICMD...\fP
.IP .IP
Execute \fICMD\fP after each timing run. This is useful for clearing disk caches, Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
for example. for example.
@@ -144,8 +129,8 @@ run after the corresponding benchmark command.
.IP .IP
Execute \fICMD\fP after the completion of all benchmarking runs for each individual Execute \fICMD\fP after the completion of all benchmarking runs for each individual
command to be benchmarked. This is useful if the commands to be benchmarked command to be benchmarked. This is useful if the commands to be benchmarked
produce artifacts that need to be cleaned up. It only runs once per series of produce artifacts that need to be cleaned up. It only runs once a series of
benchmark runs, as opposed to the \fB\-\-conclude\fR option which runs after benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
every run. every run.
.HP .HP
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP \fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
@@ -156,14 +141,14 @@ string '{\fIVAR\fP}' in each command by the current parameter value.
.RS .RS
Example: Example:
.RS .RS
\fBhyperfine\fR \fB\-\-prepare\fR 'make clean' \fB\-P\fR threads 1 8 'make \-j {threads}' \fBhyperfine\fR \fB\-P\fR threads 1 8 'make \-j {threads}'
.RE .RE
.RE .RE
.IP .IP
This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'. This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'.
.IP .IP
To have the value increase following different patterns, use shell To have the value increase following different patterns, use shell
arithmetic. arithmetics.
.IP .IP
.RS .RS
Example: Example:
@@ -199,11 +184,11 @@ Replaces the string '{\fIVAR\fP}' in each command by the current parameter value
.RS .RS
Example: Example:
.RS .RS
\fBhyperfine\fR \fB\-L\fR compiler g++,clang++ '{compiler} \-O2 main.cpp' \fBhyperfine\fR \fB\-L\fR compiler gcc,clang '{compiler} \-O2 main.cpp'
.RE .RE
.RE .RE
.IP .IP
This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'. This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'.
.IP .IP
The option can be specified multiple times to run benchmarks for all The option can be specified multiple times to run benchmarks for all
possible parameter combinations. possible parameter combinations.
@@ -222,13 +207,9 @@ things like "sleep 0.1; sleep 0.2" are not possible without a shell.
.IP .IP
An alias for '\-\-shell=none'. An alias for '\-\-shell=none'.
.HP .HP
\fB\-i\fR, \fB\-\-ignore\-failure\fR[=\fIMODE\fP] \fB\-i\fR, \fB\-\-ignore\-failure\fR
.IP .IP
Ignore failures of the benchmarked programs. Without a value or with Ignore non\-zero exit codes of the benchmarked programs.
\fBall\-non\-zero\fR, all non\-zero exit codes are ignored.
Alternatively, provide a comma\-separated list of exit codes to ignore,
for example \fB\-\-ignore\-failure=1,2\fR.
An explicit value must be attached to the option with an equals sign.
.HP .HP
\fB\-\-style\fR \fITYPE\fP \fB\-\-style\fR \fITYPE\fP
.IP .IP
@@ -328,9 +309,9 @@ Read from \fI\,/dev/null\/\fP (the default).
Read the input from the given file. Read the input from the given file.
.RE .RE
.HP .HP
\fB\-n\fR, \fB\-\-command\-name\fR \fINAME\fP \fB\-n\fR, \fB\-\-command\-name\fR \fiNAME\fP
.IP .IP
Give a meaningful \fINAME\fP to a command. This can be specified multiple times Give a meaningful \fiNAME\fP to a command. This can be specified multiple times
if several commands are benchmarked. if several commands are benchmarked.
.HP .HP
\fB\-h\fR, \fB\-\-help\fR \fB\-h\fR, \fB\-\-help\fR
@@ -382,6 +363,7 @@ Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fB\-\-conclude\fR,
'echo command n={n}' 'echo command n={n}'
.fi .fi
.RE .RE
.RE
.SH AUTHOR .SH AUTHOR
.LP .LP
David Peter <mail@david-peter.de> David Peter <mail@david-peter.de>
+2 -2
View File
@@ -1,4 +1,4 @@
This folder contains scripts that can be used in combination with hyperfine's `--export-json` option. This folder contains scripts that can be used in combination with hyperfines `--export-json` option.
### Example: ### Example:
@@ -7,7 +7,7 @@ hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
uv run plot_whisker.py sleep.json uv run plot_whisker.py sleep.json
``` ```
### Prerequisites ### Pre-requisites
Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
commands above from this directory. commands above from this directory.
+4 -7
View File
@@ -7,7 +7,8 @@
# /// # ///
"""This script performs Welch's t-test on a JSON export file with two """This script performs Welch's t-test on a JSON export file with two
benchmark results to test for a difference in their population mean runtimes.""" benchmark results to test whether or not the two distributions are
the same."""
import argparse import argparse
import json import json
@@ -39,10 +40,6 @@ print(f"t = {t:.3}, p = {p:.3}")
print() print()
if dispose: if dispose:
print( print(f"There is a difference between the two benchmarks (p < {th}).")
f"A statistically significant difference in mean runtime was detected (p < {th})."
)
else: else:
print( print(f"The two benchmarks are almost the same (p >= {th}).")
f"No statistically significant difference in mean runtime was detected (p >= {th})."
)
+39 -20
View File
@@ -7,11 +7,11 @@ use crate::options::{
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell, CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
}; };
use crate::output::progress_bar::get_progress_bar; use crate::output::progress_bar::get_progress_bar;
use crate::timer::execute_and_measure; use crate::timer::{execute_and_measure, TimerResult};
use crate::util::randomized_environment_offset; use crate::util::randomized_environment_offset;
use crate::util::units::Second; use crate::util::units::Second;
use super::measurement::Measurement; use super::timing_result::TimingResult;
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use statistical::mean; use statistical::mean;
@@ -40,7 +40,7 @@ pub trait Executor {
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement>; ) -> Result<(TimingResult, ExitStatus)>;
/// Perform a calibration of this executor. For example, /// Perform a calibration of this executor. For example,
/// when running commands through a shell, we need to /// when running commands through a shell, we need to
@@ -62,7 +62,7 @@ fn run_command_and_measure_common(
command_input_policy: &CommandInputPolicy, command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy, command_output_policy: &CommandOutputPolicy,
command_name: &str, command_name: &str,
) -> Result<Measurement> { ) -> Result<TimerResult> {
let stdin = command_input_policy.get_stdin()?; let stdin = command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?; let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
command.stdin(stdin).stdout(stdout).stderr(stderr); command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -136,15 +136,25 @@ impl Executor for RawExecutor<'_> {
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<(TimingResult, ExitStatus)> {
run_command_and_measure_common( let result = run_command_and_measure_common(
command.get_command()?, command.get_command()?,
iteration, iteration,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy, &self.options.command_input_policy,
output_policy, output_policy,
&command.get_command_line(), &command.get_command_line(),
) )?;
Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
} }
fn calibrate(&mut self) -> Result<()> { fn calibrate(&mut self) -> Result<()> {
@@ -159,7 +169,7 @@ impl Executor for RawExecutor<'_> {
pub struct ShellExecutor<'a> { pub struct ShellExecutor<'a> {
options: &'a Options, options: &'a Options,
shell: &'a Shell, shell: &'a Shell,
shell_spawning_time: Option<Measurement>, shell_spawning_time: Option<TimingResult>,
} }
impl<'a> ShellExecutor<'a> { impl<'a> ShellExecutor<'a> {
@@ -179,7 +189,7 @@ impl Executor for ShellExecutor<'_> {
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<(TimingResult, ExitStatus)> {
let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
let mut command_builder = self.shell.command(); let mut command_builder = self.shell.command();
command_builder.arg(if on_windows_cmd { "/C" } else { "-c" }); command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
@@ -208,7 +218,15 @@ impl Executor for ShellExecutor<'_> {
result.time_system = (result.time_system - spawning_time.time_system).max(0.0); result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
} }
Ok(result) Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
} }
/// Measure the average shell spawning time /// Measure the average shell spawning time
@@ -250,7 +268,7 @@ impl Executor for ShellExecutor<'_> {
shell_cmd shell_cmd
); );
} }
Ok(r) => { Ok((r, _)) => {
times_real.push(r.time_real); times_real.push(r.time_real);
times_user.push(r.time_user); times_user.push(r.time_user);
times_system.push(r.time_system); times_system.push(r.time_system);
@@ -266,12 +284,11 @@ impl Executor for ShellExecutor<'_> {
bar.finish_and_clear() bar.finish_and_clear()
} }
self.shell_spawning_time = Some(Measurement { self.shell_spawning_time = Some(TimingResult {
time_real: mean(&times_real), time_real: mean(&times_real),
time_user: mean(&times_user), time_user: mean(&times_user),
time_system: mean(&times_system), time_system: mean(&times_system),
memory_usage_byte: 0, memory_usage_byte: 0,
status: ExitStatus::default(),
}); });
Ok(()) Ok(())
@@ -309,7 +326,7 @@ impl Executor for MockExecutor {
_iteration: BenchmarkIteration, _iteration: BenchmarkIteration,
_command_failure_action: Option<CmdFailureAction>, _command_failure_action: Option<CmdFailureAction>,
_output_policy: &CommandOutputPolicy, _output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<(TimingResult, ExitStatus)> {
#[cfg(unix)] #[cfg(unix)]
let status = { let status = {
use std::os::unix::process::ExitStatusExt; use std::os::unix::process::ExitStatusExt;
@@ -322,13 +339,15 @@ impl Executor for MockExecutor {
ExitStatus::from_raw(0) ExitStatus::from_raw(0)
}; };
Ok(Measurement { Ok((
time_real: Self::extract_time(command.get_command_line()), TimingResult {
time_user: 0.0, time_real: Self::extract_time(command.get_command_line()),
time_system: 0.0, time_user: 0.0,
memory_usage_byte: 0, time_system: 0.0,
memory_usage_byte: 0,
},
status, status,
}) ))
} }
fn calibrate(&mut self) -> Result<()> { fn calibrate(&mut self) -> Result<()> {
+14 -13
View File
@@ -1,8 +1,8 @@
pub mod benchmark_result; pub mod benchmark_result;
pub mod executor; pub mod executor;
pub mod measurement;
pub mod relative_speed; pub mod relative_speed;
pub mod scheduler; pub mod scheduler;
pub mod timing_result;
use std::cmp; use std::cmp;
use std::io::{self, Write}; use std::io::{self, Write};
@@ -22,7 +22,7 @@ use crate::util::exit_code::extract_exit_code;
use crate::util::min_max::{max, min}; use crate::util::min_max::{max, min};
use crate::util::units::Second; use crate::util::units::Second;
use benchmark_result::BenchmarkResult; use benchmark_result::BenchmarkResult;
use measurement::Measurement; use timing_result::TimingResult;
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use colored::*; use colored::*;
@@ -61,7 +61,7 @@ impl<'a> Benchmark<'a> {
command: &Command<'_>, command: &Command<'_>,
error_output: &'static str, error_output: &'static str,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
self.executor self.executor
.run_command_and_measure( .run_command_and_measure(
command, command,
@@ -69,6 +69,7 @@ impl<'a> Benchmark<'a> {
Some(CmdFailureAction::RaiseError), Some(CmdFailureAction::RaiseError),
output_policy, output_policy,
) )
.map(|r| r.0)
.map_err(|_| anyhow!(error_output)) .map_err(|_| anyhow!(error_output))
} }
@@ -77,7 +78,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>, parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let command = self let command = self
.options .options
.setup_command .setup_command
@@ -98,7 +99,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>, parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let command = self let command = self
.options .options
.cleanup_command .cleanup_command
@@ -119,7 +120,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
command: &Command<'_>, command: &Command<'_>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let error_output = "The preparation command terminated with a non-zero exit code. \ let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored."; Append ' || true' to the command if you are sure that this can be ignored.";
@@ -131,7 +132,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
command: &Command<'_>, command: &Command<'_>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let error_output = "The conclusion command terminated with a non-zero exit code. \ let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored."; Append ' || true' to the command if you are sure that this can be ignored.";
@@ -246,13 +247,13 @@ impl<'a> Benchmark<'a> {
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
// Initial timing run // Initial timing run
let res = self.executor.run_command_and_measure( let (res, status) = self.executor.run_command_and_measure(
self.command, self.command,
BenchmarkIteration::Benchmark(0), BenchmarkIteration::Benchmark(0),
None, None,
output_policy, output_policy,
)?; )?;
let success = res.status.success(); let success = status.success();
let conclusion_result = run_conclusion_command()?; let conclusion_result = run_conclusion_command()?;
let conclusion_overhead = let conclusion_overhead =
@@ -283,7 +284,7 @@ impl<'a> Benchmark<'a> {
times_user.push(res.time_user); times_user.push(res.time_user);
times_system.push(res.time_system); times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte); memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(res.status)); exit_codes.push(extract_exit_code(status));
all_succeeded = all_succeeded && success; all_succeeded = all_succeeded && success;
@@ -308,19 +309,19 @@ impl<'a> Benchmark<'a> {
bar.set_message(msg.to_owned()) bar.set_message(msg.to_owned())
} }
let res = self.executor.run_command_and_measure( let (res, status) = self.executor.run_command_and_measure(
self.command, self.command,
BenchmarkIteration::Benchmark(i + 1), BenchmarkIteration::Benchmark(i + 1),
None, None,
output_policy, output_policy,
)?; )?;
let success = res.status.success(); let success = status.success();
times_real.push(res.time_real); times_real.push(res.time_real);
times_user.push(res.time_user); times_user.push(res.time_user);
times_system.push(res.time_system); times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte); memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(res.status)); exit_codes.push(extract_exit_code(status));
all_succeeded = all_succeeded && success; all_succeeded = all_succeeded && success;
@@ -1,10 +1,8 @@
use std::process::ExitStatus;
use crate::util::units::Second; use crate::util::units::Second;
/// Performance measurements and exit status from running a single command /// Results from timing a single command
#[derive(Debug, Default, Copy, Clone)] #[derive(Debug, Default, Copy, Clone)]
pub struct Measurement { pub struct TimingResult {
/// Wall clock time /// Wall clock time
pub time_real: Second, pub time_real: Second,
@@ -16,7 +14,4 @@ pub struct Measurement {
/// Maximum amount of memory used, in bytes /// Maximum amount of memory used, in bytes
pub memory_usage_byte: u64, pub memory_usage_byte: u64,
/// The exit status of the process
pub status: ExitStatus,
} }
+3 -7
View File
@@ -41,7 +41,6 @@ fn build_command() -> Command {
.short('w') .short('w')
.value_name("NUM") .value_name("NUM")
.action(ArgAction::Set) .action(ArgAction::Set)
.value_parser(clap::value_parser!(u64))
.help( .help(
"Perform NUM warmup runs before the actual benchmark. This can be used \ "Perform NUM warmup runs before the actual benchmark. This can be used \
to fill (disk) caches for I/O-heavy programs.", to fill (disk) caches for I/O-heavy programs.",
@@ -53,7 +52,6 @@ fn build_command() -> Command {
.short('m') .short('m')
.action(ArgAction::Set) .action(ArgAction::Set)
.value_name("NUM") .value_name("NUM")
.value_parser(clap::value_parser!(u64).range(1..))
.help("Perform at least NUM runs for each command (default: 10)."), .help("Perform at least NUM runs for each command (default: 10)."),
) )
.arg( .arg(
@@ -62,7 +60,6 @@ fn build_command() -> Command {
.short('M') .short('M')
.action(ArgAction::Set) .action(ArgAction::Set)
.value_name("NUM") .value_name("NUM")
.value_parser(clap::value_parser!(u64).range(1..))
.help("Perform at most NUM runs for each command. By default, there is no limit."), .help("Perform at most NUM runs for each command. By default, there is no limit."),
) )
.arg( .arg(
@@ -72,7 +69,6 @@ fn build_command() -> Command {
.short('r') .short('r')
.action(ArgAction::Set) .action(ArgAction::Set)
.value_name("NUM") .value_name("NUM")
.value_parser(clap::value_parser!(u64).range(1..))
.help("Perform exactly NUM runs for each command. If this option is not specified, \ .help("Perform exactly NUM runs for each command. If this option is not specified, \
hyperfine automatically determines the number of runs."), hyperfine automatically determines the number of runs."),
) )
@@ -165,7 +161,7 @@ fn build_command() -> Command {
.help( .help(
"Perform benchmark runs for each value in the range MIN..MAX. Replaces the \ "Perform benchmark runs for each value in the range MIN..MAX. Replaces the \
string '{VAR}' in each command by the current parameter value.\n\n \ string '{VAR}' in each command by the current parameter value.\n\n \
Example: hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\ Example: hyperfine -P threads 1 8 'make -j {threads}'\n\n\
This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\ This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
To have the value increase following different patterns, use shell arithmetics.\n\n \ To have the value increase following different patterns, use shell arithmetics.\n\n \
Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\ Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\
@@ -198,8 +194,8 @@ fn build_command() -> Command {
.help( .help(
"Perform benchmark runs for each value in the comma-separated list VALUES. \ "Perform benchmark runs for each value in the comma-separated list VALUES. \
Replaces the string '{VAR}' in each command by the current parameter value\ Replaces the string '{VAR}' in each command by the current parameter value\
.\n\nExample: hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'\n\n\ .\n\nExample: hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\n\n\
This performs benchmarks for 'g++ -O2 main.cpp' and 'clang++ -O2 main.cpp'.\n\n\ This performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp'.\n\n\
The option can be specified multiple times to run benchmarks for all \ The option can be specified multiple times to run benchmarks for all \
possible parameter combinations.\n" possible parameter combinations.\n"
), ),
+14 -7
View File
@@ -275,15 +275,22 @@ impl Default for Options {
impl Options { impl Options {
pub fn from_cli_arguments<'a>(matches: &ArgMatches) -> Result<Self, OptionsError<'a>> { pub fn from_cli_arguments<'a>(matches: &ArgMatches) -> Result<Self, OptionsError<'a>> {
let mut options = Self::default(); let mut options = Self::default();
options.warmup_count = matches let param_to_u64 = |param| {
.get_one::<u64>("warmup") matches
.copied() .get_one::<String>(param)
.unwrap_or(options.warmup_count); .map(|n| {
n.parse::<u64>()
.map_err(|e| OptionsError::IntParsingError(param, e))
})
.transpose()
};
let mut min_runs = matches.get_one::<u64>("min-runs").copied(); options.warmup_count = param_to_u64("warmup")?.unwrap_or(options.warmup_count);
let mut max_runs = matches.get_one::<u64>("max-runs").copied();
if let Some(&runs) = matches.get_one::<u64>("runs") { let mut min_runs = param_to_u64("min-runs")?;
let mut max_runs = param_to_u64("max-runs")?;
if let Some(runs) = param_to_u64("runs")? {
min_runs = Some(runs); min_runs = Some(runs);
max_runs = Some(runs); max_runs = Some(runs);
} }
+15 -4
View File
@@ -16,11 +16,11 @@ use std::os::fd::AsFd;
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
use windows_sys::Win32::System::Threading::CREATE_SUSPENDED; use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
use crate::benchmark::measurement::Measurement; use crate::util::units::Second;
use wall_clock_timer::WallClockTimer; use wall_clock_timer::WallClockTimer;
use std::io::Read; use std::io::Read;
use std::process::{ChildStdout, Command}; use std::process::{ChildStdout, Command, ExitStatus};
use anyhow::Result; use anyhow::Result;
@@ -37,6 +37,17 @@ struct CPUTimes {
pub memory_usage_byte: u64, pub memory_usage_byte: u64,
} }
/// Used to indicate the result of running a command
#[derive(Debug, Copy, Clone)]
pub struct TimerResult {
pub time_real: Second,
pub time_user: Second,
pub time_system: Second,
pub memory_usage_byte: u64,
/// The exit status of the process
pub status: ExitStatus,
}
/// Discard the output of a child process. /// Discard the output of a child process.
fn discard(output: ChildStdout) { fn discard(output: ChildStdout) {
const CHUNK_SIZE: usize = 64 << 10; const CHUNK_SIZE: usize = 64 << 10;
@@ -69,7 +80,7 @@ fn discard(output: ChildStdout) {
} }
/// Execute the given command and return a timing summary /// Execute the given command and return a timing summary
pub fn execute_and_measure(mut command: Command) -> Result<Measurement> { pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
#[cfg(not(windows))] #[cfg(not(windows))]
let cpu_timer = self::unix_timer::CPUTimer::start(); let cpu_timer = self::unix_timer::CPUTimer::start();
@@ -100,7 +111,7 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
let time_real = wallclock_timer.stop(); let time_real = wallclock_timer.stop();
let (time_user, time_system, memory_usage_byte) = cpu_timer.stop(); let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
Ok(Measurement { Ok(TimerResult {
time_real, time_real,
time_user, time_user,
time_system, time_system,
-26
View File
@@ -74,32 +74,6 @@ fn exits_quietly_when_stdout_is_closed() {
} }
} }
#[test]
fn fails_with_zero_runs() {
for option in ["--runs", "--min-runs", "--max-runs"] {
hyperfine()
.arg(option)
.arg("0")
.arg("echo dummy benchmark")
.assert()
.code(2)
.stderr(predicate::str::contains(format!(
"invalid value '0' for '{option} <NUM>'"
)));
}
}
#[test]
fn min_runs_of_one_still_performs_one_run() {
hyperfine_debug()
.arg("--min-runs=1")
.arg("--warmup=0")
.arg("sleep 4")
.assert()
.success()
.stdout(predicate::str::contains("Time (abs ≡)"));
}
#[test] #[test]
fn can_run_commands_without_a_shell() { fn can_run_commands_without_a_shell() {
hyperfine() hyperfine()