Compare commits

...
Author SHA1 Message Date
David Peter 9b97116caa Clarify that the default Unix shell is found through PATH 2026-10-02 11:29:15 +02:00
Marshall Porter 1f5a2ff70c ci: add Linux ARM64 musl release target (#924) 2026-10-02 11:26:01 +02:00
c0b3834277 Handle broken pipe errors on stdout gracefully (#932)
Replace `println!` calls on the results paths with `writeln!` on
`io::stdout()` and propagate the `io::Error` to `main`, where
`ErrorKind::BrokenPipe` is turned into a quiet `exit(0)`. This matches
the conventional behavior of command-line tools when their output pipe
is closed early (e.g. `hyperfine ... | head -n 0`), and fixes panics in
all run-output paths: the per-benchmark header and results, the speed
comparison summary, and `-`-style exports to stdout. The warnings and
"Note" outputs to stderr are converted the same way.


---------

Co-authored-by: Mathjk <Mathjk@users.noreply.github.com>
Co-authored-by: David Peter <mail@david-peter.de>
2026-10-02 11:18:56 +02:00
David Peter 4d073e08ae Dependency and MSRV update (#934) 2026-10-02 11:05:02 +02:00
David Peter 4b561defef Use cargo_bin! to locate the integration test binary (#933) 2026-10-02 10:54:51 +02:00
AlexandDavid Peter 27a079e0df docs: load aliases before parsing their benchmark invocation (#929)
* docs: load aliases before parsing their benchmark invocation

* Remove the alias example

---------

Co-authored-by: David Peter <mail@david-peter.de>
2026-10-02 10:50:46 +02:00
David Peter 148632c5f9 Remove shell alias example from README 2026-10-02 10:45:59 +02:00
Kobi Hikri 09f02e4f59 docs: update stale version metadata (README.md) 2026-10-02 10:42:06 +02:00
Kobi Hikri c30ac680a3 docs: update stale version metadata (CITATION.cff) 2026-10-02 10:42:06 +02:00
David Peter f12f3d9f86 Remove sponsors 2026-04-30 10:23:22 +02:00
David Peter 327d5f4d91 Update README 2026-02-14 14:01:52 +01:00
David Peter 975fe108c4 Bump version to 1.20.0 2025-11-18 09:34:01 +01:00
dependabot[bot] 001dcce10e Bump actions/checkout from 4 to 5
Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-18 09:20:55 +01:00
dependabot[bot] 6447ab3a4e Bump clap from 4.5.47 to 4.5.48
Bumps [clap](https://github.com/clap-rs/clap) from 4.5.47 to 4.5.48.
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.47...clap_complete-v4.5.48)

---
updated-dependencies:
- dependency-name: clap
  dependency-version: 4.5.48
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-18 09:20:35 +01:00
Sylvestre Ledru ae4c8073db Debian: Hyperfine is now available in stable 2025-11-18 09:20:24 +01:00
dependabot[bot] 57ef17cdcb Bump tempfile from 3.14.0 to 3.23.0
Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.14.0 to 3.23.0.
- [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md)
- [Commits](https://github.com/Stebalien/tempfile/compare/v3.14.0...v3.23.0)

---
updated-dependencies:
- dependency-name: tempfile
  dependency-version: 3.23.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-18 09:20:01 +01:00
Anti Räis 8b6224059c fix typo in docs 2025-11-18 09:19:49 +01:00
David Peter b5a6860d96 Support list of exit codes in --ignore-failure 2025-11-18 09:19:27 +01:00
David Peter 16348dcf35 Update README 2025-09-23 19:38:55 +02:00
David Peter de07c77387 Formatting 2025-09-04 08:59:11 +02:00
David Peter b20df39f0f Update README 2025-09-04 08:57:49 +02:00
David Peter 6f95f08d2f Update README 2025-09-04 08:57:34 +02:00
David Peter 214bff7065 Update README 2025-09-04 08:56:31 +02:00
David Peter fae8b0ef64 .contains(…) 2025-09-03 20:52:28 +02:00
David Peter 98d262c497 Fix warnings 2025-09-03 20:51:44 +02:00
Ravi Kant Sharma 6b8fa5fd3d Update ubuntu install instructions 2025-09-03 20:50:19 +02:00
Niklas Dewally 2166c9f02e Add option to name reference command
Add --reference-name option to give a reference command a name.

This was discussed in the PR that added --reference, #744, but
was unfortunately never implemented.
2025-09-03 20:45:39 +02:00
Teodoro Freund 835fc43ea1 Fixed bug on naming individual commands with parameter scan 2025-09-03 20:44:07 +02:00
24 changed files with 832 additions and 612 deletions

No files matched your search

+5 -4
View File
@@ -18,7 +18,7 @@ jobs:
name: Extract crate metadata
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- name: Extract crate information
id: crate_metadata
run: |
@@ -49,7 +49,7 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- run: cargo fmt -- --check
min_version:
@@ -58,7 +58,7 @@ jobs:
needs: crate_metadata
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@v5
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
uses: dtolnay/rust-toolchain@master
@@ -79,6 +79,7 @@ jobs:
matrix:
job:
- { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
- { target: aarch64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }
- { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }
- { target: i686-pc-windows-msvc , os: windows-2022 }
@@ -94,7 +95,7 @@ jobs:
BUILD_CMD: cargo
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@v5
- name: Install prerequisites
shell: bash
+17
View File
@@ -1,3 +1,20 @@
# v1.20.0
## Features
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
## Bugfixes
- Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr)
## Other
- Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos)
# v1.19.0
## Features
+2 -2
View File
@@ -13,5 +13,5 @@ authors:
repository-code: 'https://github.com/sharkdp/hyperfine'
abstract: A command-line benchmarking tool.
license: MIT
version: 1.16.1
date-released: '2023-03-21'
version: 1.20.0
date-released: '2025-11-18'
Generated
+438 -473
View File
File diff suppressed because it is too large. Load diff
+16 -15
View File
@@ -7,25 +7,26 @@ license = "MIT OR Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "1.19.0"
version = "1.20.0"
edition = "2018"
build = "build.rs"
rust-version = "1.88.0"
rust-version = "1.97.0"
[features]
# Use the nightly feature windows_process_extensions_main_thread_handle
windows_process_extensions_main_thread_handle = []
[dependencies]
colored = "2.1"
colored = "3.1"
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
indicatif = "=0.17.4"
statistical = "1.0"
csv = "1.3"
csv = "1.4"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
rust_decimal = "1.36"
rand = "0.8"
shell-words = "1.0"
rust_decimal = "1.43"
rand = "0.10"
shell-words = "1.1"
thiserror = "2.0"
anyhow = "1.0"
@@ -33,7 +34,7 @@ anyhow = "1.0"
libc = "0.2"
[target.'cfg(windows)'.dependencies]
windows-sys = { version = "0.59", features = [
windows-sys = { version = "0.61", features = [
"Win32_Foundation",
"Win32_Security",
"Win32_System_JobObjects",
@@ -42,10 +43,10 @@ windows-sys = { version = "0.59", features = [
] }
[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
once_cell = "1.19"
once_cell = "1.21"
[target.'cfg(target_os="linux")'.dependencies]
nix = { version = "0.29", features = ["zerocopy"] }
nix = { version = "0.31", features = ["zerocopy"] }
[dependencies.clap]
version = "4"
@@ -62,18 +63,18 @@ features = [
[dev-dependencies]
approx = "0.5"
assert_cmd = "2.0"
insta = { version = "1.41.1", features = ["yaml"] }
assert_cmd = "2.2"
insta = { version = "1.48", features = ["yaml"] }
predicates = "3.1"
tempfile = "3.14"
tempfile = "3.27"
[profile.dev.package]
insta.opt-level = 3
similar.opt-level = 3
[build-dependencies]
clap = "4.5.47"
clap_complete = "4.2.1"
clap = "4.6"
clap_complete = "4.6"
[profile.release]
lto = true
+13 -24
View File
@@ -10,18 +10,6 @@ A command-line benchmarking tool.
![hyperfine](https://i.imgur.com/z19OYxE.gif)
### Sponsors
A special *thank you* goes to our biggest <a href="doc/sponsors.md">sponsor</a>:<br>
<a href="https://www.warp.dev/hyperfine">
<img src="doc/sponsors/warp-logo.png" width="200" alt="Warp">
<br>
<strong>Warp, the intelligent terminal</strong>
<br>
<sub>Available on MacOS, Linux, Windows</sub>
</a>
## Features
* Statistical analysis across multiple runs.
@@ -101,7 +89,7 @@ hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
### Intermediate shell
By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
By default, commands are executed using `sh` on Unix (resolved through `PATH`) or `cmd.exe` on Windows.
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
```sh
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
@@ -120,7 +108,7 @@ hyperfine -N 'grep TODO /home/user'
```
### Shell functions and aliases
### Shell functions
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
@@ -130,13 +118,10 @@ export -f my_function
hyperfine --shell=bash my_function
```
Otherwise, inline them into or source them from the benchmarked program:
Otherwise, inline the function into the benchmarked command:
```sh
hyperfine 'my_function() { sleep 1; }; my_function'
echo 'alias my_alias="sleep 1"' > /tmp/my_alias.sh
hyperfine '. /tmp/my_alias.sh; my_alias'
```
### Exporting results
@@ -179,11 +164,15 @@ like `--warmup`, `--prepare <cmd>`, `--setup <cmd>` or `--cleanup <cmd>`:
### On Ubuntu
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
On Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine):
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb
sudo dpkg -i hyperfine_1.19.0_amd64.deb
apt install hyperfine
```
Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb
sudo dpkg -i hyperfine_1.20.0_amd64.deb
```
### On Fedora
@@ -210,7 +199,7 @@ pacman -S hyperfine
### On Debian Linux
On Debian Linux, hyperfine can be installed [from the testing repositories](https://packages.debian.org/testing/main/hyperfine):
On Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine):
```
apt install hyperfine
```
@@ -313,7 +302,7 @@ Hyperfine can be installed from source via [cargo](https://doc.rust-lang.org/car
cargo install --locked hyperfine
```
Make sure that you use Rust 1.76 or newer.
Make sure that you use Rust 1.97 or newer.
### From binaries (Linux, macOS, Windows)
+1 -1
View File
@@ -131,7 +131,7 @@ Execute \fICMD\fP after the completion of all benchmarking runs for each individ
command to be benchmarked. This is useful if the commands to be benchmarked
produce artifacts that need to be cleaned up. It only runs once a series of
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
ever run.
every run.
.HP
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
.IP
-14
View File
@@ -1,14 +0,0 @@
## Sponsors
`hyperfine` development is sponsored by many individuals and companies. Thank you very much!
Please note, that being sponsored does not affect the individuality of the `hyperfine`
project or affect the maintainers' actions in any way.
We remain impartial and continue to assess pull requests solely on merit - the
features added, bugs solved, and effect on the overall complexity of the code.
No issue will have a different priority based on sponsorship status of the
reporter.
Contributions from anybody are most welcomed.
If you want to see our biggest sponsors, check the top of [`README.md`](../README.md#sponsors).
Binary file not shown.

Before

Width:  |  Height:  |  Size: 130 KiB

+36 -18
View File
@@ -79,23 +79,41 @@ fn run_command_and_measure_common(
let result = execute_and_measure(command)
.with_context(|| format!("Failed to run command '{command_name}'"))?;
if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
let when = match iteration {
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=result.status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
if !result.status.success() {
use crate::util::exit_code::extract_exit_code;
),
);
let should_fail = match command_failure_action {
CmdFailureAction::RaiseError => true,
CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
// Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(result.status) {
!codes.contains(&exit_code)
} else {
// If we can't extract an exit code, treat it as a failure
true
}
}
};
if should_fail {
let when = match iteration {
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=result.status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
),
);
}
}
Ok(result)
@@ -122,7 +140,7 @@ impl Executor for RawExecutor<'_> {
let result = run_command_and_measure_common(
command.get_command()?,
iteration,
command_failure_action.unwrap_or(self.options.command_failure_action),
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
@@ -187,7 +205,7 @@ impl Executor for ShellExecutor<'_> {
let mut result = run_command_and_measure_common(
command_builder,
iteration,
command_failure_action.unwrap_or(self.options.command_failure_action),
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
+19 -11
View File
@@ -5,6 +5,7 @@ pub mod scheduler;
pub mod timing_result;
use std::cmp;
use std::io::{self, Write};
use crate::benchmark::executor::BenchmarkIteration;
use crate::command::Command;
@@ -12,6 +13,7 @@ use crate::options::{
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
};
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
use crate::output::console_writeln;
use crate::output::format::{format_duration, format_duration_unit};
use crate::output::progress_bar::get_progress_bar;
use crate::output::warnings::{OutlierWarningOptions, Warnings};
@@ -140,12 +142,13 @@ impl<'a> Benchmark<'a> {
/// Run the benchmark for a single command
pub fn run(&self) -> Result<BenchmarkResult> {
if self.options.output_style != OutputStyleOption::Disabled {
println!(
console_writeln!(
io::stdout(),
"{}{}: {}",
"Benchmark ".bold(),
(self.number + 1).to_string().bold(),
self.command.get_name_with_unused_parameters(),
);
)?;
}
let mut times_real: Vec<Second> = vec![];
@@ -358,19 +361,22 @@ impl<'a> Benchmark<'a> {
let system_str = format_duration(system_mean, Some(time_unit));
if self.options.output_style != OutputStyleOption::Disabled {
let mut stdout = io::stdout().lock();
if times_real.len() == 1 {
println!(
console_writeln!(
stdout,
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
"abs".green().bold(),
mean_str.green().bold(),
" ", // alignment
user_str.blue(),
system_str.blue()
);
)?;
} else {
let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
println!(
console_writeln!(
stdout,
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
"mean".green().bold(),
"σ".green(),
@@ -378,16 +384,17 @@ impl<'a> Benchmark<'a> {
stddev_str.green(),
user_str.blue(),
system_str.blue()
);
)?;
println!(
console_writeln!(
stdout,
" Range ({} … {}): {:>8} … {:>8} {}",
"min".cyan(),
"max".purple(),
min_str.cyan(),
max_str.purple(),
num_str.dimmed()
);
)?;
}
}
@@ -430,15 +437,16 @@ impl<'a> Benchmark<'a> {
}
if !warnings.is_empty() {
eprintln!(" ");
let mut stderr = io::stderr().lock();
console_writeln!(stderr, " ")?;
for warning in &warnings {
eprintln!(" {}: {}", "Warning".yellow(), warning);
console_writeln!(stderr, " {}: {}", "Warning".yellow(), warning)?;
}
}
if self.options.output_style != OutputStyleOption::Disabled {
println!(" ");
console_writeln!(io::stdout(), " ")?;
}
self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
+2 -2
View File
@@ -98,7 +98,7 @@ pub fn compute_with_check_from_reference<'a>(
pub fn compute_with_check(
results: &[BenchmarkResult],
sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
let fastest = fastest_of(results);
if fastest.mean == 0.0 {
@@ -112,7 +112,7 @@ pub fn compute_with_check(
pub fn compute(
results: &[BenchmarkResult],
sort_order: SortOrder,
) -> Vec<BenchmarkResultWithRelativeSpeed> {
) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
let fastest = fastest_of(results);
compute_relative_speeds(results, fastest, sort_order)
+25 -16
View File
@@ -3,10 +3,12 @@ use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
use super::{relative_speed, Benchmark};
use colored::*;
use std::cmp::Ordering;
use std::io::{self, Write};
use crate::command::{Command, Commands};
use crate::export::ExportManager;
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
use crate::output::console_writeln;
use anyhow::Result;
@@ -42,7 +44,7 @@ impl<'a> Scheduler<'a> {
.options
.reference_command
.as_ref()
.map(|cmd| Command::new(None, cmd));
.map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
executor.calibrate()?;
@@ -58,13 +60,13 @@ impl<'a> Scheduler<'a> {
Ok(())
}
pub fn print_relative_speed_comparison(&self) {
pub fn print_relative_speed_comparison(&self) -> Result<()> {
if self.options.output_style == OutputStyleOption::Disabled {
return;
return Ok(());
}
if self.results.len() < 2 {
return;
return Ok(());
}
let reference = self
@@ -79,17 +81,19 @@ impl<'a> Scheduler<'a> {
reference,
self.options.sort_order_speed_comparison,
) {
let mut stdout = io::stdout().lock();
match self.options.sort_order_speed_comparison {
SortOrder::MeanTime => {
println!("{}", "Summary".bold());
console_writeln!(stdout, "{}", "Summary".bold())?;
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
let others = annotated_results.iter().filter(|r| !r.is_reference);
println!(
console_writeln!(
stdout,
" {} ran",
reference.result.command_with_unused_parameters.cyan()
);
)?;
for item in others {
let stddev = if let Some(stddev) = item.relative_speed_stddev {
@@ -114,18 +118,20 @@ impl<'a> Scheduler<'a> {
stddev
),
};
println!(
console_writeln!(
stdout,
"{} {}",
comparator,
&item.result.command_with_unused_parameters.magenta()
);
item.result.command_with_unused_parameters.magenta()
)?;
}
}
SortOrder::Command => {
println!("{}", "Relative speed comparison".bold());
console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
for item in annotated_results {
println!(
console_writeln!(
stdout,
" {}{} {}",
format!("{:10.2}", item.relative_speed).bold().green(),
if item.is_reference {
@@ -135,13 +141,14 @@ impl<'a> Scheduler<'a> {
} else {
" ".into()
},
&item.result.command_with_unused_parameters,
);
item.result.command_with_unused_parameters,
)?;
}
}
}
} else {
eprintln!(
console_writeln!(
io::stderr(),
"{}: The benchmark comparison could not be computed as some benchmark times are zero. \
This could be caused by background interference during the initial calibration phase \
of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
@@ -149,8 +156,10 @@ impl<'a> Scheduler<'a> {
--shell=none/-N option. If it does not help either, you command is most likely too fast \
to be accurately benchmarked by hyperfine.",
"Note".bold().red()
);
)?;
}
Ok(())
}
pub fn final_export(&self) -> Result<()> {
+16 -2
View File
@@ -96,6 +96,14 @@ fn build_command() -> Command {
If this is unset, results are compared with the fastest command as reference."
)
)
.arg(
Arg::new("reference-name")
.long("reference-name")
.action(ArgAction::Set)
.value_name("CMD")
.help("Give a meaningful name to the reference command.")
.requires("reference")
)
.arg(
Arg::new("prepare")
.long("prepare")
@@ -218,9 +226,15 @@ fn build_command() -> Command {
.arg(
Arg::new("ignore-failure")
.long("ignore-failure")
.action(ArgAction::SetTrue)
.action(ArgAction::Set)
.value_name("MODE")
.num_args(0..=1)
.default_missing_value("all-non-zero")
.require_equals(true)
.short('i')
.help("Ignore non-zero exit codes of the benchmarked programs."),
.help("Ignore failures of the benchmarked programs. Without a value or with \
'all-non-zero', all non-zero exit codes are ignored. You can also provide \
a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2)."),
)
.arg(
Arg::new("style")
+49 -12
View File
@@ -158,7 +158,9 @@ impl<'a> Commands<'a> {
});
let args: Vec<_> = args.map(|v| v.as_str()).collect::<Vec<_>>();
let param_names_and_values: Vec<(&str, Vec<String>)> = args
.chunks_exact(2)
.as_chunks::<2>()
.0
.iter()
.map(|pair| {
let name = pair[0];
let list_str = pair[1];
@@ -169,7 +171,7 @@ impl<'a> Commands<'a> {
let duplicates =
Self::find_duplicates(param_names_and_values.iter().map(|(name, _)| *name));
if !duplicates.is_empty() {
bail!("Duplicate parameter names: {}", &duplicates.join(", "));
bail!("Duplicate parameter names: {}", duplicates.join(", "));
}
}
@@ -277,18 +279,8 @@ impl<'a> Commands<'a> {
command_strings: Vec<&'b str>,
) -> Result<Vec<Command<'b>>, ParameterScanError> {
let param_range = RangeStep::new(param_min, param_max, step)?;
let param_count = param_range.size_hint().1.unwrap();
let command_name_count = command_names.len();
// `--command-name` should appear exactly once or exactly B times,
// where B is the total number of benchmarks.
if command_name_count > 1 && command_name_count != param_count {
return Err(ParameterScanError::UnexpectedCommandNameCount(
command_name_count,
param_count,
));
}
let mut i = 0;
let mut commands = vec![];
for value in param_range {
@@ -305,6 +297,17 @@ impl<'a> Commands<'a> {
i += 1;
}
}
// `--command-name` should appear exactly once or exactly B times,
// where B is the total number of benchmarks.
let command_count = commands.len();
if command_name_count > 1 && command_name_count != command_count {
return Err(ParameterScanError::UnexpectedCommandNameCount(
command_name_count,
command_count,
));
}
Ok(commands)
}
@@ -471,6 +474,40 @@ fn test_build_parameter_scan_commands() {
assert_eq!(commands[1].get_command_line(), "echo 2");
}
#[test]
fn test_build_parameter_scan_commands_named() {
use crate::cli::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
"echo {val}",
"sleep {val}",
"--parameter-scan",
"val",
"1",
"2",
"--parameter-step-size",
"1",
"--command-name",
"echo-1",
"--command-name",
"sleep-1",
"--command-name",
"echo-2",
"--command-name",
"sleep-2",
]);
let commands = Commands::from_cli_arguments(&matches).unwrap().0;
assert_eq!(commands.len(), 4);
assert_eq!(commands[0].get_name(), "echo-1");
assert_eq!(commands[0].get_command_line(), "echo 1");
assert_eq!(commands[1].get_name(), "sleep-1");
assert_eq!(commands[1].get_command_line(), "sleep 1");
assert_eq!(commands[2].get_name(), "echo-2");
assert_eq!(commands[2].get_command_line(), "echo 2");
assert_eq!(commands[3].get_name(), "sleep-2");
assert_eq!(commands[3].get_command_line(), "sleep 2");
}
#[test]
fn test_parameter_scan_commands_int() {
let commands = Commands::build_parameter_scan_commands(
+4
View File
@@ -3,6 +3,10 @@ use std::num::{self, ParseFloatError, ParseIntError};
use rust_decimal::Error as DecimalError;
use thiserror::Error;
#[derive(Debug, Error)]
#[error(transparent)]
pub struct ConsoleOutputError(pub std::io::Error);
#[derive(Debug, Error)]
pub enum ParameterScanError {
#[error("Error while parsing parameter scan arguments ({0})")]
+5 -3
View File
@@ -1,5 +1,5 @@
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::io::{self, Write};
mod asciidoc;
mod csv;
@@ -18,6 +18,7 @@ use self::orgmode::OrgmodeExporter;
use crate::benchmark::benchmark_result::BenchmarkResult;
use crate::options::SortOrder;
use crate::output::console_writeln;
use crate::util::units::Unit;
use anyhow::{Context, Result};
@@ -144,8 +145,9 @@ impl ExportManager {
}
ExportTarget::Stdout => {
if !intermediate {
println!();
println!("{}", String::from_utf8(content()?).unwrap());
let mut stdout = io::stdout().lock();
console_writeln!(stdout)?;
console_writeln!(stdout, "{}", String::from_utf8(content()?).unwrap())?;
}
}
}
+1 -1
View File
@@ -9,7 +9,7 @@ impl MarkupExporter for OrgmodeExporter {
format!(
"| {} | {} |\n",
cells.first().unwrap(),
&cells[1..].join(" | ")
cells[1..].join(" | ")
)
}
+16 -2
View File
@@ -4,10 +4,12 @@
)]
use std::env;
use std::io::{self, Write};
use benchmark::scheduler::Scheduler;
use cli::get_cli_arguments;
use command::Commands;
use error::ConsoleOutputError;
use export::ExportManager;
use options::Options;
@@ -44,17 +46,29 @@ fn run() -> Result<()> {
let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
scheduler.run_benchmarks()?;
scheduler.print_relative_speed_comparison();
scheduler.print_relative_speed_comparison()?;
scheduler.final_export()?;
Ok(())
}
fn caused_by_broken_console_pipe(error: &anyhow::Error) -> bool {
error
.downcast_ref::<ConsoleOutputError>()
.is_some_and(|e| e.0.kind() == io::ErrorKind::BrokenPipe)
}
fn main() {
match run() {
Ok(_) => {}
Err(e) => {
eprintln!("{} {:#}", "Error:".red(), e);
// Exit quietly when a reader stops consuming console output early.
if caused_by_broken_console_pipe(&e) {
std::process::exit(0);
}
// The write to stderr can itself fail if stderr is closed; the
// error message cannot be shown in that case anyway.
let _ = writeln!(io::stderr(), "{} {:#}", "Error:".red(), e);
std::process::exit(1);
}
}
+29 -7
View File
@@ -67,13 +67,16 @@ impl Shell {
}
/// Action to take when an executed command fails.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CmdFailureAction {
/// Exit with an error message
RaiseError,
/// Simply ignore the non-zero exit code
Ignore,
/// Ignore all non-zero exit codes
IgnoreAllFailures,
/// Ignore specific exit codes
IgnoreSpecificFailures(Vec<i32>),
}
/// Output style type option
@@ -208,6 +211,9 @@ pub struct Options {
// Command to use as a reference for relative speed comparison
pub reference_command: Option<String>,
// Name of the reference command
pub reference_name: Option<String>,
/// Command(s) to run before each timing run
pub preparation_command: Option<Vec<String>>,
@@ -250,6 +256,7 @@ impl Default for Options {
min_benchmarking_time: 3.0,
command_failure_action: CmdFailureAction::RaiseError,
reference_command: None,
reference_name: None,
preparation_command: None,
conclusion_command: None,
setup_command: None,
@@ -310,6 +317,9 @@ impl Options {
options.setup_command = matches.get_one::<String>("setup").map(String::from);
options.reference_command = matches.get_one::<String>("reference").map(String::from);
options.reference_name = matches
.get_one::<String>("reference-name")
.map(String::from);
options.preparation_command = matches
.get_many::<String>("prepare")
@@ -354,8 +364,7 @@ impl Options {
_ => {
if options
.command_output_policies
.iter()
.any(|policy| *policy == CommandOutputPolicy::Inherit)
.contains(&CommandOutputPolicy::Inherit)
|| !io::stdout().is_terminal()
{
OutputStyleOption::Basic
@@ -409,8 +418,21 @@ impl Options {
}
};
if matches.get_flag("ignore-failure") {
options.command_failure_action = CmdFailureAction::Ignore;
if let Some(mode) = matches.get_one::<String>("ignore-failure") {
options.command_failure_action = match mode.as_str() {
"all-non-zero" | "" => CmdFailureAction::IgnoreAllFailures,
codes => {
let exit_codes: Result<Vec<i32>, _> = codes
.split(',')
.map(|s| {
s.trim()
.parse::<i32>()
.map_err(|e| OptionsError::IntParsingError("ignore-failure", e))
})
.collect();
CmdFailureAction::IgnoreSpecificFailures(exit_codes?)
}
};
}
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
+10
View File
@@ -1,3 +1,13 @@
pub mod format;
pub mod progress_bar;
pub mod warnings;
/// Mark console write errors so broken pipes can be handled separately from file errors.
macro_rules! console_writeln {
($writer:expr $(, $($args:tt)*)?) => {
writeln!($writer $(, $($args)*)?)
.map_err($crate::error::ConsoleOutputError)
};
}
pub(crate) use console_writeln;
+1 -1
View File
@@ -4,5 +4,5 @@
/// [1] Mytkowicz, 2009. Producing Wrong Data Without Doing Anything Obviously Wrong!.
/// Sigplan Notices - SIGPLAN. 44. 265-276. 10.1145/1508284.1508275.
pub fn value() -> String {
"X".repeat(rand::random::<usize>() % 4096usize)
"X".repeat(rand::random_range(0..4096))
}
+1 -3
View File
@@ -1,9 +1,7 @@
use std::process::Command;
use assert_cmd::cargo::CommandCargoExt;
pub fn hyperfine_raw_command() -> Command {
let mut cmd = Command::cargo_bin("hyperfine").unwrap();
let mut cmd = Command::new(assert_cmd::cargo::cargo_bin!("hyperfine"));
cmd.current_dir("tests/");
cmd
}
+126 -1
View File
@@ -1,5 +1,5 @@
mod common;
use common::hyperfine;
use common::{hyperfine, hyperfine_raw_command};
use predicates::prelude::*;
@@ -33,6 +33,47 @@ fn one_run_is_supported() {
.success();
}
/// Regression test: hyperfine must not panic when writing to a closed
/// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process
/// that terminates early). A `BrokenPipe` error should result in a quiet
/// exit with code 0 instead of a `println!` panic.
#[test]
fn exits_quietly_when_stdout_is_closed() {
use std::process::Stdio;
for args in [
vec!["--runs=1", "echo dummy benchmark"],
vec!["--runs=1", "--export-json=-", "echo dummy benchmark"],
vec!["--runs=1", "echo dummy benchmark", "echo second benchmark"],
] {
// A pipe whose read end has been closed before hyperfine is even
// started: every write to it will fail with `BrokenPipe`.
let (reader, writer) = std::io::pipe().expect("failed to create pipe");
drop(reader);
let output = hyperfine_raw_command()
.args(&args)
.stdout(writer)
.stderr(Stdio::piped())
.spawn()
.expect("failed to spawn hyperfine")
.wait_with_output()
.expect("failed to wait for hyperfine");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
!stderr.contains("panicked"),
"hyperfine panicked while writing to a closed pipe:\n{}",
stderr
);
assert!(
output.status.success(),
"hyperfine did not exit cleanly; stderr:\n{}",
stderr
);
}
}
#[test]
fn can_run_commands_without_a_shell() {
hyperfine()
@@ -277,6 +318,78 @@ fn can_run_failing_commands_with_ignore_failure_option() {
.success();
}
#[cfg(unix)]
#[test]
fn can_ignore_specific_exit_codes() {
// Test that specifying exit code 1 ignores it
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1")
.arg("exit 1")
.assert()
.success();
// Test that other exit codes still fail
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1")
.arg("exit 2")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command terminated with non-zero exit code 2",
));
}
#[cfg(unix)]
#[test]
fn can_ignore_multiple_exit_codes() {
// Test that all specified exit codes are ignored
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 1")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 2")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 3")
.assert()
.success();
// Test that other exit codes still fail
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 4")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command terminated with non-zero exit code 4",
));
}
#[cfg(unix)]
#[test]
fn ignore_failure_with_all_non_zero() {
// Test explicit "all-non-zero" mode
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=all-non-zero")
.arg("exit 5")
.assert()
.success();
}
#[test]
fn shows_output_of_benchmarked_command() {
hyperfine()
@@ -499,6 +612,18 @@ fn shows_benchmark_comparison_relative_to_reference() {
);
}
#[test]
fn shows_reference_name() {
hyperfine_debug()
.arg("--reference=sleep 2.0")
.arg("--reference-name=refabc123")
.arg("sleep 1.0")
.arg("sleep 3.0")
.assert()
.success()
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
}
#[test]
fn performs_all_benchmarks_in_parameter_scan() {
hyperfine_debug()