Compare commits

..
Author SHA1 Message Date
David Peter ad0e162abc Prepare release v2.0.0-alpha.1 2026-10-06 15:55:32 +02:00
6 changed files with 86 additions and 107 deletions

No files matched your search

+10 -9
View File
@@ -1,16 +1,18 @@
# hyperfine
[![CICD](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml/badge.svg)](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml)
[![Version info](https://img.shields.io/crates/v/hyperfine.svg)](https://crates.io/crates/hyperfine)
[中文](https://github.com/chinanf-boy/hyperfine-zh)
A command-line benchmarking tool.
<img width="839" height="205" alt="hyperfine demo" src="https://github.com/user-attachments/assets/13c20b1d-39c1-4da1-8fcb-0c775dd032f5" />
*(hyperfine in action, benchmarking [`mypy`](https://mypy-lang.org/) and [`ty`](https://ty.dev/))*
**Demo**: Benchmarking [`fd`](https://github.com/sharkdp/fd) and
[`find`](https://www.gnu.org/software/findutils/):
![hyperfine](https://i.imgur.com/z19OYxE.gif)
## Features
* Statistical analysis across multiple runs.
* Support for various performance metrics and hardware counters.
* Support for arbitrary shell commands.
* Constant feedback about the benchmark progress and current estimates.
* Warmup runs can be executed before the actual benchmark.
@@ -108,7 +110,7 @@ hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log'
### Intermediate shell
By default, benchmarked commands, including warmup runs, are executed directly, without an intermediate shell (`--shell=none`).
By default, commands are executed directly, without an intermediate shell (`--shell=none`).
Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported.
Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted.
This avoids shell startup overhead and the noise from correcting for it, especially for fast commands
@@ -125,11 +127,10 @@ You can also select a specific shell with `--shell <SHELL>`:
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
```
The `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands always run through a shell.
They use the shell selected by `--shell`, or the platform shell (`sh` on Unix, `cmd.exe` on Windows)
when `--shell` is omitted or set to `none`.
The shell setting applies to all commands, including `--setup`, `--prepare`, `--conclude`, and `--cleanup`.
If any of these commands need shell syntax, enable a shell explicitly.
When benchmarked commands use a shell, hyperfine *corrects for the shell spawning time*. It runs the shell with an
When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an
empty command multiple times to measure its startup time, then subtracts this time from each
time measurement.
+4 -6
View File
@@ -194,16 +194,14 @@ possible parameter combinations.
.HP
\fB\-\-shell\fR \fISHELL\fP
.IP
Set the shell to use for executing benchmarked commands, including warmup runs
(default: none). This can be the name or the path to the shell executable,
or a full command line
Set the shell to use for executing commands (default: none). This can be
the name or the path to the shell executable, or a full command line
like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly
select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be
set to "none" to disable the shell. In this case, benchmarked commands will be
set to "none" to disable the shell. In this case, commands will be
executed directly. They can still have arguments, but more complex
things like "sleep 0.1; sleep 0.2" are not possible without a shell.
Setup, prepare, conclude, and cleanup commands always use a shell: the selected
shell, or the platform shell when this option is omitted or set to "none".
This option also applies to setup, prepare, conclude, and cleanup commands.
.HP
\fB\-S\fR
.IP
+20 -51
View File
@@ -5,7 +5,9 @@ use std::process::ExitStatus;
use crate::benchmark::measurement::Measurement;
use crate::benchmark::measurement::Measurements;
use crate::command::Command;
use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell};
use crate::options::{
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
};
use crate::output::progress_bar::get_progress_bar;
use crate::quantity::{second, Information, Time, Zero};
use crate::timer::execute_and_measure;
@@ -40,21 +42,6 @@ pub trait Executor {
output_policy: &CommandOutputPolicy,
) -> Result<Measurement>;
/// Run a lifecycle command through a shell, always treating failures as errors.
fn run_intermediate_command_and_measure(
&self,
command: &Command<'_>,
iteration: BenchmarkIteration,
output_policy: &CommandOutputPolicy,
) -> Result<Measurement> {
self.run_command_and_measure(
command,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
}
/// Perform a calibration of this executor. For example,
/// when running commands through a shell, we need to
/// measure the shell spawning time separately in order
@@ -71,13 +58,13 @@ pub trait Executor {
fn run_command_and_measure_common(
mut command: std::process::Command,
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>,
options: &Options,
command_failure_action: CmdFailureAction,
command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy,
command_name: &str,
collect_hardware_counters: bool,
) -> Result<Measurement> {
let stdin = options.command_input_policy.get_stdin()?;
let stdin = command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -96,14 +83,10 @@ fn run_command_and_measure_common(
if !measurement.exit_status.success() {
use crate::util::exit_code::extract_exit_code;
let show_ignore_failure_hint = command_failure_action.is_none();
let should_fail = match command_failure_action
.as_ref()
.unwrap_or(&options.command_failure_action)
{
let should_fail = match command_failure_action {
CmdFailureAction::RaiseError => true,
CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(codes) => {
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
// Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(measurement.exit_status) {
!codes.contains(&exit_code)
@@ -122,17 +105,15 @@ fn run_command_and_measure_common(
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
let cause = measurement.exit_status.code().map_or_else(
|| "was terminated by a signal".to_owned(),
|code| format!("terminated with non-zero exit code {code}"),
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=measurement.exit_status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
),
);
let hint = if show_ignore_failure_hint {
" Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong."
} else {
" Use the '--show-output' option to debug what went wrong."
};
bail!("Command '{command_name}' {cause} in {when}.{hint}");
}
}
@@ -160,26 +141,14 @@ impl Executor for RawExecutor<'_> {
run_command_and_measure_common(
command.get_command()?,
iteration,
command_failure_action,
self.options,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
matches!(iteration, BenchmarkIteration::Benchmark(_)),
)
}
fn run_intermediate_command_and_measure(
&self,
command: &Command<'_>,
iteration: BenchmarkIteration,
output_policy: &CommandOutputPolicy,
) -> Result<Measurement> {
// Keep shell startup in the hook's duration: raw benchmarks have no
// calibrated shell overhead to add back when estimating the run count.
ShellExecutor::new(&Shell::platform_default(), self.options)
.run_intermediate_command_and_measure(command, iteration, output_policy)
}
fn calibrate(&mut self) -> Result<()> {
Ok(())
}
@@ -228,8 +197,8 @@ impl Executor for ShellExecutor<'_> {
let mut measurement = run_command_and_measure_common(
command_builder,
iteration,
command_failure_action,
self.options,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
false,
+28 -16
View File
@@ -11,7 +11,9 @@ use crate::benchmark::benchmark_result::Parameter;
use crate::benchmark::executor::BenchmarkIteration;
use crate::benchmark::measurement::{Measurement, Measurements};
use crate::command::Command;
use crate::options::{CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption};
use crate::options::{
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
};
use crate::outlier_detection::OUTLIER_THRESHOLD;
use crate::output::console_writeln;
use crate::output::progress_bar::{
@@ -53,24 +55,22 @@ impl<'a> Benchmark<'a> {
}
}
/// Run setup, cleanup, preparation, or conclusion commands
/// Run setup, cleanup, or preparation commands
fn run_intermediate_command(
&self,
command: &Command<'_>,
kind: &'static str,
error_output: &'static str,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.executor
.run_intermediate_command_and_measure(command, iteration, output_policy)
.map_err(|error| {
let hint = if cfg!(windows) {
"Append ' || exit /b 0' to the command if this failure can be ignored."
} else {
"Append ' || true' to the command if this failure can be ignored."
};
anyhow!("The {kind} command failed: {error:#} {hint}")
})
.run_command_and_measure(
command,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
.map_err(|_| anyhow!(error_output))
}
/// Run the command specified by `--setup`.
@@ -85,11 +85,14 @@ impl<'a> Benchmark<'a> {
.as_ref()
.map(|setup_command| Command::new_parametrized(None, setup_command, parameters));
let error_output = "The setup command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| {
self.run_intermediate_command(
&cmd,
"setup",
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
@@ -110,11 +113,14 @@ impl<'a> Benchmark<'a> {
.as_ref()
.map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters));
let error_output = "The cleanup command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| {
self.run_intermediate_command(
&cmd,
"cleanup",
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
@@ -130,7 +136,10 @@ impl<'a> Benchmark<'a> {
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.run_intermediate_command(command, "preparation", output_policy, iteration)
let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
/// Run the command specified by `--conclude`.
@@ -140,7 +149,10 @@ impl<'a> Benchmark<'a> {
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.run_intermediate_command(command, "conclusion", output_policy, iteration)
let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
fn validate_measurement(&self, measurement: &Measurement) -> Result<()> {
+5 -6
View File
@@ -236,15 +236,14 @@ fn build_command() -> Command {
.value_name("SHELL")
.overrides_with("shell")
.value_hint(ValueHint::CommandString)
.help("Set the shell to use for executing benchmarked commands, including warmup runs \
(default: none). This can be the name or the path to the shell executable, or a full command line \
.help("Set the shell to use for executing commands (default: none). This can be the \
name or the path to the shell executable, or a full command line \
like \"bash --norc\". It can also be set to \"default\" to explicitly select \
the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \
\"none\" to disable the shell. In this case, benchmarked commands will be executed \
\"none\" to disable the shell. In this case, commands will be executed \
directly. They can still have arguments, but more complex things like \
\"sleep 0.1; sleep 0.2\" are not possible without a shell. Setup, prepare, conclude, \
and cleanup commands always use a shell: the selected shell, or the platform \
shell when this option is omitted or set to \"none\".")
\"sleep 0.1; sleep 0.2\" are not possible without a shell. This option also \
applies to setup, prepare, conclude, and cleanup commands.")
)
.arg(
Arg::new("default-shell")
+19 -19
View File
@@ -275,7 +275,7 @@ fn fails_for_unknown_command() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577' terminated with non-zero exit code",
"Command terminated with non-zero exit code",
));
}
@@ -298,15 +298,12 @@ fn fails_for_failing_command_without_shell() {
hyperfine()
.arg("--shell=none")
.arg("--runs=1")
.arg("--command-name=named benchmark")
.args(["--parameter-list", "status", "0,1"])
.arg("test {status} -eq 0")
.arg("false")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'test 1 -eq 0' terminated with non-zero exit code 1 in the first benchmark run.",
))
.stderr(predicate::str::contains("--ignore-failure"));
"Command terminated with non-zero exit code",
));
}
#[test]
@@ -318,7 +315,9 @@ fn fails_for_unknown_setup_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The setup command failed:"));
.stderr(predicate::str::contains(
"The setup command terminated with a non-zero exit code.",
));
}
#[test]
@@ -330,7 +329,9 @@ fn fails_for_unknown_cleanup_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The cleanup command failed:"));
.stderr(predicate::str::contains(
"The cleanup command terminated with a non-zero exit code.",
));
}
#[test]
@@ -341,7 +342,9 @@ fn fails_for_unknown_prepare_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The preparation command failed:"));
.stderr(predicate::str::contains(
"The preparation command terminated with a non-zero exit code.",
));
}
#[test]
@@ -352,7 +355,9 @@ fn fails_for_unknown_conclude_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The conclusion command failed:"));
.stderr(predicate::str::contains(
"The conclusion command terminated with a non-zero exit code.",
));
}
#[cfg(unix)]
@@ -363,7 +368,7 @@ fn can_run_failing_commands_with_ignore_failure_option() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'false' terminated with non-zero exit code",
"Command terminated with non-zero exit code",
));
hyperfine()
@@ -395,7 +400,7 @@ fn can_ignore_specific_exit_codes() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'exit 2' terminated with non-zero exit code 2",
"Command terminated with non-zero exit code 2",
));
}
@@ -436,7 +441,7 @@ fn can_ignore_multiple_exit_codes() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'exit 4' terminated with non-zero exit code 4",
"Command terminated with non-zero exit code 4",
));
}
@@ -472,14 +477,9 @@ fn runs_commands_using_user_defined_shell() {
.arg("--show-output")
.arg("--shell")
.arg("echo 'custom_shell' '--shell-arg'")
.arg("--setup=echo setup")
.arg("--prepare=echo prepare")
.arg("--conclude=echo conclude")
.arg("--cleanup=echo cleanup")
.arg("echo benchmark")
.assert()
.success()
.stdout(predicate::str::contains("custom_shell --shell-arg").count(5))
.stdout(
predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or(
predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),