Compare commits

..
Author SHA1 Message Date
David Peter 4f62157cc9 Help text 2026-10-06 15:43:19 +02:00
David Peter 15108de79e CHANGELOG 2026-10-06 15:29:18 +02:00
David Peter 75d16c2239 More improvements 2026-10-06 15:19:54 +02:00
David Peter a947ba1d06 Simplifications 2026-10-06 15:09:25 +02:00
David Peter 1032204d7e README changes 2026-10-06 15:07:04 +02:00
David Peter 4f893b505e Simplifications 2026-10-06 13:34:03 +02:00
David Peter 715994847f Simplifications 2026-10-06 13:20:18 +02:00
David Peter 33edbaa784 Manpage 2026-10-06 13:06:39 +02:00
David Peter a027c1eb22 Updates 2026-10-06 12:54:39 +02:00
David Peter 55dfe6722c Better defaults 2026-10-06 12:53:42 +02:00
David Peter 908b33cc1d Add support for showing other metrics in the terminal 2026-10-06 12:53:42 +02:00
11 changed files with 100 additions and 131 deletions

No files matched your search

+1 -5
View File
@@ -222,8 +222,6 @@ jobs:
DPKG_CONFLICTS=${{ needs.crate_metadata.outputs.name }}-musl
case ${{ matrix.job.target }} in *-musl*) DPKG_BASENAME=${{ needs.crate_metadata.outputs.name }}-musl ; DPKG_CONFLICTS=${{ needs.crate_metadata.outputs.name }} ;; esac;
DPKG_VERSION=${{ needs.crate_metadata.outputs.version }}
# Debian prereleases must sort before the corresponding stable version.
DPKG_VERSION=${DPKG_VERSION/-/\~}
unset DPKG_ARCH
case ${{ matrix.job.target }} in
@@ -342,8 +340,6 @@ jobs:
uses: softprops/action-gh-release@v3
if: steps.is-release.outputs.IS_RELEASE
with:
prerelease: ${{ contains(needs.crate_metadata.outputs.version, '-') }}
make_latest: ${{ contains(needs.crate_metadata.outputs.version, '-') && 'false' || 'true' }}
files: |
${{ steps.package.outputs.PKG_PATH }}
${{ steps.debian-package.outputs.DPKG_PATH }}
@@ -354,7 +350,7 @@ jobs:
name: Publish to Winget
runs-on: ubuntu-latest
needs: build
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
if: startsWith(github.ref, 'refs/tags/v')
steps:
- uses: vedantmgoyal2009/winget-releaser@v2
with:
+6 -10
View File
@@ -1,25 +1,21 @@
# v2.0.0-alpha.1
This is the first alpha release of hyperfine 2.0. Command-line options and export formats may change before the stable release. See the breaking changes below when migrating from hyperfine 1.x.
# Unreleased
## Features
- Add support for alternative performance metrics (peak memory usage, CPU cycles, instructions, cache misses, branch misses, ...). By default, hyperfine will now display wall-clock time and peak memory usage (wall-clock time only on Windows),
- Add support for alternative performance metrics (peak memory usage, CPU cycles, instructions, cache misses, branch misses, ...). By default, hyperfine will now display wall-clock time and peak memory usage,
but users can use the new `--metrics` option to select different performance metrics.
- The terminal output now shows an overview of all selected performance metrics, including relative changes.
- Python plotting and analysis scripts support the new JSON format and can select performance metrics with `--metric`, see #981 and #984 (@sharkdp).
- The terminal output now shows an overview of all measured performance metrics, including relative changes.
## Breaking changes
- Benchmarked commands are now executed directly by default, without an intermediate shell (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell <SHELL>`.
- Benchmarked commands are now always executed directly, without an intermediate shell (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell <SHELL>`.
- The JSON format for `--export-json` has changed (schema version 2). The new format now includes metadata, per-run measurements, and statistical summaries for all measured quantities, see #790 (@sharkdp). Code that reads those exported JSON files must be updated. For example, `results[i].mean` is now `results[i].summary.time_wall_clock.mean`, and `results[i].times` is replaced by `results[i].measurements[j].time_wall_clock.value`. The new structure looks like this:
```python
{
"schema_version": 2,
"primary_metric": "time_wall_clock",
"metadata": {
"hyperfine_version": "2.0.0-alpha.1",
"hyperfine_version": "1.21.0",
"start_time": "2026-10-06T12:00:00Z",
"platform": {"os": "Linux", "architecture": "x86_64"}
},
@@ -59,7 +55,7 @@ This is the first alpha release of hyperfine 2.0. Command-line options and expor
- `--time-unit`/`-u` has been removed. Units can now be selected using the `--metrics METRIC[:UNIT],…` option, e.g. `--metrics time_wall_clock:ms,memory_peak_resident:MiB`.
- `--sort` has been removed. It complicated hyperfine significantly and doesn't make too much sense with the new output format.
- `--reference` and `--reference-name` have been removed (for now). The first command is always considered as the reference command.
- The format of Markdown, AsciiDoc, org-mode and CSV exports has also been changed. These formats contain only the primary metric (the first metric selected by `--metrics`). CSV now includes columns with the name and the unit of the primary metric.
- The format of Markdown, AsciiDoc, org-mode and CSV exports has also been changed. CSV now includes columns with the name and the unit of the primary metric.
# v1.21.0
+2 -2
View File
@@ -13,5 +13,5 @@ authors:
repository-code: 'https://github.com/sharkdp/hyperfine'
abstract: A command-line benchmarking tool.
license: MIT
version: 2.0.0-alpha.1
date-released: '2026-10-06'
version: 1.21.0
date-released: '2026-10-05'
Generated
+1 -1
View File
@@ -377,7 +377,7 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
[[package]]
name = "hyperfine"
version = "2.0.0-alpha.1"
version = "1.21.0"
dependencies = [
"anyhow",
"approx",
+1 -1
View File
@@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "2.0.0-alpha.1"
version = "1.21.0"
edition = "2018"
build = "build.rs"
rust-version = "1.97.0"
+13 -14
View File
@@ -1,16 +1,18 @@
# hyperfine
[![CICD](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml/badge.svg)](https://github.com/sharkdp/hyperfine/actions/workflows/CICD.yml)
[![Version info](https://img.shields.io/crates/v/hyperfine.svg)](https://crates.io/crates/hyperfine)
[中文](https://github.com/chinanf-boy/hyperfine-zh)
A command-line benchmarking tool.
<img width="839" height="205" alt="hyperfine demo" src="https://github.com/user-attachments/assets/13c20b1d-39c1-4da1-8fcb-0c775dd032f5" />
*(hyperfine in action, benchmarking [`mypy`](https://mypy-lang.org/) and [`ty`](https://ty.dev/))*
**Demo**: Benchmarking [`fd`](https://github.com/sharkdp/fd) and
[`find`](https://www.gnu.org/software/findutils/):
![hyperfine](https://i.imgur.com/z19OYxE.gif)
## Features
* Statistical analysis across multiple runs.
* Support for various performance metrics and hardware counters.
* Support for arbitrary shell commands.
* Constant feedback about the benchmark progress and current estimates.
* Warmup runs can be executed before the actual benchmark.
@@ -108,7 +110,7 @@ hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log'
### Intermediate shell
By default, benchmarked commands, including warmup runs, are executed directly, without an intermediate shell (`--shell=none`).
By default, commands are executed directly, without an intermediate shell (`--shell=none`).
Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported.
Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted.
This avoids shell startup overhead and the noise from correcting for it, especially for fast commands
@@ -125,11 +127,10 @@ You can also select a specific shell with `--shell <SHELL>`:
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
```
The `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands always run through a shell.
They use the shell selected by `--shell`, or the platform shell (`sh` on Unix, `cmd.exe` on Windows)
when `--shell` is omitted or set to `none`.
The shell setting applies to all commands, including `--setup`, `--prepare`, `--conclude`, and `--cleanup`.
If any of these commands need shell syntax, enable a shell explicitly.
When benchmarked commands use a shell, hyperfine *corrects for the shell spawning time*. It runs the shell with an
When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an
empty command multiple times to measure its startup time, then subtracts this time from each
time measurement.
@@ -188,9 +189,7 @@ The following metrics are available:
- Linux/macOS: Peak resident set size (RSS). This is the largest per-process peak among the
command and child processes (whose usage is collected when their parents wait for them),
*not the simultaneous total memory of the full process tree*. On Linux, measuring commands
with a very small peak RSS (below the peak inherited from hyperfine at startup, which can
be a few MiB) will currently result in that inherited value being reported instead.
*not the simultaneous total memory of the full process tree*.
- Windows: Currently not supported.
- **`cpu_cycles`**: CPU cycles consumed.
@@ -219,7 +218,7 @@ The following metrics are available:
- Linux: Same scope as `cpu_cycles`.
- macOS/Windows: Currently not supported.
Note that hardware counters are not available when a shell is enabled (`--shell=..`).
Note that hardware counters are not available if a `--shell` is used.
Time measurements support units `ns`, `us`, `ms`, `s`, `min`, and `h`. Memory measurements
support `B`, `kB`, `MB`, `GB`, `TB`, `KiB`, `MiB`, `GiB`, and `TiB`. Hardware counters
@@ -236,7 +235,7 @@ You can also use a preset to select a group of metrics:
### Exporting results
Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. Non-JSON formats
contain only the primary metric, which is the first metric selected by `--metrics`.
contain only the primary metric.
#### Markdown
+4 -6
View File
@@ -194,16 +194,14 @@ possible parameter combinations.
.HP
\fB\-\-shell\fR \fISHELL\fP
.IP
Set the shell to use for executing benchmarked commands, including warmup runs
(default: none). This can be the name or the path to the shell executable,
or a full command line
Set the shell to use for executing commands (default: none). This can be
the name or the path to the shell executable, or a full command line
like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly
select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be
set to "none" to disable the shell. In this case, benchmarked commands will be
set to "none" to disable the shell. In this case, commands will be
executed directly. They can still have arguments, but more complex
things like "sleep 0.1; sleep 0.2" are not possible without a shell.
Setup, prepare, conclude, and cleanup commands always use a shell: the selected
shell, or the platform shell when this option is omitted or set to "none".
This option also applies to setup, prepare, conclude, and cleanup commands.
.HP
\fB\-S\fR
.IP
+20 -51
View File
@@ -5,7 +5,9 @@ use std::process::ExitStatus;
use crate::benchmark::measurement::Measurement;
use crate::benchmark::measurement::Measurements;
use crate::command::Command;
use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell};
use crate::options::{
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
};
use crate::output::progress_bar::get_progress_bar;
use crate::quantity::{second, Information, Time, Zero};
use crate::timer::execute_and_measure;
@@ -40,21 +42,6 @@ pub trait Executor {
output_policy: &CommandOutputPolicy,
) -> Result<Measurement>;
/// Run a lifecycle command through a shell, always treating failures as errors.
fn run_intermediate_command_and_measure(
&self,
command: &Command<'_>,
iteration: BenchmarkIteration,
output_policy: &CommandOutputPolicy,
) -> Result<Measurement> {
self.run_command_and_measure(
command,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
}
/// Perform a calibration of this executor. For example,
/// when running commands through a shell, we need to
/// measure the shell spawning time separately in order
@@ -71,13 +58,13 @@ pub trait Executor {
fn run_command_and_measure_common(
mut command: std::process::Command,
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>,
options: &Options,
command_failure_action: CmdFailureAction,
command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy,
command_name: &str,
collect_hardware_counters: bool,
) -> Result<Measurement> {
let stdin = options.command_input_policy.get_stdin()?;
let stdin = command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -96,14 +83,10 @@ fn run_command_and_measure_common(
if !measurement.exit_status.success() {
use crate::util::exit_code::extract_exit_code;
let show_ignore_failure_hint = command_failure_action.is_none();
let should_fail = match command_failure_action
.as_ref()
.unwrap_or(&options.command_failure_action)
{
let should_fail = match command_failure_action {
CmdFailureAction::RaiseError => true,
CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(codes) => {
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
// Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(measurement.exit_status) {
!codes.contains(&exit_code)
@@ -122,17 +105,15 @@ fn run_command_and_measure_common(
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
let cause = measurement.exit_status.code().map_or_else(
|| "was terminated by a signal".to_owned(),
|code| format!("terminated with non-zero exit code {code}"),
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=measurement.exit_status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
),
);
let hint = if show_ignore_failure_hint {
" Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong."
} else {
" Use the '--show-output' option to debug what went wrong."
};
bail!("Command '{command_name}' {cause} in {when}.{hint}");
}
}
@@ -160,26 +141,14 @@ impl Executor for RawExecutor<'_> {
run_command_and_measure_common(
command.get_command()?,
iteration,
command_failure_action,
self.options,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
matches!(iteration, BenchmarkIteration::Benchmark(_)),
)
}
fn run_intermediate_command_and_measure(
&self,
command: &Command<'_>,
iteration: BenchmarkIteration,
output_policy: &CommandOutputPolicy,
) -> Result<Measurement> {
// Keep shell startup in the hook's duration: raw benchmarks have no
// calibrated shell overhead to add back when estimating the run count.
ShellExecutor::new(&Shell::platform_default(), self.options)
.run_intermediate_command_and_measure(command, iteration, output_policy)
}
fn calibrate(&mut self) -> Result<()> {
Ok(())
}
@@ -228,8 +197,8 @@ impl Executor for ShellExecutor<'_> {
let mut measurement = run_command_and_measure_common(
command_builder,
iteration,
command_failure_action,
self.options,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
false,
+28 -16
View File
@@ -11,7 +11,9 @@ use crate::benchmark::benchmark_result::Parameter;
use crate::benchmark::executor::BenchmarkIteration;
use crate::benchmark::measurement::{Measurement, Measurements};
use crate::command::Command;
use crate::options::{CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption};
use crate::options::{
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
};
use crate::outlier_detection::OUTLIER_THRESHOLD;
use crate::output::console_writeln;
use crate::output::progress_bar::{
@@ -53,24 +55,22 @@ impl<'a> Benchmark<'a> {
}
}
/// Run setup, cleanup, preparation, or conclusion commands
/// Run setup, cleanup, or preparation commands
fn run_intermediate_command(
&self,
command: &Command<'_>,
kind: &'static str,
error_output: &'static str,
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.executor
.run_intermediate_command_and_measure(command, iteration, output_policy)
.map_err(|error| {
let hint = if cfg!(windows) {
"Append ' || exit /b 0' to the command if this failure can be ignored."
} else {
"Append ' || true' to the command if this failure can be ignored."
};
anyhow!("The {kind} command failed: {error:#} {hint}")
})
.run_command_and_measure(
command,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
.map_err(|_| anyhow!(error_output))
}
/// Run the command specified by `--setup`.
@@ -85,11 +85,14 @@ impl<'a> Benchmark<'a> {
.as_ref()
.map(|setup_command| Command::new_parametrized(None, setup_command, parameters));
let error_output = "The setup command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| {
self.run_intermediate_command(
&cmd,
"setup",
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
@@ -110,11 +113,14 @@ impl<'a> Benchmark<'a> {
.as_ref()
.map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters));
let error_output = "The cleanup command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command
.map(|cmd| {
self.run_intermediate_command(
&cmd,
"cleanup",
error_output,
output_policy,
BenchmarkIteration::NonBenchmarkRun,
)
@@ -130,7 +136,10 @@ impl<'a> Benchmark<'a> {
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.run_intermediate_command(command, "preparation", output_policy, iteration)
let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
/// Run the command specified by `--conclude`.
@@ -140,7 +149,10 @@ impl<'a> Benchmark<'a> {
output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration,
) -> Result<Measurement> {
self.run_intermediate_command(command, "conclusion", output_policy, iteration)
let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy, iteration)
}
fn validate_measurement(&self, measurement: &Measurement) -> Result<()> {
+5 -6
View File
@@ -236,15 +236,14 @@ fn build_command() -> Command {
.value_name("SHELL")
.overrides_with("shell")
.value_hint(ValueHint::CommandString)
.help("Set the shell to use for executing benchmarked commands, including warmup runs \
(default: none). This can be the name or the path to the shell executable, or a full command line \
.help("Set the shell to use for executing commands (default: none). This can be the \
name or the path to the shell executable, or a full command line \
like \"bash --norc\". It can also be set to \"default\" to explicitly select \
the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \
\"none\" to disable the shell. In this case, benchmarked commands will be executed \
\"none\" to disable the shell. In this case, commands will be executed \
directly. They can still have arguments, but more complex things like \
\"sleep 0.1; sleep 0.2\" are not possible without a shell. Setup, prepare, conclude, \
and cleanup commands always use a shell: the selected shell, or the platform \
shell when this option is omitted or set to \"none\".")
\"sleep 0.1; sleep 0.2\" are not possible without a shell. This option also \
applies to setup, prepare, conclude, and cleanup commands.")
)
.arg(
Arg::new("default-shell")
+19 -19
View File
@@ -275,7 +275,7 @@ fn fails_for_unknown_command() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577' terminated with non-zero exit code",
"Command terminated with non-zero exit code",
));
}
@@ -298,15 +298,12 @@ fn fails_for_failing_command_without_shell() {
hyperfine()
.arg("--shell=none")
.arg("--runs=1")
.arg("--command-name=named benchmark")
.args(["--parameter-list", "status", "0,1"])
.arg("test {status} -eq 0")
.arg("false")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'test 1 -eq 0' terminated with non-zero exit code 1 in the first benchmark run.",
))
.stderr(predicate::str::contains("--ignore-failure"));
"Command terminated with non-zero exit code",
));
}
#[test]
@@ -318,7 +315,9 @@ fn fails_for_unknown_setup_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The setup command failed:"));
.stderr(predicate::str::contains(
"The setup command terminated with a non-zero exit code.",
));
}
#[test]
@@ -330,7 +329,9 @@ fn fails_for_unknown_cleanup_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The cleanup command failed:"));
.stderr(predicate::str::contains(
"The cleanup command terminated with a non-zero exit code.",
));
}
#[test]
@@ -341,7 +342,9 @@ fn fails_for_unknown_prepare_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The preparation command failed:"));
.stderr(predicate::str::contains(
"The preparation command terminated with a non-zero exit code.",
));
}
#[test]
@@ -352,7 +355,9 @@ fn fails_for_unknown_conclude_command() {
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains("The conclusion command failed:"));
.stderr(predicate::str::contains(
"The conclusion command terminated with a non-zero exit code.",
));
}
#[cfg(unix)]
@@ -363,7 +368,7 @@ fn can_run_failing_commands_with_ignore_failure_option() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'false' terminated with non-zero exit code",
"Command terminated with non-zero exit code",
));
hyperfine()
@@ -395,7 +400,7 @@ fn can_ignore_specific_exit_codes() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'exit 2' terminated with non-zero exit code 2",
"Command terminated with non-zero exit code 2",
));
}
@@ -436,7 +441,7 @@ fn can_ignore_multiple_exit_codes() {
.assert()
.failure()
.stderr(predicate::str::contains(
"Command 'exit 4' terminated with non-zero exit code 4",
"Command terminated with non-zero exit code 4",
));
}
@@ -472,14 +477,9 @@ fn runs_commands_using_user_defined_shell() {
.arg("--show-output")
.arg("--shell")
.arg("echo 'custom_shell' '--shell-arg'")
.arg("--setup=echo setup")
.arg("--prepare=echo prepare")
.arg("--conclude=echo conclude")
.arg("--cleanup=echo cleanup")
.arg("echo benchmark")
.assert()
.success()
.stdout(predicate::str::contains("custom_shell --shell-arg").count(5))
.stdout(
predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or(
predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),