Compare commits

..
35 Commits
Author SHA1 Message Date
David Peter 975fe108c4 Bump version to 1.20.0 2025-11-18 09:34:01 +01:00
dependabot[bot] 001dcce10e Bump actions/checkout from 4 to 5
Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-18 09:20:55 +01:00
dependabot[bot] 6447ab3a4e Bump clap from 4.5.47 to 4.5.48
Bumps [clap](https://github.com/clap-rs/clap) from 4.5.47 to 4.5.48.
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.47...clap_complete-v4.5.48)

---
updated-dependencies:
- dependency-name: clap
  dependency-version: 4.5.48
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-18 09:20:35 +01:00
Sylvestre Ledru ae4c8073db Debian: Hyperfine is now available in stable 2025-11-18 09:20:24 +01:00
dependabot[bot] 57ef17cdcb Bump tempfile from 3.14.0 to 3.23.0
Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.14.0 to 3.23.0.
- [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md)
- [Commits](https://github.com/Stebalien/tempfile/compare/v3.14.0...v3.23.0)

---
updated-dependencies:
- dependency-name: tempfile
  dependency-version: 3.23.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-18 09:20:01 +01:00
Anti Räis 8b6224059c fix typo in docs 2025-11-18 09:19:49 +01:00
David Peter b5a6860d96 Support list of exit codes in --ignore-failure 2025-11-18 09:19:27 +01:00
David Peter 16348dcf35 Update README 2025-09-23 19:38:55 +02:00
David Peter de07c77387 Formatting 2025-09-04 08:59:11 +02:00
David Peter b20df39f0f Update README 2025-09-04 08:57:49 +02:00
David Peter 6f95f08d2f Update README 2025-09-04 08:57:34 +02:00
David Peter 214bff7065 Update README 2025-09-04 08:56:31 +02:00
David Peter fae8b0ef64 .contains(…) 2025-09-03 20:52:28 +02:00
David Peter 98d262c497 Fix warnings 2025-09-03 20:51:44 +02:00
Ravi Kant Sharma 6b8fa5fd3d Update ubuntu install instructions 2025-09-03 20:50:19 +02:00
Niklas Dewally 2166c9f02e Add option to name reference command
Add --reference-name option to give a reference command a name.

This was discussed in the PR that added --reference, #744, but
was unfortunately never implemented.
2025-09-03 20:45:39 +02:00
Teodoro Freund 835fc43ea1 Fixed bug on naming individual commands with parameter scan 2025-09-03 20:44:07 +02:00
dependabot[bot] 5e32684411 Bump predicates from 3.1.2 to 3.1.3
Bumps [predicates](https://github.com/assert-rs/predicates-rs) from 3.1.2 to 3.1.3.
- [Changelog](https://github.com/assert-rs/predicates-rs/blob/master/CHANGELOG.md)
- [Commits](https://github.com/assert-rs/predicates-rs/compare/v3.1.2...v3.1.3)

---
updated-dependencies:
- dependency-name: predicates
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-03 20:41:16 +02:00
dependabot[bot] c6fd0395b9 Bump clap from 4.5.23 to 4.5.37
Bumps [clap](https://github.com/clap-rs/clap) from 4.5.23 to 4.5.37.
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.23...clap_complete-v4.5.37)

---
updated-dependencies:
- dependency-name: clap
  dependency-version: 4.5.37
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-03 20:40:58 +02:00
David Peter f8db978653 Fix CI 2025-09-03 20:34:31 +02:00
Minh Tran 648819a1ad doc(manpage): add missing --conclude header 2025-09-03 20:31:00 +02:00
Martin Liska 3cedcc38d0 Address comments 2025-03-14 21:04:11 +01:00
Martin Liska ee4e4d04b2 Add plot_benchmarks scripts
The script is intended for a collection of hyperfine JSON files where
multiple benchmarks were run for a selected group of commands.
2025-03-14 21:04:11 +01:00
David Peter 92cc4f2d79 Update README 2025-03-09 19:27:22 +01:00
Lars Kappert 3b0918511a Add link to Venz 2025-01-10 22:52:18 +01:00
Ritvik 3bd38f275e fix(tests): restrict 'cat' tests to unix environments. Fixes #776 (#777)
Tests using the 'cat' command are specific
to unix-like environments and
fail on unsupported platforms.
2025-01-04 22:27:16 +01:00
dependabot[bot] f6794c313c Bump anyhow from 1.0.93 to 1.0.95
Bumps [anyhow](https://github.com/dtolnay/anyhow) from 1.0.93 to 1.0.95.
- [Release notes](https://github.com/dtolnay/anyhow/releases)
- [Commits](https://github.com/dtolnay/anyhow/compare/1.0.93...1.0.95)

---
updated-dependencies:
- dependency-name: anyhow
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-04 16:33:50 +01:00
dependabot[bot] efe3b53767 Bump clap from 4.5.20 to 4.5.23
Bumps [clap](https://github.com/clap-rs/clap) from 4.5.20 to 4.5.23.
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.20...clap_complete-v4.5.23)

---
updated-dependencies:
- dependency-name: clap
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-01-04 16:33:41 +01:00
David Peter 582f2ed960 Update macOS runners 2024-12-29 14:10:01 +01:00
David Peter ebcf794d97 Clippy 2024-12-29 14:07:21 +01:00
David Peter 038f369420 SortOrder field 2024-12-29 14:07:21 +01:00
David Peter 0705d6c7d0 Basic scheduler test 2024-12-29 14:07:21 +01:00
David Peter 0d2e3a4dd7 Simplify CSV tests 2024-12-29 14:07:21 +01:00
David Peter da399ab654 Move more tests to insta 2024-12-29 14:07:21 +01:00
David Peter 7a9d04d35c Use insta for exporter tests 2024-12-29 14:07:21 +01:00
17 changed files with 776 additions and 318 deletions

No files matched your search

+18 -18
View File
@@ -18,7 +18,7 @@ jobs:
name: Extract crate metadata
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- name: Extract crate information
id: crate_metadata
run: |
@@ -44,21 +44,21 @@ jobs:
ensure_cargo_fmt:
name: Ensure 'cargo fmt' has been run
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- run: cargo fmt -- --check
min_version:
name: Minimum supported rust version
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
needs: crate_metadata
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@v5
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
uses: dtolnay/rust-toolchain@master
@@ -78,23 +78,23 @@ jobs:
fail-fast: false
matrix:
job:
- { target: aarch64-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
- { target: aarch64-apple-darwin , os: macos-14 }
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-20.04, use-cross: true }
- { target: arm-unknown-linux-musleabihf, os: ubuntu-20.04, use-cross: true }
- { target: i686-pc-windows-msvc , os: windows-2019 }
- { target: i686-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
- { target: i686-unknown-linux-musl , os: ubuntu-20.04, use-cross: true }
- { target: x86_64-apple-darwin , os: macos-12 }
# - { target: x86_64-pc-windows-gnu , os: windows-2019 }
- { target: x86_64-pc-windows-msvc , os: windows-2019 }
- { target: x86_64-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
- { target: x86_64-unknown-linux-musl , os: ubuntu-20.04, use-cross: true }
- { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }
- { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }
- { target: i686-pc-windows-msvc , os: windows-2022 }
- { target: i686-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
- { target: i686-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
- { target: x86_64-apple-darwin , os: macos-15 }
- { target: aarch64-apple-darwin , os: macos-15 }
# - { target: x86_64-pc-windows-gnu , os: windows-2022 }
- { target: x86_64-pc-windows-msvc , os: windows-2022 }
- { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
- { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
env:
BUILD_CMD: cargo
steps:
- name: Checkout source code
uses: actions/checkout@v4
uses: actions/checkout@v5
- name: Install prerequisites
shell: bash
+17
View File
@@ -1,3 +1,20 @@
# v1.20.0
## Features
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
## Bugfixes
- Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr)
## Other
- Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos)
# v1.19.0
## Features
Generated
+381 -247
View File
File diff suppressed because it is too large. Load diff
+5 -5
View File
@@ -7,10 +7,10 @@ license = "MIT OR Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "1.19.0"
version = "1.20.0"
edition = "2018"
build = "build.rs"
rust-version = "1.76.0"
rust-version = "1.88.0"
[features]
# Use the nightly feature windows_process_extensions_main_thread_handle
@@ -64,15 +64,15 @@ features = [
approx = "0.5"
assert_cmd = "2.0"
insta = { version = "1.41.1", features = ["yaml"] }
predicates = "3.0"
tempfile = "3.14"
predicates = "3.1"
tempfile = "3.23"
[profile.dev.package]
insta.opt-level = 3
similar.opt-level = 3
[build-dependencies]
clap = "4.4.12"
clap = "4.5.48"
clap_complete = "4.2.1"
[profile.release]
+35 -5
View File
@@ -10,6 +10,29 @@ A command-line benchmarking tool.
![hyperfine](https://i.imgur.com/z19OYxE.gif)
### Sponsors
A special *thank you* goes to our biggest <a href="doc/sponsors.md">sponsors</a>:<br>
<a href="https://www.warp.dev/hyperfine">
<img src="doc/sponsors/warp-logo.png" width="200" alt="Warp">
<br>
<strong>Warp, the intelligent terminal</strong>
<br>
<sub>Available on MacOS, Linux, Windows</sub>
</a>
<br>
<br>
<a href="https://tuple.app/hyperfine">
<img src="doc/sponsors/tuple-logo.png" width="200" alt="Tuple">
<br>
<strong>Tuple, the premier screen sharing app for developers</strong>
<br>
<sub>Available for MacOS &amp; Windows</sub>
</a>
## Features
* Statistical analysis across multiple runs.
@@ -167,11 +190,15 @@ like `--warmup`, `--prepare <cmd>`, `--setup <cmd>` or `--cleanup <cmd>`:
### On Ubuntu
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
On Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine):
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb
sudo dpkg -i hyperfine_1.19.0_amd64.deb
apt install hyperfine
```
Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb
sudo dpkg -i hyperfine_1.20.0_amd64.deb
```
### On Fedora
@@ -198,7 +225,7 @@ pacman -S hyperfine
### On Debian Linux
On Debian Linux, hyperfine can be installed [from the testing repositories](https://packages.debian.org/testing/main/hyperfine):
On Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine):
```
apt install hyperfine
```
@@ -319,6 +346,9 @@ visualize changes in benchmark timings across your Git history.
[Bencher](https://github.com/bencherdev/bencher) is a continuous benchmarking tool that supports `hyperfine` to
track benchmarks and catch performance regressions in CI.
Drop hyperfine JSON outputs onto the [Venz](https://try.venz.dev) chart to visualize the results,
and manage hyperfine configurations.
Make sure to check out the [`scripts` folder](https://github.com/sharkdp/hyperfine/tree/master/scripts)
in this repository for a set of tools to work with `hyperfine` benchmark results.
+2 -1
View File
@@ -117,6 +117,7 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
once for each command. In the latter case, each preparation command will be
run prior to the corresponding benchmark command.
.HP
\fB\-\-conclude\fR \fICMD...\fP
.IP
Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
for example.
@@ -130,7 +131,7 @@ Execute \fICMD\fP after the completion of all benchmarking runs for each individ
command to be benchmarked. This is useful if the commands to be benchmarked
produce artifacts that need to be cleaned up. It only runs once a series of
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
ever run.
every run.
.HP
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
.IP
+14
View File
@@ -0,0 +1,14 @@
## Sponsors
`hyperfine` development is sponsored by many individuals and companies. Thank you very much!
Please note, that being sponsored does not affect the individuality of the `hyperfine`
project or affect the maintainers' actions in any way.
We remain impartial and continue to assess pull requests solely on merit - the
features added, bugs solved, and effect on the overall complexity of the code.
No issue will have a different priority based on sponsorship status of the
reporter.
Contributions from anybody are most welcomed.
If you want to see our biggest sponsors, check the top of [`README.md`](../README.md#sponsors).
Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 130 KiB

+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env python
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "matplotlib",
# "pyqt6",
# "numpy",
# ]
# ///
"""
This script shows `hyperfine` benchmark results as a bar plot grouped by command.
Note all the input files must contain results for all commands.
"""
import argparse
import json
import pathlib
import matplotlib.pyplot as plt
import numpy as np
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
)
parser.add_argument("--title", help="Plot Title")
parser.add_argument(
"--benchmark-names", nargs="+", help="Names of the benchmark groups"
)
parser.add_argument("-o", "--output", help="Save image to the given filename")
args = parser.parse_args()
commands = None
data = []
inputs = []
if args.benchmark_names:
assert len(args.files) == len(
args.benchmark_names
), "Number of benchmark names must match the number of input files."
for i, filename in enumerate(args.files):
with open(filename) as f:
results = json.load(f)["results"]
benchmark_commands = [b["command"] for b in results]
if commands is None:
commands = benchmark_commands
else:
assert (
commands == benchmark_commands
), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
data.append([round(b["mean"], 2) for b in results])
if args.benchmark_names:
inputs.append(args.benchmark_names[i])
else:
inputs.append(filename.stem)
data = np.transpose(data)
x = np.arange(len(inputs)) # the label locations
width = 0.25 # the width of the bars
fig, ax = plt.subplots(layout="constrained")
fig.set_figheight(5)
fig.set_figwidth(10)
for i, command in enumerate(commands):
offset = width * (i + 1)
rects = ax.bar(x + offset, data[i], width, label=command)
ax.set_xticks(x + 0.5, inputs)
ax.grid(visible=True, axis="y")
if args.title:
plt.title(args.title)
plt.xlabel("Benchmark")
plt.ylabel("Time [s]")
plt.legend(title="Command")
if args.output:
plt.savefig(args.output)
else:
plt.show()
+36 -18
View File
@@ -79,23 +79,41 @@ fn run_command_and_measure_common(
let result = execute_and_measure(command)
.with_context(|| format!("Failed to run command '{command_name}'"))?;
if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
let when = match iteration {
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=result.status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
if !result.status.success() {
use crate::util::exit_code::extract_exit_code;
),
);
let should_fail = match command_failure_action {
CmdFailureAction::RaiseError => true,
CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
// Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(result.status) {
!codes.contains(&exit_code)
} else {
// If we can't extract an exit code, treat it as a failure
true
}
}
};
if should_fail {
let when = match iteration {
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=result.status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
),
);
}
}
Ok(result)
@@ -122,7 +140,7 @@ impl Executor for RawExecutor<'_> {
let result = run_command_and_measure_common(
command.get_command()?,
iteration,
command_failure_action.unwrap_or(self.options.command_failure_action),
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
@@ -187,7 +205,7 @@ impl Executor for ShellExecutor<'_> {
let mut result = run_command_and_measure_common(
command_builder,
iteration,
command_failure_action.unwrap_or(self.options.command_failure_action),
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
+2 -2
View File
@@ -98,7 +98,7 @@ pub fn compute_with_check_from_reference<'a>(
pub fn compute_with_check(
results: &[BenchmarkResult],
sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
let fastest = fastest_of(results);
if fastest.mean == 0.0 {
@@ -112,7 +112,7 @@ pub fn compute_with_check(
pub fn compute(
results: &[BenchmarkResult],
sort_order: SortOrder,
) -> Vec<BenchmarkResultWithRelativeSpeed> {
) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
let fastest = fastest_of(results);
compute_relative_speeds(results, fastest, sort_order)
+1 -1
View File
@@ -42,7 +42,7 @@ impl<'a> Scheduler<'a> {
.options
.reference_command
.as_ref()
.map(|cmd| Command::new(None, cmd));
.map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
executor.calibrate()?;
+16 -2
View File
@@ -96,6 +96,14 @@ fn build_command() -> Command {
If this is unset, results are compared with the fastest command as reference."
)
)
.arg(
Arg::new("reference-name")
.long("reference-name")
.action(ArgAction::Set)
.value_name("CMD")
.help("Give a meaningful name to the reference command.")
.requires("reference")
)
.arg(
Arg::new("prepare")
.long("prepare")
@@ -218,9 +226,15 @@ fn build_command() -> Command {
.arg(
Arg::new("ignore-failure")
.long("ignore-failure")
.action(ArgAction::SetTrue)
.action(ArgAction::Set)
.value_name("MODE")
.num_args(0..=1)
.default_missing_value("all-non-zero")
.require_equals(true)
.short('i')
.help("Ignore non-zero exit codes of the benchmarked programs."),
.help("Ignore failures of the benchmarked programs. Without a value or with \
'all-non-zero', all non-zero exit codes are ignored. You can also provide \
a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2)."),
)
.arg(
Arg::new("style")
+45 -10
View File
@@ -277,18 +277,8 @@ impl<'a> Commands<'a> {
command_strings: Vec<&'b str>,
) -> Result<Vec<Command<'b>>, ParameterScanError> {
let param_range = RangeStep::new(param_min, param_max, step)?;
let param_count = param_range.size_hint().1.unwrap();
let command_name_count = command_names.len();
// `--command-name` should appear exactly once or exactly B times,
// where B is the total number of benchmarks.
if command_name_count > 1 && command_name_count != param_count {
return Err(ParameterScanError::UnexpectedCommandNameCount(
command_name_count,
param_count,
));
}
let mut i = 0;
let mut commands = vec![];
for value in param_range {
@@ -305,6 +295,17 @@ impl<'a> Commands<'a> {
i += 1;
}
}
// `--command-name` should appear exactly once or exactly B times,
// where B is the total number of benchmarks.
let command_count = commands.len();
if command_name_count > 1 && command_name_count != command_count {
return Err(ParameterScanError::UnexpectedCommandNameCount(
command_name_count,
command_count,
));
}
Ok(commands)
}
@@ -471,6 +472,40 @@ fn test_build_parameter_scan_commands() {
assert_eq!(commands[1].get_command_line(), "echo 2");
}
#[test]
fn test_build_parameter_scan_commands_named() {
use crate::cli::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
"echo {val}",
"sleep {val}",
"--parameter-scan",
"val",
"1",
"2",
"--parameter-step-size",
"1",
"--command-name",
"echo-1",
"--command-name",
"sleep-1",
"--command-name",
"echo-2",
"--command-name",
"sleep-2",
]);
let commands = Commands::from_cli_arguments(&matches).unwrap().0;
assert_eq!(commands.len(), 4);
assert_eq!(commands[0].get_name(), "echo-1");
assert_eq!(commands[0].get_command_line(), "echo 1");
assert_eq!(commands[1].get_name(), "sleep-1");
assert_eq!(commands[1].get_command_line(), "sleep 1");
assert_eq!(commands[2].get_name(), "echo-2");
assert_eq!(commands[2].get_command_line(), "echo 2");
assert_eq!(commands[3].get_name(), "sleep-2");
assert_eq!(commands[3].get_command_line(), "sleep 2");
}
#[test]
fn test_parameter_scan_commands_int() {
let commands = Commands::build_parameter_scan_commands(
+29 -7
View File
@@ -67,13 +67,16 @@ impl Shell {
}
/// Action to take when an executed command fails.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CmdFailureAction {
/// Exit with an error message
RaiseError,
/// Simply ignore the non-zero exit code
Ignore,
/// Ignore all non-zero exit codes
IgnoreAllFailures,
/// Ignore specific exit codes
IgnoreSpecificFailures(Vec<i32>),
}
/// Output style type option
@@ -208,6 +211,9 @@ pub struct Options {
// Command to use as a reference for relative speed comparison
pub reference_command: Option<String>,
// Name of the reference command
pub reference_name: Option<String>,
/// Command(s) to run before each timing run
pub preparation_command: Option<Vec<String>>,
@@ -250,6 +256,7 @@ impl Default for Options {
min_benchmarking_time: 3.0,
command_failure_action: CmdFailureAction::RaiseError,
reference_command: None,
reference_name: None,
preparation_command: None,
conclusion_command: None,
setup_command: None,
@@ -310,6 +317,9 @@ impl Options {
options.setup_command = matches.get_one::<String>("setup").map(String::from);
options.reference_command = matches.get_one::<String>("reference").map(String::from);
options.reference_name = matches
.get_one::<String>("reference-name")
.map(String::from);
options.preparation_command = matches
.get_many::<String>("prepare")
@@ -354,8 +364,7 @@ impl Options {
_ => {
if options
.command_output_policies
.iter()
.any(|policy| *policy == CommandOutputPolicy::Inherit)
.contains(&CommandOutputPolicy::Inherit)
|| !io::stdout().is_terminal()
{
OutputStyleOption::Basic
@@ -409,8 +418,21 @@ impl Options {
}
};
if matches.get_flag("ignore-failure") {
options.command_failure_action = CmdFailureAction::Ignore;
if let Some(mode) = matches.get_one::<String>("ignore-failure") {
options.command_failure_action = match mode.as_str() {
"all-non-zero" | "" => CmdFailureAction::IgnoreAllFailures,
codes => {
let exit_codes: Result<Vec<i32>, _> = codes
.split(',')
.map(|s| {
s.trim()
.parse::<i32>()
.map_err(|e| OptionsError::IntParsingError("ignore-failure", e))
})
.collect();
CmdFailureAction::IgnoreSpecificFailures(exit_codes?)
}
};
}
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
+92 -2
View File
@@ -3,6 +3,12 @@ use common::hyperfine;
use predicates::prelude::*;
/// Platform-specific I/O utility.
/// - On Unix-like systems, defaults to `cat`.
/// - On Windows, uses `findstr` as an alternative.
/// See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
pub fn hyperfine_debug() -> assert_cmd::Command {
let mut cmd = hyperfine();
cmd.arg("--debug-mode");
@@ -271,6 +277,78 @@ fn can_run_failing_commands_with_ignore_failure_option() {
.success();
}
#[cfg(unix)]
#[test]
fn can_ignore_specific_exit_codes() {
// Test that specifying exit code 1 ignores it
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1")
.arg("exit 1")
.assert()
.success();
// Test that other exit codes still fail
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1")
.arg("exit 2")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command terminated with non-zero exit code 2",
));
}
#[cfg(unix)]
#[test]
fn can_ignore_multiple_exit_codes() {
// Test that all specified exit codes are ignored
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 1")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 2")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 3")
.assert()
.success();
// Test that other exit codes still fail
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=1,2,3")
.arg("exit 4")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command terminated with non-zero exit code 4",
));
}
#[cfg(unix)]
#[test]
fn ignore_failure_with_all_non_zero() {
// Test explicit "all-non-zero" mode
hyperfine()
.arg("--runs=1")
.arg("--ignore-failure=all-non-zero")
.arg("exit 5")
.assert()
.success();
}
#[test]
fn shows_output_of_benchmarked_command() {
hyperfine()
@@ -306,7 +384,7 @@ fn can_pass_input_to_command_from_a_file() {
.arg("--runs=1")
.arg("--input=example_input_file.txt")
.arg("--show-output")
.arg("cat")
.arg(STDIN_READ_COMMAND)
.assert()
.success()
.stdout(predicate::str::contains("This text is part of a file"));
@@ -318,7 +396,7 @@ fn fails_if_invalid_stdin_data_file_provided() {
.arg("--runs=1")
.arg("--input=example_non_existent_file.txt")
.arg("--show-output")
.arg("cat")
.arg(STDIN_READ_COMMAND)
.assert()
.failure()
.stderr(predicate::str::contains(
@@ -493,6 +571,18 @@ fn shows_benchmark_comparison_relative_to_reference() {
);
}
#[test]
fn shows_reference_name() {
hyperfine_debug()
.arg("--reference=sleep 2.0")
.arg("--reference-name=refabc123")
.arg("sleep 1.0")
.arg("sleep 3.0")
.assert()
.success()
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
}
#[test]
fn performs_all_benchmarks_in_parameter_scan() {
hyperfine_debug()