mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-17 18:01:30 +00:00
Compare commits
35
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
975fe108c4 | ||
|
|
001dcce10e | ||
|
|
6447ab3a4e | ||
|
|
ae4c8073db | ||
|
|
57ef17cdcb | ||
|
|
8b6224059c | ||
|
|
b5a6860d96 | ||
|
|
16348dcf35 | ||
|
|
de07c77387 | ||
|
|
b20df39f0f | ||
|
|
6f95f08d2f | ||
|
|
214bff7065 | ||
|
|
fae8b0ef64 | ||
|
|
98d262c497 | ||
|
|
6b8fa5fd3d | ||
|
|
2166c9f02e | ||
|
|
835fc43ea1 | ||
|
|
5e32684411 | ||
|
|
c6fd0395b9 | ||
|
|
f8db978653 | ||
|
|
648819a1ad | ||
|
|
3cedcc38d0 | ||
|
|
ee4e4d04b2 | ||
|
|
92cc4f2d79 | ||
|
|
3b0918511a | ||
|
|
3bd38f275e | ||
|
|
f6794c313c | ||
|
|
efe3b53767 | ||
|
|
582f2ed960 | ||
|
|
ebcf794d97 | ||
|
|
038f369420 | ||
|
|
0705d6c7d0 | ||
|
|
0d2e3a4dd7 | ||
|
|
da399ab654 | ||
|
|
7a9d04d35c |
No files matched your search
+18
-18
@@ -18,7 +18,7 @@ jobs:
|
||||
name: Extract crate metadata
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v5
|
||||
- name: Extract crate information
|
||||
id: crate_metadata
|
||||
run: |
|
||||
@@ -44,21 +44,21 @@ jobs:
|
||||
|
||||
ensure_cargo_fmt:
|
||||
name: Ensure 'cargo fmt' has been run
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: rustfmt
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v5
|
||||
- run: cargo fmt -- --check
|
||||
|
||||
min_version:
|
||||
name: Minimum supported rust version
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-24.04
|
||||
needs: crate_metadata
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
|
||||
uses: dtolnay/rust-toolchain@master
|
||||
@@ -78,23 +78,23 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job:
|
||||
- { target: aarch64-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: aarch64-apple-darwin , os: macos-14 }
|
||||
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: arm-unknown-linux-musleabihf, os: ubuntu-20.04, use-cross: true }
|
||||
- { target: i686-pc-windows-msvc , os: windows-2019 }
|
||||
- { target: i686-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: i686-unknown-linux-musl , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: x86_64-apple-darwin , os: macos-12 }
|
||||
# - { target: x86_64-pc-windows-gnu , os: windows-2019 }
|
||||
- { target: x86_64-pc-windows-msvc , os: windows-2019 }
|
||||
- { target: x86_64-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: x86_64-unknown-linux-musl , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }
|
||||
- { target: i686-pc-windows-msvc , os: windows-2022 }
|
||||
- { target: i686-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: i686-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: x86_64-apple-darwin , os: macos-15 }
|
||||
- { target: aarch64-apple-darwin , os: macos-15 }
|
||||
# - { target: x86_64-pc-windows-gnu , os: windows-2022 }
|
||||
- { target: x86_64-pc-windows-msvc , os: windows-2022 }
|
||||
- { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||
env:
|
||||
BUILD_CMD: cargo
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Install prerequisites
|
||||
shell: bash
|
||||
|
||||
@@ -1,3 +1,20 @@
|
||||
# v1.20.0
|
||||
|
||||
## Features
|
||||
|
||||
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
|
||||
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
|
||||
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
|
||||
- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
|
||||
|
||||
## Bugfixes
|
||||
|
||||
- Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr)
|
||||
|
||||
## Other
|
||||
|
||||
- Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos)
|
||||
|
||||
# v1.19.0
|
||||
|
||||
## Features
|
||||
|
||||
Generated
+381
-247
File diff suppressed because it is too large.
Load diff
+5
-5
@@ -7,10 +7,10 @@ license = "MIT OR Apache-2.0"
|
||||
name = "hyperfine"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/sharkdp/hyperfine"
|
||||
version = "1.19.0"
|
||||
version = "1.20.0"
|
||||
edition = "2018"
|
||||
build = "build.rs"
|
||||
rust-version = "1.76.0"
|
||||
rust-version = "1.88.0"
|
||||
|
||||
[features]
|
||||
# Use the nightly feature windows_process_extensions_main_thread_handle
|
||||
@@ -64,15 +64,15 @@ features = [
|
||||
approx = "0.5"
|
||||
assert_cmd = "2.0"
|
||||
insta = { version = "1.41.1", features = ["yaml"] }
|
||||
predicates = "3.0"
|
||||
tempfile = "3.14"
|
||||
predicates = "3.1"
|
||||
tempfile = "3.23"
|
||||
|
||||
[profile.dev.package]
|
||||
insta.opt-level = 3
|
||||
similar.opt-level = 3
|
||||
|
||||
[build-dependencies]
|
||||
clap = "4.4.12"
|
||||
clap = "4.5.48"
|
||||
clap_complete = "4.2.1"
|
||||
|
||||
[profile.release]
|
||||
|
||||
@@ -10,6 +10,29 @@ A command-line benchmarking tool.
|
||||
|
||||

|
||||
|
||||
### Sponsors
|
||||
|
||||
A special *thank you* goes to our biggest <a href="doc/sponsors.md">sponsors</a>:<br>
|
||||
|
||||
<a href="https://www.warp.dev/hyperfine">
|
||||
<img src="doc/sponsors/warp-logo.png" width="200" alt="Warp">
|
||||
<br>
|
||||
<strong>Warp, the intelligent terminal</strong>
|
||||
<br>
|
||||
<sub>Available on MacOS, Linux, Windows</sub>
|
||||
</a>
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
<a href="https://tuple.app/hyperfine">
|
||||
<img src="doc/sponsors/tuple-logo.png" width="200" alt="Tuple">
|
||||
<br>
|
||||
<strong>Tuple, the premier screen sharing app for developers</strong>
|
||||
<br>
|
||||
<sub>Available for MacOS & Windows</sub>
|
||||
</a>
|
||||
|
||||
## Features
|
||||
|
||||
* Statistical analysis across multiple runs.
|
||||
@@ -167,11 +190,15 @@ like `--warmup`, `--prepare <cmd>`, `--setup <cmd>` or `--cleanup <cmd>`:
|
||||
|
||||
### On Ubuntu
|
||||
|
||||
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
|
||||
and install it via `dpkg`:
|
||||
On Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine):
|
||||
```
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.19.0_amd64.deb
|
||||
apt install hyperfine
|
||||
```
|
||||
|
||||
Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:
|
||||
```
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.20.0_amd64.deb
|
||||
```
|
||||
|
||||
### On Fedora
|
||||
@@ -198,7 +225,7 @@ pacman -S hyperfine
|
||||
|
||||
### On Debian Linux
|
||||
|
||||
On Debian Linux, hyperfine can be installed [from the testing repositories](https://packages.debian.org/testing/main/hyperfine):
|
||||
On Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine):
|
||||
```
|
||||
apt install hyperfine
|
||||
```
|
||||
@@ -319,6 +346,9 @@ visualize changes in benchmark timings across your Git history.
|
||||
[Bencher](https://github.com/bencherdev/bencher) is a continuous benchmarking tool that supports `hyperfine` to
|
||||
track benchmarks and catch performance regressions in CI.
|
||||
|
||||
Drop hyperfine JSON outputs onto the [Venz](https://try.venz.dev) chart to visualize the results,
|
||||
and manage hyperfine configurations.
|
||||
|
||||
Make sure to check out the [`scripts` folder](https://github.com/sharkdp/hyperfine/tree/master/scripts)
|
||||
in this repository for a set of tools to work with `hyperfine` benchmark results.
|
||||
|
||||
|
||||
+2
-1
@@ -117,6 +117,7 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
|
||||
once for each command. In the latter case, each preparation command will be
|
||||
run prior to the corresponding benchmark command.
|
||||
.HP
|
||||
\fB\-\-conclude\fR \fICMD...\fP
|
||||
.IP
|
||||
Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
|
||||
for example.
|
||||
@@ -130,7 +131,7 @@ Execute \fICMD\fP after the completion of all benchmarking runs for each individ
|
||||
command to be benchmarked. This is useful if the commands to be benchmarked
|
||||
produce artifacts that need to be cleaned up. It only runs once a series of
|
||||
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
|
||||
ever run.
|
||||
every run.
|
||||
.HP
|
||||
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
|
||||
.IP
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
## Sponsors
|
||||
|
||||
`hyperfine` development is sponsored by many individuals and companies. Thank you very much!
|
||||
|
||||
Please note, that being sponsored does not affect the individuality of the `hyperfine`
|
||||
project or affect the maintainers' actions in any way.
|
||||
We remain impartial and continue to assess pull requests solely on merit - the
|
||||
features added, bugs solved, and effect on the overall complexity of the code.
|
||||
No issue will have a different priority based on sponsorship status of the
|
||||
reporter.
|
||||
|
||||
Contributions from anybody are most welcomed.
|
||||
|
||||
If you want to see our biggest sponsors, check the top of [`README.md`](../README.md#sponsors).
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 130 KiB |
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "matplotlib",
|
||||
# "pyqt6",
|
||||
# "numpy",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
"""
|
||||
This script shows `hyperfine` benchmark results as a bar plot grouped by command.
|
||||
Note all the input files must contain results for all commands.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
|
||||
)
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument(
|
||||
"--benchmark-names", nargs="+", help="Names of the benchmark groups"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
commands = None
|
||||
data = []
|
||||
inputs = []
|
||||
|
||||
if args.benchmark_names:
|
||||
assert len(args.files) == len(
|
||||
args.benchmark_names
|
||||
), "Number of benchmark names must match the number of input files."
|
||||
|
||||
for i, filename in enumerate(args.files):
|
||||
with open(filename) as f:
|
||||
results = json.load(f)["results"]
|
||||
benchmark_commands = [b["command"] for b in results]
|
||||
if commands is None:
|
||||
commands = benchmark_commands
|
||||
else:
|
||||
assert (
|
||||
commands == benchmark_commands
|
||||
), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
|
||||
data.append([round(b["mean"], 2) for b in results])
|
||||
if args.benchmark_names:
|
||||
inputs.append(args.benchmark_names[i])
|
||||
else:
|
||||
inputs.append(filename.stem)
|
||||
|
||||
data = np.transpose(data)
|
||||
x = np.arange(len(inputs)) # the label locations
|
||||
width = 0.25 # the width of the bars
|
||||
|
||||
fig, ax = plt.subplots(layout="constrained")
|
||||
fig.set_figheight(5)
|
||||
fig.set_figwidth(10)
|
||||
for i, command in enumerate(commands):
|
||||
offset = width * (i + 1)
|
||||
rects = ax.bar(x + offset, data[i], width, label=command)
|
||||
|
||||
ax.set_xticks(x + 0.5, inputs)
|
||||
ax.grid(visible=True, axis="y")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.xlabel("Benchmark")
|
||||
plt.ylabel("Time [s]")
|
||||
plt.legend(title="Command")
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
+36
-18
@@ -79,23 +79,41 @@ fn run_command_and_measure_common(
|
||||
let result = execute_and_measure(command)
|
||||
.with_context(|| format!("Failed to run command '{command_name}'"))?;
|
||||
|
||||
if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
|
||||
let when = match iteration {
|
||||
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
|
||||
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
|
||||
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
|
||||
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
|
||||
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
|
||||
};
|
||||
bail!(
|
||||
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
||||
Alternatively, use the '--show-output' option to debug what went wrong.",
|
||||
cause=result.status.code().map_or(
|
||||
"The process has been terminated by a signal".into(),
|
||||
|c| format!("Command terminated with non-zero exit code {c}")
|
||||
if !result.status.success() {
|
||||
use crate::util::exit_code::extract_exit_code;
|
||||
|
||||
),
|
||||
);
|
||||
let should_fail = match command_failure_action {
|
||||
CmdFailureAction::RaiseError => true,
|
||||
CmdFailureAction::IgnoreAllFailures => false,
|
||||
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
|
||||
// Only fail if the exit code is not in the list of codes to ignore
|
||||
if let Some(exit_code) = extract_exit_code(result.status) {
|
||||
!codes.contains(&exit_code)
|
||||
} else {
|
||||
// If we can't extract an exit code, treat it as a failure
|
||||
true
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if should_fail {
|
||||
let when = match iteration {
|
||||
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
|
||||
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
|
||||
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
|
||||
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
|
||||
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
|
||||
};
|
||||
bail!(
|
||||
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
||||
Alternatively, use the '--show-output' option to debug what went wrong.",
|
||||
cause=result.status.code().map_or(
|
||||
"The process has been terminated by a signal".into(),
|
||||
|c| format!("Command terminated with non-zero exit code {c}")
|
||||
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
@@ -122,7 +140,7 @@ impl Executor for RawExecutor<'_> {
|
||||
let result = run_command_and_measure_common(
|
||||
command.get_command()?,
|
||||
iteration,
|
||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
||||
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||
&self.options.command_input_policy,
|
||||
output_policy,
|
||||
&command.get_command_line(),
|
||||
@@ -187,7 +205,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
let mut result = run_command_and_measure_common(
|
||||
command_builder,
|
||||
iteration,
|
||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
||||
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||
&self.options.command_input_policy,
|
||||
output_policy,
|
||||
&command.get_command_line(),
|
||||
|
||||
@@ -98,7 +98,7 @@ pub fn compute_with_check_from_reference<'a>(
|
||||
pub fn compute_with_check(
|
||||
results: &[BenchmarkResult],
|
||||
sort_order: SortOrder,
|
||||
) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
|
||||
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
|
||||
let fastest = fastest_of(results);
|
||||
|
||||
if fastest.mean == 0.0 {
|
||||
@@ -112,7 +112,7 @@ pub fn compute_with_check(
|
||||
pub fn compute(
|
||||
results: &[BenchmarkResult],
|
||||
sort_order: SortOrder,
|
||||
) -> Vec<BenchmarkResultWithRelativeSpeed> {
|
||||
) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
|
||||
let fastest = fastest_of(results);
|
||||
|
||||
compute_relative_speeds(results, fastest, sort_order)
|
||||
|
||||
@@ -42,7 +42,7 @@ impl<'a> Scheduler<'a> {
|
||||
.options
|
||||
.reference_command
|
||||
.as_ref()
|
||||
.map(|cmd| Command::new(None, cmd));
|
||||
.map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
|
||||
|
||||
executor.calibrate()?;
|
||||
|
||||
|
||||
+16
-2
@@ -96,6 +96,14 @@ fn build_command() -> Command {
|
||||
If this is unset, results are compared with the fastest command as reference."
|
||||
)
|
||||
)
|
||||
.arg(
|
||||
Arg::new("reference-name")
|
||||
.long("reference-name")
|
||||
.action(ArgAction::Set)
|
||||
.value_name("CMD")
|
||||
.help("Give a meaningful name to the reference command.")
|
||||
.requires("reference")
|
||||
)
|
||||
.arg(
|
||||
Arg::new("prepare")
|
||||
.long("prepare")
|
||||
@@ -218,9 +226,15 @@ fn build_command() -> Command {
|
||||
.arg(
|
||||
Arg::new("ignore-failure")
|
||||
.long("ignore-failure")
|
||||
.action(ArgAction::SetTrue)
|
||||
.action(ArgAction::Set)
|
||||
.value_name("MODE")
|
||||
.num_args(0..=1)
|
||||
.default_missing_value("all-non-zero")
|
||||
.require_equals(true)
|
||||
.short('i')
|
||||
.help("Ignore non-zero exit codes of the benchmarked programs."),
|
||||
.help("Ignore failures of the benchmarked programs. Without a value or with \
|
||||
'all-non-zero', all non-zero exit codes are ignored. You can also provide \
|
||||
a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2)."),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("style")
|
||||
|
||||
+45
-10
@@ -277,18 +277,8 @@ impl<'a> Commands<'a> {
|
||||
command_strings: Vec<&'b str>,
|
||||
) -> Result<Vec<Command<'b>>, ParameterScanError> {
|
||||
let param_range = RangeStep::new(param_min, param_max, step)?;
|
||||
let param_count = param_range.size_hint().1.unwrap();
|
||||
let command_name_count = command_names.len();
|
||||
|
||||
// `--command-name` should appear exactly once or exactly B times,
|
||||
// where B is the total number of benchmarks.
|
||||
if command_name_count > 1 && command_name_count != param_count {
|
||||
return Err(ParameterScanError::UnexpectedCommandNameCount(
|
||||
command_name_count,
|
||||
param_count,
|
||||
));
|
||||
}
|
||||
|
||||
let mut i = 0;
|
||||
let mut commands = vec![];
|
||||
for value in param_range {
|
||||
@@ -305,6 +295,17 @@ impl<'a> Commands<'a> {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// `--command-name` should appear exactly once or exactly B times,
|
||||
// where B is the total number of benchmarks.
|
||||
let command_count = commands.len();
|
||||
if command_name_count > 1 && command_name_count != command_count {
|
||||
return Err(ParameterScanError::UnexpectedCommandNameCount(
|
||||
command_name_count,
|
||||
command_count,
|
||||
));
|
||||
}
|
||||
|
||||
Ok(commands)
|
||||
}
|
||||
|
||||
@@ -471,6 +472,40 @@ fn test_build_parameter_scan_commands() {
|
||||
assert_eq!(commands[1].get_command_line(), "echo 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_parameter_scan_commands_named() {
|
||||
use crate::cli::get_cli_arguments;
|
||||
let matches = get_cli_arguments(vec![
|
||||
"hyperfine",
|
||||
"echo {val}",
|
||||
"sleep {val}",
|
||||
"--parameter-scan",
|
||||
"val",
|
||||
"1",
|
||||
"2",
|
||||
"--parameter-step-size",
|
||||
"1",
|
||||
"--command-name",
|
||||
"echo-1",
|
||||
"--command-name",
|
||||
"sleep-1",
|
||||
"--command-name",
|
||||
"echo-2",
|
||||
"--command-name",
|
||||
"sleep-2",
|
||||
]);
|
||||
let commands = Commands::from_cli_arguments(&matches).unwrap().0;
|
||||
assert_eq!(commands.len(), 4);
|
||||
assert_eq!(commands[0].get_name(), "echo-1");
|
||||
assert_eq!(commands[0].get_command_line(), "echo 1");
|
||||
assert_eq!(commands[1].get_name(), "sleep-1");
|
||||
assert_eq!(commands[1].get_command_line(), "sleep 1");
|
||||
assert_eq!(commands[2].get_name(), "echo-2");
|
||||
assert_eq!(commands[2].get_command_line(), "echo 2");
|
||||
assert_eq!(commands[3].get_name(), "sleep-2");
|
||||
assert_eq!(commands[3].get_command_line(), "sleep 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parameter_scan_commands_int() {
|
||||
let commands = Commands::build_parameter_scan_commands(
|
||||
|
||||
+29
-7
@@ -67,13 +67,16 @@ impl Shell {
|
||||
}
|
||||
|
||||
/// Action to take when an executed command fails.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CmdFailureAction {
|
||||
/// Exit with an error message
|
||||
RaiseError,
|
||||
|
||||
/// Simply ignore the non-zero exit code
|
||||
Ignore,
|
||||
/// Ignore all non-zero exit codes
|
||||
IgnoreAllFailures,
|
||||
|
||||
/// Ignore specific exit codes
|
||||
IgnoreSpecificFailures(Vec<i32>),
|
||||
}
|
||||
|
||||
/// Output style type option
|
||||
@@ -208,6 +211,9 @@ pub struct Options {
|
||||
// Command to use as a reference for relative speed comparison
|
||||
pub reference_command: Option<String>,
|
||||
|
||||
// Name of the reference command
|
||||
pub reference_name: Option<String>,
|
||||
|
||||
/// Command(s) to run before each timing run
|
||||
pub preparation_command: Option<Vec<String>>,
|
||||
|
||||
@@ -250,6 +256,7 @@ impl Default for Options {
|
||||
min_benchmarking_time: 3.0,
|
||||
command_failure_action: CmdFailureAction::RaiseError,
|
||||
reference_command: None,
|
||||
reference_name: None,
|
||||
preparation_command: None,
|
||||
conclusion_command: None,
|
||||
setup_command: None,
|
||||
@@ -310,6 +317,9 @@ impl Options {
|
||||
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
||||
|
||||
options.reference_command = matches.get_one::<String>("reference").map(String::from);
|
||||
options.reference_name = matches
|
||||
.get_one::<String>("reference-name")
|
||||
.map(String::from);
|
||||
|
||||
options.preparation_command = matches
|
||||
.get_many::<String>("prepare")
|
||||
@@ -354,8 +364,7 @@ impl Options {
|
||||
_ => {
|
||||
if options
|
||||
.command_output_policies
|
||||
.iter()
|
||||
.any(|policy| *policy == CommandOutputPolicy::Inherit)
|
||||
.contains(&CommandOutputPolicy::Inherit)
|
||||
|| !io::stdout().is_terminal()
|
||||
{
|
||||
OutputStyleOption::Basic
|
||||
@@ -409,8 +418,21 @@ impl Options {
|
||||
}
|
||||
};
|
||||
|
||||
if matches.get_flag("ignore-failure") {
|
||||
options.command_failure_action = CmdFailureAction::Ignore;
|
||||
if let Some(mode) = matches.get_one::<String>("ignore-failure") {
|
||||
options.command_failure_action = match mode.as_str() {
|
||||
"all-non-zero" | "" => CmdFailureAction::IgnoreAllFailures,
|
||||
codes => {
|
||||
let exit_codes: Result<Vec<i32>, _> = codes
|
||||
.split(',')
|
||||
.map(|s| {
|
||||
s.trim()
|
||||
.parse::<i32>()
|
||||
.map_err(|e| OptionsError::IntParsingError("ignore-failure", e))
|
||||
})
|
||||
.collect();
|
||||
CmdFailureAction::IgnoreSpecificFailures(exit_codes?)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
|
||||
|
||||
@@ -3,6 +3,12 @@ use common::hyperfine;
|
||||
|
||||
use predicates::prelude::*;
|
||||
|
||||
/// Platform-specific I/O utility.
|
||||
/// - On Unix-like systems, defaults to `cat`.
|
||||
/// - On Windows, uses `findstr` as an alternative.
|
||||
/// See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
|
||||
const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
|
||||
|
||||
pub fn hyperfine_debug() -> assert_cmd::Command {
|
||||
let mut cmd = hyperfine();
|
||||
cmd.arg("--debug-mode");
|
||||
@@ -271,6 +277,78 @@ fn can_run_failing_commands_with_ignore_failure_option() {
|
||||
.success();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn can_ignore_specific_exit_codes() {
|
||||
// Test that specifying exit code 1 ignores it
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1")
|
||||
.arg("exit 1")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
// Test that other exit codes still fail
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1")
|
||||
.arg("exit 2")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Command terminated with non-zero exit code 2",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn can_ignore_multiple_exit_codes() {
|
||||
// Test that all specified exit codes are ignored
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 1")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 2")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 3")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
// Test that other exit codes still fail
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 4")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Command terminated with non-zero exit code 4",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn ignore_failure_with_all_non_zero() {
|
||||
// Test explicit "all-non-zero" mode
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=all-non-zero")
|
||||
.arg("exit 5")
|
||||
.assert()
|
||||
.success();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_output_of_benchmarked_command() {
|
||||
hyperfine()
|
||||
@@ -306,7 +384,7 @@ fn can_pass_input_to_command_from_a_file() {
|
||||
.arg("--runs=1")
|
||||
.arg("--input=example_input_file.txt")
|
||||
.arg("--show-output")
|
||||
.arg("cat")
|
||||
.arg(STDIN_READ_COMMAND)
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("This text is part of a file"));
|
||||
@@ -318,7 +396,7 @@ fn fails_if_invalid_stdin_data_file_provided() {
|
||||
.arg("--runs=1")
|
||||
.arg("--input=example_non_existent_file.txt")
|
||||
.arg("--show-output")
|
||||
.arg("cat")
|
||||
.arg(STDIN_READ_COMMAND)
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
@@ -493,6 +571,18 @@ fn shows_benchmark_comparison_relative_to_reference() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_reference_name() {
|
||||
hyperfine_debug()
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("--reference-name=refabc123")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
|
||||
Reference in new issue
Block a user