mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-14 08:21:39 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7d3c7bb81e | ||
|
|
eb1a3212f7 | ||
|
|
72aa8b40fe | ||
|
|
74070e2357 | ||
|
|
7004997397 | ||
|
|
7ea8e9f215 |
No files matched your search
+18
-19
@@ -18,7 +18,7 @@ jobs:
|
||||
name: Extract crate metadata
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v4
|
||||
- name: Extract crate information
|
||||
id: crate_metadata
|
||||
run: |
|
||||
@@ -44,21 +44,21 @@ jobs:
|
||||
|
||||
ensure_cargo_fmt:
|
||||
name: Ensure 'cargo fmt' has been run
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: rustfmt
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v4
|
||||
- run: cargo fmt -- --check
|
||||
|
||||
min_version:
|
||||
name: Minimum supported rust version
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-20.04
|
||||
needs: crate_metadata
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
|
||||
uses: dtolnay/rust-toolchain@master
|
||||
@@ -78,24 +78,23 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job:
|
||||
- { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: aarch64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }
|
||||
- { target: i686-pc-windows-msvc , os: windows-2022 }
|
||||
- { target: i686-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: i686-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: x86_64-apple-darwin , os: macos-15 }
|
||||
- { target: aarch64-apple-darwin , os: macos-15 }
|
||||
# - { target: x86_64-pc-windows-gnu , os: windows-2022 }
|
||||
- { target: x86_64-pc-windows-msvc , os: windows-2022 }
|
||||
- { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: aarch64-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: aarch64-apple-darwin , os: macos-14 }
|
||||
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: arm-unknown-linux-musleabihf, os: ubuntu-20.04, use-cross: true }
|
||||
- { target: i686-pc-windows-msvc , os: windows-2019 }
|
||||
- { target: i686-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: i686-unknown-linux-musl , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: x86_64-apple-darwin , os: macos-12 }
|
||||
# - { target: x86_64-pc-windows-gnu , os: windows-2019 }
|
||||
- { target: x86_64-pc-windows-msvc , os: windows-2019 }
|
||||
- { target: x86_64-unknown-linux-gnu , os: ubuntu-20.04, use-cross: true }
|
||||
- { target: x86_64-unknown-linux-musl , os: ubuntu-20.04, use-cross: true }
|
||||
env:
|
||||
BUILD_CMD: cargo
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install prerequisites
|
||||
shell: bash
|
||||
|
||||
@@ -1,20 +1,3 @@
|
||||
# v1.20.0
|
||||
|
||||
## Features
|
||||
|
||||
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
|
||||
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
|
||||
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
|
||||
- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
|
||||
|
||||
## Bugfixes
|
||||
|
||||
- Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr)
|
||||
|
||||
## Other
|
||||
|
||||
- Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos)
|
||||
|
||||
# v1.19.0
|
||||
|
||||
## Features
|
||||
|
||||
+2
-2
@@ -13,5 +13,5 @@ authors:
|
||||
repository-code: 'https://github.com/sharkdp/hyperfine'
|
||||
abstract: A command-line benchmarking tool.
|
||||
license: MIT
|
||||
version: 1.20.0
|
||||
date-released: '2025-11-18'
|
||||
version: 1.16.1
|
||||
date-released: '2023-03-21'
|
||||
Generated
+483
-448
File diff suppressed because it is too large.
Load diff
+16
-17
@@ -7,26 +7,25 @@ license = "MIT OR Apache-2.0"
|
||||
name = "hyperfine"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/sharkdp/hyperfine"
|
||||
version = "1.20.0"
|
||||
version = "1.19.0"
|
||||
edition = "2018"
|
||||
build = "build.rs"
|
||||
rust-version = "1.97.0"
|
||||
rust-version = "1.76.0"
|
||||
|
||||
[features]
|
||||
# Use the nightly feature windows_process_extensions_main_thread_handle
|
||||
windows_process_extensions_main_thread_handle = []
|
||||
|
||||
[dependencies]
|
||||
colored = "3.1"
|
||||
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
|
||||
colored = "2.1"
|
||||
indicatif = "=0.17.4"
|
||||
statistical = "1.0"
|
||||
csv = "1.4"
|
||||
csv = "1.3"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
rust_decimal = "1.43"
|
||||
rand = "0.10"
|
||||
shell-words = "1.1"
|
||||
rust_decimal = "1.36"
|
||||
rand = "0.8"
|
||||
shell-words = "1.0"
|
||||
thiserror = "2.0"
|
||||
anyhow = "1.0"
|
||||
|
||||
@@ -34,7 +33,7 @@ anyhow = "1.0"
|
||||
libc = "0.2"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = [
|
||||
windows-sys = { version = "0.59", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_Security",
|
||||
"Win32_System_JobObjects",
|
||||
@@ -43,10 +42,10 @@ windows-sys = { version = "0.61", features = [
|
||||
] }
|
||||
|
||||
[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
|
||||
once_cell = "1.21"
|
||||
once_cell = "1.19"
|
||||
|
||||
[target.'cfg(target_os="linux")'.dependencies]
|
||||
nix = { version = "0.31", features = ["zerocopy"] }
|
||||
nix = { version = "0.29", features = ["zerocopy"] }
|
||||
|
||||
[dependencies.clap]
|
||||
version = "4"
|
||||
@@ -63,18 +62,18 @@ features = [
|
||||
|
||||
[dev-dependencies]
|
||||
approx = "0.5"
|
||||
assert_cmd = "2.2"
|
||||
insta = { version = "1.48", features = ["yaml"] }
|
||||
predicates = "3.1"
|
||||
tempfile = "3.27"
|
||||
assert_cmd = "2.0"
|
||||
insta = { version = "1.41.1", features = ["yaml"] }
|
||||
predicates = "3.0"
|
||||
tempfile = "3.14"
|
||||
|
||||
[profile.dev.package]
|
||||
insta.opt-level = 3
|
||||
similar.opt-level = 3
|
||||
|
||||
[build-dependencies]
|
||||
clap = "4.6"
|
||||
clap_complete = "4.6"
|
||||
clap = "4.4.12"
|
||||
clap_complete = "4.2.1"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
|
||||
@@ -104,11 +104,11 @@ If you want to run a benchmark *without an intermediate shell*, you can use the
|
||||
option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
|
||||
produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case.
|
||||
```
|
||||
hyperfine -N 'grep -R TODO /home/user'
|
||||
hyperfine -N 'grep TODO /home/user'
|
||||
```
|
||||
|
||||
|
||||
### Shell functions
|
||||
### Shell functions and aliases
|
||||
|
||||
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
|
||||
|
||||
@@ -118,10 +118,13 @@ export -f my_function
|
||||
hyperfine --shell=bash my_function
|
||||
```
|
||||
|
||||
Otherwise, inline the function into the benchmarked command:
|
||||
Otherwise, inline them into or source them from the benchmarked program:
|
||||
|
||||
```sh
|
||||
hyperfine 'my_function() { sleep 1; }; my_function'
|
||||
|
||||
echo 'alias my_alias="sleep 1"' > /tmp/my_alias.sh
|
||||
hyperfine '. /tmp/my_alias.sh; my_alias'
|
||||
```
|
||||
|
||||
### Exporting results
|
||||
@@ -164,15 +167,11 @@ like `--warmup`, `--prepare <cmd>`, `--setup <cmd>` or `--cleanup <cmd>`:
|
||||
|
||||
### On Ubuntu
|
||||
|
||||
On Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine):
|
||||
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
|
||||
and install it via `dpkg`:
|
||||
```
|
||||
apt install hyperfine
|
||||
```
|
||||
|
||||
Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:
|
||||
```
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.20.0_amd64.deb
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.19.0_amd64.deb
|
||||
```
|
||||
|
||||
### On Fedora
|
||||
@@ -199,7 +198,7 @@ pacman -S hyperfine
|
||||
|
||||
### On Debian Linux
|
||||
|
||||
On Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine):
|
||||
On Debian Linux, hyperfine can be installed [from the testing repositories](https://packages.debian.org/testing/main/hyperfine):
|
||||
```
|
||||
apt install hyperfine
|
||||
```
|
||||
@@ -302,7 +301,7 @@ Hyperfine can be installed from source via [cargo](https://doc.rust-lang.org/car
|
||||
cargo install --locked hyperfine
|
||||
```
|
||||
|
||||
Make sure that you use Rust 1.97 or newer.
|
||||
Make sure that you use Rust 1.76 or newer.
|
||||
|
||||
### From binaries (Linux, macOS, Windows)
|
||||
|
||||
@@ -320,9 +319,6 @@ visualize changes in benchmark timings across your Git history.
|
||||
[Bencher](https://github.com/bencherdev/bencher) is a continuous benchmarking tool that supports `hyperfine` to
|
||||
track benchmarks and catch performance regressions in CI.
|
||||
|
||||
Drop hyperfine JSON outputs onto the [Venz](https://try.venz.dev) chart to visualize the results,
|
||||
and manage hyperfine configurations.
|
||||
|
||||
Make sure to check out the [`scripts` folder](https://github.com/sharkdp/hyperfine/tree/master/scripts)
|
||||
in this repository for a set of tools to work with `hyperfine` benchmark results.
|
||||
|
||||
|
||||
+1
-2
@@ -117,7 +117,6 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
|
||||
once for each command. In the latter case, each preparation command will be
|
||||
run prior to the corresponding benchmark command.
|
||||
.HP
|
||||
\fB\-\-conclude\fR \fICMD...\fP
|
||||
.IP
|
||||
Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
|
||||
for example.
|
||||
@@ -131,7 +130,7 @@ Execute \fICMD\fP after the completion of all benchmarking runs for each individ
|
||||
command to be benchmarked. This is useful if the commands to be benchmarked
|
||||
produce artifacts that need to be cleaned up. It only runs once a series of
|
||||
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
|
||||
every run.
|
||||
ever run.
|
||||
.HP
|
||||
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
|
||||
.IP
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "matplotlib",
|
||||
# "pyqt6",
|
||||
# "numpy",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
"""
|
||||
This script shows `hyperfine` benchmark results as a bar plot grouped by command.
|
||||
Note all the input files must contain results for all commands.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
|
||||
)
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument(
|
||||
"--benchmark-names", nargs="+", help="Names of the benchmark groups"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
commands = None
|
||||
data = []
|
||||
inputs = []
|
||||
|
||||
if args.benchmark_names:
|
||||
assert len(args.files) == len(
|
||||
args.benchmark_names
|
||||
), "Number of benchmark names must match the number of input files."
|
||||
|
||||
for i, filename in enumerate(args.files):
|
||||
with open(filename) as f:
|
||||
results = json.load(f)["results"]
|
||||
benchmark_commands = [b["command"] for b in results]
|
||||
if commands is None:
|
||||
commands = benchmark_commands
|
||||
else:
|
||||
assert (
|
||||
commands == benchmark_commands
|
||||
), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
|
||||
data.append([round(b["mean"], 2) for b in results])
|
||||
if args.benchmark_names:
|
||||
inputs.append(args.benchmark_names[i])
|
||||
else:
|
||||
inputs.append(filename.stem)
|
||||
|
||||
data = np.transpose(data)
|
||||
x = np.arange(len(inputs)) # the label locations
|
||||
width = 0.25 # the width of the bars
|
||||
|
||||
fig, ax = plt.subplots(layout="constrained")
|
||||
fig.set_figheight(5)
|
||||
fig.set_figwidth(10)
|
||||
for i, command in enumerate(commands):
|
||||
offset = width * (i + 1)
|
||||
rects = ax.bar(x + offset, data[i], width, label=command)
|
||||
|
||||
ax.set_xticks(x + 0.5, inputs)
|
||||
ax.grid(visible=True, axis="y")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.xlabel("Benchmark")
|
||||
plt.ylabel("Time [s]")
|
||||
plt.legend(title="Command")
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
+17
-35
@@ -79,41 +79,23 @@ fn run_command_and_measure_common(
|
||||
let result = execute_and_measure(command)
|
||||
.with_context(|| format!("Failed to run command '{command_name}'"))?;
|
||||
|
||||
if !result.status.success() {
|
||||
use crate::util::exit_code::extract_exit_code;
|
||||
|
||||
let should_fail = match command_failure_action {
|
||||
CmdFailureAction::RaiseError => true,
|
||||
CmdFailureAction::IgnoreAllFailures => false,
|
||||
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
|
||||
// Only fail if the exit code is not in the list of codes to ignore
|
||||
if let Some(exit_code) = extract_exit_code(result.status) {
|
||||
!codes.contains(&exit_code)
|
||||
} else {
|
||||
// If we can't extract an exit code, treat it as a failure
|
||||
true
|
||||
}
|
||||
}
|
||||
if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
|
||||
let when = match iteration {
|
||||
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
|
||||
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
|
||||
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
|
||||
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
|
||||
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
|
||||
};
|
||||
bail!(
|
||||
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
||||
Alternatively, use the '--show-output' option to debug what went wrong.",
|
||||
cause=result.status.code().map_or(
|
||||
"The process has been terminated by a signal".into(),
|
||||
|c| format!("Command terminated with non-zero exit code {c}")
|
||||
|
||||
if should_fail {
|
||||
let when = match iteration {
|
||||
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
|
||||
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
|
||||
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
|
||||
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
|
||||
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
|
||||
};
|
||||
bail!(
|
||||
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
||||
Alternatively, use the '--show-output' option to debug what went wrong.",
|
||||
cause=result.status.code().map_or(
|
||||
"The process has been terminated by a signal".into(),
|
||||
|c| format!("Command terminated with non-zero exit code {c}")
|
||||
|
||||
),
|
||||
);
|
||||
}
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
@@ -140,7 +122,7 @@ impl Executor for RawExecutor<'_> {
|
||||
let result = run_command_and_measure_common(
|
||||
command.get_command()?,
|
||||
iteration,
|
||||
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
||||
&self.options.command_input_policy,
|
||||
output_policy,
|
||||
&command.get_command_line(),
|
||||
@@ -205,7 +187,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
let mut result = run_command_and_measure_common(
|
||||
command_builder,
|
||||
iteration,
|
||||
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
||||
&self.options.command_input_policy,
|
||||
output_policy,
|
||||
&command.get_command_line(),
|
||||
|
||||
+11
-19
@@ -5,7 +5,6 @@ pub mod scheduler;
|
||||
pub mod timing_result;
|
||||
|
||||
use std::cmp;
|
||||
use std::io::{self, Write};
|
||||
|
||||
use crate::benchmark::executor::BenchmarkIteration;
|
||||
use crate::command::Command;
|
||||
@@ -13,7 +12,6 @@ use crate::options::{
|
||||
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
|
||||
};
|
||||
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
|
||||
use crate::output::console_writeln;
|
||||
use crate::output::format::{format_duration, format_duration_unit};
|
||||
use crate::output::progress_bar::get_progress_bar;
|
||||
use crate::output::warnings::{OutlierWarningOptions, Warnings};
|
||||
@@ -142,13 +140,12 @@ impl<'a> Benchmark<'a> {
|
||||
/// Run the benchmark for a single command
|
||||
pub fn run(&self) -> Result<BenchmarkResult> {
|
||||
if self.options.output_style != OutputStyleOption::Disabled {
|
||||
console_writeln!(
|
||||
io::stdout(),
|
||||
println!(
|
||||
"{}{}: {}",
|
||||
"Benchmark ".bold(),
|
||||
(self.number + 1).to_string().bold(),
|
||||
self.command.get_name_with_unused_parameters(),
|
||||
)?;
|
||||
);
|
||||
}
|
||||
|
||||
let mut times_real: Vec<Second> = vec![];
|
||||
@@ -361,22 +358,19 @@ impl<'a> Benchmark<'a> {
|
||||
let system_str = format_duration(system_mean, Some(time_unit));
|
||||
|
||||
if self.options.output_style != OutputStyleOption::Disabled {
|
||||
let mut stdout = io::stdout().lock();
|
||||
if times_real.len() == 1 {
|
||||
console_writeln!(
|
||||
stdout,
|
||||
println!(
|
||||
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
|
||||
"abs".green().bold(),
|
||||
mean_str.green().bold(),
|
||||
" ", // alignment
|
||||
user_str.blue(),
|
||||
system_str.blue()
|
||||
)?;
|
||||
);
|
||||
} else {
|
||||
let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
|
||||
|
||||
console_writeln!(
|
||||
stdout,
|
||||
println!(
|
||||
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
|
||||
"mean".green().bold(),
|
||||
"σ".green(),
|
||||
@@ -384,17 +378,16 @@ impl<'a> Benchmark<'a> {
|
||||
stddev_str.green(),
|
||||
user_str.blue(),
|
||||
system_str.blue()
|
||||
)?;
|
||||
);
|
||||
|
||||
console_writeln!(
|
||||
stdout,
|
||||
println!(
|
||||
" Range ({} … {}): {:>8} … {:>8} {}",
|
||||
"min".cyan(),
|
||||
"max".purple(),
|
||||
min_str.cyan(),
|
||||
max_str.purple(),
|
||||
num_str.dimmed()
|
||||
)?;
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,16 +430,15 @@ impl<'a> Benchmark<'a> {
|
||||
}
|
||||
|
||||
if !warnings.is_empty() {
|
||||
let mut stderr = io::stderr().lock();
|
||||
console_writeln!(stderr, " ")?;
|
||||
eprintln!(" ");
|
||||
|
||||
for warning in &warnings {
|
||||
console_writeln!(stderr, " {}: {}", "Warning".yellow(), warning)?;
|
||||
eprintln!(" {}: {}", "Warning".yellow(), warning);
|
||||
}
|
||||
}
|
||||
|
||||
if self.options.output_style != OutputStyleOption::Disabled {
|
||||
console_writeln!(io::stdout(), " ")?;
|
||||
println!(" ");
|
||||
}
|
||||
|
||||
self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
|
||||
|
||||
@@ -98,7 +98,7 @@ pub fn compute_with_check_from_reference<'a>(
|
||||
pub fn compute_with_check(
|
||||
results: &[BenchmarkResult],
|
||||
sort_order: SortOrder,
|
||||
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
|
||||
) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
|
||||
let fastest = fastest_of(results);
|
||||
|
||||
if fastest.mean == 0.0 {
|
||||
@@ -112,7 +112,7 @@ pub fn compute_with_check(
|
||||
pub fn compute(
|
||||
results: &[BenchmarkResult],
|
||||
sort_order: SortOrder,
|
||||
) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
|
||||
) -> Vec<BenchmarkResultWithRelativeSpeed> {
|
||||
let fastest = fastest_of(results);
|
||||
|
||||
compute_relative_speeds(results, fastest, sort_order)
|
||||
|
||||
+16
-25
@@ -3,12 +3,10 @@ use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
|
||||
use super::{relative_speed, Benchmark};
|
||||
use colored::*;
|
||||
use std::cmp::Ordering;
|
||||
use std::io::{self, Write};
|
||||
|
||||
use crate::command::{Command, Commands};
|
||||
use crate::export::ExportManager;
|
||||
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
|
||||
use crate::output::console_writeln;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
@@ -44,7 +42,7 @@ impl<'a> Scheduler<'a> {
|
||||
.options
|
||||
.reference_command
|
||||
.as_ref()
|
||||
.map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
|
||||
.map(|cmd| Command::new(None, cmd));
|
||||
|
||||
executor.calibrate()?;
|
||||
|
||||
@@ -60,13 +58,13 @@ impl<'a> Scheduler<'a> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn print_relative_speed_comparison(&self) -> Result<()> {
|
||||
pub fn print_relative_speed_comparison(&self) {
|
||||
if self.options.output_style == OutputStyleOption::Disabled {
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
|
||||
if self.results.len() < 2 {
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
|
||||
let reference = self
|
||||
@@ -81,19 +79,17 @@ impl<'a> Scheduler<'a> {
|
||||
reference,
|
||||
self.options.sort_order_speed_comparison,
|
||||
) {
|
||||
let mut stdout = io::stdout().lock();
|
||||
match self.options.sort_order_speed_comparison {
|
||||
SortOrder::MeanTime => {
|
||||
console_writeln!(stdout, "{}", "Summary".bold())?;
|
||||
println!("{}", "Summary".bold());
|
||||
|
||||
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
|
||||
let others = annotated_results.iter().filter(|r| !r.is_reference);
|
||||
|
||||
console_writeln!(
|
||||
stdout,
|
||||
println!(
|
||||
" {} ran",
|
||||
reference.result.command_with_unused_parameters.cyan()
|
||||
)?;
|
||||
);
|
||||
|
||||
for item in others {
|
||||
let stddev = if let Some(stddev) = item.relative_speed_stddev {
|
||||
@@ -118,20 +114,18 @@ impl<'a> Scheduler<'a> {
|
||||
stddev
|
||||
),
|
||||
};
|
||||
console_writeln!(
|
||||
stdout,
|
||||
println!(
|
||||
"{} {}",
|
||||
comparator,
|
||||
item.result.command_with_unused_parameters.magenta()
|
||||
)?;
|
||||
&item.result.command_with_unused_parameters.magenta()
|
||||
);
|
||||
}
|
||||
}
|
||||
SortOrder::Command => {
|
||||
console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
|
||||
println!("{}", "Relative speed comparison".bold());
|
||||
|
||||
for item in annotated_results {
|
||||
console_writeln!(
|
||||
stdout,
|
||||
println!(
|
||||
" {}{} {}",
|
||||
format!("{:10.2}", item.relative_speed).bold().green(),
|
||||
if item.is_reference {
|
||||
@@ -141,14 +135,13 @@ impl<'a> Scheduler<'a> {
|
||||
} else {
|
||||
" ".into()
|
||||
},
|
||||
item.result.command_with_unused_parameters,
|
||||
)?;
|
||||
&item.result.command_with_unused_parameters,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console_writeln!(
|
||||
io::stderr(),
|
||||
eprintln!(
|
||||
"{}: The benchmark comparison could not be computed as some benchmark times are zero. \
|
||||
This could be caused by background interference during the initial calibration phase \
|
||||
of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
|
||||
@@ -156,10 +149,8 @@ impl<'a> Scheduler<'a> {
|
||||
--shell=none/-N option. If it does not help either, you command is most likely too fast \
|
||||
to be accurately benchmarked by hyperfine.",
|
||||
"Note".bold().red()
|
||||
)?;
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn final_export(&self) -> Result<()> {
|
||||
|
||||
+2
-16
@@ -96,14 +96,6 @@ fn build_command() -> Command {
|
||||
If this is unset, results are compared with the fastest command as reference."
|
||||
)
|
||||
)
|
||||
.arg(
|
||||
Arg::new("reference-name")
|
||||
.long("reference-name")
|
||||
.action(ArgAction::Set)
|
||||
.value_name("CMD")
|
||||
.help("Give a meaningful name to the reference command.")
|
||||
.requires("reference")
|
||||
)
|
||||
.arg(
|
||||
Arg::new("prepare")
|
||||
.long("prepare")
|
||||
@@ -226,15 +218,9 @@ fn build_command() -> Command {
|
||||
.arg(
|
||||
Arg::new("ignore-failure")
|
||||
.long("ignore-failure")
|
||||
.action(ArgAction::Set)
|
||||
.value_name("MODE")
|
||||
.num_args(0..=1)
|
||||
.default_missing_value("all-non-zero")
|
||||
.require_equals(true)
|
||||
.action(ArgAction::SetTrue)
|
||||
.short('i')
|
||||
.help("Ignore failures of the benchmarked programs. Without a value or with \
|
||||
'all-non-zero', all non-zero exit codes are ignored. You can also provide \
|
||||
a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2)."),
|
||||
.help("Ignore non-zero exit codes of the benchmarked programs."),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("style")
|
||||
|
||||
+12
-49
@@ -158,9 +158,7 @@ impl<'a> Commands<'a> {
|
||||
});
|
||||
let args: Vec<_> = args.map(|v| v.as_str()).collect::<Vec<_>>();
|
||||
let param_names_and_values: Vec<(&str, Vec<String>)> = args
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.chunks_exact(2)
|
||||
.map(|pair| {
|
||||
let name = pair[0];
|
||||
let list_str = pair[1];
|
||||
@@ -171,7 +169,7 @@ impl<'a> Commands<'a> {
|
||||
let duplicates =
|
||||
Self::find_duplicates(param_names_and_values.iter().map(|(name, _)| *name));
|
||||
if !duplicates.is_empty() {
|
||||
bail!("Duplicate parameter names: {}", duplicates.join(", "));
|
||||
bail!("Duplicate parameter names: {}", &duplicates.join(", "));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,8 +277,18 @@ impl<'a> Commands<'a> {
|
||||
command_strings: Vec<&'b str>,
|
||||
) -> Result<Vec<Command<'b>>, ParameterScanError> {
|
||||
let param_range = RangeStep::new(param_min, param_max, step)?;
|
||||
let param_count = param_range.size_hint().1.unwrap();
|
||||
let command_name_count = command_names.len();
|
||||
|
||||
// `--command-name` should appear exactly once or exactly B times,
|
||||
// where B is the total number of benchmarks.
|
||||
if command_name_count > 1 && command_name_count != param_count {
|
||||
return Err(ParameterScanError::UnexpectedCommandNameCount(
|
||||
command_name_count,
|
||||
param_count,
|
||||
));
|
||||
}
|
||||
|
||||
let mut i = 0;
|
||||
let mut commands = vec![];
|
||||
for value in param_range {
|
||||
@@ -297,17 +305,6 @@ impl<'a> Commands<'a> {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// `--command-name` should appear exactly once or exactly B times,
|
||||
// where B is the total number of benchmarks.
|
||||
let command_count = commands.len();
|
||||
if command_name_count > 1 && command_name_count != command_count {
|
||||
return Err(ParameterScanError::UnexpectedCommandNameCount(
|
||||
command_name_count,
|
||||
command_count,
|
||||
));
|
||||
}
|
||||
|
||||
Ok(commands)
|
||||
}
|
||||
|
||||
@@ -474,40 +471,6 @@ fn test_build_parameter_scan_commands() {
|
||||
assert_eq!(commands[1].get_command_line(), "echo 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_parameter_scan_commands_named() {
|
||||
use crate::cli::get_cli_arguments;
|
||||
let matches = get_cli_arguments(vec![
|
||||
"hyperfine",
|
||||
"echo {val}",
|
||||
"sleep {val}",
|
||||
"--parameter-scan",
|
||||
"val",
|
||||
"1",
|
||||
"2",
|
||||
"--parameter-step-size",
|
||||
"1",
|
||||
"--command-name",
|
||||
"echo-1",
|
||||
"--command-name",
|
||||
"sleep-1",
|
||||
"--command-name",
|
||||
"echo-2",
|
||||
"--command-name",
|
||||
"sleep-2",
|
||||
]);
|
||||
let commands = Commands::from_cli_arguments(&matches).unwrap().0;
|
||||
assert_eq!(commands.len(), 4);
|
||||
assert_eq!(commands[0].get_name(), "echo-1");
|
||||
assert_eq!(commands[0].get_command_line(), "echo 1");
|
||||
assert_eq!(commands[1].get_name(), "sleep-1");
|
||||
assert_eq!(commands[1].get_command_line(), "sleep 1");
|
||||
assert_eq!(commands[2].get_name(), "echo-2");
|
||||
assert_eq!(commands[2].get_command_line(), "echo 2");
|
||||
assert_eq!(commands[3].get_name(), "sleep-2");
|
||||
assert_eq!(commands[3].get_command_line(), "sleep 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parameter_scan_commands_int() {
|
||||
let commands = Commands::build_parameter_scan_commands(
|
||||
|
||||
@@ -3,10 +3,6 @@ use std::num::{self, ParseFloatError, ParseIntError};
|
||||
use rust_decimal::Error as DecimalError;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
#[error(transparent)]
|
||||
pub struct ConsoleOutputError(pub std::io::Error);
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ParameterScanError {
|
||||
#[error("Error while parsing parameter scan arguments ({0})")]
|
||||
|
||||
+3
-5
@@ -1,5 +1,5 @@
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::io::Write;
|
||||
|
||||
mod asciidoc;
|
||||
mod csv;
|
||||
@@ -18,7 +18,6 @@ use self::orgmode::OrgmodeExporter;
|
||||
|
||||
use crate::benchmark::benchmark_result::BenchmarkResult;
|
||||
use crate::options::SortOrder;
|
||||
use crate::output::console_writeln;
|
||||
use crate::util::units::Unit;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
@@ -145,9 +144,8 @@ impl ExportManager {
|
||||
}
|
||||
ExportTarget::Stdout => {
|
||||
if !intermediate {
|
||||
let mut stdout = io::stdout().lock();
|
||||
console_writeln!(stdout)?;
|
||||
console_writeln!(stdout, "{}", String::from_utf8(content()?).unwrap())?;
|
||||
println!();
|
||||
println!("{}", String::from_utf8(content()?).unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ impl MarkupExporter for OrgmodeExporter {
|
||||
format!(
|
||||
"| {} | {} |\n",
|
||||
cells.first().unwrap(),
|
||||
cells[1..].join(" | ")
|
||||
&cells[1..].join(" | ")
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+2
-16
@@ -4,12 +4,10 @@
|
||||
)]
|
||||
|
||||
use std::env;
|
||||
use std::io::{self, Write};
|
||||
|
||||
use benchmark::scheduler::Scheduler;
|
||||
use cli::get_cli_arguments;
|
||||
use command::Commands;
|
||||
use error::ConsoleOutputError;
|
||||
use export::ExportManager;
|
||||
use options::Options;
|
||||
|
||||
@@ -46,29 +44,17 @@ fn run() -> Result<()> {
|
||||
|
||||
let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
|
||||
scheduler.run_benchmarks()?;
|
||||
scheduler.print_relative_speed_comparison()?;
|
||||
scheduler.print_relative_speed_comparison();
|
||||
scheduler.final_export()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn caused_by_broken_console_pipe(error: &anyhow::Error) -> bool {
|
||||
error
|
||||
.downcast_ref::<ConsoleOutputError>()
|
||||
.is_some_and(|e| e.0.kind() == io::ErrorKind::BrokenPipe)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
match run() {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
// Exit quietly when a reader stops consuming console output early.
|
||||
if caused_by_broken_console_pipe(&e) {
|
||||
std::process::exit(0);
|
||||
}
|
||||
// The write to stderr can itself fail if stderr is closed; the
|
||||
// error message cannot be shown in that case anyway.
|
||||
let _ = writeln!(io::stderr(), "{} {:#}", "Error:".red(), e);
|
||||
eprintln!("{} {:#}", "Error:".red(), e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
+7
-29
@@ -67,16 +67,13 @@ impl Shell {
|
||||
}
|
||||
|
||||
/// Action to take when an executed command fails.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CmdFailureAction {
|
||||
/// Exit with an error message
|
||||
RaiseError,
|
||||
|
||||
/// Ignore all non-zero exit codes
|
||||
IgnoreAllFailures,
|
||||
|
||||
/// Ignore specific exit codes
|
||||
IgnoreSpecificFailures(Vec<i32>),
|
||||
/// Simply ignore the non-zero exit code
|
||||
Ignore,
|
||||
}
|
||||
|
||||
/// Output style type option
|
||||
@@ -211,9 +208,6 @@ pub struct Options {
|
||||
// Command to use as a reference for relative speed comparison
|
||||
pub reference_command: Option<String>,
|
||||
|
||||
// Name of the reference command
|
||||
pub reference_name: Option<String>,
|
||||
|
||||
/// Command(s) to run before each timing run
|
||||
pub preparation_command: Option<Vec<String>>,
|
||||
|
||||
@@ -256,7 +250,6 @@ impl Default for Options {
|
||||
min_benchmarking_time: 3.0,
|
||||
command_failure_action: CmdFailureAction::RaiseError,
|
||||
reference_command: None,
|
||||
reference_name: None,
|
||||
preparation_command: None,
|
||||
conclusion_command: None,
|
||||
setup_command: None,
|
||||
@@ -317,9 +310,6 @@ impl Options {
|
||||
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
||||
|
||||
options.reference_command = matches.get_one::<String>("reference").map(String::from);
|
||||
options.reference_name = matches
|
||||
.get_one::<String>("reference-name")
|
||||
.map(String::from);
|
||||
|
||||
options.preparation_command = matches
|
||||
.get_many::<String>("prepare")
|
||||
@@ -364,7 +354,8 @@ impl Options {
|
||||
_ => {
|
||||
if options
|
||||
.command_output_policies
|
||||
.contains(&CommandOutputPolicy::Inherit)
|
||||
.iter()
|
||||
.any(|policy| *policy == CommandOutputPolicy::Inherit)
|
||||
|| !io::stdout().is_terminal()
|
||||
{
|
||||
OutputStyleOption::Basic
|
||||
@@ -418,21 +409,8 @@ impl Options {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(mode) = matches.get_one::<String>("ignore-failure") {
|
||||
options.command_failure_action = match mode.as_str() {
|
||||
"all-non-zero" | "" => CmdFailureAction::IgnoreAllFailures,
|
||||
codes => {
|
||||
let exit_codes: Result<Vec<i32>, _> = codes
|
||||
.split(',')
|
||||
.map(|s| {
|
||||
s.trim()
|
||||
.parse::<i32>()
|
||||
.map_err(|e| OptionsError::IntParsingError("ignore-failure", e))
|
||||
})
|
||||
.collect();
|
||||
CmdFailureAction::IgnoreSpecificFailures(exit_codes?)
|
||||
}
|
||||
};
|
||||
if matches.get_flag("ignore-failure") {
|
||||
options.command_failure_action = CmdFailureAction::Ignore;
|
||||
}
|
||||
|
||||
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
|
||||
|
||||
@@ -1,13 +1,3 @@
|
||||
pub mod format;
|
||||
pub mod progress_bar;
|
||||
pub mod warnings;
|
||||
|
||||
/// Mark console write errors so broken pipes can be handled separately from file errors.
|
||||
macro_rules! console_writeln {
|
||||
($writer:expr $(, $($args:tt)*)?) => {
|
||||
writeln!($writer $(, $($args)*)?)
|
||||
.map_err($crate::error::ConsoleOutputError)
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) use console_writeln;
|
||||
@@ -4,5 +4,5 @@
|
||||
/// [1] Mytkowicz, 2009. Producing Wrong Data Without Doing Anything Obviously Wrong!.
|
||||
/// Sigplan Notices - SIGPLAN. 44. 265-276. 10.1145/1508284.1508275.
|
||||
pub fn value() -> String {
|
||||
"X".repeat(rand::random_range(0..4096))
|
||||
"X".repeat(rand::random::<usize>() % 4096usize)
|
||||
}
|
||||
+3
-1
@@ -1,7 +1,9 @@
|
||||
use std::process::Command;
|
||||
|
||||
use assert_cmd::cargo::CommandCargoExt;
|
||||
|
||||
pub fn hyperfine_raw_command() -> Command {
|
||||
let mut cmd = Command::new(assert_cmd::cargo::cargo_bin!("hyperfine"));
|
||||
let mut cmd = Command::cargo_bin("hyperfine").unwrap();
|
||||
cmd.current_dir("tests/");
|
||||
cmd
|
||||
}
|
||||
|
||||
+3
-134
@@ -1,14 +1,8 @@
|
||||
mod common;
|
||||
use common::{hyperfine, hyperfine_raw_command};
|
||||
use common::hyperfine;
|
||||
|
||||
use predicates::prelude::*;
|
||||
|
||||
/// Platform-specific I/O utility.
|
||||
/// - On Unix-like systems, defaults to `cat`.
|
||||
/// - On Windows, uses `findstr` as an alternative.
|
||||
/// See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
|
||||
const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
|
||||
|
||||
pub fn hyperfine_debug() -> assert_cmd::Command {
|
||||
let mut cmd = hyperfine();
|
||||
cmd.arg("--debug-mode");
|
||||
@@ -33,47 +27,6 @@ fn one_run_is_supported() {
|
||||
.success();
|
||||
}
|
||||
|
||||
/// Regression test: hyperfine must not panic when writing to a closed
|
||||
/// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process
|
||||
/// that terminates early). A `BrokenPipe` error should result in a quiet
|
||||
/// exit with code 0 instead of a `println!` panic.
|
||||
#[test]
|
||||
fn exits_quietly_when_stdout_is_closed() {
|
||||
use std::process::Stdio;
|
||||
|
||||
for args in [
|
||||
vec!["--runs=1", "echo dummy benchmark"],
|
||||
vec!["--runs=1", "--export-json=-", "echo dummy benchmark"],
|
||||
vec!["--runs=1", "echo dummy benchmark", "echo second benchmark"],
|
||||
] {
|
||||
// A pipe whose read end has been closed before hyperfine is even
|
||||
// started: every write to it will fail with `BrokenPipe`.
|
||||
let (reader, writer) = std::io::pipe().expect("failed to create pipe");
|
||||
drop(reader);
|
||||
|
||||
let output = hyperfine_raw_command()
|
||||
.args(&args)
|
||||
.stdout(writer)
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("failed to spawn hyperfine")
|
||||
.wait_with_output()
|
||||
.expect("failed to wait for hyperfine");
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
assert!(
|
||||
!stderr.contains("panicked"),
|
||||
"hyperfine panicked while writing to a closed pipe:\n{}",
|
||||
stderr
|
||||
);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"hyperfine did not exit cleanly; stderr:\n{}",
|
||||
stderr
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn can_run_commands_without_a_shell() {
|
||||
hyperfine()
|
||||
@@ -318,78 +271,6 @@ fn can_run_failing_commands_with_ignore_failure_option() {
|
||||
.success();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn can_ignore_specific_exit_codes() {
|
||||
// Test that specifying exit code 1 ignores it
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1")
|
||||
.arg("exit 1")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
// Test that other exit codes still fail
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1")
|
||||
.arg("exit 2")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Command terminated with non-zero exit code 2",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn can_ignore_multiple_exit_codes() {
|
||||
// Test that all specified exit codes are ignored
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 1")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 2")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 3")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
// Test that other exit codes still fail
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=1,2,3")
|
||||
.arg("exit 4")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Command terminated with non-zero exit code 4",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn ignore_failure_with_all_non_zero() {
|
||||
// Test explicit "all-non-zero" mode
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--ignore-failure=all-non-zero")
|
||||
.arg("exit 5")
|
||||
.assert()
|
||||
.success();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_output_of_benchmarked_command() {
|
||||
hyperfine()
|
||||
@@ -425,7 +306,7 @@ fn can_pass_input_to_command_from_a_file() {
|
||||
.arg("--runs=1")
|
||||
.arg("--input=example_input_file.txt")
|
||||
.arg("--show-output")
|
||||
.arg(STDIN_READ_COMMAND)
|
||||
.arg("cat")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("This text is part of a file"));
|
||||
@@ -437,7 +318,7 @@ fn fails_if_invalid_stdin_data_file_provided() {
|
||||
.arg("--runs=1")
|
||||
.arg("--input=example_non_existent_file.txt")
|
||||
.arg("--show-output")
|
||||
.arg(STDIN_READ_COMMAND)
|
||||
.arg("cat")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
@@ -612,18 +493,6 @@ fn shows_benchmark_comparison_relative_to_reference() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_reference_name() {
|
||||
hyperfine_debug()
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("--reference-name=refabc123")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
|
||||
Reference in new issue
Block a user