mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-20 03:11:28 +00:00
Compare commits
29
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9cc442567a | ||
|
|
352c504d3f | ||
|
|
1f5a2ff70c | ||
|
|
c0b3834277 | ||
|
|
4d073e08ae | ||
|
|
4b561defef | ||
|
|
27a079e0df | ||
|
|
148632c5f9 | ||
|
|
09f02e4f59 | ||
|
|
c30ac680a3 | ||
|
|
f12f3d9f86 | ||
|
|
327d5f4d91 | ||
|
|
975fe108c4 | ||
|
|
001dcce10e | ||
|
|
6447ab3a4e | ||
|
|
ae4c8073db | ||
|
|
57ef17cdcb | ||
|
|
8b6224059c | ||
|
|
b5a6860d96 | ||
|
|
16348dcf35 | ||
|
|
de07c77387 | ||
|
|
b20df39f0f | ||
|
|
6f95f08d2f | ||
|
|
214bff7065 | ||
|
|
fae8b0ef64 | ||
|
|
98d262c497 | ||
|
|
6b8fa5fd3d | ||
|
|
2166c9f02e | ||
|
|
835fc43ea1 |
No files matched your search
@@ -18,7 +18,7 @@ jobs:
|
|||||||
name: Extract crate metadata
|
name: Extract crate metadata
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v5
|
||||||
- name: Extract crate information
|
- name: Extract crate information
|
||||||
id: crate_metadata
|
id: crate_metadata
|
||||||
run: |
|
run: |
|
||||||
@@ -49,7 +49,7 @@ jobs:
|
|||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
components: rustfmt
|
components: rustfmt
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v5
|
||||||
- run: cargo fmt -- --check
|
- run: cargo fmt -- --check
|
||||||
|
|
||||||
min_version:
|
min_version:
|
||||||
@@ -58,7 +58,7 @@ jobs:
|
|||||||
needs: crate_metadata
|
needs: crate_metadata
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout source code
|
- name: Checkout source code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v5
|
||||||
|
|
||||||
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
|
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
|
||||||
uses: dtolnay/rust-toolchain@master
|
uses: dtolnay/rust-toolchain@master
|
||||||
@@ -79,6 +79,7 @@ jobs:
|
|||||||
matrix:
|
matrix:
|
||||||
job:
|
job:
|
||||||
- { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
- { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||||
|
- { target: aarch64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||||
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }
|
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true }
|
||||||
- { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }
|
- { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true }
|
||||||
- { target: i686-pc-windows-msvc , os: windows-2022 }
|
- { target: i686-pc-windows-msvc , os: windows-2022 }
|
||||||
@@ -94,7 +95,7 @@ jobs:
|
|||||||
BUILD_CMD: cargo
|
BUILD_CMD: cargo
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout source code
|
- name: Checkout source code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v5
|
||||||
|
|
||||||
- name: Install prerequisites
|
- name: Install prerequisites
|
||||||
shell: bash
|
shell: bash
|
||||||
|
|||||||
@@ -1,3 +1,20 @@
|
|||||||
|
# v1.20.0
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
|
||||||
|
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
|
||||||
|
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
|
||||||
|
- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
|
||||||
|
|
||||||
|
## Bugfixes
|
||||||
|
|
||||||
|
- Fix bug where naming individual commands with parameter scan was not working correctly, see #794 (@teofr)
|
||||||
|
|
||||||
|
## Other
|
||||||
|
|
||||||
|
- Restrict `cat` tests to Unix environments, see #776 and #777 (@ritvikos)
|
||||||
|
|
||||||
# v1.19.0
|
# v1.19.0
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|||||||
+2
-2
@@ -13,5 +13,5 @@ authors:
|
|||||||
repository-code: 'https://github.com/sharkdp/hyperfine'
|
repository-code: 'https://github.com/sharkdp/hyperfine'
|
||||||
abstract: A command-line benchmarking tool.
|
abstract: A command-line benchmarking tool.
|
||||||
license: MIT
|
license: MIT
|
||||||
version: 1.16.1
|
version: 1.20.0
|
||||||
date-released: '2023-03-21'
|
date-released: '2025-11-18'
|
||||||
Generated
+438
-473
File diff suppressed because it is too large.
Load diff
+16
-15
@@ -7,25 +7,26 @@ license = "MIT OR Apache-2.0"
|
|||||||
name = "hyperfine"
|
name = "hyperfine"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
repository = "https://github.com/sharkdp/hyperfine"
|
repository = "https://github.com/sharkdp/hyperfine"
|
||||||
version = "1.19.0"
|
version = "1.20.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
build = "build.rs"
|
build = "build.rs"
|
||||||
rust-version = "1.88.0"
|
rust-version = "1.97.0"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
# Use the nightly feature windows_process_extensions_main_thread_handle
|
# Use the nightly feature windows_process_extensions_main_thread_handle
|
||||||
windows_process_extensions_main_thread_handle = []
|
windows_process_extensions_main_thread_handle = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
colored = "2.1"
|
colored = "3.1"
|
||||||
|
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
|
||||||
indicatif = "=0.17.4"
|
indicatif = "=0.17.4"
|
||||||
statistical = "1.0"
|
statistical = "1.0"
|
||||||
csv = "1.3"
|
csv = "1.4"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
rust_decimal = "1.36"
|
rust_decimal = "1.43"
|
||||||
rand = "0.8"
|
rand = "0.10"
|
||||||
shell-words = "1.0"
|
shell-words = "1.1"
|
||||||
thiserror = "2.0"
|
thiserror = "2.0"
|
||||||
anyhow = "1.0"
|
anyhow = "1.0"
|
||||||
|
|
||||||
@@ -33,7 +34,7 @@ anyhow = "1.0"
|
|||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
|
|
||||||
[target.'cfg(windows)'.dependencies]
|
[target.'cfg(windows)'.dependencies]
|
||||||
windows-sys = { version = "0.59", features = [
|
windows-sys = { version = "0.61", features = [
|
||||||
"Win32_Foundation",
|
"Win32_Foundation",
|
||||||
"Win32_Security",
|
"Win32_Security",
|
||||||
"Win32_System_JobObjects",
|
"Win32_System_JobObjects",
|
||||||
@@ -42,10 +43,10 @@ windows-sys = { version = "0.59", features = [
|
|||||||
] }
|
] }
|
||||||
|
|
||||||
[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
|
[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
|
||||||
once_cell = "1.19"
|
once_cell = "1.21"
|
||||||
|
|
||||||
[target.'cfg(target_os="linux")'.dependencies]
|
[target.'cfg(target_os="linux")'.dependencies]
|
||||||
nix = { version = "0.29", features = ["zerocopy"] }
|
nix = { version = "0.31", features = ["zerocopy"] }
|
||||||
|
|
||||||
[dependencies.clap]
|
[dependencies.clap]
|
||||||
version = "4"
|
version = "4"
|
||||||
@@ -62,18 +63,18 @@ features = [
|
|||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
approx = "0.5"
|
approx = "0.5"
|
||||||
assert_cmd = "2.0"
|
assert_cmd = "2.2"
|
||||||
insta = { version = "1.41.1", features = ["yaml"] }
|
insta = { version = "1.48", features = ["yaml"] }
|
||||||
predicates = "3.1"
|
predicates = "3.1"
|
||||||
tempfile = "3.14"
|
tempfile = "3.27"
|
||||||
|
|
||||||
[profile.dev.package]
|
[profile.dev.package]
|
||||||
insta.opt-level = 3
|
insta.opt-level = 3
|
||||||
similar.opt-level = 3
|
similar.opt-level = 3
|
||||||
|
|
||||||
[build-dependencies]
|
[build-dependencies]
|
||||||
clap = "4.5.47"
|
clap = "4.6"
|
||||||
clap_complete = "4.2.1"
|
clap_complete = "4.6"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = true
|
lto = true
|
||||||
|
|||||||
@@ -10,18 +10,6 @@ A command-line benchmarking tool.
|
|||||||
|
|
||||||

|

|
||||||
|
|
||||||
### Sponsors
|
|
||||||
|
|
||||||
A special *thank you* goes to our biggest <a href="doc/sponsors.md">sponsor</a>:<br>
|
|
||||||
|
|
||||||
<a href="https://www.warp.dev/hyperfine">
|
|
||||||
<img src="doc/sponsors/warp-logo.png" width="200" alt="Warp">
|
|
||||||
<br>
|
|
||||||
<strong>Warp, the intelligent terminal</strong>
|
|
||||||
<br>
|
|
||||||
<sub>Available on MacOS, Linux, Windows</sub>
|
|
||||||
</a>
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
* Statistical analysis across multiple runs.
|
* Statistical analysis across multiple runs.
|
||||||
@@ -120,7 +108,7 @@ hyperfine -N 'grep TODO /home/user'
|
|||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
### Shell functions and aliases
|
### Shell functions
|
||||||
|
|
||||||
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
|
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
|
||||||
|
|
||||||
@@ -130,13 +118,10 @@ export -f my_function
|
|||||||
hyperfine --shell=bash my_function
|
hyperfine --shell=bash my_function
|
||||||
```
|
```
|
||||||
|
|
||||||
Otherwise, inline them into or source them from the benchmarked program:
|
Otherwise, inline the function into the benchmarked command:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
hyperfine 'my_function() { sleep 1; }; my_function'
|
hyperfine 'my_function() { sleep 1; }; my_function'
|
||||||
|
|
||||||
echo 'alias my_alias="sleep 1"' > /tmp/my_alias.sh
|
|
||||||
hyperfine '. /tmp/my_alias.sh; my_alias'
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Exporting results
|
### Exporting results
|
||||||
@@ -179,11 +164,15 @@ like `--warmup`, `--prepare <cmd>`, `--setup <cmd>` or `--cleanup <cmd>`:
|
|||||||
|
|
||||||
### On Ubuntu
|
### On Ubuntu
|
||||||
|
|
||||||
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
|
On Ubuntu, hyperfine can be installed [from the official repositories](https://launchpad.net/ubuntu/+source/rust-hyperfine):
|
||||||
and install it via `dpkg`:
|
|
||||||
```
|
```
|
||||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb
|
apt install hyperfine
|
||||||
sudo dpkg -i hyperfine_1.19.0_amd64.deb
|
```
|
||||||
|
|
||||||
|
Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:
|
||||||
|
```
|
||||||
|
wget https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb
|
||||||
|
sudo dpkg -i hyperfine_1.20.0_amd64.deb
|
||||||
```
|
```
|
||||||
|
|
||||||
### On Fedora
|
### On Fedora
|
||||||
@@ -210,7 +199,7 @@ pacman -S hyperfine
|
|||||||
|
|
||||||
### On Debian Linux
|
### On Debian Linux
|
||||||
|
|
||||||
On Debian Linux, hyperfine can be installed [from the testing repositories](https://packages.debian.org/testing/main/hyperfine):
|
On Debian Linux, hyperfine can be installed [from the official repositories](https://packages.debian.org/hyperfine):
|
||||||
```
|
```
|
||||||
apt install hyperfine
|
apt install hyperfine
|
||||||
```
|
```
|
||||||
@@ -313,7 +302,7 @@ Hyperfine can be installed from source via [cargo](https://doc.rust-lang.org/car
|
|||||||
cargo install --locked hyperfine
|
cargo install --locked hyperfine
|
||||||
```
|
```
|
||||||
|
|
||||||
Make sure that you use Rust 1.76 or newer.
|
Make sure that you use Rust 1.97 or newer.
|
||||||
|
|
||||||
### From binaries (Linux, macOS, Windows)
|
### From binaries (Linux, macOS, Windows)
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -131,7 +131,7 @@ Execute \fICMD\fP after the completion of all benchmarking runs for each individ
|
|||||||
command to be benchmarked. This is useful if the commands to be benchmarked
|
command to be benchmarked. This is useful if the commands to be benchmarked
|
||||||
produce artifacts that need to be cleaned up. It only runs once a series of
|
produce artifacts that need to be cleaned up. It only runs once a series of
|
||||||
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
|
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
|
||||||
ever run.
|
every run.
|
||||||
.HP
|
.HP
|
||||||
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
|
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
|
||||||
.IP
|
.IP
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
## Sponsors
|
|
||||||
|
|
||||||
`hyperfine` development is sponsored by many individuals and companies. Thank you very much!
|
|
||||||
|
|
||||||
Please note, that being sponsored does not affect the individuality of the `hyperfine`
|
|
||||||
project or affect the maintainers' actions in any way.
|
|
||||||
We remain impartial and continue to assess pull requests solely on merit - the
|
|
||||||
features added, bugs solved, and effect on the overall complexity of the code.
|
|
||||||
No issue will have a different priority based on sponsorship status of the
|
|
||||||
reporter.
|
|
||||||
|
|
||||||
Contributions from anybody are most welcomed.
|
|
||||||
|
|
||||||
If you want to see our biggest sponsors, check the top of [`README.md`](../README.md#sponsors).
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 130 KiB |
+6
-15
@@ -4,23 +4,14 @@ This folder contains scripts that can be used in combination with hyperfines `--
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
|
hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
|
||||||
./plot_whisker.py sleep.json
|
uv run plot_whisker.py sleep.json
|
||||||
```
|
```
|
||||||
|
|
||||||
### Pre-requisites
|
### Pre-requisites
|
||||||
|
|
||||||
To make these scripts work, you will need `numpy`, `matplotlib` and `scipy`.
|
Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
|
||||||
|
commands above from this directory.
|
||||||
|
|
||||||
If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/)
|
The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
|
||||||
inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx),
|
inline script metadata. `uv run` automatically installs these dependencies in an
|
||||||
you can directly run the scripts using
|
isolated environment before running the script.
|
||||||
|
|
||||||
```bash
|
|
||||||
uv run plot_whisker.py sleep.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Otherwise, install the dependencies via your system package manager or using `pip`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pip install numpy matplotlib scipy # pip3, if you are using python3
|
|
||||||
```
|
|
||||||
+36
-18
@@ -79,23 +79,41 @@ fn run_command_and_measure_common(
|
|||||||
let result = execute_and_measure(command)
|
let result = execute_and_measure(command)
|
||||||
.with_context(|| format!("Failed to run command '{command_name}'"))?;
|
.with_context(|| format!("Failed to run command '{command_name}'"))?;
|
||||||
|
|
||||||
if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
|
if !result.status.success() {
|
||||||
let when = match iteration {
|
use crate::util::exit_code::extract_exit_code;
|
||||||
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
|
|
||||||
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
|
|
||||||
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
|
|
||||||
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
|
|
||||||
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
|
|
||||||
};
|
|
||||||
bail!(
|
|
||||||
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
|
||||||
Alternatively, use the '--show-output' option to debug what went wrong.",
|
|
||||||
cause=result.status.code().map_or(
|
|
||||||
"The process has been terminated by a signal".into(),
|
|
||||||
|c| format!("Command terminated with non-zero exit code {c}")
|
|
||||||
|
|
||||||
),
|
let should_fail = match command_failure_action {
|
||||||
);
|
CmdFailureAction::RaiseError => true,
|
||||||
|
CmdFailureAction::IgnoreAllFailures => false,
|
||||||
|
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
|
||||||
|
// Only fail if the exit code is not in the list of codes to ignore
|
||||||
|
if let Some(exit_code) = extract_exit_code(result.status) {
|
||||||
|
!codes.contains(&exit_code)
|
||||||
|
} else {
|
||||||
|
// If we can't extract an exit code, treat it as a failure
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if should_fail {
|
||||||
|
let when = match iteration {
|
||||||
|
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
|
||||||
|
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
|
||||||
|
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
|
||||||
|
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
|
||||||
|
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
|
||||||
|
};
|
||||||
|
bail!(
|
||||||
|
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
||||||
|
Alternatively, use the '--show-output' option to debug what went wrong.",
|
||||||
|
cause=result.status.code().map_or(
|
||||||
|
"The process has been terminated by a signal".into(),
|
||||||
|
|c| format!("Command terminated with non-zero exit code {c}")
|
||||||
|
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
@@ -122,7 +140,7 @@ impl Executor for RawExecutor<'_> {
|
|||||||
let result = run_command_and_measure_common(
|
let result = run_command_and_measure_common(
|
||||||
command.get_command()?,
|
command.get_command()?,
|
||||||
iteration,
|
iteration,
|
||||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||||
&self.options.command_input_policy,
|
&self.options.command_input_policy,
|
||||||
output_policy,
|
output_policy,
|
||||||
&command.get_command_line(),
|
&command.get_command_line(),
|
||||||
@@ -187,7 +205,7 @@ impl Executor for ShellExecutor<'_> {
|
|||||||
let mut result = run_command_and_measure_common(
|
let mut result = run_command_and_measure_common(
|
||||||
command_builder,
|
command_builder,
|
||||||
iteration,
|
iteration,
|
||||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||||
&self.options.command_input_policy,
|
&self.options.command_input_policy,
|
||||||
output_policy,
|
output_policy,
|
||||||
&command.get_command_line(),
|
&command.get_command_line(),
|
||||||
|
|||||||
+19
-11
@@ -5,6 +5,7 @@ pub mod scheduler;
|
|||||||
pub mod timing_result;
|
pub mod timing_result;
|
||||||
|
|
||||||
use std::cmp;
|
use std::cmp;
|
||||||
|
use std::io::{self, Write};
|
||||||
|
|
||||||
use crate::benchmark::executor::BenchmarkIteration;
|
use crate::benchmark::executor::BenchmarkIteration;
|
||||||
use crate::command::Command;
|
use crate::command::Command;
|
||||||
@@ -12,6 +13,7 @@ use crate::options::{
|
|||||||
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
|
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
|
||||||
};
|
};
|
||||||
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
|
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
|
||||||
|
use crate::output::console_writeln;
|
||||||
use crate::output::format::{format_duration, format_duration_unit};
|
use crate::output::format::{format_duration, format_duration_unit};
|
||||||
use crate::output::progress_bar::get_progress_bar;
|
use crate::output::progress_bar::get_progress_bar;
|
||||||
use crate::output::warnings::{OutlierWarningOptions, Warnings};
|
use crate::output::warnings::{OutlierWarningOptions, Warnings};
|
||||||
@@ -140,12 +142,13 @@ impl<'a> Benchmark<'a> {
|
|||||||
/// Run the benchmark for a single command
|
/// Run the benchmark for a single command
|
||||||
pub fn run(&self) -> Result<BenchmarkResult> {
|
pub fn run(&self) -> Result<BenchmarkResult> {
|
||||||
if self.options.output_style != OutputStyleOption::Disabled {
|
if self.options.output_style != OutputStyleOption::Disabled {
|
||||||
println!(
|
console_writeln!(
|
||||||
|
io::stdout(),
|
||||||
"{}{}: {}",
|
"{}{}: {}",
|
||||||
"Benchmark ".bold(),
|
"Benchmark ".bold(),
|
||||||
(self.number + 1).to_string().bold(),
|
(self.number + 1).to_string().bold(),
|
||||||
self.command.get_name_with_unused_parameters(),
|
self.command.get_name_with_unused_parameters(),
|
||||||
);
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut times_real: Vec<Second> = vec![];
|
let mut times_real: Vec<Second> = vec![];
|
||||||
@@ -358,19 +361,22 @@ impl<'a> Benchmark<'a> {
|
|||||||
let system_str = format_duration(system_mean, Some(time_unit));
|
let system_str = format_duration(system_mean, Some(time_unit));
|
||||||
|
|
||||||
if self.options.output_style != OutputStyleOption::Disabled {
|
if self.options.output_style != OutputStyleOption::Disabled {
|
||||||
|
let mut stdout = io::stdout().lock();
|
||||||
if times_real.len() == 1 {
|
if times_real.len() == 1 {
|
||||||
println!(
|
console_writeln!(
|
||||||
|
stdout,
|
||||||
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
|
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
|
||||||
"abs".green().bold(),
|
"abs".green().bold(),
|
||||||
mean_str.green().bold(),
|
mean_str.green().bold(),
|
||||||
" ", // alignment
|
" ", // alignment
|
||||||
user_str.blue(),
|
user_str.blue(),
|
||||||
system_str.blue()
|
system_str.blue()
|
||||||
);
|
)?;
|
||||||
} else {
|
} else {
|
||||||
let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
|
let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
|
||||||
|
|
||||||
println!(
|
console_writeln!(
|
||||||
|
stdout,
|
||||||
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
|
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
|
||||||
"mean".green().bold(),
|
"mean".green().bold(),
|
||||||
"σ".green(),
|
"σ".green(),
|
||||||
@@ -378,16 +384,17 @@ impl<'a> Benchmark<'a> {
|
|||||||
stddev_str.green(),
|
stddev_str.green(),
|
||||||
user_str.blue(),
|
user_str.blue(),
|
||||||
system_str.blue()
|
system_str.blue()
|
||||||
);
|
)?;
|
||||||
|
|
||||||
println!(
|
console_writeln!(
|
||||||
|
stdout,
|
||||||
" Range ({} … {}): {:>8} … {:>8} {}",
|
" Range ({} … {}): {:>8} … {:>8} {}",
|
||||||
"min".cyan(),
|
"min".cyan(),
|
||||||
"max".purple(),
|
"max".purple(),
|
||||||
min_str.cyan(),
|
min_str.cyan(),
|
||||||
max_str.purple(),
|
max_str.purple(),
|
||||||
num_str.dimmed()
|
num_str.dimmed()
|
||||||
);
|
)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -430,15 +437,16 @@ impl<'a> Benchmark<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !warnings.is_empty() {
|
if !warnings.is_empty() {
|
||||||
eprintln!(" ");
|
let mut stderr = io::stderr().lock();
|
||||||
|
console_writeln!(stderr, " ")?;
|
||||||
|
|
||||||
for warning in &warnings {
|
for warning in &warnings {
|
||||||
eprintln!(" {}: {}", "Warning".yellow(), warning);
|
console_writeln!(stderr, " {}: {}", "Warning".yellow(), warning)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.options.output_style != OutputStyleOption::Disabled {
|
if self.options.output_style != OutputStyleOption::Disabled {
|
||||||
println!(" ");
|
console_writeln!(io::stdout(), " ")?;
|
||||||
}
|
}
|
||||||
|
|
||||||
self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
|
self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ pub fn compute_with_check_from_reference<'a>(
|
|||||||
pub fn compute_with_check(
|
pub fn compute_with_check(
|
||||||
results: &[BenchmarkResult],
|
results: &[BenchmarkResult],
|
||||||
sort_order: SortOrder,
|
sort_order: SortOrder,
|
||||||
) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
|
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
|
||||||
let fastest = fastest_of(results);
|
let fastest = fastest_of(results);
|
||||||
|
|
||||||
if fastest.mean == 0.0 {
|
if fastest.mean == 0.0 {
|
||||||
@@ -112,7 +112,7 @@ pub fn compute_with_check(
|
|||||||
pub fn compute(
|
pub fn compute(
|
||||||
results: &[BenchmarkResult],
|
results: &[BenchmarkResult],
|
||||||
sort_order: SortOrder,
|
sort_order: SortOrder,
|
||||||
) -> Vec<BenchmarkResultWithRelativeSpeed> {
|
) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
|
||||||
let fastest = fastest_of(results);
|
let fastest = fastest_of(results);
|
||||||
|
|
||||||
compute_relative_speeds(results, fastest, sort_order)
|
compute_relative_speeds(results, fastest, sort_order)
|
||||||
|
|||||||
+25
-16
@@ -3,10 +3,12 @@ use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
|
|||||||
use super::{relative_speed, Benchmark};
|
use super::{relative_speed, Benchmark};
|
||||||
use colored::*;
|
use colored::*;
|
||||||
use std::cmp::Ordering;
|
use std::cmp::Ordering;
|
||||||
|
use std::io::{self, Write};
|
||||||
|
|
||||||
use crate::command::{Command, Commands};
|
use crate::command::{Command, Commands};
|
||||||
use crate::export::ExportManager;
|
use crate::export::ExportManager;
|
||||||
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
|
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
|
||||||
|
use crate::output::console_writeln;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
@@ -42,7 +44,7 @@ impl<'a> Scheduler<'a> {
|
|||||||
.options
|
.options
|
||||||
.reference_command
|
.reference_command
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|cmd| Command::new(None, cmd));
|
.map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
|
||||||
|
|
||||||
executor.calibrate()?;
|
executor.calibrate()?;
|
||||||
|
|
||||||
@@ -58,13 +60,13 @@ impl<'a> Scheduler<'a> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn print_relative_speed_comparison(&self) {
|
pub fn print_relative_speed_comparison(&self) -> Result<()> {
|
||||||
if self.options.output_style == OutputStyleOption::Disabled {
|
if self.options.output_style == OutputStyleOption::Disabled {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.results.len() < 2 {
|
if self.results.len() < 2 {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let reference = self
|
let reference = self
|
||||||
@@ -79,17 +81,19 @@ impl<'a> Scheduler<'a> {
|
|||||||
reference,
|
reference,
|
||||||
self.options.sort_order_speed_comparison,
|
self.options.sort_order_speed_comparison,
|
||||||
) {
|
) {
|
||||||
|
let mut stdout = io::stdout().lock();
|
||||||
match self.options.sort_order_speed_comparison {
|
match self.options.sort_order_speed_comparison {
|
||||||
SortOrder::MeanTime => {
|
SortOrder::MeanTime => {
|
||||||
println!("{}", "Summary".bold());
|
console_writeln!(stdout, "{}", "Summary".bold())?;
|
||||||
|
|
||||||
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
|
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
|
||||||
let others = annotated_results.iter().filter(|r| !r.is_reference);
|
let others = annotated_results.iter().filter(|r| !r.is_reference);
|
||||||
|
|
||||||
println!(
|
console_writeln!(
|
||||||
|
stdout,
|
||||||
" {} ran",
|
" {} ran",
|
||||||
reference.result.command_with_unused_parameters.cyan()
|
reference.result.command_with_unused_parameters.cyan()
|
||||||
);
|
)?;
|
||||||
|
|
||||||
for item in others {
|
for item in others {
|
||||||
let stddev = if let Some(stddev) = item.relative_speed_stddev {
|
let stddev = if let Some(stddev) = item.relative_speed_stddev {
|
||||||
@@ -114,18 +118,20 @@ impl<'a> Scheduler<'a> {
|
|||||||
stddev
|
stddev
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
println!(
|
console_writeln!(
|
||||||
|
stdout,
|
||||||
"{} {}",
|
"{} {}",
|
||||||
comparator,
|
comparator,
|
||||||
&item.result.command_with_unused_parameters.magenta()
|
item.result.command_with_unused_parameters.magenta()
|
||||||
);
|
)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
SortOrder::Command => {
|
SortOrder::Command => {
|
||||||
println!("{}", "Relative speed comparison".bold());
|
console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
|
||||||
|
|
||||||
for item in annotated_results {
|
for item in annotated_results {
|
||||||
println!(
|
console_writeln!(
|
||||||
|
stdout,
|
||||||
" {}{} {}",
|
" {}{} {}",
|
||||||
format!("{:10.2}", item.relative_speed).bold().green(),
|
format!("{:10.2}", item.relative_speed).bold().green(),
|
||||||
if item.is_reference {
|
if item.is_reference {
|
||||||
@@ -135,13 +141,14 @@ impl<'a> Scheduler<'a> {
|
|||||||
} else {
|
} else {
|
||||||
" ".into()
|
" ".into()
|
||||||
},
|
},
|
||||||
&item.result.command_with_unused_parameters,
|
item.result.command_with_unused_parameters,
|
||||||
);
|
)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
eprintln!(
|
console_writeln!(
|
||||||
|
io::stderr(),
|
||||||
"{}: The benchmark comparison could not be computed as some benchmark times are zero. \
|
"{}: The benchmark comparison could not be computed as some benchmark times are zero. \
|
||||||
This could be caused by background interference during the initial calibration phase \
|
This could be caused by background interference during the initial calibration phase \
|
||||||
of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
|
of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
|
||||||
@@ -149,8 +156,10 @@ impl<'a> Scheduler<'a> {
|
|||||||
--shell=none/-N option. If it does not help either, you command is most likely too fast \
|
--shell=none/-N option. If it does not help either, you command is most likely too fast \
|
||||||
to be accurately benchmarked by hyperfine.",
|
to be accurately benchmarked by hyperfine.",
|
||||||
"Note".bold().red()
|
"Note".bold().red()
|
||||||
);
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn final_export(&self) -> Result<()> {
|
pub fn final_export(&self) -> Result<()> {
|
||||||
|
|||||||
+16
-2
@@ -96,6 +96,14 @@ fn build_command() -> Command {
|
|||||||
If this is unset, results are compared with the fastest command as reference."
|
If this is unset, results are compared with the fastest command as reference."
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::new("reference-name")
|
||||||
|
.long("reference-name")
|
||||||
|
.action(ArgAction::Set)
|
||||||
|
.value_name("CMD")
|
||||||
|
.help("Give a meaningful name to the reference command.")
|
||||||
|
.requires("reference")
|
||||||
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::new("prepare")
|
Arg::new("prepare")
|
||||||
.long("prepare")
|
.long("prepare")
|
||||||
@@ -218,9 +226,15 @@ fn build_command() -> Command {
|
|||||||
.arg(
|
.arg(
|
||||||
Arg::new("ignore-failure")
|
Arg::new("ignore-failure")
|
||||||
.long("ignore-failure")
|
.long("ignore-failure")
|
||||||
.action(ArgAction::SetTrue)
|
.action(ArgAction::Set)
|
||||||
|
.value_name("MODE")
|
||||||
|
.num_args(0..=1)
|
||||||
|
.default_missing_value("all-non-zero")
|
||||||
|
.require_equals(true)
|
||||||
.short('i')
|
.short('i')
|
||||||
.help("Ignore non-zero exit codes of the benchmarked programs."),
|
.help("Ignore failures of the benchmarked programs. Without a value or with \
|
||||||
|
'all-non-zero', all non-zero exit codes are ignored. You can also provide \
|
||||||
|
a comma-separated list of exit codes to ignore (e.g., --ignore-failure=1,2)."),
|
||||||
)
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::new("style")
|
Arg::new("style")
|
||||||
|
|||||||
+49
-12
@@ -158,7 +158,9 @@ impl<'a> Commands<'a> {
|
|||||||
});
|
});
|
||||||
let args: Vec<_> = args.map(|v| v.as_str()).collect::<Vec<_>>();
|
let args: Vec<_> = args.map(|v| v.as_str()).collect::<Vec<_>>();
|
||||||
let param_names_and_values: Vec<(&str, Vec<String>)> = args
|
let param_names_and_values: Vec<(&str, Vec<String>)> = args
|
||||||
.chunks_exact(2)
|
.as_chunks::<2>()
|
||||||
|
.0
|
||||||
|
.iter()
|
||||||
.map(|pair| {
|
.map(|pair| {
|
||||||
let name = pair[0];
|
let name = pair[0];
|
||||||
let list_str = pair[1];
|
let list_str = pair[1];
|
||||||
@@ -169,7 +171,7 @@ impl<'a> Commands<'a> {
|
|||||||
let duplicates =
|
let duplicates =
|
||||||
Self::find_duplicates(param_names_and_values.iter().map(|(name, _)| *name));
|
Self::find_duplicates(param_names_and_values.iter().map(|(name, _)| *name));
|
||||||
if !duplicates.is_empty() {
|
if !duplicates.is_empty() {
|
||||||
bail!("Duplicate parameter names: {}", &duplicates.join(", "));
|
bail!("Duplicate parameter names: {}", duplicates.join(", "));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -277,18 +279,8 @@ impl<'a> Commands<'a> {
|
|||||||
command_strings: Vec<&'b str>,
|
command_strings: Vec<&'b str>,
|
||||||
) -> Result<Vec<Command<'b>>, ParameterScanError> {
|
) -> Result<Vec<Command<'b>>, ParameterScanError> {
|
||||||
let param_range = RangeStep::new(param_min, param_max, step)?;
|
let param_range = RangeStep::new(param_min, param_max, step)?;
|
||||||
let param_count = param_range.size_hint().1.unwrap();
|
|
||||||
let command_name_count = command_names.len();
|
let command_name_count = command_names.len();
|
||||||
|
|
||||||
// `--command-name` should appear exactly once or exactly B times,
|
|
||||||
// where B is the total number of benchmarks.
|
|
||||||
if command_name_count > 1 && command_name_count != param_count {
|
|
||||||
return Err(ParameterScanError::UnexpectedCommandNameCount(
|
|
||||||
command_name_count,
|
|
||||||
param_count,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut i = 0;
|
let mut i = 0;
|
||||||
let mut commands = vec![];
|
let mut commands = vec![];
|
||||||
for value in param_range {
|
for value in param_range {
|
||||||
@@ -305,6 +297,17 @@ impl<'a> Commands<'a> {
|
|||||||
i += 1;
|
i += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// `--command-name` should appear exactly once or exactly B times,
|
||||||
|
// where B is the total number of benchmarks.
|
||||||
|
let command_count = commands.len();
|
||||||
|
if command_name_count > 1 && command_name_count != command_count {
|
||||||
|
return Err(ParameterScanError::UnexpectedCommandNameCount(
|
||||||
|
command_name_count,
|
||||||
|
command_count,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
Ok(commands)
|
Ok(commands)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -471,6 +474,40 @@ fn test_build_parameter_scan_commands() {
|
|||||||
assert_eq!(commands[1].get_command_line(), "echo 2");
|
assert_eq!(commands[1].get_command_line(), "echo 2");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_parameter_scan_commands_named() {
|
||||||
|
use crate::cli::get_cli_arguments;
|
||||||
|
let matches = get_cli_arguments(vec![
|
||||||
|
"hyperfine",
|
||||||
|
"echo {val}",
|
||||||
|
"sleep {val}",
|
||||||
|
"--parameter-scan",
|
||||||
|
"val",
|
||||||
|
"1",
|
||||||
|
"2",
|
||||||
|
"--parameter-step-size",
|
||||||
|
"1",
|
||||||
|
"--command-name",
|
||||||
|
"echo-1",
|
||||||
|
"--command-name",
|
||||||
|
"sleep-1",
|
||||||
|
"--command-name",
|
||||||
|
"echo-2",
|
||||||
|
"--command-name",
|
||||||
|
"sleep-2",
|
||||||
|
]);
|
||||||
|
let commands = Commands::from_cli_arguments(&matches).unwrap().0;
|
||||||
|
assert_eq!(commands.len(), 4);
|
||||||
|
assert_eq!(commands[0].get_name(), "echo-1");
|
||||||
|
assert_eq!(commands[0].get_command_line(), "echo 1");
|
||||||
|
assert_eq!(commands[1].get_name(), "sleep-1");
|
||||||
|
assert_eq!(commands[1].get_command_line(), "sleep 1");
|
||||||
|
assert_eq!(commands[2].get_name(), "echo-2");
|
||||||
|
assert_eq!(commands[2].get_command_line(), "echo 2");
|
||||||
|
assert_eq!(commands[3].get_name(), "sleep-2");
|
||||||
|
assert_eq!(commands[3].get_command_line(), "sleep 2");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parameter_scan_commands_int() {
|
fn test_parameter_scan_commands_int() {
|
||||||
let commands = Commands::build_parameter_scan_commands(
|
let commands = Commands::build_parameter_scan_commands(
|
||||||
|
|||||||
@@ -3,6 +3,10 @@ use std::num::{self, ParseFloatError, ParseIntError};
|
|||||||
use rust_decimal::Error as DecimalError;
|
use rust_decimal::Error as DecimalError;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
#[error(transparent)]
|
||||||
|
pub struct ConsoleOutputError(pub std::io::Error);
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum ParameterScanError {
|
pub enum ParameterScanError {
|
||||||
#[error("Error while parsing parameter scan arguments ({0})")]
|
#[error("Error while parsing parameter scan arguments ({0})")]
|
||||||
|
|||||||
+5
-3
@@ -1,5 +1,5 @@
|
|||||||
use std::fs::{File, OpenOptions};
|
use std::fs::{File, OpenOptions};
|
||||||
use std::io::Write;
|
use std::io::{self, Write};
|
||||||
|
|
||||||
mod asciidoc;
|
mod asciidoc;
|
||||||
mod csv;
|
mod csv;
|
||||||
@@ -18,6 +18,7 @@ use self::orgmode::OrgmodeExporter;
|
|||||||
|
|
||||||
use crate::benchmark::benchmark_result::BenchmarkResult;
|
use crate::benchmark::benchmark_result::BenchmarkResult;
|
||||||
use crate::options::SortOrder;
|
use crate::options::SortOrder;
|
||||||
|
use crate::output::console_writeln;
|
||||||
use crate::util::units::Unit;
|
use crate::util::units::Unit;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
@@ -144,8 +145,9 @@ impl ExportManager {
|
|||||||
}
|
}
|
||||||
ExportTarget::Stdout => {
|
ExportTarget::Stdout => {
|
||||||
if !intermediate {
|
if !intermediate {
|
||||||
println!();
|
let mut stdout = io::stdout().lock();
|
||||||
println!("{}", String::from_utf8(content()?).unwrap());
|
console_writeln!(stdout)?;
|
||||||
|
console_writeln!(stdout, "{}", String::from_utf8(content()?).unwrap())?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ impl MarkupExporter for OrgmodeExporter {
|
|||||||
format!(
|
format!(
|
||||||
"| {} | {} |\n",
|
"| {} | {} |\n",
|
||||||
cells.first().unwrap(),
|
cells.first().unwrap(),
|
||||||
&cells[1..].join(" | ")
|
cells[1..].join(" | ")
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+16
-2
@@ -4,10 +4,12 @@
|
|||||||
)]
|
)]
|
||||||
|
|
||||||
use std::env;
|
use std::env;
|
||||||
|
use std::io::{self, Write};
|
||||||
|
|
||||||
use benchmark::scheduler::Scheduler;
|
use benchmark::scheduler::Scheduler;
|
||||||
use cli::get_cli_arguments;
|
use cli::get_cli_arguments;
|
||||||
use command::Commands;
|
use command::Commands;
|
||||||
|
use error::ConsoleOutputError;
|
||||||
use export::ExportManager;
|
use export::ExportManager;
|
||||||
use options::Options;
|
use options::Options;
|
||||||
|
|
||||||
@@ -44,17 +46,29 @@ fn run() -> Result<()> {
|
|||||||
|
|
||||||
let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
|
let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
|
||||||
scheduler.run_benchmarks()?;
|
scheduler.run_benchmarks()?;
|
||||||
scheduler.print_relative_speed_comparison();
|
scheduler.print_relative_speed_comparison()?;
|
||||||
scheduler.final_export()?;
|
scheduler.final_export()?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn caused_by_broken_console_pipe(error: &anyhow::Error) -> bool {
|
||||||
|
error
|
||||||
|
.downcast_ref::<ConsoleOutputError>()
|
||||||
|
.is_some_and(|e| e.0.kind() == io::ErrorKind::BrokenPipe)
|
||||||
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
match run() {
|
match run() {
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("{} {:#}", "Error:".red(), e);
|
// Exit quietly when a reader stops consuming console output early.
|
||||||
|
if caused_by_broken_console_pipe(&e) {
|
||||||
|
std::process::exit(0);
|
||||||
|
}
|
||||||
|
// The write to stderr can itself fail if stderr is closed; the
|
||||||
|
// error message cannot be shown in that case anyway.
|
||||||
|
let _ = writeln!(io::stderr(), "{} {:#}", "Error:".red(), e);
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+29
-7
@@ -67,13 +67,16 @@ impl Shell {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Action to take when an executed command fails.
|
/// Action to take when an executed command fails.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum CmdFailureAction {
|
pub enum CmdFailureAction {
|
||||||
/// Exit with an error message
|
/// Exit with an error message
|
||||||
RaiseError,
|
RaiseError,
|
||||||
|
|
||||||
/// Simply ignore the non-zero exit code
|
/// Ignore all non-zero exit codes
|
||||||
Ignore,
|
IgnoreAllFailures,
|
||||||
|
|
||||||
|
/// Ignore specific exit codes
|
||||||
|
IgnoreSpecificFailures(Vec<i32>),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Output style type option
|
/// Output style type option
|
||||||
@@ -208,6 +211,9 @@ pub struct Options {
|
|||||||
// Command to use as a reference for relative speed comparison
|
// Command to use as a reference for relative speed comparison
|
||||||
pub reference_command: Option<String>,
|
pub reference_command: Option<String>,
|
||||||
|
|
||||||
|
// Name of the reference command
|
||||||
|
pub reference_name: Option<String>,
|
||||||
|
|
||||||
/// Command(s) to run before each timing run
|
/// Command(s) to run before each timing run
|
||||||
pub preparation_command: Option<Vec<String>>,
|
pub preparation_command: Option<Vec<String>>,
|
||||||
|
|
||||||
@@ -250,6 +256,7 @@ impl Default for Options {
|
|||||||
min_benchmarking_time: 3.0,
|
min_benchmarking_time: 3.0,
|
||||||
command_failure_action: CmdFailureAction::RaiseError,
|
command_failure_action: CmdFailureAction::RaiseError,
|
||||||
reference_command: None,
|
reference_command: None,
|
||||||
|
reference_name: None,
|
||||||
preparation_command: None,
|
preparation_command: None,
|
||||||
conclusion_command: None,
|
conclusion_command: None,
|
||||||
setup_command: None,
|
setup_command: None,
|
||||||
@@ -310,6 +317,9 @@ impl Options {
|
|||||||
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
||||||
|
|
||||||
options.reference_command = matches.get_one::<String>("reference").map(String::from);
|
options.reference_command = matches.get_one::<String>("reference").map(String::from);
|
||||||
|
options.reference_name = matches
|
||||||
|
.get_one::<String>("reference-name")
|
||||||
|
.map(String::from);
|
||||||
|
|
||||||
options.preparation_command = matches
|
options.preparation_command = matches
|
||||||
.get_many::<String>("prepare")
|
.get_many::<String>("prepare")
|
||||||
@@ -354,8 +364,7 @@ impl Options {
|
|||||||
_ => {
|
_ => {
|
||||||
if options
|
if options
|
||||||
.command_output_policies
|
.command_output_policies
|
||||||
.iter()
|
.contains(&CommandOutputPolicy::Inherit)
|
||||||
.any(|policy| *policy == CommandOutputPolicy::Inherit)
|
|
||||||
|| !io::stdout().is_terminal()
|
|| !io::stdout().is_terminal()
|
||||||
{
|
{
|
||||||
OutputStyleOption::Basic
|
OutputStyleOption::Basic
|
||||||
@@ -409,8 +418,21 @@ impl Options {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if matches.get_flag("ignore-failure") {
|
if let Some(mode) = matches.get_one::<String>("ignore-failure") {
|
||||||
options.command_failure_action = CmdFailureAction::Ignore;
|
options.command_failure_action = match mode.as_str() {
|
||||||
|
"all-non-zero" | "" => CmdFailureAction::IgnoreAllFailures,
|
||||||
|
codes => {
|
||||||
|
let exit_codes: Result<Vec<i32>, _> = codes
|
||||||
|
.split(',')
|
||||||
|
.map(|s| {
|
||||||
|
s.trim()
|
||||||
|
.parse::<i32>()
|
||||||
|
.map_err(|e| OptionsError::IntParsingError("ignore-failure", e))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
CmdFailureAction::IgnoreSpecificFailures(exit_codes?)
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
|
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
|
||||||
|
|||||||
@@ -1,3 +1,13 @@
|
|||||||
pub mod format;
|
pub mod format;
|
||||||
pub mod progress_bar;
|
pub mod progress_bar;
|
||||||
pub mod warnings;
|
pub mod warnings;
|
||||||
|
|
||||||
|
/// Mark console write errors so broken pipes can be handled separately from file errors.
|
||||||
|
macro_rules! console_writeln {
|
||||||
|
($writer:expr $(, $($args:tt)*)?) => {
|
||||||
|
writeln!($writer $(, $($args)*)?)
|
||||||
|
.map_err($crate::error::ConsoleOutputError)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) use console_writeln;
|
||||||
@@ -4,5 +4,5 @@
|
|||||||
/// [1] Mytkowicz, 2009. Producing Wrong Data Without Doing Anything Obviously Wrong!.
|
/// [1] Mytkowicz, 2009. Producing Wrong Data Without Doing Anything Obviously Wrong!.
|
||||||
/// Sigplan Notices - SIGPLAN. 44. 265-276. 10.1145/1508284.1508275.
|
/// Sigplan Notices - SIGPLAN. 44. 265-276. 10.1145/1508284.1508275.
|
||||||
pub fn value() -> String {
|
pub fn value() -> String {
|
||||||
"X".repeat(rand::random::<usize>() % 4096usize)
|
"X".repeat(rand::random_range(0..4096))
|
||||||
}
|
}
|
||||||
+1
-3
@@ -1,9 +1,7 @@
|
|||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
use assert_cmd::cargo::CommandCargoExt;
|
|
||||||
|
|
||||||
pub fn hyperfine_raw_command() -> Command {
|
pub fn hyperfine_raw_command() -> Command {
|
||||||
let mut cmd = Command::cargo_bin("hyperfine").unwrap();
|
let mut cmd = Command::new(assert_cmd::cargo::cargo_bin!("hyperfine"));
|
||||||
cmd.current_dir("tests/");
|
cmd.current_dir("tests/");
|
||||||
cmd
|
cmd
|
||||||
}
|
}
|
||||||
|
|||||||
+126
-1
@@ -1,5 +1,5 @@
|
|||||||
mod common;
|
mod common;
|
||||||
use common::hyperfine;
|
use common::{hyperfine, hyperfine_raw_command};
|
||||||
|
|
||||||
use predicates::prelude::*;
|
use predicates::prelude::*;
|
||||||
|
|
||||||
@@ -33,6 +33,47 @@ fn one_run_is_supported() {
|
|||||||
.success();
|
.success();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Regression test: hyperfine must not panic when writing to a closed
|
||||||
|
/// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process
|
||||||
|
/// that terminates early). A `BrokenPipe` error should result in a quiet
|
||||||
|
/// exit with code 0 instead of a `println!` panic.
|
||||||
|
#[test]
|
||||||
|
fn exits_quietly_when_stdout_is_closed() {
|
||||||
|
use std::process::Stdio;
|
||||||
|
|
||||||
|
for args in [
|
||||||
|
vec!["--runs=1", "echo dummy benchmark"],
|
||||||
|
vec!["--runs=1", "--export-json=-", "echo dummy benchmark"],
|
||||||
|
vec!["--runs=1", "echo dummy benchmark", "echo second benchmark"],
|
||||||
|
] {
|
||||||
|
// A pipe whose read end has been closed before hyperfine is even
|
||||||
|
// started: every write to it will fail with `BrokenPipe`.
|
||||||
|
let (reader, writer) = std::io::pipe().expect("failed to create pipe");
|
||||||
|
drop(reader);
|
||||||
|
|
||||||
|
let output = hyperfine_raw_command()
|
||||||
|
.args(&args)
|
||||||
|
.stdout(writer)
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.expect("failed to spawn hyperfine")
|
||||||
|
.wait_with_output()
|
||||||
|
.expect("failed to wait for hyperfine");
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
!stderr.contains("panicked"),
|
||||||
|
"hyperfine panicked while writing to a closed pipe:\n{}",
|
||||||
|
stderr
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"hyperfine did not exit cleanly; stderr:\n{}",
|
||||||
|
stderr
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn can_run_commands_without_a_shell() {
|
fn can_run_commands_without_a_shell() {
|
||||||
hyperfine()
|
hyperfine()
|
||||||
@@ -277,6 +318,78 @@ fn can_run_failing_commands_with_ignore_failure_option() {
|
|||||||
.success();
|
.success();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn can_ignore_specific_exit_codes() {
|
||||||
|
// Test that specifying exit code 1 ignores it
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=1")
|
||||||
|
.arg("exit 1")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
|
// Test that other exit codes still fail
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=1")
|
||||||
|
.arg("exit 2")
|
||||||
|
.assert()
|
||||||
|
.failure()
|
||||||
|
.stderr(predicate::str::contains(
|
||||||
|
"Command terminated with non-zero exit code 2",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn can_ignore_multiple_exit_codes() {
|
||||||
|
// Test that all specified exit codes are ignored
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=1,2,3")
|
||||||
|
.arg("exit 1")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=1,2,3")
|
||||||
|
.arg("exit 2")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=1,2,3")
|
||||||
|
.arg("exit 3")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
|
// Test that other exit codes still fail
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=1,2,3")
|
||||||
|
.arg("exit 4")
|
||||||
|
.assert()
|
||||||
|
.failure()
|
||||||
|
.stderr(predicate::str::contains(
|
||||||
|
"Command terminated with non-zero exit code 4",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn ignore_failure_with_all_non_zero() {
|
||||||
|
// Test explicit "all-non-zero" mode
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--ignore-failure=all-non-zero")
|
||||||
|
.arg("exit 5")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn shows_output_of_benchmarked_command() {
|
fn shows_output_of_benchmarked_command() {
|
||||||
hyperfine()
|
hyperfine()
|
||||||
@@ -499,6 +612,18 @@ fn shows_benchmark_comparison_relative_to_reference() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shows_reference_name() {
|
||||||
|
hyperfine_debug()
|
||||||
|
.arg("--reference=sleep 2.0")
|
||||||
|
.arg("--reference-name=refabc123")
|
||||||
|
.arg("sleep 1.0")
|
||||||
|
.arg("sleep 3.0")
|
||||||
|
.assert()
|
||||||
|
.success()
|
||||||
|
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn performs_all_benchmarks_in_parameter_scan() {
|
fn performs_all_benchmarks_in_parameter_scan() {
|
||||||
hyperfine_debug()
|
hyperfine_debug()
|
||||||
|
|||||||
Reference in new issue
Block a user