Compare commits

..
Author SHA1 Message Date
David Peter 02157eb6ca Add histogram image 2022-03-05 12:51:01 +01:00
David Peter 65932dd5ce Update warning text 2022-03-05 12:47:30 +01:00
David Peter 851e9b8dfa Update documentation 2022-03-05 12:39:00 +01:00
David Peter 5c676a9626 Bump version to v1.13.0 2022-03-05 12:10:38 +01:00
David Peter f1557d47fb Add plot_progression.py 2022-03-05 12:03:29 +01:00
David Peter bcd3705ce0 Add number of runs to advanced_statistics.py 2022-03-05 11:53:59 +01:00
David Peter 176275ecab Update dependencies 2022-03-05 11:53:45 +01:00
Vin Singh 56cce61077 Add scipy to scripts README.md pre-requisites 2022-03-03 20:37:09 +01:00
dependabot[bot] 770e23bc1b Bump clap from 3.1.1 to 3.1.3
Bumps [clap](https://github.com/clap-rs/clap) from 3.1.1 to 3.1.3.
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/v3.1.1...v3.1.3)

---
updated-dependencies:
- dependency-name: clap
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-03-01 08:43:19 +01:00
David Peter 0601a1971e Add --t-min/--t-max to histogram script 2022-02-27 20:11:13 +01:00
David Peter f742329f1e Adjust image size 2022-02-27 17:04:00 +01:00
David Peter d13454a0d4 Add additional integration test for failing command 2022-02-22 22:51:09 +01:00
David Peter e666b01e09 Add simple integation test for --shell=none 2022-02-22 22:51:09 +01:00
David Peter e38ac46179 Add CHANGELOG entry 2022-02-22 22:51:09 +01:00
David Peter 365ff2584d Switch to --shell=none/-N, do not change the default 2022-02-22 22:51:09 +01:00
David Peter d9418e0731 Use raw executor by default, skip intermediate shell
This adds a new "raw executor" (next to the "shell executor") that
allows hyperfine to execute commands directly without any intermediate
shell.

The command line is split into tokens (using the `shell-words` crate),
and according to POSIX rules. The first token is taken as the executable,
and the rest as arguments.

The new executor is enabled by default. In order to select the shell
executor, users will have to pass `--shell=default`.

This allows us to reduce measurement noise and to benchmark very quick
commands. It also decreases the time to run benchmarks, as we don't need
the calibration phase.

Also, it allows one to make sure that the executed command is not
implemented as a shell builtin. For example `hyperfine true`
and `hyperfine --shell=default true` return different times due
to the fact that `bash` executes `true` as a NOP.

Co-authored: Ciprian Dorin Craciun <ciprian@volution.ro>
2022-02-22 22:51:09 +01:00
David Peter f2f1dd9d40 Update dependencies 2022-02-22 14:04:07 +01:00
David Peter 308bf5cd26 Extract common functionality 2022-02-22 13:24:05 +01:00
David Peter f19223ba9d Simplify 2022-02-22 13:24:05 +01:00
David Peter 7fd4aea495 Rename to TimerResult 2022-02-22 13:24:05 +01:00
David Peter 55f0ef6637 Move wall clock timer into execute_and_measure 2022-02-22 13:24:05 +01:00
David Peter e001530f61 Change argument order 2022-02-22 13:24:05 +01:00
David Peter d27cbceed2 Add integration test for --show-output 2022-02-22 13:24:05 +01:00
David Peter 9b8f2a9ee2 Move Stdio extraction to enum 2022-02-22 13:24:05 +01:00
David Peter e1da8ba00f Further simplify execute_and_measure 2022-02-22 13:24:05 +01:00
David Peter 0ffc6d7cc0 Move to timer module 2022-02-22 13:24:05 +01:00
David Peter 1f3943e7d1 Move custom functionality to executor 2022-02-22 13:24:05 +01:00
David Peter f822455d99 Rename functions 2022-02-22 13:24:05 +01:00
David Peter 7c929f7e48 Rename to run_command_and_measure 2022-02-22 13:24:05 +01:00
David Peter 826449bc8f Rename to get_command_line 2022-02-22 13:24:05 +01:00
21 changed files with 426 additions and 188 deletions

No files matched your search

+27 -5
View File
@@ -2,8 +2,35 @@
## Features
## Changes
## Bugfixes
## Other
## Packaging
# v1.13.0
## Features
- Added a new `--shell=none`/`-N` option to disable the intermediate
shell for executing the benchmarked commands. Hyperfine normally
measures and subtracts the shell spawning time, but the intermediate
shell always introduces a certain level of measurement noise. Using
`--shell=none`/`-N` allows users to benchmark very fast commands
(with a runtime on the order of a few milliseconds). See #336, #429,
and #487 (@cipriancraciun and @sharkdp)
- Added `--setup` (`s`) option that can be used to run `make all` or
similar. It runs once per set of tests, like `--cleanup` (`c`).
- Added new `plot_progression.py` script to debug background interference
effects.
## Changes
@@ -12,16 +39,11 @@
- The environment offset randomization is now also available on Windows,
see #484
## Bugfixes
## Other
- Improved documentation and test coverage, cleaned up code base for
future improvements.
## Packaging
# v1.12.0
Generated
+46 -53
View File
@@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.53"
version = "1.0.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0"
checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd"
[[package]]
name = "approx"
@@ -59,15 +59,18 @@ dependencies = [
[[package]]
name = "autocfg"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
dependencies = [
"autocfg 1.1.0",
]
[[package]]
name = "autocfg"
version = "1.0.1"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bitflags"
@@ -95,9 +98,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "3.0.14"
version = "3.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63edc3f163b3c71ec8aa23f9bd6070f77edbf3d1d198b164afa90ff00e4ec62"
checksum = "ced1892c55c910c1219e98d6fc8d71f6bddba7905866ce740066d8bfea859312"
dependencies = [
"atty",
"bitflags",
@@ -112,9 +115,9 @@ dependencies = [
[[package]]
name = "clap_complete"
version = "3.0.5"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4dabb7e2f006497e1da045feaa512acf0686f76b68d94925da2d9422dcb521"
checksum = "df6f3613c0a3cddfd78b41b10203eb322cb29b600cbdf808a7d3db95691b8e25"
dependencies = [
"clap",
]
@@ -224,9 +227,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "getrandom"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
dependencies = [
"cfg-if",
"libc",
@@ -250,7 +253,7 @@ dependencies = [
[[package]]
name = "hyperfine"
version = "1.12.0"
version = "1.13.0"
dependencies = [
"anyhow",
"approx",
@@ -263,7 +266,7 @@ dependencies = [
"indicatif",
"libc",
"predicates",
"rand 0.8.4",
"rand 0.8.5",
"rust_decimal",
"serde",
"serde_json",
@@ -280,7 +283,7 @@ version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
"hashbrown",
]
@@ -334,9 +337,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.117"
version = "0.2.119"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c"
checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
[[package]]
name = "memchr"
@@ -370,7 +373,7 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
"num-integer",
"num-traits",
]
@@ -381,7 +384,7 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
"num-traits",
]
@@ -391,7 +394,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
"num-traits",
]
@@ -401,7 +404,7 @@ version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
"num-integer",
"num-traits",
]
@@ -412,7 +415,7 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
"num-bigint",
"num-integer",
"num-traits",
@@ -424,7 +427,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg 1.0.1",
"autocfg 1.1.0",
]
[[package]]
@@ -435,9 +438,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.9.0"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
[[package]]
name = "os_str_bytes"
@@ -508,11 +511,11 @@ version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.7",
"autocfg 0.1.8",
"libc",
"rand_chacha 0.1.1",
"rand_core 0.4.2",
"rand_hc 0.1.0",
"rand_hc",
"rand_isaac",
"rand_jitter",
"rand_os",
@@ -523,14 +526,13 @@ dependencies = [
[[package]]
name = "rand"
version = "0.8.4"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.3",
"rand_hc 0.3.1",
]
[[package]]
@@ -539,7 +541,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg 0.1.7",
"autocfg 0.1.8",
"rand_core 0.3.1",
]
@@ -586,15 +588,6 @@ dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_hc"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
dependencies = [
"rand_core 0.6.3",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
@@ -635,7 +628,7 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg 0.1.7",
"autocfg 0.1.8",
"rand_core 0.4.2",
]
@@ -659,9 +652,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.10"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c"
dependencies = [
"bitflags",
]
@@ -700,9 +693,9 @@ dependencies = [
[[package]]
name = "rust_decimal"
version = "1.21.0"
version = "1.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4214023b1223d02a4aad9f0bb9828317634a56530870a2eaf7200a99c0c10f68"
checksum = "d37baa70cf8662d2ba1c1868c5983dda16ef32b105cce41fb5c47e72936a90b3"
dependencies = [
"arrayvec",
"num-traits",
@@ -737,9 +730,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.78"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085"
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
dependencies = [
"itoa 1.0.1",
"ryu",
@@ -748,9 +741,9 @@ dependencies = [
[[package]]
name = "shell-words"
version = "1.0.0"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074"
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
[[package]]
name = "statistical"
@@ -795,9 +788,9 @@ dependencies = [
[[package]]
name = "termcolor"
version = "1.1.2"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
dependencies = [
"winapi-util",
]
@@ -820,9 +813,9 @@ checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
[[package]]
name = "textwrap"
version = "0.14.2"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
dependencies = [
"terminal_size",
]
+2 -2
View File
@@ -7,7 +7,7 @@ license = "MIT/Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "1.12.0"
version = "1.13.0"
edition = "2018"
build = "build.rs"
@@ -45,7 +45,7 @@ tempfile = "3.3"
[build-dependencies]
clap = "3"
atty = "0.2"
clap_complete = "3.0"
clap_complete = "3"
[profile.release]
lto = true
+44 -12
View File
@@ -24,7 +24,7 @@ A command-line benchmarking tool.
## Usage
### Basic benchmark
### Basic benchmarks
To run a benchmark, you can simply call `hyperfine <command>...`. The argument(s) can be any
shell command. For example:
@@ -33,19 +33,24 @@ hyperfine 'sleep 0.3'
```
Hyperfine will automatically determine the number of runs to perform for each command. By default,
it will perform *at least* 10 benchmarking runs. To change this, you can use the `-m`/`--min-runs`
option:
it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
you can use the `-r`/`--runs` option:
``` bash
hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
hyperfine --runs 5 'sleep 0.3'
```
If you want to compare the runtimes of different programs, you can pass multiple commands:
``` bash
hyperfine 'hexdump file' 'xxd file'
```
### Warmup runs and preparation commands
If the program execution time is limited by disk I/O, the benchmarking results can be heavily
influenced by disk caches and whether they are cold or warm.
For programs that perform a lot of disk I/O, the benchmarking results can be heavily influenced
by disk caches and whether they are cold or warm.
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform
a certain number of program executions before the actual benchmark:
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to
perform a certain number of program executions before the actual benchmark:
``` bash
hyperfine --warmup 3 'grep -R TODO *'
```
@@ -56,7 +61,7 @@ on Linux, you can run
``` bash
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
```
To use this specific command with Hyperfine, call `sudo -v` to temporarily gain sudo permissions
To use this specific command with hyperfine, call `sudo -v` to temporarily gain sudo permissions
and then call:
``` bash
hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *'
@@ -64,7 +69,7 @@ hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R
### Parameterized benchmarks
If you want to run a benchmark where only a single parameter is varied (say, the number of
If you want to run a series of benchmarks where a single parameter is varied (say, the number of
threads), you can use the `-P`/`--parameter-scan` option and call:
``` bash
hyperfine --prepare 'make clean' --parameter-scan num_threads 1 12 'make -j {num_threads}'
@@ -76,6 +81,33 @@ hyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}'
```
This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
option:
```
hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
```
### Intermediate shell
By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
``` bash
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
```
Note that hyperfine always *corrects for the shell spawning time*. To do this, it performs a calibration
procedure where it runs the shell with an empty command (multiple times), to measure the startup time
of the shell. It will then subtract this time from the total to show the actual time used by the command
in question.
If you want to run a benchmark *without an intermediate shell*, you can use the `-N` or `--shell=none`
option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
produce a significant amount of noise. Note that you can not use shell syntax like `*` or `~` in this case.
```
hyperfine -N 'grep TODO /home/user'
```
### Shell functions and aliases
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
@@ -120,8 +152,8 @@ The JSON output is useful if you want to analyze the benchmark results in more d
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
sudo dpkg -i hyperfine_1.12.0_amd64.deb
wget https://github.com/sharkdp/hyperfine/releases/download/v1.13.0/hyperfine_1.13.0_amd64.deb
sudo dpkg -i hyperfine_1.13.0_amd64.deb
```
### On Fedora
+3 -3
View File
@@ -2,7 +2,7 @@ use std::fs;
use clap_complete::{generate_to, Shell};
include!("src/app.rs");
include!("src/cli.rs");
fn main() {
let var = std::env::var_os("SHELL_COMPLETIONS_DIR").or_else(|| std::env::var_os("OUT_DIR"));
@@ -12,7 +12,7 @@ fn main() {
};
fs::create_dir_all(&outdir).unwrap();
let mut app = build_app();
let mut command = build_command();
for shell in [
Shell::Bash,
Shell::Fish,
@@ -20,6 +20,6 @@ fn main() {
Shell::PowerShell,
Shell::Elvish,
] {
generate_to(shell, &mut app, "hyperfine", &outdir).unwrap();
generate_to(shell, &mut command, "hyperfine", &outdir).unwrap();
}
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 233 KiB

After

Width:  |  Height:  |  Size: 94 KiB

+79 -79
View File
@@ -2,16 +2,16 @@
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="1699.0797"
height="1802.972"
viewBox="0 0 449.54818 477.03637"
width="1724.755"
height="1832.972"
viewBox="0 0 456.34143 484.97387"
version="1.1"
id="svg5"
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20, custom)"
sodipodi:docname="execution-order.svg"
inkscape:export-filename="/home/shark/Informatik/rust/hyperfine/doc/execution-order.png"
inkscape:export-xdpi="96"
inkscape:export-ydpi="96"
inkscape:export-xdpi="44.529999"
inkscape:export-ydpi="44.529999"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
@@ -27,8 +27,8 @@
inkscape:document-units="mm"
showgrid="false"
inkscape:zoom="0.38529391"
inkscape:cx="1027.7868"
inkscape:cy="991.45092"
inkscape:cx="1064.1227"
inkscape:cy="949.92418"
inkscape:window-width="1920"
inkscape:window-height="1175"
inkscape:window-x="1920"
@@ -38,19 +38,19 @@
showborder="true"
showguides="false"
inkscape:guide-bbox="true"
fit-margin-top="15"
fit-margin-left="15"
fit-margin-right="15"
fit-margin-bottom="15"
fit-margin-top="30"
fit-margin-left="30"
fit-margin-right="30"
fit-margin-bottom="30"
lock-margins="true"
units="px">
<inkscape:grid
type="xygrid"
id="grid949"
originx="332.12096"
originy="-14.030302" />
originx="334.94547"
originy="-10.061553" />
<sodipodi:guide
position="187.92302,354.80624"
position="190.74752,358.77499"
orientation="1,0"
id="guide130046" />
</sodipodi:namedview>
@@ -60,121 +60,121 @@
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(332.12094,-14.0303)">
transform="translate(334.94544,-10.06155)">
<rect
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect159433"
width="88.724548"
height="14.787424"
x="-206.65671"
y="294.47958"
ry="1.105878" />
width="80.681"
height="13.446833"
x="-226.03801"
y="147.57736"
ry="1.0056218" />
<rect
style="fill:#010101;fill-opacity:1;stroke:#000000;stroke-width:0.720253;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#010101;fill-opacity:1;stroke:#000000;stroke-width:0.654957;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect123376"
width="88.004295"
height="14.067173"
x="-290.76392"
y="277.79633"
ry="1.0550385" />
width="80.026047"
height="12.791878"
x="-302.52026"
y="132.40656"
ry="0.9593913" />
<rect
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.720253;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.654957;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect129065"
width="88.004295"
height="14.067173"
x="-290.76392"
y="311.14478"
ry="1.0550385" />
width="80.026047"
height="12.791878"
x="-302.52026"
y="162.73172"
ry="0.9593913" />
<rect
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect117266"
width="88.724548"
height="14.787424"
x="-206.37056"
y="261.36728"
ry="1.1092001" />
width="80.681"
height="13.446833"
x="-225.7778"
y="117.46693"
ry="1.0086428" />
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect117165"
width="88.724548"
height="14.787417"
x="-206.6562"
y="327.82858"
ry="1.1090568" />
width="80.681"
height="13.446827"
x="-226.03755"
y="177.90302"
ry="1.0085125" />
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect117064"
width="88.724548"
height="14.787429"
x="-206.37056"
y="244.73143"
ry="1.1090568" />
width="80.681"
height="13.446837"
x="-225.7778"
y="102.33925"
ry="1.0085125" />
<text
xml:space="preserve"
style="font-size:10.5833px;line-height:1.2;font-family:Audiowide;-inkscape-font-specification:Audiowide;white-space:pre;inline-size:199.93;stroke-width:0.264583"
x="-79.596779"
y="-115.14695"
id="text41875"
transform="matrix(1.2893387,0,0,1.2893387,-202.89709,353.8546)"><tspan
transform="matrix(1.1724504,0,0,1.1724504,-222.61922,201.56959)"><tspan
x="-79.596779"
y="-115.14695"
id="tspan5244"><tspan
id="tspan14151"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5242">hyperfine
id="tspan14149">hyperfine
</tspan></tspan><tspan
x="-79.596779"
y="-102.16832"
id="tspan5250"><tspan
id="tspan14157"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#e35a00"
id="tspan5246"> --warmup 2</tspan><tspan
id="tspan14153"> --warmup 2</tspan><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5248">
id="tspan14155">
</tspan></tspan><tspan
x="-79.596779"
y="-89.189692"
id="tspan5256"><tspan
id="tspan14163"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#00bbeb"
id="tspan5252"> --runs 3</tspan><tspan
id="tspan14159"> --runs 3</tspan><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5254">
id="tspan14161">
</tspan></tspan><tspan
x="-79.596779"
y="-76.211062"
id="tspan5260"><tspan
id="tspan14167"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5258"> --setup &lt;setup&gt;
id="tspan14165"> --setup &lt;setup&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-63.232432"
id="tspan5264"><tspan
id="tspan14171"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5262"> --prepare &lt;prepare1&gt;
id="tspan14169"> --prepare &lt;prepare1&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-50.253802"
id="tspan5270"><tspan
id="tspan14177"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#ffffff"
id="tspan5266"> &lt;command1&gt;</tspan><tspan
id="tspan14173"> &lt;command1&gt;</tspan><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5268">
id="tspan14175">
</tspan></tspan><tspan
x="-79.596779"
y="-37.275172"
id="tspan5274"><tspan
id="tspan14181"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5272"> --prepare &lt;prepare2&gt;
id="tspan14179"> --prepare &lt;prepare2&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-24.296542"
id="tspan5278"><tspan
id="tspan14185"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#ffffff"
id="tspan5276"> &lt;command2&gt;
id="tspan14183"> &lt;command2&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-11.317913"
id="tspan5282"><tspan
id="tspan14189"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5280"> --cleanup &lt;cleanup&gt;</tspan></tspan></text>
id="tspan14187"> --cleanup &lt;cleanup&gt;</tspan></tspan></text>
<rect
style="fill:#80e5ff;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect6355"
@@ -325,7 +325,7 @@
transform="translate(34.591878,14.34874)"><tspan
x="-16.304935"
y="161.32399"
id="tspan5284">3 benchmark runs</tspan></text>
id="tspan14191">3 benchmark runs</tspan></text>
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect13386"
@@ -590,7 +590,7 @@
transform="translate(34.591878,257.76541)"><tspan
x="-16.304935"
y="161.32399"
id="tspan5286">3 benchmark runs</tspan></text>
id="tspan14193">3 benchmark runs</tspan></text>
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect272454"
@@ -711,11 +711,11 @@
x="-79.596779"
y="-115.14695"
id="text283737"
transform="matrix(1.2893387,0,0,1.2893387,-225.35928,177.7847)"><tspan
transform="matrix(1.7031443,0,0,1.7031443,-192.73007,228.6586)"><tspan
x="-79.596779"
y="-115.14695"
id="tspan5290"><tspan
style="font-weight:bold;font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5288">benchmark execution order</tspan></tspan></text>
id="tspan14197"><tspan
style="font-weight:bold;font-family:'Segoe UI';-inkscape-font-specification:'Segoe UI Bold'"
id="tspan14195">benchmark execution order</tspan></tspan></text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 35 KiB

After

Width:  |  Height:  |  Size: 35 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

+2 -2
View File
@@ -9,9 +9,9 @@ This folder contains scripts that can be used in combination with hyperfines `--
### Pre-requisites
To make these scripts work, you will need to install `numpy` and `matplotlib`. Install them via
To make these scripts work, you will need to install `numpy`, `matplotlib` and `scipy`. Install them via
your package manager or `pip`:
```bash
pip install numpy matplotlib # pip3, if you are using python3
pip install numpy matplotlib scipy # pip3, if you are using python3
```
+1
View File
@@ -23,6 +23,7 @@ for command, ts in zip(commands, times):
iqr = p75 - p25
print("Command '{}'".format(command))
print(" runs: {:8d}".format(len(ts)))
print(" mean: {:8.3f} s".format(np.mean(ts)))
print(" stddev: {:8.3f} s".format(np.std(ts, ddof=1)))
print(" median: {:8.3f} s".format(np.median(ts)))
+8 -2
View File
@@ -20,6 +20,12 @@ parser.add_argument(
parser.add_argument(
"-o", "--output", help="Save image to the given filename."
)
parser.add_argument(
"--t-min", metavar="T", help="Minimum time to be displayed (seconds)"
)
parser.add_argument(
"--t-max", metavar="T", help="Maximum time to be displayed (seconds)"
)
args = parser.parse_args()
@@ -32,8 +38,8 @@ else:
labels = [b["command"] for b in results]
all_times = [b["times"] for b in results]
t_min = np.min(list(map(np.min, all_times)))
t_max = np.max(list(map(np.max, all_times)))
t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))
bins = int(args.bins) if args.bins else "auto"
histtype = args.type if args.type else "bar"
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python
"""This program shows `hyperfine` benchmark results in a sequential way
in order to debug possible background interference, caching effects,
thermal throttling and similar effects.
"""
import argparse
import json
import matplotlib.pyplot as plt
import numpy as np
def moving_average(times, num_runs):
times_padded = np.pad(
times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge"
)
kernel = np.ones(num_runs) / num_runs
return np.convolve(times_padded, kernel, mode="valid")
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("file", help="JSON file with benchmark results")
parser.add_argument("--title", help="Plot Title")
parser.add_argument("-o", "--output", help="Save image to the given filename.")
parser.add_argument(
"-w",
"--moving-average-width",
type=int,
metavar="num_runs",
help="Width of the moving-average window (default: N/5)",
)
args = parser.parse_args()
with open(args.file) as f:
results = json.load(f)["results"]
label = results[0]["command"]
times = results[0]["times"]
num = len(times)
nums = range(num)
plt.scatter(x=nums, y=times, marker=".", color="orange")
plt.ylim([0, None])
plt.xlim([-1, num])
moving_average_width = (
num // 5 if args.moving_average_width is None else args.moving_average_width
)
moving_average = moving_average(times, moving_average_width)
plt.plot(nums, moving_average, "-", color="blue")
if args.title:
plt.title(args.title)
plt.legend(labels=[label], loc="best", fontsize="medium")
plt.ylabel("Time [s]")
if args.output:
plt.savefig(args.output)
else:
plt.show()
+42
View File
@@ -64,6 +64,48 @@ fn run_command_and_measure_common(
Ok(result)
}
pub struct RawExecutor<'a> {
options: &'a Options,
}
impl<'a> RawExecutor<'a> {
pub fn new(options: &'a Options) -> Self {
RawExecutor { options }
}
}
impl<'a> Executor for RawExecutor<'a> {
fn run_command_and_measure(
&self,
command: &Command<'_>,
command_failure_action: Option<CmdFailureAction>,
) -> Result<(TimingResult, ExitStatus)> {
let result = run_command_and_measure_common(
command.get_command()?,
command_failure_action.unwrap_or(self.options.command_failure_action),
self.options.command_output_policy,
&command.get_command_line(),
)?;
Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
},
result.status,
))
}
fn calibrate(&mut self) -> Result<()> {
Ok(())
}
fn time_overhead(&self) -> Second {
0.0
}
}
pub struct ShellExecutor<'a> {
options: &'a Options,
shell: &'a Shell,
+4 -2
View File
@@ -7,7 +7,7 @@ pub mod timing_result;
use std::cmp;
use crate::command::Command;
use crate::options::{CmdFailureAction, Options, OutputStyleOption};
use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption};
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
use crate::output::format::{format_duration, format_duration_unit};
use crate::output::progress_bar::get_progress_bar;
@@ -320,7 +320,9 @@ impl<'a> Benchmark<'a> {
let mut warnings = vec![];
// Check execution time
if times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) {
if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
&& times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
{
warnings.push(Warnings::FastExecutionTime);
}
+2 -1
View File
@@ -1,7 +1,7 @@
use colored::*;
use super::benchmark_result::BenchmarkResult;
use super::executor::{Executor, MockExecutor, ShellExecutor};
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
use super::{relative_speed, Benchmark};
use crate::command::Commands;
@@ -33,6 +33,7 @@ impl<'a> Scheduler<'a> {
pub fn run_benchmarks(&mut self) -> Result<()> {
let mut executor: Box<dyn Executor> = match self.options.executor_kind {
ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),
ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
};
+27 -13
View File
@@ -1,30 +1,32 @@
use std::ffi::OsString;
use clap::{crate_version, App, AppSettings, Arg, ArgMatches};
use clap::{crate_version, AppSettings, Arg, ArgMatches, Command};
pub fn get_cli_arguments<'a, I, T>(args: I) -> ArgMatches
where
I: IntoIterator<Item = T>,
T: Into<OsString> + Clone + 'a,
{
let app = build_app();
app.get_matches_from(args)
let command = build_command();
command.get_matches_from(args)
}
/// Build the clap app for parsing command line arguments
fn build_app() -> App<'static> {
App::new("hyperfine")
/// Build the clap command for parsing command line arguments
fn build_command() -> Command<'static> {
Command::new("hyperfine")
.version(crate_version!())
.setting(AppSettings::DeriveDisplayOrder)
.setting(AppSettings::NextLineHelp)
.setting(AppSettings::HidePossibleValues)
.next_line_help(true)
.hide_possible_values(true)
.max_term_width(90)
.about("A command-line benchmarking tool.")
.arg(
Arg::new("command")
.help("Command to benchmark. This can be the name of an executable or a shell \
command like \"sleep 0.5 && echo test\". If multiple commands are given, \
hyperfine will show a comparison of the respective runtimes.")
.help("The command to benchmark. This can be the name of an executable, a command \
line like \"grep -i todo\" or a shell command like \"sleep 0.5 && echo test\". \
The latter is only available if the shell is not explicitly disabled via \
'--shell=none'. If multiple commands are given, hyperfine will show a \
comparison of the respective runtimes.")
.required(true)
.multiple_occurrences(true)
.forbid_empty_values(true),
@@ -181,7 +183,19 @@ fn build_app() -> App<'static> {
.takes_value(true)
.value_name("SHELL")
.overrides_with("shell")
.help("Set the shell to use for executing benchmarked commands."),
.help("Set the shell to use for executing benchmarked commands. This can be the \
name or the path to the shell executable, or a full command line \
like \"bash --norc\". It can also be set to \"default\" to explicitly select \
the default shell on this platform. Finally, this can also be set to \
\"none\" to disable the shell. In this case, commands will be executed \
directly. They can still have arguments, but more complex things like \
\"sleep 0.1; sleep 0.2\" are not possible without a shell.")
)
.arg(
Arg::new("no-shell")
.short('N')
.conflicts_with_all(&["shell", "debug-mode"])
.help("An alias for '--shell=none'.")
)
.arg(
Arg::new("ignore-failure")
@@ -259,5 +273,5 @@ fn build_app() -> App<'static> {
#[test]
fn verify_app() {
build_app().debug_assert();
build_command().debug_assert();
}
+19 -4
View File
@@ -15,7 +15,7 @@ use clap::ArgMatches;
use crate::parameter::tokenize::tokenize;
use crate::parameter::ParameterValue;
use anyhow::{bail, Result};
use anyhow::{bail, Context, Result};
use clap::Values;
use rust_decimal::Decimal;
@@ -64,6 +64,21 @@ impl<'a> Command<'a> {
self.replace_parameters_in(self.expression)
}
pub fn get_command(&self) -> Result<std::process::Command> {
let command_line = self.get_command_line();
let mut tokens = shell_words::split(&command_line)
.with_context(|| format!("Failed to parse command '{}'", command_line))?
.into_iter();
if let Some(program_name) = tokens.next() {
let mut command_builder = std::process::Command::new(program_name);
command_builder.args(tokens);
Ok(command_builder)
} else {
bail!("Can not execute empty command")
}
}
pub fn get_parameters(&self) -> &[(&'a str, ParameterValue)] {
&self.parameters
}
@@ -351,7 +366,7 @@ impl<'a> fmt::Display for Command<'a> {
#[test]
fn test_build_commands_cross_product() {
use crate::app::get_cli_arguments;
use crate::cli::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
@@ -389,7 +404,7 @@ fn test_build_commands_cross_product() {
#[test]
fn test_build_parameter_list_commands() {
use crate::app::get_cli_arguments;
use crate::cli::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
@@ -410,7 +425,7 @@ fn test_build_parameter_list_commands() {
#[test]
fn test_build_parameter_scan_commands() {
use crate::app::get_cli_arguments;
use crate::cli::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
"echo {val}",
+2 -2
View File
@@ -1,7 +1,7 @@
use std::env;
use app::get_cli_arguments;
use benchmark::scheduler::Scheduler;
use cli::get_cli_arguments;
use command::Commands;
use export::ExportManager;
use options::Options;
@@ -9,8 +9,8 @@ use options::Options;
use anyhow::Result;
use colored::*;
pub mod app;
pub mod benchmark;
pub mod cli;
pub mod command;
pub mod error;
pub mod export;
+12 -6
View File
@@ -134,6 +134,7 @@ impl CommandOutputPolicy {
}
pub enum ExecutorKind {
Raw,
Shell(Shell),
Mock(Option<String>),
}
@@ -281,12 +282,17 @@ impl Options {
OutputStyleOption::Disabled => {}
};
options.executor_kind = match (matches.is_present("debug-mode"), matches.value_of("shell"))
{
(false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?),
(false, None) => ExecutorKind::Shell(Shell::default()),
(true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
(true, None) => ExecutorKind::Mock(None),
options.executor_kind = if matches.is_present("no-shell") {
ExecutorKind::Raw
} else {
match (matches.is_present("debug-mode"), matches.value_of("shell")) {
(false, Some(shell)) if shell == "default" => ExecutorKind::Shell(Shell::default()),
(false, Some(shell)) if shell == "none" => ExecutorKind::Raw,
(false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?),
(false, None) => ExecutorKind::Shell(Shell::default()),
(true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
(true, None) => ExecutorKind::Mock(None),
}
};
if matches.is_present("ignore-failure") {
+4 -1
View File
@@ -17,7 +17,10 @@ impl fmt::Display for Warnings {
match *self {
Warnings::FastExecutionTime => write!(
f,
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
"Command took less than {:.0} ms to complete. Note that the results might be \
inaccurate because hyperfine can not calibrate the shell startup time much \
more precise than this limit. You can try to use the `-N`/`--shell=none` \
option to disable the shell completely.",
MIN_EXECUTION_TIME * 1e3
),
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
+40 -1
View File
@@ -10,7 +10,7 @@ pub fn hyperfine_debug() -> assert_cmd::Command {
}
#[test]
fn hyperfine_runs_successfully() {
fn runs_successfully() {
hyperfine()
.arg("--runs=2")
.arg("echo dummy benchmark")
@@ -27,6 +27,18 @@ fn one_run_is_supported() {
.success();
}
#[test]
fn can_run_commands_without_a_shell() {
hyperfine()
.arg("--runs=1")
.arg("--show-output")
.arg("--shell=none")
.arg("echo 'hello world' argument2")
.assert()
.success()
.stdout(predicate::str::contains("hello world argument2"));
}
#[test]
fn fails_with_wrong_number_of_command_name_arguments() {
hyperfine()
@@ -90,6 +102,33 @@ fn fails_for_unknown_command() {
));
}
#[test]
fn fails_for_unknown_command_without_shell() {
hyperfine()
.arg("--shell=none")
.arg("--runs=1")
.arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
.assert()
.failure()
.stderr(predicate::str::contains(
"Failed to run command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577'",
));
}
#[cfg(unix)]
#[test]
fn fails_for_failing_command_without_shell() {
hyperfine()
.arg("--shell=none")
.arg("--runs=1")
.arg("false")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command terminated with non-zero exit code",
));
}
#[test]
fn fails_for_unknown_setup_command() {
hyperfine()