Compare commits

..
21 changed files with 188 additions and 426 deletions

No files matched your search

+5 -27
View File
@@ -2,35 +2,8 @@
## Features
## Changes
## Bugfixes
## Other
## Packaging
# v1.13.0
## Features
- Added a new `--shell=none`/`-N` option to disable the intermediate
shell for executing the benchmarked commands. Hyperfine normally
measures and subtracts the shell spawning time, but the intermediate
shell always introduces a certain level of measurement noise. Using
`--shell=none`/`-N` allows users to benchmark very fast commands
(with a runtime on the order of a few milliseconds). See #336, #429,
and #487 (@cipriancraciun and @sharkdp)
- Added `--setup` (`s`) option that can be used to run `make all` or
similar. It runs once per set of tests, like `--cleanup` (`c`).
- Added new `plot_progression.py` script to debug background interference
effects.
## Changes
@@ -39,11 +12,16 @@
- The environment offset randomization is now also available on Windows,
see #484
## Bugfixes
## Other
- Improved documentation and test coverage, cleaned up code base for
future improvements.
## Packaging
# v1.12.0
Generated
+53 -46
View File
@@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.55"
version = "1.0.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd"
checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0"
[[package]]
name = "approx"
@@ -59,18 +59,15 @@ dependencies = [
[[package]]
name = "autocfg"
version = "0.1.8"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
dependencies = [
"autocfg 1.1.0",
]
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
[[package]]
name = "autocfg"
version = "1.1.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "bitflags"
@@ -98,9 +95,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "3.1.5"
version = "3.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced1892c55c910c1219e98d6fc8d71f6bddba7905866ce740066d8bfea859312"
checksum = "b63edc3f163b3c71ec8aa23f9bd6070f77edbf3d1d198b164afa90ff00e4ec62"
dependencies = [
"atty",
"bitflags",
@@ -115,9 +112,9 @@ dependencies = [
[[package]]
name = "clap_complete"
version = "3.1.1"
version = "3.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df6f3613c0a3cddfd78b41b10203eb322cb29b600cbdf808a7d3db95691b8e25"
checksum = "be4dabb7e2f006497e1da045feaa512acf0686f76b68d94925da2d9422dcb521"
dependencies = [
"clap",
]
@@ -227,9 +224,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "getrandom"
version = "0.2.5"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
dependencies = [
"cfg-if",
"libc",
@@ -253,7 +250,7 @@ dependencies = [
[[package]]
name = "hyperfine"
version = "1.13.0"
version = "1.12.0"
dependencies = [
"anyhow",
"approx",
@@ -266,7 +263,7 @@ dependencies = [
"indicatif",
"libc",
"predicates",
"rand 0.8.5",
"rand 0.8.4",
"rust_decimal",
"serde",
"serde_json",
@@ -283,7 +280,7 @@ version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
"hashbrown",
]
@@ -337,9 +334,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.119"
version = "0.2.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c"
[[package]]
name = "memchr"
@@ -373,7 +370,7 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
"num-integer",
"num-traits",
]
@@ -384,7 +381,7 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
"num-traits",
]
@@ -394,7 +391,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
"num-traits",
]
@@ -404,7 +401,7 @@ version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
"num-integer",
"num-traits",
]
@@ -415,7 +412,7 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
"num-bigint",
"num-integer",
"num-traits",
@@ -427,7 +424,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg 1.1.0",
"autocfg 1.0.1",
]
[[package]]
@@ -438,9 +435,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.10.0"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
[[package]]
name = "os_str_bytes"
@@ -511,11 +508,11 @@ version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.8",
"autocfg 0.1.7",
"libc",
"rand_chacha 0.1.1",
"rand_core 0.4.2",
"rand_hc",
"rand_hc 0.1.0",
"rand_isaac",
"rand_jitter",
"rand_os",
@@ -526,13 +523,14 @@ dependencies = [
[[package]]
name = "rand"
version = "0.8.5"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.3",
"rand_hc 0.3.1",
]
[[package]]
@@ -541,7 +539,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg 0.1.8",
"autocfg 0.1.7",
"rand_core 0.3.1",
]
@@ -588,6 +586,15 @@ dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_hc"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
dependencies = [
"rand_core 0.6.3",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
@@ -628,7 +635,7 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg 0.1.8",
"autocfg 0.1.7",
"rand_core 0.4.2",
]
@@ -652,9 +659,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.11"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c"
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
dependencies = [
"bitflags",
]
@@ -693,9 +700,9 @@ dependencies = [
[[package]]
name = "rust_decimal"
version = "1.22.0"
version = "1.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d37baa70cf8662d2ba1c1868c5983dda16ef32b105cce41fb5c47e72936a90b3"
checksum = "4214023b1223d02a4aad9f0bb9828317634a56530870a2eaf7200a99c0c10f68"
dependencies = [
"arrayvec",
"num-traits",
@@ -730,9 +737,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.79"
version = "1.0.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085"
dependencies = [
"itoa 1.0.1",
"ryu",
@@ -741,9 +748,9 @@ dependencies = [
[[package]]
name = "shell-words"
version = "1.1.0"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074"
[[package]]
name = "statistical"
@@ -788,9 +795,9 @@ dependencies = [
[[package]]
name = "termcolor"
version = "1.1.3"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
dependencies = [
"winapi-util",
]
@@ -813,9 +820,9 @@ checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
[[package]]
name = "textwrap"
version = "0.15.0"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
dependencies = [
"terminal_size",
]
+2 -2
View File
@@ -7,7 +7,7 @@ license = "MIT/Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "1.13.0"
version = "1.12.0"
edition = "2018"
build = "build.rs"
@@ -45,7 +45,7 @@ tempfile = "3.3"
[build-dependencies]
clap = "3"
atty = "0.2"
clap_complete = "3"
clap_complete = "3.0"
[profile.release]
lto = true
+12 -44
View File
@@ -24,7 +24,7 @@ A command-line benchmarking tool.
## Usage
### Basic benchmarks
### Basic benchmark
To run a benchmark, you can simply call `hyperfine <command>...`. The argument(s) can be any
shell command. For example:
@@ -33,24 +33,19 @@ hyperfine 'sleep 0.3'
```
Hyperfine will automatically determine the number of runs to perform for each command. By default,
it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
you can use the `-r`/`--runs` option:
it will perform *at least* 10 benchmarking runs. To change this, you can use the `-m`/`--min-runs`
option:
``` bash
hyperfine --runs 5 'sleep 0.3'
```
If you want to compare the runtimes of different programs, you can pass multiple commands:
``` bash
hyperfine 'hexdump file' 'xxd file'
hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
```
### Warmup runs and preparation commands
For programs that perform a lot of disk I/O, the benchmarking results can be heavily influenced
by disk caches and whether they are cold or warm.
If the program execution time is limited by disk I/O, the benchmarking results can be heavily
influenced by disk caches and whether they are cold or warm.
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to
perform a certain number of program executions before the actual benchmark:
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform
a certain number of program executions before the actual benchmark:
``` bash
hyperfine --warmup 3 'grep -R TODO *'
```
@@ -61,7 +56,7 @@ on Linux, you can run
``` bash
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
```
To use this specific command with hyperfine, call `sudo -v` to temporarily gain sudo permissions
To use this specific command with Hyperfine, call `sudo -v` to temporarily gain sudo permissions
and then call:
``` bash
hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *'
@@ -69,7 +64,7 @@ hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R
### Parameterized benchmarks
If you want to run a series of benchmarks where a single parameter is varied (say, the number of
If you want to run a benchmark where only a single parameter is varied (say, the number of
threads), you can use the `-P`/`--parameter-scan` option and call:
``` bash
hyperfine --prepare 'make clean' --parameter-scan num_threads 1 12 'make -j {num_threads}'
@@ -81,33 +76,6 @@ hyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}'
```
This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
option:
```
hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
```
### Intermediate shell
By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
``` bash
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
```
Note that hyperfine always *corrects for the shell spawning time*. To do this, it performs a calibration
procedure where it runs the shell with an empty command (multiple times), to measure the startup time
of the shell. It will then subtract this time from the total to show the actual time used by the command
in question.
If you want to run a benchmark *without an intermediate shell*, you can use the `-N` or `--shell=none`
option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
produce a significant amount of noise. Note that you can not use shell syntax like `*` or `~` in this case.
```
hyperfine -N 'grep TODO /home/user'
```
### Shell functions and aliases
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
@@ -152,8 +120,8 @@ The JSON output is useful if you want to analyze the benchmark results in more d
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.13.0/hyperfine_1.13.0_amd64.deb
sudo dpkg -i hyperfine_1.13.0_amd64.deb
wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
sudo dpkg -i hyperfine_1.12.0_amd64.deb
```
### On Fedora
+3 -3
View File
@@ -2,7 +2,7 @@ use std::fs;
use clap_complete::{generate_to, Shell};
include!("src/cli.rs");
include!("src/app.rs");
fn main() {
let var = std::env::var_os("SHELL_COMPLETIONS_DIR").or_else(|| std::env::var_os("OUT_DIR"));
@@ -12,7 +12,7 @@ fn main() {
};
fs::create_dir_all(&outdir).unwrap();
let mut command = build_command();
let mut app = build_app();
for shell in [
Shell::Bash,
Shell::Fish,
@@ -20,6 +20,6 @@ fn main() {
Shell::PowerShell,
Shell::Elvish,
] {
generate_to(shell, &mut command, "hyperfine", &outdir).unwrap();
generate_to(shell, &mut app, "hyperfine", &outdir).unwrap();
}
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 94 KiB

After

Width:  |  Height:  |  Size: 233 KiB

+79 -79
View File
@@ -2,16 +2,16 @@
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="1724.755"
height="1832.972"
viewBox="0 0 456.34143 484.97387"
width="1699.0797"
height="1802.972"
viewBox="0 0 449.54818 477.03637"
version="1.1"
id="svg5"
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20, custom)"
sodipodi:docname="execution-order.svg"
inkscape:export-filename="/home/shark/Informatik/rust/hyperfine/doc/execution-order.png"
inkscape:export-xdpi="44.529999"
inkscape:export-ydpi="44.529999"
inkscape:export-xdpi="96"
inkscape:export-ydpi="96"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
@@ -27,8 +27,8 @@
inkscape:document-units="mm"
showgrid="false"
inkscape:zoom="0.38529391"
inkscape:cx="1064.1227"
inkscape:cy="949.92418"
inkscape:cx="1027.7868"
inkscape:cy="991.45092"
inkscape:window-width="1920"
inkscape:window-height="1175"
inkscape:window-x="1920"
@@ -38,19 +38,19 @@
showborder="true"
showguides="false"
inkscape:guide-bbox="true"
fit-margin-top="30"
fit-margin-left="30"
fit-margin-right="30"
fit-margin-bottom="30"
fit-margin-top="15"
fit-margin-left="15"
fit-margin-right="15"
fit-margin-bottom="15"
lock-margins="true"
units="px">
<inkscape:grid
type="xygrid"
id="grid949"
originx="334.94547"
originy="-10.061553" />
originx="332.12096"
originy="-14.030302" />
<sodipodi:guide
position="190.74752,358.77499"
position="187.92302,354.80624"
orientation="1,0"
id="guide130046" />
</sodipodi:namedview>
@@ -60,121 +60,121 @@
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(334.94544,-10.06155)">
transform="translate(332.12094,-14.0303)">
<rect
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect159433"
width="80.681"
height="13.446833"
x="-226.03801"
y="147.57736"
ry="1.0056218" />
width="88.724548"
height="14.787424"
x="-206.65671"
y="294.47958"
ry="1.105878" />
<rect
style="fill:#010101;fill-opacity:1;stroke:#000000;stroke-width:0.654957;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#010101;fill-opacity:1;stroke:#000000;stroke-width:0.720253;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect123376"
width="80.026047"
height="12.791878"
x="-302.52026"
y="132.40656"
ry="0.9593913" />
width="88.004295"
height="14.067173"
x="-290.76392"
y="277.79633"
ry="1.0550385" />
<rect
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.654957;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.720253;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect129065"
width="80.026047"
height="12.791878"
x="-302.52026"
y="162.73172"
ry="0.9593913" />
width="88.004295"
height="14.067173"
x="-290.76392"
y="311.14478"
ry="1.0550385" />
<rect
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect117266"
width="80.681"
height="13.446833"
x="-225.7778"
y="117.46693"
ry="1.0086428" />
width="88.724548"
height="14.787424"
x="-206.37056"
y="261.36728"
ry="1.1092001" />
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect117165"
width="80.681"
height="13.446827"
x="-226.03755"
y="177.90302"
ry="1.0085125" />
width="88.724548"
height="14.787417"
x="-206.6562"
y="327.82858"
ry="1.1090568" />
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect117064"
width="80.681"
height="13.446837"
x="-225.7778"
y="102.33925"
ry="1.0085125" />
width="88.724548"
height="14.787429"
x="-206.37056"
y="244.73143"
ry="1.1090568" />
<text
xml:space="preserve"
style="font-size:10.5833px;line-height:1.2;font-family:Audiowide;-inkscape-font-specification:Audiowide;white-space:pre;inline-size:199.93;stroke-width:0.264583"
x="-79.596779"
y="-115.14695"
id="text41875"
transform="matrix(1.1724504,0,0,1.1724504,-222.61922,201.56959)"><tspan
transform="matrix(1.2893387,0,0,1.2893387,-202.89709,353.8546)"><tspan
x="-79.596779"
y="-115.14695"
id="tspan14151"><tspan
id="tspan5244"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14149">hyperfine
id="tspan5242">hyperfine
</tspan></tspan><tspan
x="-79.596779"
y="-102.16832"
id="tspan14157"><tspan
id="tspan5250"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#e35a00"
id="tspan14153"> --warmup 2</tspan><tspan
id="tspan5246"> --warmup 2</tspan><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14155">
id="tspan5248">
</tspan></tspan><tspan
x="-79.596779"
y="-89.189692"
id="tspan14163"><tspan
id="tspan5256"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#00bbeb"
id="tspan14159"> --runs 3</tspan><tspan
id="tspan5252"> --runs 3</tspan><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14161">
id="tspan5254">
</tspan></tspan><tspan
x="-79.596779"
y="-76.211062"
id="tspan14167"><tspan
id="tspan5260"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14165"> --setup &lt;setup&gt;
id="tspan5258"> --setup &lt;setup&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-63.232432"
id="tspan14171"><tspan
id="tspan5264"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14169"> --prepare &lt;prepare1&gt;
id="tspan5262"> --prepare &lt;prepare1&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-50.253802"
id="tspan14177"><tspan
id="tspan5270"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#ffffff"
id="tspan14173"> &lt;command1&gt;</tspan><tspan
id="tspan5266"> &lt;command1&gt;</tspan><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14175">
id="tspan5268">
</tspan></tspan><tspan
x="-79.596779"
y="-37.275172"
id="tspan14181"><tspan
id="tspan5274"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14179"> --prepare &lt;prepare2&gt;
id="tspan5272"> --prepare &lt;prepare2&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-24.296542"
id="tspan14185"><tspan
id="tspan5278"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#ffffff"
id="tspan14183"> &lt;command2&gt;
id="tspan5276"> &lt;command2&gt;
</tspan></tspan><tspan
x="-79.596779"
y="-11.317913"
id="tspan14189"><tspan
id="tspan5282"><tspan
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan14187"> --cleanup &lt;cleanup&gt;</tspan></tspan></text>
id="tspan5280"> --cleanup &lt;cleanup&gt;</tspan></tspan></text>
<rect
style="fill:#80e5ff;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect6355"
@@ -325,7 +325,7 @@
transform="translate(34.591878,14.34874)"><tspan
x="-16.304935"
y="161.32399"
id="tspan14191">3 benchmark runs</tspan></text>
id="tspan5284">3 benchmark runs</tspan></text>
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect13386"
@@ -590,7 +590,7 @@
transform="translate(34.591878,257.76541)"><tspan
x="-16.304935"
y="161.32399"
id="tspan14193">3 benchmark runs</tspan></text>
id="tspan5286">3 benchmark runs</tspan></text>
<rect
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
id="rect272454"
@@ -711,11 +711,11 @@
x="-79.596779"
y="-115.14695"
id="text283737"
transform="matrix(1.7031443,0,0,1.7031443,-192.73007,228.6586)"><tspan
transform="matrix(1.2893387,0,0,1.2893387,-225.35928,177.7847)"><tspan
x="-79.596779"
y="-115.14695"
id="tspan14197"><tspan
style="font-weight:bold;font-family:'Segoe UI';-inkscape-font-specification:'Segoe UI Bold'"
id="tspan14195">benchmark execution order</tspan></tspan></text>
id="tspan5290"><tspan
style="font-weight:bold;font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
id="tspan5288">benchmark execution order</tspan></tspan></text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 35 KiB

After

Width:  |  Height:  |  Size: 35 KiB

BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 11 KiB

+2 -2
View File
@@ -9,9 +9,9 @@ This folder contains scripts that can be used in combination with hyperfines `--
### Pre-requisites
To make these scripts work, you will need to install `numpy`, `matplotlib` and `scipy`. Install them via
To make these scripts work, you will need to install `numpy` and `matplotlib`. Install them via
your package manager or `pip`:
```bash
pip install numpy matplotlib scipy # pip3, if you are using python3
pip install numpy matplotlib # pip3, if you are using python3
```
-1
View File
@@ -23,7 +23,6 @@ for command, ts in zip(commands, times):
iqr = p75 - p25
print("Command '{}'".format(command))
print(" runs: {:8d}".format(len(ts)))
print(" mean: {:8.3f} s".format(np.mean(ts)))
print(" stddev: {:8.3f} s".format(np.std(ts, ddof=1)))
print(" median: {:8.3f} s".format(np.median(ts)))
+2 -8
View File
@@ -20,12 +20,6 @@ parser.add_argument(
parser.add_argument(
"-o", "--output", help="Save image to the given filename."
)
parser.add_argument(
"--t-min", metavar="T", help="Minimum time to be displayed (seconds)"
)
parser.add_argument(
"--t-max", metavar="T", help="Maximum time to be displayed (seconds)"
)
args = parser.parse_args()
@@ -38,8 +32,8 @@ else:
labels = [b["command"] for b in results]
all_times = [b["times"] for b in results]
t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))
t_min = np.min(list(map(np.min, all_times)))
t_max = np.max(list(map(np.max, all_times)))
bins = int(args.bins) if args.bins else "auto"
histtype = args.type if args.type else "bar"
-62
View File
@@ -1,62 +0,0 @@
#!/usr/bin/env python
"""This program shows `hyperfine` benchmark results in a sequential way
in order to debug possible background interference, caching effects,
thermal throttling and similar effects.
"""
import argparse
import json
import matplotlib.pyplot as plt
import numpy as np
def moving_average(times, num_runs):
times_padded = np.pad(
times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge"
)
kernel = np.ones(num_runs) / num_runs
return np.convolve(times_padded, kernel, mode="valid")
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("file", help="JSON file with benchmark results")
parser.add_argument("--title", help="Plot Title")
parser.add_argument("-o", "--output", help="Save image to the given filename.")
parser.add_argument(
"-w",
"--moving-average-width",
type=int,
metavar="num_runs",
help="Width of the moving-average window (default: N/5)",
)
args = parser.parse_args()
with open(args.file) as f:
results = json.load(f)["results"]
label = results[0]["command"]
times = results[0]["times"]
num = len(times)
nums = range(num)
plt.scatter(x=nums, y=times, marker=".", color="orange")
plt.ylim([0, None])
plt.xlim([-1, num])
moving_average_width = (
num // 5 if args.moving_average_width is None else args.moving_average_width
)
moving_average = moving_average(times, moving_average_width)
plt.plot(nums, moving_average, "-", color="blue")
if args.title:
plt.title(args.title)
plt.legend(labels=[label], loc="best", fontsize="medium")
plt.ylabel("Time [s]")
if args.output:
plt.savefig(args.output)
else:
plt.show()
+13 -27
View File
@@ -1,32 +1,30 @@
use std::ffi::OsString;
use clap::{crate_version, AppSettings, Arg, ArgMatches, Command};
use clap::{crate_version, App, AppSettings, Arg, ArgMatches};
pub fn get_cli_arguments<'a, I, T>(args: I) -> ArgMatches
where
I: IntoIterator<Item = T>,
T: Into<OsString> + Clone + 'a,
{
let command = build_command();
command.get_matches_from(args)
let app = build_app();
app.get_matches_from(args)
}
/// Build the clap command for parsing command line arguments
fn build_command() -> Command<'static> {
Command::new("hyperfine")
/// Build the clap app for parsing command line arguments
fn build_app() -> App<'static> {
App::new("hyperfine")
.version(crate_version!())
.setting(AppSettings::DeriveDisplayOrder)
.next_line_help(true)
.hide_possible_values(true)
.setting(AppSettings::NextLineHelp)
.setting(AppSettings::HidePossibleValues)
.max_term_width(90)
.about("A command-line benchmarking tool.")
.arg(
Arg::new("command")
.help("The command to benchmark. This can be the name of an executable, a command \
line like \"grep -i todo\" or a shell command like \"sleep 0.5 && echo test\". \
The latter is only available if the shell is not explicitly disabled via \
'--shell=none'. If multiple commands are given, hyperfine will show a \
comparison of the respective runtimes.")
.help("Command to benchmark. This can be the name of an executable or a shell \
command like \"sleep 0.5 && echo test\". If multiple commands are given, \
hyperfine will show a comparison of the respective runtimes.")
.required(true)
.multiple_occurrences(true)
.forbid_empty_values(true),
@@ -183,19 +181,7 @@ fn build_command() -> Command<'static> {
.takes_value(true)
.value_name("SHELL")
.overrides_with("shell")
.help("Set the shell to use for executing benchmarked commands. This can be the \
name or the path to the shell executable, or a full command line \
like \"bash --norc\". It can also be set to \"default\" to explicitly select \
the default shell on this platform. Finally, this can also be set to \
\"none\" to disable the shell. In this case, commands will be executed \
directly. They can still have arguments, but more complex things like \
\"sleep 0.1; sleep 0.2\" are not possible without a shell.")
)
.arg(
Arg::new("no-shell")
.short('N')
.conflicts_with_all(&["shell", "debug-mode"])
.help("An alias for '--shell=none'.")
.help("Set the shell to use for executing benchmarked commands."),
)
.arg(
Arg::new("ignore-failure")
@@ -273,5 +259,5 @@ fn build_command() -> Command<'static> {
#[test]
fn verify_app() {
build_command().debug_assert();
build_app().debug_assert();
}
-42
View File
@@ -64,48 +64,6 @@ fn run_command_and_measure_common(
Ok(result)
}
pub struct RawExecutor<'a> {
options: &'a Options,
}
impl<'a> RawExecutor<'a> {
pub fn new(options: &'a Options) -> Self {
RawExecutor { options }
}
}
impl<'a> Executor for RawExecutor<'a> {
fn run_command_and_measure(
&self,
command: &Command<'_>,
command_failure_action: Option<CmdFailureAction>,
) -> Result<(TimingResult, ExitStatus)> {
let result = run_command_and_measure_common(
command.get_command()?,
command_failure_action.unwrap_or(self.options.command_failure_action),
self.options.command_output_policy,
&command.get_command_line(),
)?;
Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
},
result.status,
))
}
fn calibrate(&mut self) -> Result<()> {
Ok(())
}
fn time_overhead(&self) -> Second {
0.0
}
}
pub struct ShellExecutor<'a> {
options: &'a Options,
shell: &'a Shell,
+2 -4
View File
@@ -7,7 +7,7 @@ pub mod timing_result;
use std::cmp;
use crate::command::Command;
use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption};
use crate::options::{CmdFailureAction, Options, OutputStyleOption};
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
use crate::output::format::{format_duration, format_duration_unit};
use crate::output::progress_bar::get_progress_bar;
@@ -320,9 +320,7 @@ impl<'a> Benchmark<'a> {
let mut warnings = vec![];
// Check execution time
if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
&& times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
{
if times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) {
warnings.push(Warnings::FastExecutionTime);
}
+1 -2
View File
@@ -1,7 +1,7 @@
use colored::*;
use super::benchmark_result::BenchmarkResult;
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
use super::executor::{Executor, MockExecutor, ShellExecutor};
use super::{relative_speed, Benchmark};
use crate::command::Commands;
@@ -33,7 +33,6 @@ impl<'a> Scheduler<'a> {
pub fn run_benchmarks(&mut self) -> Result<()> {
let mut executor: Box<dyn Executor> = match self.options.executor_kind {
ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),
ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
};
+4 -19
View File
@@ -15,7 +15,7 @@ use clap::ArgMatches;
use crate::parameter::tokenize::tokenize;
use crate::parameter::ParameterValue;
use anyhow::{bail, Context, Result};
use anyhow::{bail, Result};
use clap::Values;
use rust_decimal::Decimal;
@@ -64,21 +64,6 @@ impl<'a> Command<'a> {
self.replace_parameters_in(self.expression)
}
pub fn get_command(&self) -> Result<std::process::Command> {
let command_line = self.get_command_line();
let mut tokens = shell_words::split(&command_line)
.with_context(|| format!("Failed to parse command '{}'", command_line))?
.into_iter();
if let Some(program_name) = tokens.next() {
let mut command_builder = std::process::Command::new(program_name);
command_builder.args(tokens);
Ok(command_builder)
} else {
bail!("Can not execute empty command")
}
}
pub fn get_parameters(&self) -> &[(&'a str, ParameterValue)] {
&self.parameters
}
@@ -366,7 +351,7 @@ impl<'a> fmt::Display for Command<'a> {
#[test]
fn test_build_commands_cross_product() {
use crate::cli::get_cli_arguments;
use crate::app::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
@@ -404,7 +389,7 @@ fn test_build_commands_cross_product() {
#[test]
fn test_build_parameter_list_commands() {
use crate::cli::get_cli_arguments;
use crate::app::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
@@ -425,7 +410,7 @@ fn test_build_parameter_list_commands() {
#[test]
fn test_build_parameter_scan_commands() {
use crate::cli::get_cli_arguments;
use crate::app::get_cli_arguments;
let matches = get_cli_arguments(vec![
"hyperfine",
"echo {val}",
+2 -2
View File
@@ -1,7 +1,7 @@
use std::env;
use app::get_cli_arguments;
use benchmark::scheduler::Scheduler;
use cli::get_cli_arguments;
use command::Commands;
use export::ExportManager;
use options::Options;
@@ -9,8 +9,8 @@ use options::Options;
use anyhow::Result;
use colored::*;
pub mod app;
pub mod benchmark;
pub mod cli;
pub mod command;
pub mod error;
pub mod export;
+6 -12
View File
@@ -134,7 +134,6 @@ impl CommandOutputPolicy {
}
pub enum ExecutorKind {
Raw,
Shell(Shell),
Mock(Option<String>),
}
@@ -282,17 +281,12 @@ impl Options {
OutputStyleOption::Disabled => {}
};
options.executor_kind = if matches.is_present("no-shell") {
ExecutorKind::Raw
} else {
match (matches.is_present("debug-mode"), matches.value_of("shell")) {
(false, Some(shell)) if shell == "default" => ExecutorKind::Shell(Shell::default()),
(false, Some(shell)) if shell == "none" => ExecutorKind::Raw,
(false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?),
(false, None) => ExecutorKind::Shell(Shell::default()),
(true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
(true, None) => ExecutorKind::Mock(None),
}
options.executor_kind = match (matches.is_present("debug-mode"), matches.value_of("shell"))
{
(false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?),
(false, None) => ExecutorKind::Shell(Shell::default()),
(true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
(true, None) => ExecutorKind::Mock(None),
};
if matches.is_present("ignore-failure") {
+1 -4
View File
@@ -17,10 +17,7 @@ impl fmt::Display for Warnings {
match *self {
Warnings::FastExecutionTime => write!(
f,
"Command took less than {:.0} ms to complete. Note that the results might be \
inaccurate because hyperfine can not calibrate the shell startup time much \
more precise than this limit. You can try to use the `-N`/`--shell=none` \
option to disable the shell completely.",
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
MIN_EXECUTION_TIME * 1e3
),
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
+1 -40
View File
@@ -10,7 +10,7 @@ pub fn hyperfine_debug() -> assert_cmd::Command {
}
#[test]
fn runs_successfully() {
fn hyperfine_runs_successfully() {
hyperfine()
.arg("--runs=2")
.arg("echo dummy benchmark")
@@ -27,18 +27,6 @@ fn one_run_is_supported() {
.success();
}
#[test]
fn can_run_commands_without_a_shell() {
hyperfine()
.arg("--runs=1")
.arg("--show-output")
.arg("--shell=none")
.arg("echo 'hello world' argument2")
.assert()
.success()
.stdout(predicate::str::contains("hello world argument2"));
}
#[test]
fn fails_with_wrong_number_of_command_name_arguments() {
hyperfine()
@@ -102,33 +90,6 @@ fn fails_for_unknown_command() {
));
}
#[test]
fn fails_for_unknown_command_without_shell() {
hyperfine()
.arg("--shell=none")
.arg("--runs=1")
.arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
.assert()
.failure()
.stderr(predicate::str::contains(
"Failed to run command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577'",
));
}
#[cfg(unix)]
#[test]
fn fails_for_failing_command_without_shell() {
hyperfine()
.arg("--shell=none")
.arg("--runs=1")
.arg("false")
.assert()
.failure()
.stderr(predicate::str::contains(
"Command terminated with non-zero exit code",
));
}
#[test]
fn fails_for_unknown_setup_command() {
hyperfine()