mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-16 09:21:27 +00:00
Compare commits
30
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
02157eb6ca | ||
|
|
65932dd5ce | ||
|
|
851e9b8dfa | ||
|
|
5c676a9626 | ||
|
|
f1557d47fb | ||
|
|
bcd3705ce0 | ||
|
|
176275ecab | ||
|
|
56cce61077 | ||
|
|
770e23bc1b | ||
|
|
0601a1971e | ||
|
|
f742329f1e | ||
|
|
d13454a0d4 | ||
|
|
e666b01e09 | ||
|
|
e38ac46179 | ||
|
|
365ff2584d | ||
|
|
d9418e0731 | ||
|
|
f2f1dd9d40 | ||
|
|
308bf5cd26 | ||
|
|
f19223ba9d | ||
|
|
7fd4aea495 | ||
|
|
55f0ef6637 | ||
|
|
e001530f61 | ||
|
|
d27cbceed2 | ||
|
|
9b8f2a9ee2 | ||
|
|
e1da8ba00f | ||
|
|
0ffc6d7cc0 | ||
|
|
1f3943e7d1 | ||
|
|
f822455d99 | ||
|
|
7c929f7e48 | ||
|
|
826449bc8f |
No files matched your search
+27
-5
@@ -2,8 +2,35 @@
|
||||
|
||||
## Features
|
||||
|
||||
|
||||
## Changes
|
||||
|
||||
|
||||
## Bugfixes
|
||||
|
||||
|
||||
## Other
|
||||
|
||||
|
||||
## Packaging
|
||||
|
||||
|
||||
|
||||
# v1.13.0
|
||||
|
||||
## Features
|
||||
|
||||
- Added a new `--shell=none`/`-N` option to disable the intermediate
|
||||
shell for executing the benchmarked commands. Hyperfine normally
|
||||
measures and subtracts the shell spawning time, but the intermediate
|
||||
shell always introduces a certain level of measurement noise. Using
|
||||
`--shell=none`/`-N` allows users to benchmark very fast commands
|
||||
(with a runtime on the order of a few milliseconds). See #336, #429,
|
||||
and #487 (@cipriancraciun and @sharkdp)
|
||||
- Added `--setup` (`s`) option that can be used to run `make all` or
|
||||
similar. It runs once per set of tests, like `--cleanup` (`c`).
|
||||
- Added new `plot_progression.py` script to debug background interference
|
||||
effects.
|
||||
|
||||
## Changes
|
||||
|
||||
@@ -12,16 +39,11 @@
|
||||
- The environment offset randomization is now also available on Windows,
|
||||
see #484
|
||||
|
||||
## Bugfixes
|
||||
|
||||
## Other
|
||||
|
||||
- Improved documentation and test coverage, cleaned up code base for
|
||||
future improvements.
|
||||
|
||||
## Packaging
|
||||
|
||||
|
||||
|
||||
# v1.12.0
|
||||
|
||||
|
||||
Generated
+46
-53
@@ -13,9 +13,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.53"
|
||||
version = "1.0.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0"
|
||||
checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd"
|
||||
|
||||
[[package]]
|
||||
name = "approx"
|
||||
@@ -59,15 +59,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "0.1.7"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
|
||||
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
|
||||
dependencies = [
|
||||
"autocfg 1.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
@@ -95,9 +98,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "3.0.14"
|
||||
version = "3.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63edc3f163b3c71ec8aa23f9bd6070f77edbf3d1d198b164afa90ff00e4ec62"
|
||||
checksum = "ced1892c55c910c1219e98d6fc8d71f6bddba7905866ce740066d8bfea859312"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags",
|
||||
@@ -112,9 +115,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_complete"
|
||||
version = "3.0.5"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4dabb7e2f006497e1da045feaa512acf0686f76b68d94925da2d9422dcb521"
|
||||
checksum = "df6f3613c0a3cddfd78b41b10203eb322cb29b600cbdf808a7d3db95691b8e25"
|
||||
dependencies = [
|
||||
"clap",
|
||||
]
|
||||
@@ -224,9 +227,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.4"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
|
||||
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
@@ -250,7 +253,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyperfine"
|
||||
version = "1.12.0"
|
||||
version = "1.13.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"approx",
|
||||
@@ -263,7 +266,7 @@ dependencies = [
|
||||
"indicatif",
|
||||
"libc",
|
||||
"predicates",
|
||||
"rand 0.8.4",
|
||||
"rand 0.8.5",
|
||||
"rust_decimal",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -280,7 +283,7 @@ version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
@@ -334,9 +337,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.117"
|
||||
version = "0.2.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c"
|
||||
checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
@@ -370,7 +373,7 @@ version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
@@ -381,7 +384,7 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
@@ -391,7 +394,7 @@ version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
@@ -401,7 +404,7 @@ version = "0.1.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
@@ -412,7 +415,7 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
@@ -424,7 +427,7 @@ version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
|
||||
dependencies = [
|
||||
"autocfg 1.0.1",
|
||||
"autocfg 1.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -435,9 +438,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.9.0"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
|
||||
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
@@ -508,11 +511,11 @@ version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
|
||||
dependencies = [
|
||||
"autocfg 0.1.7",
|
||||
"autocfg 0.1.8",
|
||||
"libc",
|
||||
"rand_chacha 0.1.1",
|
||||
"rand_core 0.4.2",
|
||||
"rand_hc 0.1.0",
|
||||
"rand_hc",
|
||||
"rand_isaac",
|
||||
"rand_jitter",
|
||||
"rand_os",
|
||||
@@ -523,14 +526,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.4"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.1",
|
||||
"rand_core 0.6.3",
|
||||
"rand_hc 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -539,7 +541,7 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
|
||||
dependencies = [
|
||||
"autocfg 0.1.7",
|
||||
"autocfg 0.1.8",
|
||||
"rand_core 0.3.1",
|
||||
]
|
||||
|
||||
@@ -586,15 +588,6 @@ dependencies = [
|
||||
"rand_core 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
|
||||
dependencies = [
|
||||
"rand_core 0.6.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_isaac"
|
||||
version = "0.1.1"
|
||||
@@ -635,7 +628,7 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
|
||||
dependencies = [
|
||||
"autocfg 0.1.7",
|
||||
"autocfg 0.1.8",
|
||||
"rand_core 0.4.2",
|
||||
]
|
||||
|
||||
@@ -659,9 +652,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.10"
|
||||
version = "0.2.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
|
||||
checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
@@ -700,9 +693,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rust_decimal"
|
||||
version = "1.21.0"
|
||||
version = "1.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4214023b1223d02a4aad9f0bb9828317634a56530870a2eaf7200a99c0c10f68"
|
||||
checksum = "d37baa70cf8662d2ba1c1868c5983dda16ef32b105cce41fb5c47e72936a90b3"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"num-traits",
|
||||
@@ -737,9 +730,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.78"
|
||||
version = "1.0.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085"
|
||||
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
|
||||
dependencies = [
|
||||
"itoa 1.0.1",
|
||||
"ryu",
|
||||
@@ -748,9 +741,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "shell-words"
|
||||
version = "1.0.0"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074"
|
||||
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
|
||||
|
||||
[[package]]
|
||||
name = "statistical"
|
||||
@@ -795,9 +788,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.1.2"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
|
||||
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
@@ -820,9 +813,9 @@ checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.14.2"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
|
||||
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
|
||||
dependencies = [
|
||||
"terminal_size",
|
||||
]
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ license = "MIT/Apache-2.0"
|
||||
name = "hyperfine"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/sharkdp/hyperfine"
|
||||
version = "1.12.0"
|
||||
version = "1.13.0"
|
||||
edition = "2018"
|
||||
build = "build.rs"
|
||||
|
||||
@@ -45,7 +45,7 @@ tempfile = "3.3"
|
||||
[build-dependencies]
|
||||
clap = "3"
|
||||
atty = "0.2"
|
||||
clap_complete = "3.0"
|
||||
clap_complete = "3"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
@@ -24,7 +24,7 @@ A command-line benchmarking tool.
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic benchmark
|
||||
### Basic benchmarks
|
||||
|
||||
To run a benchmark, you can simply call `hyperfine <command>...`. The argument(s) can be any
|
||||
shell command. For example:
|
||||
@@ -33,19 +33,24 @@ hyperfine 'sleep 0.3'
|
||||
```
|
||||
|
||||
Hyperfine will automatically determine the number of runs to perform for each command. By default,
|
||||
it will perform *at least* 10 benchmarking runs. To change this, you can use the `-m`/`--min-runs`
|
||||
option:
|
||||
it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
|
||||
you can use the `-r`/`--runs` option:
|
||||
``` bash
|
||||
hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
|
||||
hyperfine --runs 5 'sleep 0.3'
|
||||
```
|
||||
|
||||
If you want to compare the runtimes of different programs, you can pass multiple commands:
|
||||
``` bash
|
||||
hyperfine 'hexdump file' 'xxd file'
|
||||
```
|
||||
|
||||
### Warmup runs and preparation commands
|
||||
|
||||
If the program execution time is limited by disk I/O, the benchmarking results can be heavily
|
||||
influenced by disk caches and whether they are cold or warm.
|
||||
For programs that perform a lot of disk I/O, the benchmarking results can be heavily influenced
|
||||
by disk caches and whether they are cold or warm.
|
||||
|
||||
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform
|
||||
a certain number of program executions before the actual benchmark:
|
||||
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to
|
||||
perform a certain number of program executions before the actual benchmark:
|
||||
``` bash
|
||||
hyperfine --warmup 3 'grep -R TODO *'
|
||||
```
|
||||
@@ -56,7 +61,7 @@ on Linux, you can run
|
||||
``` bash
|
||||
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
|
||||
```
|
||||
To use this specific command with Hyperfine, call `sudo -v` to temporarily gain sudo permissions
|
||||
To use this specific command with hyperfine, call `sudo -v` to temporarily gain sudo permissions
|
||||
and then call:
|
||||
``` bash
|
||||
hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *'
|
||||
@@ -64,7 +69,7 @@ hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R
|
||||
|
||||
### Parameterized benchmarks
|
||||
|
||||
If you want to run a benchmark where only a single parameter is varied (say, the number of
|
||||
If you want to run a series of benchmarks where a single parameter is varied (say, the number of
|
||||
threads), you can use the `-P`/`--parameter-scan` option and call:
|
||||
``` bash
|
||||
hyperfine --prepare 'make clean' --parameter-scan num_threads 1 12 'make -j {num_threads}'
|
||||
@@ -76,6 +81,33 @@ hyperfine --parameter-scan delay 0.3 0.7 -D 0.2 'sleep {delay}'
|
||||
```
|
||||
This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
|
||||
|
||||
For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
|
||||
option:
|
||||
```
|
||||
hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
|
||||
```
|
||||
|
||||
### Intermediate shell
|
||||
|
||||
By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
|
||||
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
|
||||
``` bash
|
||||
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
|
||||
```
|
||||
|
||||
Note that hyperfine always *corrects for the shell spawning time*. To do this, it performs a calibration
|
||||
procedure where it runs the shell with an empty command (multiple times), to measure the startup time
|
||||
of the shell. It will then subtract this time from the total to show the actual time used by the command
|
||||
in question.
|
||||
|
||||
If you want to run a benchmark *without an intermediate shell*, you can use the `-N` or `--shell=none`
|
||||
option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
|
||||
produce a significant amount of noise. Note that you can not use shell syntax like `*` or `~` in this case.
|
||||
```
|
||||
hyperfine -N 'grep TODO /home/user'
|
||||
```
|
||||
|
||||
|
||||
### Shell functions and aliases
|
||||
|
||||
If you are using bash, you can export shell functions to directly benchmark them with hyperfine:
|
||||
@@ -120,8 +152,8 @@ The JSON output is useful if you want to analyze the benchmark results in more d
|
||||
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
|
||||
and install it via `dpkg`:
|
||||
```
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.12.0_amd64.deb
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.13.0/hyperfine_1.13.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.13.0_amd64.deb
|
||||
```
|
||||
|
||||
### On Fedora
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::fs;
|
||||
|
||||
use clap_complete::{generate_to, Shell};
|
||||
|
||||
include!("src/app.rs");
|
||||
include!("src/cli.rs");
|
||||
|
||||
fn main() {
|
||||
let var = std::env::var_os("SHELL_COMPLETIONS_DIR").or_else(|| std::env::var_os("OUT_DIR"));
|
||||
@@ -12,7 +12,7 @@ fn main() {
|
||||
};
|
||||
fs::create_dir_all(&outdir).unwrap();
|
||||
|
||||
let mut app = build_app();
|
||||
let mut command = build_command();
|
||||
for shell in [
|
||||
Shell::Bash,
|
||||
Shell::Fish,
|
||||
@@ -20,6 +20,6 @@ fn main() {
|
||||
Shell::PowerShell,
|
||||
Shell::Elvish,
|
||||
] {
|
||||
generate_to(shell, &mut app, "hyperfine", &outdir).unwrap();
|
||||
generate_to(shell, &mut command, "hyperfine", &outdir).unwrap();
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 233 KiB After Width: | Height: | Size: 94 KiB |
+79
-79
@@ -2,16 +2,16 @@
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
width="1699.0797"
|
||||
height="1802.972"
|
||||
viewBox="0 0 449.54818 477.03637"
|
||||
width="1724.755"
|
||||
height="1832.972"
|
||||
viewBox="0 0 456.34143 484.97387"
|
||||
version="1.1"
|
||||
id="svg5"
|
||||
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20, custom)"
|
||||
sodipodi:docname="execution-order.svg"
|
||||
inkscape:export-filename="/home/shark/Informatik/rust/hyperfine/doc/execution-order.png"
|
||||
inkscape:export-xdpi="96"
|
||||
inkscape:export-ydpi="96"
|
||||
inkscape:export-xdpi="44.529999"
|
||||
inkscape:export-ydpi="44.529999"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
@@ -27,8 +27,8 @@
|
||||
inkscape:document-units="mm"
|
||||
showgrid="false"
|
||||
inkscape:zoom="0.38529391"
|
||||
inkscape:cx="1027.7868"
|
||||
inkscape:cy="991.45092"
|
||||
inkscape:cx="1064.1227"
|
||||
inkscape:cy="949.92418"
|
||||
inkscape:window-width="1920"
|
||||
inkscape:window-height="1175"
|
||||
inkscape:window-x="1920"
|
||||
@@ -38,19 +38,19 @@
|
||||
showborder="true"
|
||||
showguides="false"
|
||||
inkscape:guide-bbox="true"
|
||||
fit-margin-top="15"
|
||||
fit-margin-left="15"
|
||||
fit-margin-right="15"
|
||||
fit-margin-bottom="15"
|
||||
fit-margin-top="30"
|
||||
fit-margin-left="30"
|
||||
fit-margin-right="30"
|
||||
fit-margin-bottom="30"
|
||||
lock-margins="true"
|
||||
units="px">
|
||||
<inkscape:grid
|
||||
type="xygrid"
|
||||
id="grid949"
|
||||
originx="332.12096"
|
||||
originy="-14.030302" />
|
||||
originx="334.94547"
|
||||
originy="-10.061553" />
|
||||
<sodipodi:guide
|
||||
position="187.92302,354.80624"
|
||||
position="190.74752,358.77499"
|
||||
orientation="1,0"
|
||||
id="guide130046" />
|
||||
</sodipodi:namedview>
|
||||
@@ -60,121 +60,121 @@
|
||||
inkscape:label="Ebene 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(332.12094,-14.0303)">
|
||||
transform="translate(334.94544,-10.06155)">
|
||||
<rect
|
||||
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect159433"
|
||||
width="88.724548"
|
||||
height="14.787424"
|
||||
x="-206.65671"
|
||||
y="294.47958"
|
||||
ry="1.105878" />
|
||||
width="80.681"
|
||||
height="13.446833"
|
||||
x="-226.03801"
|
||||
y="147.57736"
|
||||
ry="1.0056218" />
|
||||
<rect
|
||||
style="fill:#010101;fill-opacity:1;stroke:#000000;stroke-width:0.720253;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
style="fill:#010101;fill-opacity:1;stroke:#000000;stroke-width:0.654957;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect123376"
|
||||
width="88.004295"
|
||||
height="14.067173"
|
||||
x="-290.76392"
|
||||
y="277.79633"
|
||||
ry="1.0550385" />
|
||||
width="80.026047"
|
||||
height="12.791878"
|
||||
x="-302.52026"
|
||||
y="132.40656"
|
||||
ry="0.9593913" />
|
||||
<rect
|
||||
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.720253;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.654957;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect129065"
|
||||
width="88.004295"
|
||||
height="14.067173"
|
||||
x="-290.76392"
|
||||
y="311.14478"
|
||||
ry="1.0550385" />
|
||||
width="80.026047"
|
||||
height="12.791878"
|
||||
x="-302.52026"
|
||||
y="162.73172"
|
||||
ry="0.9593913" />
|
||||
<rect
|
||||
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
style="fill:#ececec;fill-opacity:1;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect117266"
|
||||
width="88.724548"
|
||||
height="14.787424"
|
||||
x="-206.37056"
|
||||
y="261.36728"
|
||||
ry="1.1092001" />
|
||||
width="80.681"
|
||||
height="13.446833"
|
||||
x="-225.7778"
|
||||
y="117.46693"
|
||||
ry="1.0086428" />
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect117165"
|
||||
width="88.724548"
|
||||
height="14.787417"
|
||||
x="-206.6562"
|
||||
y="327.82858"
|
||||
ry="1.1090568" />
|
||||
width="80.681"
|
||||
height="13.446827"
|
||||
x="-226.03755"
|
||||
y="177.90302"
|
||||
ry="1.0085125" />
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.481194;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect117064"
|
||||
width="88.724548"
|
||||
height="14.787429"
|
||||
x="-206.37056"
|
||||
y="244.73143"
|
||||
ry="1.1090568" />
|
||||
width="80.681"
|
||||
height="13.446837"
|
||||
x="-225.7778"
|
||||
y="102.33925"
|
||||
ry="1.0085125" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-size:10.5833px;line-height:1.2;font-family:Audiowide;-inkscape-font-specification:Audiowide;white-space:pre;inline-size:199.93;stroke-width:0.264583"
|
||||
x="-79.596779"
|
||||
y="-115.14695"
|
||||
id="text41875"
|
||||
transform="matrix(1.2893387,0,0,1.2893387,-202.89709,353.8546)"><tspan
|
||||
transform="matrix(1.1724504,0,0,1.1724504,-222.61922,201.56959)"><tspan
|
||||
x="-79.596779"
|
||||
y="-115.14695"
|
||||
id="tspan5244"><tspan
|
||||
id="tspan14151"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5242">hyperfine
|
||||
id="tspan14149">hyperfine
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-102.16832"
|
||||
id="tspan5250"><tspan
|
||||
id="tspan14157"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#e35a00"
|
||||
id="tspan5246"> --warmup 2</tspan><tspan
|
||||
id="tspan14153"> --warmup 2</tspan><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5248">
|
||||
id="tspan14155">
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-89.189692"
|
||||
id="tspan5256"><tspan
|
||||
id="tspan14163"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#00bbeb"
|
||||
id="tspan5252"> --runs 3</tspan><tspan
|
||||
id="tspan14159"> --runs 3</tspan><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5254">
|
||||
id="tspan14161">
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-76.211062"
|
||||
id="tspan5260"><tspan
|
||||
id="tspan14167"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5258"> --setup <setup>
|
||||
id="tspan14165"> --setup <setup>
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-63.232432"
|
||||
id="tspan5264"><tspan
|
||||
id="tspan14171"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5262"> --prepare <prepare1>
|
||||
id="tspan14169"> --prepare <prepare1>
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-50.253802"
|
||||
id="tspan5270"><tspan
|
||||
id="tspan14177"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#ffffff"
|
||||
id="tspan5266"> <command1></tspan><tspan
|
||||
id="tspan14173"> <command1></tspan><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5268">
|
||||
id="tspan14175">
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-37.275172"
|
||||
id="tspan5274"><tspan
|
||||
id="tspan14181"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5272"> --prepare <prepare2>
|
||||
id="tspan14179"> --prepare <prepare2>
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-24.296542"
|
||||
id="tspan5278"><tspan
|
||||
id="tspan14185"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code';fill:#ffffff"
|
||||
id="tspan5276"> <command2>
|
||||
id="tspan14183"> <command2>
|
||||
</tspan></tspan><tspan
|
||||
x="-79.596779"
|
||||
y="-11.317913"
|
||||
id="tspan5282"><tspan
|
||||
id="tspan14189"><tspan
|
||||
style="font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5280"> --cleanup <cleanup></tspan></tspan></text>
|
||||
id="tspan14187"> --cleanup <cleanup></tspan></tspan></text>
|
||||
<rect
|
||||
style="fill:#80e5ff;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect6355"
|
||||
@@ -325,7 +325,7 @@
|
||||
transform="translate(34.591878,14.34874)"><tspan
|
||||
x="-16.304935"
|
||||
y="161.32399"
|
||||
id="tspan5284">3 benchmark runs</tspan></text>
|
||||
id="tspan14191">3 benchmark runs</tspan></text>
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect13386"
|
||||
@@ -590,7 +590,7 @@
|
||||
transform="translate(34.591878,257.76541)"><tspan
|
||||
x="-16.304935"
|
||||
y="161.32399"
|
||||
id="tspan5286">3 benchmark runs</tspan></text>
|
||||
id="tspan14193">3 benchmark runs</tspan></text>
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#000000;stroke-width:0.529167;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.80126;stroke-opacity:1;paint-order:stroke fill markers"
|
||||
id="rect272454"
|
||||
@@ -711,11 +711,11 @@
|
||||
x="-79.596779"
|
||||
y="-115.14695"
|
||||
id="text283737"
|
||||
transform="matrix(1.2893387,0,0,1.2893387,-225.35928,177.7847)"><tspan
|
||||
transform="matrix(1.7031443,0,0,1.7031443,-192.73007,228.6586)"><tspan
|
||||
x="-79.596779"
|
||||
y="-115.14695"
|
||||
id="tspan5290"><tspan
|
||||
style="font-weight:bold;font-family:'Fira Code';-inkscape-font-specification:'Fira Code'"
|
||||
id="tspan5288">benchmark execution order</tspan></tspan></text>
|
||||
id="tspan14197"><tspan
|
||||
style="font-weight:bold;font-family:'Segoe UI';-inkscape-font-specification:'Segoe UI Bold'"
|
||||
id="tspan14195">benchmark execution order</tspan></tspan></text>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 35 KiB After Width: | Height: | Size: 35 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
+2
-2
@@ -9,9 +9,9 @@ This folder contains scripts that can be used in combination with hyperfines `--
|
||||
|
||||
### Pre-requisites
|
||||
|
||||
To make these scripts work, you will need to install `numpy` and `matplotlib`. Install them via
|
||||
To make these scripts work, you will need to install `numpy`, `matplotlib` and `scipy`. Install them via
|
||||
your package manager or `pip`:
|
||||
|
||||
```bash
|
||||
pip install numpy matplotlib # pip3, if you are using python3
|
||||
pip install numpy matplotlib scipy # pip3, if you are using python3
|
||||
```
|
||||
@@ -23,6 +23,7 @@ for command, ts in zip(commands, times):
|
||||
iqr = p75 - p25
|
||||
|
||||
print("Command '{}'".format(command))
|
||||
print(" runs: {:8d}".format(len(ts)))
|
||||
print(" mean: {:8.3f} s".format(np.mean(ts)))
|
||||
print(" stddev: {:8.3f} s".format(np.std(ts, ddof=1)))
|
||||
print(" median: {:8.3f} s".format(np.median(ts)))
|
||||
|
||||
@@ -20,6 +20,12 @@ parser.add_argument(
|
||||
parser.add_argument(
|
||||
"-o", "--output", help="Save image to the given filename."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--t-min", metavar="T", help="Minimum time to be displayed (seconds)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--t-max", metavar="T", help="Maximum time to be displayed (seconds)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -32,8 +38,8 @@ else:
|
||||
labels = [b["command"] for b in results]
|
||||
all_times = [b["times"] for b in results]
|
||||
|
||||
t_min = np.min(list(map(np.min, all_times)))
|
||||
t_max = np.max(list(map(np.max, all_times)))
|
||||
t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
|
||||
t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))
|
||||
|
||||
bins = int(args.bins) if args.bins else "auto"
|
||||
histtype = args.type if args.type else "bar"
|
||||
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""This program shows `hyperfine` benchmark results in a sequential way
|
||||
in order to debug possible background interference, caching effects,
|
||||
thermal throttling and similar effects.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
|
||||
def moving_average(times, num_runs):
|
||||
times_padded = np.pad(
|
||||
times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge"
|
||||
)
|
||||
kernel = np.ones(num_runs) / num_runs
|
||||
return np.convolve(times_padded, kernel, mode="valid")
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
parser.add_argument(
|
||||
"-w",
|
||||
"--moving-average-width",
|
||||
type=int,
|
||||
metavar="num_runs",
|
||||
help="Width of the moving-average window (default: N/5)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.file) as f:
|
||||
results = json.load(f)["results"]
|
||||
|
||||
label = results[0]["command"]
|
||||
times = results[0]["times"]
|
||||
num = len(times)
|
||||
nums = range(num)
|
||||
|
||||
plt.scatter(x=nums, y=times, marker=".", color="orange")
|
||||
plt.ylim([0, None])
|
||||
plt.xlim([-1, num])
|
||||
|
||||
moving_average_width = (
|
||||
num // 5 if args.moving_average_width is None else args.moving_average_width
|
||||
)
|
||||
|
||||
moving_average = moving_average(times, moving_average_width)
|
||||
plt.plot(nums, moving_average, "-", color="blue")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.legend(labels=[label], loc="best", fontsize="medium")
|
||||
plt.ylabel("Time [s]")
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
@@ -64,6 +64,48 @@ fn run_command_and_measure_common(
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub struct RawExecutor<'a> {
|
||||
options: &'a Options,
|
||||
}
|
||||
|
||||
impl<'a> RawExecutor<'a> {
|
||||
pub fn new(options: &'a Options) -> Self {
|
||||
RawExecutor { options }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Executor for RawExecutor<'a> {
|
||||
fn run_command_and_measure(
|
||||
&self,
|
||||
command: &Command<'_>,
|
||||
command_failure_action: Option<CmdFailureAction>,
|
||||
) -> Result<(TimingResult, ExitStatus)> {
|
||||
let result = run_command_and_measure_common(
|
||||
command.get_command()?,
|
||||
command_failure_action.unwrap_or(self.options.command_failure_action),
|
||||
self.options.command_output_policy,
|
||||
&command.get_command_line(),
|
||||
)?;
|
||||
|
||||
Ok((
|
||||
TimingResult {
|
||||
time_real: result.time_real,
|
||||
time_user: result.time_user,
|
||||
time_system: result.time_system,
|
||||
},
|
||||
result.status,
|
||||
))
|
||||
}
|
||||
|
||||
fn calibrate(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn time_overhead(&self) -> Second {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ShellExecutor<'a> {
|
||||
options: &'a Options,
|
||||
shell: &'a Shell,
|
||||
|
||||
@@ -7,7 +7,7 @@ pub mod timing_result;
|
||||
use std::cmp;
|
||||
|
||||
use crate::command::Command;
|
||||
use crate::options::{CmdFailureAction, Options, OutputStyleOption};
|
||||
use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption};
|
||||
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
|
||||
use crate::output::format::{format_duration, format_duration_unit};
|
||||
use crate::output::progress_bar::get_progress_bar;
|
||||
@@ -320,7 +320,9 @@ impl<'a> Benchmark<'a> {
|
||||
let mut warnings = vec![];
|
||||
|
||||
// Check execution time
|
||||
if times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) {
|
||||
if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
|
||||
&& times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
|
||||
{
|
||||
warnings.push(Warnings::FastExecutionTime);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use colored::*;
|
||||
|
||||
use super::benchmark_result::BenchmarkResult;
|
||||
use super::executor::{Executor, MockExecutor, ShellExecutor};
|
||||
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
|
||||
use super::{relative_speed, Benchmark};
|
||||
|
||||
use crate::command::Commands;
|
||||
@@ -33,6 +33,7 @@ impl<'a> Scheduler<'a> {
|
||||
|
||||
pub fn run_benchmarks(&mut self) -> Result<()> {
|
||||
let mut executor: Box<dyn Executor> = match self.options.executor_kind {
|
||||
ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),
|
||||
ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),
|
||||
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
|
||||
};
|
||||
|
||||
+27
-13
@@ -1,30 +1,32 @@
|
||||
use std::ffi::OsString;
|
||||
|
||||
use clap::{crate_version, App, AppSettings, Arg, ArgMatches};
|
||||
use clap::{crate_version, AppSettings, Arg, ArgMatches, Command};
|
||||
|
||||
pub fn get_cli_arguments<'a, I, T>(args: I) -> ArgMatches
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: Into<OsString> + Clone + 'a,
|
||||
{
|
||||
let app = build_app();
|
||||
app.get_matches_from(args)
|
||||
let command = build_command();
|
||||
command.get_matches_from(args)
|
||||
}
|
||||
|
||||
/// Build the clap app for parsing command line arguments
|
||||
fn build_app() -> App<'static> {
|
||||
App::new("hyperfine")
|
||||
/// Build the clap command for parsing command line arguments
|
||||
fn build_command() -> Command<'static> {
|
||||
Command::new("hyperfine")
|
||||
.version(crate_version!())
|
||||
.setting(AppSettings::DeriveDisplayOrder)
|
||||
.setting(AppSettings::NextLineHelp)
|
||||
.setting(AppSettings::HidePossibleValues)
|
||||
.next_line_help(true)
|
||||
.hide_possible_values(true)
|
||||
.max_term_width(90)
|
||||
.about("A command-line benchmarking tool.")
|
||||
.arg(
|
||||
Arg::new("command")
|
||||
.help("Command to benchmark. This can be the name of an executable or a shell \
|
||||
command like \"sleep 0.5 && echo test\". If multiple commands are given, \
|
||||
hyperfine will show a comparison of the respective runtimes.")
|
||||
.help("The command to benchmark. This can be the name of an executable, a command \
|
||||
line like \"grep -i todo\" or a shell command like \"sleep 0.5 && echo test\". \
|
||||
The latter is only available if the shell is not explicitly disabled via \
|
||||
'--shell=none'. If multiple commands are given, hyperfine will show a \
|
||||
comparison of the respective runtimes.")
|
||||
.required(true)
|
||||
.multiple_occurrences(true)
|
||||
.forbid_empty_values(true),
|
||||
@@ -181,7 +183,19 @@ fn build_app() -> App<'static> {
|
||||
.takes_value(true)
|
||||
.value_name("SHELL")
|
||||
.overrides_with("shell")
|
||||
.help("Set the shell to use for executing benchmarked commands."),
|
||||
.help("Set the shell to use for executing benchmarked commands. This can be the \
|
||||
name or the path to the shell executable, or a full command line \
|
||||
like \"bash --norc\". It can also be set to \"default\" to explicitly select \
|
||||
the default shell on this platform. Finally, this can also be set to \
|
||||
\"none\" to disable the shell. In this case, commands will be executed \
|
||||
directly. They can still have arguments, but more complex things like \
|
||||
\"sleep 0.1; sleep 0.2\" are not possible without a shell.")
|
||||
)
|
||||
.arg(
|
||||
Arg::new("no-shell")
|
||||
.short('N')
|
||||
.conflicts_with_all(&["shell", "debug-mode"])
|
||||
.help("An alias for '--shell=none'.")
|
||||
)
|
||||
.arg(
|
||||
Arg::new("ignore-failure")
|
||||
@@ -259,5 +273,5 @@ fn build_app() -> App<'static> {
|
||||
|
||||
#[test]
|
||||
fn verify_app() {
|
||||
build_app().debug_assert();
|
||||
build_command().debug_assert();
|
||||
}
|
||||
+19
-4
@@ -15,7 +15,7 @@ use clap::ArgMatches;
|
||||
use crate::parameter::tokenize::tokenize;
|
||||
use crate::parameter::ParameterValue;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use anyhow::{bail, Context, Result};
|
||||
use clap::Values;
|
||||
use rust_decimal::Decimal;
|
||||
|
||||
@@ -64,6 +64,21 @@ impl<'a> Command<'a> {
|
||||
self.replace_parameters_in(self.expression)
|
||||
}
|
||||
|
||||
pub fn get_command(&self) -> Result<std::process::Command> {
|
||||
let command_line = self.get_command_line();
|
||||
let mut tokens = shell_words::split(&command_line)
|
||||
.with_context(|| format!("Failed to parse command '{}'", command_line))?
|
||||
.into_iter();
|
||||
|
||||
if let Some(program_name) = tokens.next() {
|
||||
let mut command_builder = std::process::Command::new(program_name);
|
||||
command_builder.args(tokens);
|
||||
Ok(command_builder)
|
||||
} else {
|
||||
bail!("Can not execute empty command")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_parameters(&self) -> &[(&'a str, ParameterValue)] {
|
||||
&self.parameters
|
||||
}
|
||||
@@ -351,7 +366,7 @@ impl<'a> fmt::Display for Command<'a> {
|
||||
|
||||
#[test]
|
||||
fn test_build_commands_cross_product() {
|
||||
use crate::app::get_cli_arguments;
|
||||
use crate::cli::get_cli_arguments;
|
||||
|
||||
let matches = get_cli_arguments(vec![
|
||||
"hyperfine",
|
||||
@@ -389,7 +404,7 @@ fn test_build_commands_cross_product() {
|
||||
|
||||
#[test]
|
||||
fn test_build_parameter_list_commands() {
|
||||
use crate::app::get_cli_arguments;
|
||||
use crate::cli::get_cli_arguments;
|
||||
|
||||
let matches = get_cli_arguments(vec![
|
||||
"hyperfine",
|
||||
@@ -410,7 +425,7 @@ fn test_build_parameter_list_commands() {
|
||||
|
||||
#[test]
|
||||
fn test_build_parameter_scan_commands() {
|
||||
use crate::app::get_cli_arguments;
|
||||
use crate::cli::get_cli_arguments;
|
||||
let matches = get_cli_arguments(vec![
|
||||
"hyperfine",
|
||||
"echo {val}",
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
use std::env;
|
||||
|
||||
use app::get_cli_arguments;
|
||||
use benchmark::scheduler::Scheduler;
|
||||
use cli::get_cli_arguments;
|
||||
use command::Commands;
|
||||
use export::ExportManager;
|
||||
use options::Options;
|
||||
@@ -9,8 +9,8 @@ use options::Options;
|
||||
use anyhow::Result;
|
||||
use colored::*;
|
||||
|
||||
pub mod app;
|
||||
pub mod benchmark;
|
||||
pub mod cli;
|
||||
pub mod command;
|
||||
pub mod error;
|
||||
pub mod export;
|
||||
|
||||
+12
-6
@@ -134,6 +134,7 @@ impl CommandOutputPolicy {
|
||||
}
|
||||
|
||||
pub enum ExecutorKind {
|
||||
Raw,
|
||||
Shell(Shell),
|
||||
Mock(Option<String>),
|
||||
}
|
||||
@@ -281,12 +282,17 @@ impl Options {
|
||||
OutputStyleOption::Disabled => {}
|
||||
};
|
||||
|
||||
options.executor_kind = match (matches.is_present("debug-mode"), matches.value_of("shell"))
|
||||
{
|
||||
(false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?),
|
||||
(false, None) => ExecutorKind::Shell(Shell::default()),
|
||||
(true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
|
||||
(true, None) => ExecutorKind::Mock(None),
|
||||
options.executor_kind = if matches.is_present("no-shell") {
|
||||
ExecutorKind::Raw
|
||||
} else {
|
||||
match (matches.is_present("debug-mode"), matches.value_of("shell")) {
|
||||
(false, Some(shell)) if shell == "default" => ExecutorKind::Shell(Shell::default()),
|
||||
(false, Some(shell)) if shell == "none" => ExecutorKind::Raw,
|
||||
(false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?),
|
||||
(false, None) => ExecutorKind::Shell(Shell::default()),
|
||||
(true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
|
||||
(true, None) => ExecutorKind::Mock(None),
|
||||
}
|
||||
};
|
||||
|
||||
if matches.is_present("ignore-failure") {
|
||||
|
||||
@@ -17,7 +17,10 @@ impl fmt::Display for Warnings {
|
||||
match *self {
|
||||
Warnings::FastExecutionTime => write!(
|
||||
f,
|
||||
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
|
||||
"Command took less than {:.0} ms to complete. Note that the results might be \
|
||||
inaccurate because hyperfine can not calibrate the shell startup time much \
|
||||
more precise than this limit. You can try to use the `-N`/`--shell=none` \
|
||||
option to disable the shell completely.",
|
||||
MIN_EXECUTION_TIME * 1e3
|
||||
),
|
||||
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
|
||||
|
||||
@@ -10,7 +10,7 @@ pub fn hyperfine_debug() -> assert_cmd::Command {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hyperfine_runs_successfully() {
|
||||
fn runs_successfully() {
|
||||
hyperfine()
|
||||
.arg("--runs=2")
|
||||
.arg("echo dummy benchmark")
|
||||
@@ -27,6 +27,18 @@ fn one_run_is_supported() {
|
||||
.success();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn can_run_commands_without_a_shell() {
|
||||
hyperfine()
|
||||
.arg("--runs=1")
|
||||
.arg("--show-output")
|
||||
.arg("--shell=none")
|
||||
.arg("echo 'hello world' argument2")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("hello world argument2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fails_with_wrong_number_of_command_name_arguments() {
|
||||
hyperfine()
|
||||
@@ -90,6 +102,33 @@ fn fails_for_unknown_command() {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fails_for_unknown_command_without_shell() {
|
||||
hyperfine()
|
||||
.arg("--shell=none")
|
||||
.arg("--runs=1")
|
||||
.arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Failed to run command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577'",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn fails_for_failing_command_without_shell() {
|
||||
hyperfine()
|
||||
.arg("--shell=none")
|
||||
.arg("--runs=1")
|
||||
.arg("false")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Command terminated with non-zero exit code",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fails_for_unknown_setup_command() {
|
||||
hyperfine()
|
||||
|
||||
Reference in new issue
Block a user