mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-14 08:21:39 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 David Peter
|
9c4612401a | ||
|
David Peter
|
cdbca1bf1e |
No files matched your search
@@ -89,7 +89,6 @@ jobs:
|
||||
- { target: aarch64-apple-darwin , os: macos-15 }
|
||||
# - { target: x86_64-pc-windows-gnu , os: windows-2022 }
|
||||
- { target: x86_64-pc-windows-msvc , os: windows-2022 }
|
||||
- { target: aarch64-pc-windows-msvc , os: windows-11-arm }
|
||||
- { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
|
||||
- { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
|
||||
env:
|
||||
@@ -161,7 +160,7 @@ jobs:
|
||||
run: |
|
||||
# test only library unit tests and binary for arm-type targets
|
||||
unset CARGO_TEST_OPTIONS
|
||||
unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ needs.crate_metadata.outputs.bin-name }}" ;; esac;
|
||||
unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ steps.bin.outputs.BIN_NAME }}" ;; esac;
|
||||
echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Run tests
|
||||
|
||||
+5
-5
@@ -5,7 +5,7 @@
|
||||
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
|
||||
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
|
||||
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
|
||||
- Python scripts: Add new `plot_benchmark_comparison.py` script for plotting collections of benchmarks, see #806 (@marxin)
|
||||
- Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
|
||||
|
||||
## Bugfixes
|
||||
|
||||
@@ -99,7 +99,7 @@
|
||||
## Other
|
||||
|
||||
- Thanks to @berombau for working on dependency upgrades, see #584
|
||||
- Fixed installation on Windows, see #595 and #596 (@AntoniosBarotsis)
|
||||
- Fixed installationm on Windows, see #595 and #596 (@AntoniosBarotsis)
|
||||
|
||||
|
||||
# v1.15.0
|
||||
@@ -244,10 +244,10 @@ Package authors: note that Hyperfine now comes with a set of shell completion fi
|
||||
- The new `--parameter-list <VAR> <VALUES>` option can be used to run
|
||||
a parametrized benchmark on a user-specified list of values.
|
||||
This is similar to `--parameter-scan <VAR> <MIN> <MAX>`, but doesn't
|
||||
necessarily require numeric arguments.
|
||||
necessarily required numeric arguments.
|
||||
|
||||
``` bash
|
||||
hyperfine --parameter-list compiler "g++,clang++" \
|
||||
hyperfine --parameter-list compiler "gcc,clang" \
|
||||
"{compiler} -O2 main.cpp"
|
||||
```
|
||||
|
||||
@@ -415,7 +415,7 @@ for reviewing pull requests and for giving very valuable feedback.
|
||||
## Other
|
||||
|
||||
- [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2).
|
||||
- Ubuntu/Debian packages are now available.
|
||||
- Ubuntu/Debian packages are now are available.
|
||||
|
||||
# v0.2.0
|
||||
|
||||
|
||||
Generated
+247
-3
@@ -97,12 +97,27 @@ dependencies = [
|
||||
"wait-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
@@ -216,6 +231,15 @@ version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
||||
|
||||
[[package]]
|
||||
name = "cloudabi"
|
||||
version = "0.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
@@ -328,6 +352,12 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-cprng"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -391,6 +421,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"shell-words",
|
||||
"statistical",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -477,7 +508,7 @@ version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -489,13 +520,79 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
|
||||
|
||||
[[package]]
|
||||
name = "num"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-iter",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-iter"
|
||||
version = "0.1.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"autocfg 1.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -600,6 +697,25 @@ version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
|
||||
dependencies = [
|
||||
"autocfg 0.1.8",
|
||||
"libc",
|
||||
"rand_chacha 0.1.1",
|
||||
"rand_core 0.4.3",
|
||||
"rand_hc",
|
||||
"rand_isaac",
|
||||
"rand_jitter",
|
||||
"rand_os",
|
||||
"rand_pcg",
|
||||
"rand_xorshift",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.8"
|
||||
@@ -632,6 +748,16 @@ dependencies = [
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
|
||||
dependencies = [
|
||||
"autocfg 0.1.8",
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
@@ -652,6 +778,21 @@ dependencies = [
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96f815e01bbd9678b50d927f79aa1cf3ffdfdb1b9787317c1284dadb894ad0e8"
|
||||
dependencies = [
|
||||
"rand_core 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e5937858e6fd18cd595d558f90bb5de3b72ae23f9e3763af0e805949b04ef60"
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
@@ -676,6 +817,77 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_isaac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_jitter"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_core 0.4.3",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_os"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
|
||||
dependencies = [
|
||||
"cloudabi",
|
||||
"fuchsia-cprng",
|
||||
"libc",
|
||||
"rand_core 0.4.3",
|
||||
"rdrand",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_pcg"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
|
||||
dependencies = [
|
||||
"autocfg 0.1.8",
|
||||
"rand_core 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_xorshift"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rdrand"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
@@ -728,7 +940,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
@@ -802,6 +1014,16 @@ version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
|
||||
|
||||
[[package]]
|
||||
name = "statistical"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49d57902bb128e5e38b5218d3681215ae3e322d99f65d5420e9849730d2ea372"
|
||||
dependencies = [
|
||||
"num",
|
||||
"rand 0.6.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
@@ -1003,6 +1225,28 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
|
||||
@@ -20,6 +20,7 @@ windows_process_extensions_main_thread_handle = []
|
||||
colored = "3.1"
|
||||
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
|
||||
indicatif = "=0.17.4"
|
||||
statistical = "1.0"
|
||||
csv = "1.4"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
|
||||
@@ -33,9 +33,8 @@ hyperfine 'sleep 0.3'
|
||||
```
|
||||
|
||||
Hyperfine will automatically determine the number of runs to perform for each command. By default,
|
||||
it will perform *at least* 10 benchmarking runs and estimate a run count targeting roughly
|
||||
3 seconds, including shell overhead and preparation and conclusion commands. To set an exact
|
||||
number of runs, you can use the `-r`/`--runs` option:
|
||||
it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
|
||||
you can use the `-r`/`--runs` option:
|
||||
```sh
|
||||
hyperfine --runs 5 'sleep 0.3'
|
||||
```
|
||||
@@ -57,8 +56,8 @@ hyperfine --warmup 3 'grep -R TODO *'
|
||||
```
|
||||
|
||||
Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`
|
||||
option to run a special command before *each* timing run. For example, to clear Linux filesystem caches,
|
||||
you can run
|
||||
option to run a special command before *each* timing run. For example, to clear harddisk caches
|
||||
on Linux, you can run
|
||||
```sh
|
||||
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
|
||||
```
|
||||
@@ -85,12 +84,12 @@ This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
|
||||
For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
|
||||
option:
|
||||
```
|
||||
hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'
|
||||
hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
|
||||
```
|
||||
|
||||
### Intermediate shell
|
||||
|
||||
By default, commands are executed using `sh` on Unix (resolved through `PATH`) or `cmd.exe` on Windows.
|
||||
By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
|
||||
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
|
||||
```sh
|
||||
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
|
||||
@@ -213,6 +212,13 @@ cave resolve -x repository/rust
|
||||
cave resolve -x hyperfine
|
||||
```
|
||||
|
||||
### On Funtoo Linux
|
||||
|
||||
On Funtoo Linux, hyperfine can be installed [from core-kit](https://github.com/funtoo/core-kit/tree/1.4-release/app-benchmarks/hyperfine):
|
||||
```
|
||||
emerge app-benchmarks/hyperfine
|
||||
```
|
||||
|
||||
### On NixOS
|
||||
|
||||
On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine):
|
||||
@@ -328,7 +334,7 @@ The name *hyperfine* was chosen in reference to the hyperfine levels of caesium
|
||||
|
||||
## Citing hyperfine
|
||||
|
||||
Thank you for considering citing hyperfine in your research work. Please see the information
|
||||
Thank you for considering to cite hyperfine in your research work. Please see the information
|
||||
in the sidebar on how to properly cite hyperfine.
|
||||
|
||||
## License
|
||||
|
||||
+14
-32
@@ -3,8 +3,7 @@
|
||||
hyperfine \- command\-line benchmarking tool
|
||||
.SH SYNOPSIS
|
||||
.B hyperfine
|
||||
.RB [ \-hVN ]
|
||||
.RB [ \-\-ignore\-failure[=MODE] ]
|
||||
.RB [ \-ihVN ]
|
||||
.RB [ \-\-warmup
|
||||
.IR NUM ]
|
||||
.RB [ \-\-min\-runs
|
||||
@@ -15,10 +14,6 @@ hyperfine \- command\-line benchmarking tool
|
||||
.IR NUM ]
|
||||
.RB [ \-\-setup
|
||||
.IR CMD ]
|
||||
.RB [ \-\-reference
|
||||
.IR CMD ]
|
||||
.RB [ \-\-reference\-name
|
||||
.IR NAME ]
|
||||
.RB [ \-\-prepare
|
||||
.IR CMD ]
|
||||
.RB [ \-\-conclude
|
||||
@@ -110,20 +105,10 @@ Perform exactly \fINUM\fP runs for each command. If this option is not specified
|
||||
\fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP
|
||||
.IP
|
||||
Execute \fICMD\fP once before each set of timing runs. This is useful
|
||||
for compiling your software with the provided parameters, or to do any
|
||||
for compiling your software or with the provided parameters, or to do any
|
||||
other work that should happen once before a series of benchmark runs,
|
||||
not every time as would happen with the \fB\-\-prepare\fR option.
|
||||
.HP
|
||||
\fB\-\-reference\fR \fICMD\fP
|
||||
.IP
|
||||
Benchmark \fICMD\fP as the reference for the relative comparison of results.
|
||||
If this option is not specified, the fastest command is used as the reference.
|
||||
.HP
|
||||
\fB\-\-reference\-name\fR \fINAME\fP
|
||||
.IP
|
||||
Give a meaningful \fINAME\fP to the reference command.
|
||||
Requires \fB\-\-reference\fR.
|
||||
.HP
|
||||
\fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP
|
||||
.IP
|
||||
Execute \fICMD\fP before each timing run. This is useful for clearing disk caches,
|
||||
@@ -132,7 +117,7 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
|
||||
once for each command. In the latter case, each preparation command will be
|
||||
run prior to the corresponding benchmark command.
|
||||
.HP
|
||||
\fB\-C\fR, \fB\-\-conclude\fR \fICMD...\fP
|
||||
\fB\-\-conclude\fR \fICMD...\fP
|
||||
.IP
|
||||
Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
|
||||
for example.
|
||||
@@ -144,8 +129,8 @@ run after the corresponding benchmark command.
|
||||
.IP
|
||||
Execute \fICMD\fP after the completion of all benchmarking runs for each individual
|
||||
command to be benchmarked. This is useful if the commands to be benchmarked
|
||||
produce artifacts that need to be cleaned up. It only runs once per series of
|
||||
benchmark runs, as opposed to the \fB\-\-conclude\fR option which runs after
|
||||
produce artifacts that need to be cleaned up. It only runs once a series of
|
||||
benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
|
||||
every run.
|
||||
.HP
|
||||
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
|
||||
@@ -156,14 +141,14 @@ string '{\fIVAR\fP}' in each command by the current parameter value.
|
||||
.RS
|
||||
Example:
|
||||
.RS
|
||||
\fBhyperfine\fR \fB\-\-prepare\fR 'make clean' \fB\-P\fR threads 1 8 'make \-j {threads}'
|
||||
\fBhyperfine\fR \fB\-P\fR threads 1 8 'make \-j {threads}'
|
||||
.RE
|
||||
.RE
|
||||
.IP
|
||||
This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'.
|
||||
.IP
|
||||
To have the value increase following different patterns, use shell
|
||||
arithmetic.
|
||||
arithmetics.
|
||||
.IP
|
||||
.RS
|
||||
Example:
|
||||
@@ -199,11 +184,11 @@ Replaces the string '{\fIVAR\fP}' in each command by the current parameter value
|
||||
.RS
|
||||
Example:
|
||||
.RS
|
||||
\fBhyperfine\fR \fB\-L\fR compiler g++,clang++ '{compiler} \-O2 main.cpp'
|
||||
\fBhyperfine\fR \fB\-L\fR compiler gcc,clang '{compiler} \-O2 main.cpp'
|
||||
.RE
|
||||
.RE
|
||||
.IP
|
||||
This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'.
|
||||
This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'.
|
||||
.IP
|
||||
The option can be specified multiple times to run benchmarks for all
|
||||
possible parameter combinations.
|
||||
@@ -222,13 +207,9 @@ things like "sleep 0.1; sleep 0.2" are not possible without a shell.
|
||||
.IP
|
||||
An alias for '\-\-shell=none'.
|
||||
.HP
|
||||
\fB\-i\fR, \fB\-\-ignore\-failure\fR[=\fIMODE\fP]
|
||||
\fB\-i\fR, \fB\-\-ignore\-failure\fR
|
||||
.IP
|
||||
Ignore failures of the benchmarked programs. Without a value or with
|
||||
\fBall\-non\-zero\fR, all non\-zero exit codes are ignored.
|
||||
Alternatively, provide a comma\-separated list of exit codes to ignore,
|
||||
for example \fB\-\-ignore\-failure=1,2\fR.
|
||||
An explicit value must be attached to the option with an equals sign.
|
||||
Ignore non\-zero exit codes of the benchmarked programs.
|
||||
.HP
|
||||
\fB\-\-style\fR \fITYPE\fP
|
||||
.IP
|
||||
@@ -328,9 +309,9 @@ Read from \fI\,/dev/null\/\fP (the default).
|
||||
Read the input from the given file.
|
||||
.RE
|
||||
.HP
|
||||
\fB\-n\fR, \fB\-\-command\-name\fR \fINAME\fP
|
||||
\fB\-n\fR, \fB\-\-command\-name\fR \fiNAME\fP
|
||||
.IP
|
||||
Give a meaningful \fINAME\fP to a command. This can be specified multiple times
|
||||
Give a meaningful \fiNAME\fP to a command. This can be specified multiple times
|
||||
if several commands are benchmarked.
|
||||
.HP
|
||||
\fB\-h\fR, \fB\-\-help\fR
|
||||
@@ -382,6 +363,7 @@ Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fB\-\-conclude\fR,
|
||||
'echo command n={n}'
|
||||
.fi
|
||||
.RE
|
||||
.RE
|
||||
.SH AUTHOR
|
||||
.LP
|
||||
David Peter <mail@david-peter.de>
|
||||
|
||||
+17
-8
@@ -1,17 +1,26 @@
|
||||
This folder contains scripts that can be used in combination with hyperfine's `--export-json` option.
|
||||
This folder contains scripts that can be used in combination with hyperfines `--export-json` option.
|
||||
|
||||
### Example:
|
||||
|
||||
```bash
|
||||
hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
|
||||
./plot_whisker.py sleep.json
|
||||
```
|
||||
|
||||
### Pre-requisites
|
||||
|
||||
To make these scripts work, you will need `numpy`, `matplotlib` and `scipy`.
|
||||
|
||||
If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/)
|
||||
inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx),
|
||||
you can directly run the scripts using
|
||||
|
||||
```bash
|
||||
uv run plot_whisker.py sleep.json
|
||||
```
|
||||
|
||||
### Prerequisites
|
||||
Otherwise, install the dependencies via your system package manager or using `pip`:
|
||||
|
||||
Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
|
||||
commands above from this directory.
|
||||
|
||||
The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
|
||||
inline script metadata. `uv run` automatically installs these dependencies in an
|
||||
isolated environment before running the script.
|
||||
```bash
|
||||
pip install numpy matplotlib scipy # pip3, if you are using python3
|
||||
```
|
||||
@@ -7,7 +7,8 @@
|
||||
# ///
|
||||
|
||||
"""This script performs Welch's t-test on a JSON export file with two
|
||||
benchmark results to test for a difference in their population mean runtimes."""
|
||||
benchmark results to test whether or not the two distributions are
|
||||
the same."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
@@ -39,10 +40,6 @@ print(f"t = {t:.3}, p = {p:.3}")
|
||||
print()
|
||||
|
||||
if dispose:
|
||||
print(
|
||||
f"A statistically significant difference in mean runtime was detected (p < {th})."
|
||||
)
|
||||
print(f"There is a difference between the two benchmarks (p < {th}).")
|
||||
else:
|
||||
print(
|
||||
f"No statistically significant difference in mean runtime was detected (p >= {th})."
|
||||
)
|
||||
print(f"The two benchmarks are almost the same (p >= {th}).")
|
||||
+40
-21
@@ -7,14 +7,14 @@ use crate::options::{
|
||||
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
|
||||
};
|
||||
use crate::output::progress_bar::get_progress_bar;
|
||||
use crate::timer::execute_and_measure;
|
||||
use crate::timer::{execute_and_measure, TimerResult};
|
||||
use crate::util::randomized_environment_offset;
|
||||
use crate::util::statistics::mean;
|
||||
use crate::util::units::Second;
|
||||
|
||||
use super::measurement::Measurement;
|
||||
use super::timing_result::TimingResult;
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use statistical::mean;
|
||||
|
||||
pub enum BenchmarkIteration {
|
||||
NonBenchmarkRun,
|
||||
@@ -40,7 +40,7 @@ pub trait Executor {
|
||||
iteration: BenchmarkIteration,
|
||||
command_failure_action: Option<CmdFailureAction>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement>;
|
||||
) -> Result<(TimingResult, ExitStatus)>;
|
||||
|
||||
/// Perform a calibration of this executor. For example,
|
||||
/// when running commands through a shell, we need to
|
||||
@@ -62,7 +62,7 @@ fn run_command_and_measure_common(
|
||||
command_input_policy: &CommandInputPolicy,
|
||||
command_output_policy: &CommandOutputPolicy,
|
||||
command_name: &str,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<TimerResult> {
|
||||
let stdin = command_input_policy.get_stdin()?;
|
||||
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
|
||||
command.stdin(stdin).stdout(stdout).stderr(stderr);
|
||||
@@ -136,15 +136,25 @@ impl Executor for RawExecutor<'_> {
|
||||
iteration: BenchmarkIteration,
|
||||
command_failure_action: Option<CmdFailureAction>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
run_command_and_measure_common(
|
||||
) -> Result<(TimingResult, ExitStatus)> {
|
||||
let result = run_command_and_measure_common(
|
||||
command.get_command()?,
|
||||
iteration,
|
||||
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
|
||||
&self.options.command_input_policy,
|
||||
output_policy,
|
||||
&command.get_command_line(),
|
||||
)
|
||||
)?;
|
||||
|
||||
Ok((
|
||||
TimingResult {
|
||||
time_real: result.time_real,
|
||||
time_user: result.time_user,
|
||||
time_system: result.time_system,
|
||||
memory_usage_byte: result.memory_usage_byte,
|
||||
},
|
||||
result.status,
|
||||
))
|
||||
}
|
||||
|
||||
fn calibrate(&mut self) -> Result<()> {
|
||||
@@ -159,7 +169,7 @@ impl Executor for RawExecutor<'_> {
|
||||
pub struct ShellExecutor<'a> {
|
||||
options: &'a Options,
|
||||
shell: &'a Shell,
|
||||
shell_spawning_time: Option<Measurement>,
|
||||
shell_spawning_time: Option<TimingResult>,
|
||||
}
|
||||
|
||||
impl<'a> ShellExecutor<'a> {
|
||||
@@ -179,7 +189,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
iteration: BenchmarkIteration,
|
||||
command_failure_action: Option<CmdFailureAction>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<(TimingResult, ExitStatus)> {
|
||||
let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
|
||||
let mut command_builder = self.shell.command();
|
||||
command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
|
||||
@@ -208,7 +218,15 @@ impl Executor for ShellExecutor<'_> {
|
||||
result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
Ok((
|
||||
TimingResult {
|
||||
time_real: result.time_real,
|
||||
time_user: result.time_user,
|
||||
time_system: result.time_system,
|
||||
memory_usage_byte: result.memory_usage_byte,
|
||||
},
|
||||
result.status,
|
||||
))
|
||||
}
|
||||
|
||||
/// Measure the average shell spawning time
|
||||
@@ -250,7 +268,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
shell_cmd
|
||||
);
|
||||
}
|
||||
Ok(r) => {
|
||||
Ok((r, _)) => {
|
||||
times_real.push(r.time_real);
|
||||
times_user.push(r.time_user);
|
||||
times_system.push(r.time_system);
|
||||
@@ -266,12 +284,11 @@ impl Executor for ShellExecutor<'_> {
|
||||
bar.finish_and_clear()
|
||||
}
|
||||
|
||||
self.shell_spawning_time = Some(Measurement {
|
||||
self.shell_spawning_time = Some(TimingResult {
|
||||
time_real: mean(×_real),
|
||||
time_user: mean(×_user),
|
||||
time_system: mean(×_system),
|
||||
memory_usage_byte: 0,
|
||||
status: ExitStatus::default(),
|
||||
});
|
||||
|
||||
Ok(())
|
||||
@@ -309,7 +326,7 @@ impl Executor for MockExecutor {
|
||||
_iteration: BenchmarkIteration,
|
||||
_command_failure_action: Option<CmdFailureAction>,
|
||||
_output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<(TimingResult, ExitStatus)> {
|
||||
#[cfg(unix)]
|
||||
let status = {
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
@@ -322,13 +339,15 @@ impl Executor for MockExecutor {
|
||||
ExitStatus::from_raw(0)
|
||||
};
|
||||
|
||||
Ok(Measurement {
|
||||
time_real: Self::extract_time(command.get_command_line()),
|
||||
time_user: 0.0,
|
||||
time_system: 0.0,
|
||||
memory_usage_byte: 0,
|
||||
Ok((
|
||||
TimingResult {
|
||||
time_real: Self::extract_time(command.get_command_line()),
|
||||
time_user: 0.0,
|
||||
time_system: 0.0,
|
||||
memory_usage_byte: 0,
|
||||
},
|
||||
status,
|
||||
})
|
||||
))
|
||||
}
|
||||
|
||||
fn calibrate(&mut self) -> Result<()> {
|
||||
|
||||
+19
-17
@@ -1,8 +1,8 @@
|
||||
pub mod benchmark_result;
|
||||
pub mod executor;
|
||||
pub mod measurement;
|
||||
pub mod relative_speed;
|
||||
pub mod scheduler;
|
||||
pub mod timing_result;
|
||||
|
||||
use std::cmp;
|
||||
use std::io::{self, Write};
|
||||
@@ -19,13 +19,14 @@ use crate::output::progress_bar::get_progress_bar;
|
||||
use crate::output::warnings::{OutlierWarningOptions, Warnings};
|
||||
use crate::parameter::ParameterNameAndValue;
|
||||
use crate::util::exit_code::extract_exit_code;
|
||||
use crate::util::statistics::{max, mean, median, min, standard_deviation};
|
||||
use crate::util::min_max::{max, min};
|
||||
use crate::util::units::Second;
|
||||
use benchmark_result::BenchmarkResult;
|
||||
use measurement::Measurement;
|
||||
use timing_result::TimingResult;
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use colored::*;
|
||||
use statistical::{mean, median, standard_deviation};
|
||||
|
||||
use self::executor::Executor;
|
||||
|
||||
@@ -60,7 +61,7 @@ impl<'a> Benchmark<'a> {
|
||||
command: &Command<'_>,
|
||||
error_output: &'static str,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<TimingResult> {
|
||||
self.executor
|
||||
.run_command_and_measure(
|
||||
command,
|
||||
@@ -68,6 +69,7 @@ impl<'a> Benchmark<'a> {
|
||||
Some(CmdFailureAction::RaiseError),
|
||||
output_policy,
|
||||
)
|
||||
.map(|r| r.0)
|
||||
.map_err(|_| anyhow!(error_output))
|
||||
}
|
||||
|
||||
@@ -76,7 +78,7 @@ impl<'a> Benchmark<'a> {
|
||||
&self,
|
||||
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<TimingResult> {
|
||||
let command = self
|
||||
.options
|
||||
.setup_command
|
||||
@@ -97,7 +99,7 @@ impl<'a> Benchmark<'a> {
|
||||
&self,
|
||||
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<TimingResult> {
|
||||
let command = self
|
||||
.options
|
||||
.cleanup_command
|
||||
@@ -118,7 +120,7 @@ impl<'a> Benchmark<'a> {
|
||||
&self,
|
||||
command: &Command<'_>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<TimingResult> {
|
||||
let error_output = "The preparation command terminated with a non-zero exit code. \
|
||||
Append ' || true' to the command if you are sure that this can be ignored.";
|
||||
|
||||
@@ -130,7 +132,7 @@ impl<'a> Benchmark<'a> {
|
||||
&self,
|
||||
command: &Command<'_>,
|
||||
output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
) -> Result<TimingResult> {
|
||||
let error_output = "The conclusion command terminated with a non-zero exit code. \
|
||||
Append ' || true' to the command if you are sure that this can be ignored.";
|
||||
|
||||
@@ -245,13 +247,13 @@ impl<'a> Benchmark<'a> {
|
||||
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
|
||||
|
||||
// Initial timing run
|
||||
let res = self.executor.run_command_and_measure(
|
||||
let (res, status) = self.executor.run_command_and_measure(
|
||||
self.command,
|
||||
BenchmarkIteration::Benchmark(0),
|
||||
None,
|
||||
output_policy,
|
||||
)?;
|
||||
let success = res.status.success();
|
||||
let success = status.success();
|
||||
|
||||
let conclusion_result = run_conclusion_command()?;
|
||||
let conclusion_overhead =
|
||||
@@ -282,7 +284,7 @@ impl<'a> Benchmark<'a> {
|
||||
times_user.push(res.time_user);
|
||||
times_system.push(res.time_system);
|
||||
memory_usage_byte.push(res.memory_usage_byte);
|
||||
exit_codes.push(extract_exit_code(res.status));
|
||||
exit_codes.push(extract_exit_code(status));
|
||||
|
||||
all_succeeded = all_succeeded && success;
|
||||
|
||||
@@ -307,19 +309,19 @@ impl<'a> Benchmark<'a> {
|
||||
bar.set_message(msg.to_owned())
|
||||
}
|
||||
|
||||
let res = self.executor.run_command_and_measure(
|
||||
let (res, status) = self.executor.run_command_and_measure(
|
||||
self.command,
|
||||
BenchmarkIteration::Benchmark(i + 1),
|
||||
None,
|
||||
output_policy,
|
||||
)?;
|
||||
let success = res.status.success();
|
||||
let success = status.success();
|
||||
|
||||
times_real.push(res.time_real);
|
||||
times_user.push(res.time_user);
|
||||
times_system.push(res.time_system);
|
||||
memory_usage_byte.push(res.memory_usage_byte);
|
||||
exit_codes.push(extract_exit_code(res.status));
|
||||
exit_codes.push(extract_exit_code(status));
|
||||
|
||||
all_succeeded = all_succeeded && success;
|
||||
|
||||
@@ -338,13 +340,13 @@ impl<'a> Benchmark<'a> {
|
||||
let t_num = times_real.len();
|
||||
let t_mean = mean(×_real);
|
||||
let t_stddev = if times_real.len() > 1 {
|
||||
Some(standard_deviation(×_real, t_mean))
|
||||
Some(standard_deviation(×_real, Some(t_mean)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let t_median = median(×_real);
|
||||
let t_min = min(times_real.iter().copied());
|
||||
let t_max = max(times_real.iter().copied());
|
||||
let t_min = min(×_real);
|
||||
let t_max = max(×_real);
|
||||
|
||||
let user_mean = mean(×_user);
|
||||
let system_mean = mean(×_system);
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
use std::process::ExitStatus;
|
||||
|
||||
use crate::util::units::Second;
|
||||
|
||||
/// Performance measurements and exit status from running a single command
|
||||
/// Results from timing a single command
|
||||
#[derive(Debug, Default, Copy, Clone)]
|
||||
pub struct Measurement {
|
||||
pub struct TimingResult {
|
||||
/// Wall clock time
|
||||
pub time_real: Second,
|
||||
|
||||
@@ -16,7 +14,4 @@ pub struct Measurement {
|
||||
|
||||
/// Maximum amount of memory used, in bytes
|
||||
pub memory_usage_byte: u64,
|
||||
|
||||
/// The exit status of the process
|
||||
pub status: ExitStatus,
|
||||
}
|
||||
+3
-7
@@ -41,7 +41,6 @@ fn build_command() -> Command {
|
||||
.short('w')
|
||||
.value_name("NUM")
|
||||
.action(ArgAction::Set)
|
||||
.value_parser(clap::value_parser!(u64))
|
||||
.help(
|
||||
"Perform NUM warmup runs before the actual benchmark. This can be used \
|
||||
to fill (disk) caches for I/O-heavy programs.",
|
||||
@@ -53,7 +52,6 @@ fn build_command() -> Command {
|
||||
.short('m')
|
||||
.action(ArgAction::Set)
|
||||
.value_name("NUM")
|
||||
.value_parser(clap::value_parser!(u64).range(1..))
|
||||
.help("Perform at least NUM runs for each command (default: 10)."),
|
||||
)
|
||||
.arg(
|
||||
@@ -62,7 +60,6 @@ fn build_command() -> Command {
|
||||
.short('M')
|
||||
.action(ArgAction::Set)
|
||||
.value_name("NUM")
|
||||
.value_parser(clap::value_parser!(u64).range(1..))
|
||||
.help("Perform at most NUM runs for each command. By default, there is no limit."),
|
||||
)
|
||||
.arg(
|
||||
@@ -72,7 +69,6 @@ fn build_command() -> Command {
|
||||
.short('r')
|
||||
.action(ArgAction::Set)
|
||||
.value_name("NUM")
|
||||
.value_parser(clap::value_parser!(u64).range(1..))
|
||||
.help("Perform exactly NUM runs for each command. If this option is not specified, \
|
||||
hyperfine automatically determines the number of runs."),
|
||||
)
|
||||
@@ -165,7 +161,7 @@ fn build_command() -> Command {
|
||||
.help(
|
||||
"Perform benchmark runs for each value in the range MIN..MAX. Replaces the \
|
||||
string '{VAR}' in each command by the current parameter value.\n\n \
|
||||
Example: hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\
|
||||
Example: hyperfine -P threads 1 8 'make -j {threads}'\n\n\
|
||||
This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
|
||||
To have the value increase following different patterns, use shell arithmetics.\n\n \
|
||||
Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\
|
||||
@@ -198,8 +194,8 @@ fn build_command() -> Command {
|
||||
.help(
|
||||
"Perform benchmark runs for each value in the comma-separated list VALUES. \
|
||||
Replaces the string '{VAR}' in each command by the current parameter value\
|
||||
.\n\nExample: hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'\n\n\
|
||||
This performs benchmarks for 'g++ -O2 main.cpp' and 'clang++ -O2 main.cpp'.\n\n\
|
||||
.\n\nExample: hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\n\n\
|
||||
This performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp'.\n\n\
|
||||
The option can be specified multiple times to run benchmarks for all \
|
||||
possible parameter combinations.\n"
|
||||
),
|
||||
|
||||
+14
-7
@@ -275,15 +275,22 @@ impl Default for Options {
|
||||
impl Options {
|
||||
pub fn from_cli_arguments<'a>(matches: &ArgMatches) -> Result<Self, OptionsError<'a>> {
|
||||
let mut options = Self::default();
|
||||
options.warmup_count = matches
|
||||
.get_one::<u64>("warmup")
|
||||
.copied()
|
||||
.unwrap_or(options.warmup_count);
|
||||
let param_to_u64 = |param| {
|
||||
matches
|
||||
.get_one::<String>(param)
|
||||
.map(|n| {
|
||||
n.parse::<u64>()
|
||||
.map_err(|e| OptionsError::IntParsingError(param, e))
|
||||
})
|
||||
.transpose()
|
||||
};
|
||||
|
||||
let mut min_runs = matches.get_one::<u64>("min-runs").copied();
|
||||
let mut max_runs = matches.get_one::<u64>("max-runs").copied();
|
||||
options.warmup_count = param_to_u64("warmup")?.unwrap_or(options.warmup_count);
|
||||
|
||||
if let Some(&runs) = matches.get_one::<u64>("runs") {
|
||||
let mut min_runs = param_to_u64("min-runs")?;
|
||||
let mut max_runs = param_to_u64("max-runs")?;
|
||||
|
||||
if let Some(runs) = param_to_u64("runs")? {
|
||||
min_runs = Some(runs);
|
||||
max_runs = Some(runs);
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
|
||||
//! Ph.D., Editor.
|
||||
|
||||
use crate::util::statistics::median;
|
||||
use statistical::median;
|
||||
|
||||
/// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
|
||||
/// converts the MAD to an estimator for the standard deviation. The second factor is the number
|
||||
|
||||
+15
-4
@@ -16,11 +16,11 @@ use std::os::fd::AsFd;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
|
||||
|
||||
use crate::benchmark::measurement::Measurement;
|
||||
use crate::util::units::Second;
|
||||
use wall_clock_timer::WallClockTimer;
|
||||
|
||||
use std::io::Read;
|
||||
use std::process::{ChildStdout, Command};
|
||||
use std::process::{ChildStdout, Command, ExitStatus};
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
@@ -37,6 +37,17 @@ struct CPUTimes {
|
||||
pub memory_usage_byte: u64,
|
||||
}
|
||||
|
||||
/// Used to indicate the result of running a command
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct TimerResult {
|
||||
pub time_real: Second,
|
||||
pub time_user: Second,
|
||||
pub time_system: Second,
|
||||
pub memory_usage_byte: u64,
|
||||
/// The exit status of the process
|
||||
pub status: ExitStatus,
|
||||
}
|
||||
|
||||
/// Discard the output of a child process.
|
||||
fn discard(output: ChildStdout) {
|
||||
const CHUNK_SIZE: usize = 64 << 10;
|
||||
@@ -69,7 +80,7 @@ fn discard(output: ChildStdout) {
|
||||
}
|
||||
|
||||
/// Execute the given command and return a timing summary
|
||||
pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
|
||||
pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
|
||||
#[cfg(not(windows))]
|
||||
let cpu_timer = self::unix_timer::CPUTimer::start();
|
||||
|
||||
@@ -100,7 +111,7 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
|
||||
let time_real = wallclock_timer.stop();
|
||||
let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
|
||||
|
||||
Ok(Measurement {
|
||||
Ok(TimerResult {
|
||||
time_real,
|
||||
time_user,
|
||||
time_system,
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
/// A max function for f64's without NaNs
|
||||
pub fn max(vals: &[f64]) -> f64 {
|
||||
*vals
|
||||
.iter()
|
||||
.max_by(|a, b| a.partial_cmp(b).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// A min function for f64's without NaNs
|
||||
pub fn min(vals: &[f64]) -> f64 {
|
||||
*vals
|
||||
.iter()
|
||||
.min_by(|a, b| a.partial_cmp(b).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max() {
|
||||
let assert_float_eq = |a: f64, b: f64| {
|
||||
assert!((a - b).abs() < f64::EPSILON);
|
||||
};
|
||||
|
||||
assert_float_eq(1.0, max(&[1.0]));
|
||||
assert_float_eq(-1.0, max(&[-1.0]));
|
||||
assert_float_eq(-1.0, max(&[-2.0, -1.0]));
|
||||
assert_float_eq(1.0, max(&[-1.0, 1.0]));
|
||||
assert_float_eq(1.0, max(&[-1.0, 1.0, 0.0]));
|
||||
}
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
pub mod exit_code;
|
||||
pub mod min_max;
|
||||
pub mod number;
|
||||
pub mod randomized_environment_offset;
|
||||
pub mod statistics;
|
||||
pub mod units;
|
||||
@@ -1,113 +0,0 @@
|
||||
//! Statistics for non-empty samples of finite measurements.
|
||||
|
||||
/// A min function that assumes no NaNs and at least one element
|
||||
pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
|
||||
values
|
||||
.into_iter()
|
||||
.min_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
|
||||
.expect("'min' requires at least one element")
|
||||
}
|
||||
|
||||
/// A max function that assumes no NaNs and at least one element
|
||||
pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
|
||||
values
|
||||
.into_iter()
|
||||
.max_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
|
||||
.expect("'max' requires at least one element")
|
||||
}
|
||||
|
||||
/// Arithmetic mean, summing measurements in their original order.
|
||||
pub fn mean(values: &[f64]) -> f64 {
|
||||
values.iter().fold(0.0, |sum, value| sum + value) / values.len() as f64
|
||||
}
|
||||
|
||||
/// Median without modifying the input sample.
|
||||
pub fn median(values: &[f64]) -> f64 {
|
||||
assert!(
|
||||
!values.is_empty(),
|
||||
"median requires at least one measurement"
|
||||
);
|
||||
let mut sorted = values.to_vec();
|
||||
sorted.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
|
||||
|
||||
let mid = sorted.len() / 2;
|
||||
if sorted.len() % 2 == 1 {
|
||||
sorted[mid]
|
||||
} else {
|
||||
(sorted[mid] + sorted[mid - 1]) / 2.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Sample standard deviation, using an already computed mean.
|
||||
pub fn standard_deviation(values: &[f64], mean: f64) -> f64 {
|
||||
assert!(
|
||||
values.len() > 1,
|
||||
"standard deviation requires at least two measurements"
|
||||
);
|
||||
let sum = values
|
||||
.iter()
|
||||
.map(|value| (value - mean) * (value - mean))
|
||||
.fold(0.0, |sum, deviation| sum + deviation);
|
||||
assert!(sum >= 0.0, "invalid sum of squared deviations");
|
||||
|
||||
// Preserve the operation order of statistical 1.0: dividing by n - 1
|
||||
// before taking the square root also preserves exported floating-point values.
|
||||
(sum / (values.len() as f64 - 1.0)).sqrt()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use approx::assert_relative_eq;
|
||||
|
||||
#[test]
|
||||
fn test_min() {
|
||||
assert_eq!(1.0, min([1.0]));
|
||||
assert_eq!(-1.0, min([-1.0]));
|
||||
assert_eq!(-2.0, min([-2.0, -1.0]));
|
||||
assert_eq!(-1.0, min([-1.0, 1.0]));
|
||||
assert_eq!(-1.0, min([1.0, -1.0, 0.0]));
|
||||
|
||||
let values = vec![1024.0, 2.0, 3.0];
|
||||
assert_eq!(*min(&values), 2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max() {
|
||||
assert_eq!(1.0, max([1.0]));
|
||||
assert_eq!(-1.0, max([-1.0]));
|
||||
assert_eq!(-1.0, max([-2.0, -1.0]));
|
||||
assert_eq!(1.0, max([-1.0, 1.0]));
|
||||
assert_eq!(1.0, max([-1.0, 1.0, 0.0]));
|
||||
|
||||
let values = vec![1.0, 2048.0, 3.0];
|
||||
assert_eq!(*max(&values), 2048.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mean() {
|
||||
assert_eq!(1.0, mean(&[1.0]));
|
||||
assert_relative_eq!(2.0, mean(&[1.0, 3.0]));
|
||||
|
||||
let values = [0.1, 0.2, 0.6];
|
||||
assert_relative_eq!(mean(&values), 0.3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_median() {
|
||||
assert_eq!(1.0, median(&[1.0]));
|
||||
assert_relative_eq!(2.0, median(&[1.0, 3.0]));
|
||||
|
||||
let values = [0.1, 0.2, 0.6];
|
||||
assert_relative_eq!(median(&values), 0.2);
|
||||
|
||||
let values = [0.1, 0.2, 0.3, 0.6];
|
||||
assert_relative_eq!(median(&values), 0.25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_standard_deviation() {
|
||||
let values = [0.1, 0.2, 0.3];
|
||||
assert_relative_eq!(standard_deviation(&values, mean(&values)), 0.1);
|
||||
}
|
||||
}
|
||||
@@ -74,32 +74,6 @@ fn exits_quietly_when_stdout_is_closed() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fails_with_zero_runs() {
|
||||
for option in ["--runs", "--min-runs", "--max-runs"] {
|
||||
hyperfine()
|
||||
.arg(option)
|
||||
.arg("0")
|
||||
.arg("echo dummy benchmark")
|
||||
.assert()
|
||||
.code(2)
|
||||
.stderr(predicate::str::contains(format!(
|
||||
"invalid value '0' for '{option} <NUM>'"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_runs_of_one_still_performs_one_run() {
|
||||
hyperfine_debug()
|
||||
.arg("--min-runs=1")
|
||||
.arg("--warmup=0")
|
||||
.arg("sleep 4")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Time (abs ≡)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn can_run_commands_without_a_shell() {
|
||||
hyperfine()
|
||||
|
||||
Reference in new issue
Block a user