Compare commits

..
Author SHA1 Message Date
David Peter aba7bdec07 Use C++ compiler drivers in parameter-list examples 2026-10-02 11:29:13 +02:00
19 changed files with 421 additions and 264 deletions

No files matched your search

+1 -2
View File
@@ -89,7 +89,6 @@ jobs:
- { target: aarch64-apple-darwin , os: macos-15 } - { target: aarch64-apple-darwin , os: macos-15 }
# - { target: x86_64-pc-windows-gnu , os: windows-2022 } # - { target: x86_64-pc-windows-gnu , os: windows-2022 }
- { target: x86_64-pc-windows-msvc , os: windows-2022 } - { target: x86_64-pc-windows-msvc , os: windows-2022 }
- { target: aarch64-pc-windows-msvc , os: windows-11-arm }
- { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } - { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true }
- { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } - { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true }
env: env:
@@ -161,7 +160,7 @@ jobs:
run: | run: |
# test only library unit tests and binary for arm-type targets # test only library unit tests and binary for arm-type targets
unset CARGO_TEST_OPTIONS unset CARGO_TEST_OPTIONS
unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ needs.crate_metadata.outputs.bin-name }}" ;; esac; unset CARGO_TEST_OPTIONS ; case ${{ matrix.job.target }} in arm-* | aarch64-*) CARGO_TEST_OPTIONS="--bin ${{ steps.bin.outputs.BIN_NAME }}" ;; esac;
echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT
- name: Run tests - name: Run tests
+4 -4
View File
@@ -5,7 +5,7 @@
- Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally) - Add `--reference-name` option to give a meaningful name to the reference command, see #808 (@niklasdewally)
- The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp) - The `--ignore-failure` option now supports a comma-separated list of exit codes to ignore (e.g., `--ignore-failure=1,2`), see #836 (@sharkdp)
- Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp) - Python scripts: Add `--time-unit` option to `advanced_statistics.py` (@sharkdp)
- Python scripts: Add new `plot_benchmark_comparison.py` script for plotting collections of benchmarks, see #806 (@marxin) - Python scripts: Add new `plot_benchmarks.py` script for plotting collections of benchmarks, see #806 (@marxin)
## Bugfixes ## Bugfixes
@@ -99,7 +99,7 @@
## Other ## Other
- Thanks to @berombau for working on dependency upgrades, see #584 - Thanks to @berombau for working on dependency upgrades, see #584
- Fixed installation on Windows, see #595 and #596 (@AntoniosBarotsis) - Fixed installationm on Windows, see #595 and #596 (@AntoniosBarotsis)
# v1.15.0 # v1.15.0
@@ -244,7 +244,7 @@ Package authors: note that Hyperfine now comes with a set of shell completion fi
- The new `--parameter-list <VAR> <VALUES>` option can be used to run - The new `--parameter-list <VAR> <VALUES>` option can be used to run
a parametrized benchmark on a user-specified list of values. a parametrized benchmark on a user-specified list of values.
This is similar to `--parameter-scan <VAR> <MIN> <MAX>`, but doesn't This is similar to `--parameter-scan <VAR> <MIN> <MAX>`, but doesn't
necessarily require numeric arguments. necessarily required numeric arguments.
``` bash ``` bash
hyperfine --parameter-list compiler "g++,clang++" \ hyperfine --parameter-list compiler "g++,clang++" \
@@ -415,7 +415,7 @@ for reviewing pull requests and for giving very valuable feedback.
## Other ## Other
- [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2). - [Arch Linux package](https://aur.archlinux.org/packages/hyperfine) for Hyperfine (@jD91mZM2).
- Ubuntu/Debian packages are now available. - Ubuntu/Debian packages are now are available.
# v0.2.0 # v0.2.0
Generated
+247 -3
View File
@@ -97,12 +97,27 @@ dependencies = [
"wait-timeout", "wait-timeout",
] ]
[[package]]
name = "autocfg"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
dependencies = [
"autocfg 1.5.1",
]
[[package]] [[package]]
name = "autocfg" name = "autocfg"
version = "1.5.1" version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.13.2" version = "2.13.2"
@@ -216,6 +231,15 @@ version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags 1.3.2",
]
[[package]] [[package]]
name = "colorchoice" name = "colorchoice"
version = "1.0.5" version = "1.0.5"
@@ -328,6 +352,12 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.17" version = "0.2.17"
@@ -391,6 +421,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"shell-words", "shell-words",
"statistical",
"tempfile", "tempfile",
"thiserror", "thiserror",
"windows-sys 0.61.2", "windows-sys 0.61.2",
@@ -477,7 +508,7 @@ version = "0.31.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.2",
"cfg-if", "cfg-if",
"cfg_aliases", "cfg_aliases",
"libc", "libc",
@@ -489,13 +520,79 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
[[package]]
name = "num"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg 1.5.1",
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
"autocfg 1.5.1",
"num-traits",
]
[[package]]
name = "num-integer"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg 1.5.1",
"num-bigint",
"num-integer",
"num-traits",
]
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.19" version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [ dependencies = [
"autocfg", "autocfg 1.5.1",
] ]
[[package]] [[package]]
@@ -600,6 +697,25 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.8",
"libc",
"rand_chacha 0.1.1",
"rand_core 0.4.3",
"rand_hc",
"rand_isaac",
"rand_jitter",
"rand_os",
"rand_pcg",
"rand_xorshift",
"winapi",
]
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.8.8" version = "0.8.8"
@@ -632,6 +748,16 @@ dependencies = [
"rand_core 0.10.1", "rand_core 0.10.1",
] ]
[[package]]
name = "rand_chacha"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg 0.1.8",
"rand_core 0.3.2",
]
[[package]] [[package]]
name = "rand_chacha" name = "rand_chacha"
version = "0.3.1" version = "0.3.1"
@@ -652,6 +778,21 @@ dependencies = [
"rand_core 0.9.5", "rand_core 0.9.5",
] ]
[[package]]
name = "rand_core"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96f815e01bbd9678b50d927f79aa1cf3ffdfdb1b9787317c1284dadb894ad0e8"
dependencies = [
"rand_core 0.4.3",
]
[[package]]
name = "rand_core"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e5937858e6fd18cd595d558f90bb5de3b72ae23f9e3763af0e805949b04ef60"
[[package]] [[package]]
name = "rand_core" name = "rand_core"
version = "0.6.4" version = "0.6.4"
@@ -676,6 +817,77 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rand_hc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "rand_jitter"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
dependencies = [
"libc",
"rand_core 0.4.3",
"winapi",
]
[[package]]
name = "rand_os"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
dependencies = [
"cloudabi",
"fuchsia-cprng",
"libc",
"rand_core 0.4.3",
"rdrand",
"winapi",
]
[[package]]
name = "rand_pcg"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg 0.1.8",
"rand_core 0.4.3",
]
[[package]]
name = "rand_xorshift"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.2",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.13.1" version = "1.13.1"
@@ -728,7 +940,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.2",
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
@@ -802,6 +1014,16 @@ version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]]
name = "statistical"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49d57902bb128e5e38b5218d3681215ae3e322d99f65d5420e9849730d2ea372"
dependencies = [
"num",
"rand 0.6.5",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.11.1" version = "0.11.1"
@@ -1003,6 +1225,28 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "windows-link" name = "windows-link"
version = "0.2.1" version = "0.2.1"
+1
View File
@@ -20,6 +20,7 @@ windows_process_extensions_main_thread_handle = []
colored = "3.1" colored = "3.1"
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580 # Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
indicatif = "=0.17.4" indicatif = "=0.17.4"
statistical = "1.0"
csv = "1.4" csv = "1.4"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
+14 -8
View File
@@ -33,9 +33,8 @@ hyperfine 'sleep 0.3'
``` ```
Hyperfine will automatically determine the number of runs to perform for each command. By default, Hyperfine will automatically determine the number of runs to perform for each command. By default,
it will perform *at least* 10 benchmarking runs and estimate a run count targeting roughly it will perform *at least* 10 benchmarking runs and measure for at least 3 seconds. To change this,
3 seconds, including shell overhead and preparation and conclusion commands. To set an exact you can use the `-r`/`--runs` option:
number of runs, you can use the `-r`/`--runs` option:
```sh ```sh
hyperfine --runs 5 'sleep 0.3' hyperfine --runs 5 'sleep 0.3'
``` ```
@@ -57,8 +56,8 @@ hyperfine --warmup 3 'grep -R TODO *'
``` ```
Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare` Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`
option to run a special command before *each* timing run. For example, to clear Linux filesystem caches, option to run a special command before *each* timing run. For example, to clear harddisk caches
you can run on Linux, you can run
```sh ```sh
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
``` ```
@@ -90,7 +89,7 @@ hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'
### Intermediate shell ### Intermediate shell
By default, commands are executed using `sh` on Unix (resolved through `PATH`) or `cmd.exe` on Windows. By default, commands are executed using a predefined shell (`/bin/sh` on Unix, `cmd.exe` on Windows).
If you want to use a different shell, you can use the `-S, --shell <SHELL>` option: If you want to use a different shell, you can use the `-S, --shell <SHELL>` option:
```sh ```sh
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done' hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
@@ -105,7 +104,7 @@ If you want to run a benchmark *without an intermediate shell*, you can use the
option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would
produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case. produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case.
``` ```
hyperfine -N 'grep -R TODO /home/user' hyperfine -N 'grep TODO /home/user'
``` ```
@@ -213,6 +212,13 @@ cave resolve -x repository/rust
cave resolve -x hyperfine cave resolve -x hyperfine
``` ```
### On Funtoo Linux
On Funtoo Linux, hyperfine can be installed [from core-kit](https://github.com/funtoo/core-kit/tree/1.4-release/app-benchmarks/hyperfine):
```
emerge app-benchmarks/hyperfine
```
### On NixOS ### On NixOS
On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine): On NixOS, hyperfine can be installed [from the official repositories](https://nixos.org/nixos/packages.html?query=hyperfine):
@@ -328,7 +334,7 @@ The name *hyperfine* was chosen in reference to the hyperfine levels of caesium
## Citing hyperfine ## Citing hyperfine
Thank you for considering citing hyperfine in your research work. Please see the information Thank you for considering to cite hyperfine in your research work. Please see the information
in the sidebar on how to properly cite hyperfine. in the sidebar on how to properly cite hyperfine.
## License ## License
+12 -30
View File
@@ -3,8 +3,7 @@
hyperfine \- command\-line benchmarking tool hyperfine \- command\-line benchmarking tool
.SH SYNOPSIS .SH SYNOPSIS
.B hyperfine .B hyperfine
.RB [ \-hVN ] .RB [ \-ihVN ]
.RB [ \-\-ignore\-failure[=MODE] ]
.RB [ \-\-warmup .RB [ \-\-warmup
.IR NUM ] .IR NUM ]
.RB [ \-\-min\-runs .RB [ \-\-min\-runs
@@ -15,10 +14,6 @@ hyperfine \- command\-line benchmarking tool
.IR NUM ] .IR NUM ]
.RB [ \-\-setup .RB [ \-\-setup
.IR CMD ] .IR CMD ]
.RB [ \-\-reference
.IR CMD ]
.RB [ \-\-reference\-name
.IR NAME ]
.RB [ \-\-prepare .RB [ \-\-prepare
.IR CMD ] .IR CMD ]
.RB [ \-\-conclude .RB [ \-\-conclude
@@ -110,20 +105,10 @@ Perform exactly \fINUM\fP runs for each command. If this option is not specified
\fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP \fB\-s\fR, \fB\-\-setup\fR \fICMD...\fP
.IP .IP
Execute \fICMD\fP once before each set of timing runs. This is useful Execute \fICMD\fP once before each set of timing runs. This is useful
for compiling your software with the provided parameters, or to do any for compiling your software or with the provided parameters, or to do any
other work that should happen once before a series of benchmark runs, other work that should happen once before a series of benchmark runs,
not every time as would happen with the \fB\-\-prepare\fR option. not every time as would happen with the \fB\-\-prepare\fR option.
.HP .HP
\fB\-\-reference\fR \fICMD\fP
.IP
Benchmark \fICMD\fP as the reference for the relative comparison of results.
If this option is not specified, the fastest command is used as the reference.
.HP
\fB\-\-reference\-name\fR \fINAME\fP
.IP
Give a meaningful \fINAME\fP to the reference command.
Requires \fB\-\-reference\fR.
.HP
\fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP
.IP .IP
Execute \fICMD\fP before each timing run. This is useful for clearing disk caches, Execute \fICMD\fP before each timing run. This is useful for clearing disk caches,
@@ -132,7 +117,7 @@ The \fB\-\-prepare\fR option can be specified once for all commands or multiple
once for each command. In the latter case, each preparation command will be once for each command. In the latter case, each preparation command will be
run prior to the corresponding benchmark command. run prior to the corresponding benchmark command.
.HP .HP
\fB\-C\fR, \fB\-\-conclude\fR \fICMD...\fP \fB\-\-conclude\fR \fICMD...\fP
.IP .IP
Execute \fICMD\fP after each timing run. This is useful for clearing disk caches, Execute \fICMD\fP after each timing run. This is useful for clearing disk caches,
for example. for example.
@@ -144,8 +129,8 @@ run after the corresponding benchmark command.
.IP .IP
Execute \fICMD\fP after the completion of all benchmarking runs for each individual Execute \fICMD\fP after the completion of all benchmarking runs for each individual
command to be benchmarked. This is useful if the commands to be benchmarked command to be benchmarked. This is useful if the commands to be benchmarked
produce artifacts that need to be cleaned up. It only runs once per series of produce artifacts that need to be cleaned up. It only runs once a series of
benchmark runs, as opposed to the \fB\-\-conclude\fR option which runs after benchmark runs, as opposed to \fB\-\-conclude\fR option which runs after
every run. every run.
.HP .HP
\fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP \fB\-P\fR, \fB\-\-parameter\-scan\fR \fIVAR\fP \fIMIN\fP \fIMAX\fP
@@ -156,14 +141,14 @@ string '{\fIVAR\fP}' in each command by the current parameter value.
.RS .RS
Example: Example:
.RS .RS
\fBhyperfine\fR \fB\-\-prepare\fR 'make clean' \fB\-P\fR threads 1 8 'make \-j {threads}' \fBhyperfine\fR \fB\-P\fR threads 1 8 'make \-j {threads}'
.RE .RE
.RE .RE
.IP .IP
This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'. This performs benchmarks for 'make \-j 1', 'make \-j 2', ..., 'make \-j 8'.
.IP .IP
To have the value increase following different patterns, use shell To have the value increase following different patterns, use shell
arithmetic. arithmetics.
.IP .IP
.RS .RS
Example: Example:
@@ -222,13 +207,9 @@ things like "sleep 0.1; sleep 0.2" are not possible without a shell.
.IP .IP
An alias for '\-\-shell=none'. An alias for '\-\-shell=none'.
.HP .HP
\fB\-i\fR, \fB\-\-ignore\-failure\fR[=\fIMODE\fP] \fB\-i\fR, \fB\-\-ignore\-failure\fR
.IP .IP
Ignore failures of the benchmarked programs. Without a value or with Ignore non\-zero exit codes of the benchmarked programs.
\fBall\-non\-zero\fR, all non\-zero exit codes are ignored.
Alternatively, provide a comma\-separated list of exit codes to ignore,
for example \fB\-\-ignore\-failure=1,2\fR.
An explicit value must be attached to the option with an equals sign.
.HP .HP
\fB\-\-style\fR \fITYPE\fP \fB\-\-style\fR \fITYPE\fP
.IP .IP
@@ -328,9 +309,9 @@ Read from \fI\,/dev/null\/\fP (the default).
Read the input from the given file. Read the input from the given file.
.RE .RE
.HP .HP
\fB\-n\fR, \fB\-\-command\-name\fR \fINAME\fP \fB\-n\fR, \fB\-\-command\-name\fR \fiNAME\fP
.IP .IP
Give a meaningful \fINAME\fP to a command. This can be specified multiple times Give a meaningful \fiNAME\fP to a command. This can be specified multiple times
if several commands are benchmarked. if several commands are benchmarked.
.HP .HP
\fB\-h\fR, \fB\-\-help\fR \fB\-h\fR, \fB\-\-help\fR
@@ -382,6 +363,7 @@ Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fB\-\-conclude\fR,
'echo command n={n}' 'echo command n={n}'
.fi .fi
.RE .RE
.RE
.SH AUTHOR .SH AUTHOR
.LP .LP
David Peter <mail@david-peter.de> David Peter <mail@david-peter.de>
+17 -8
View File
@@ -1,17 +1,26 @@
This folder contains scripts that can be used in combination with hyperfine's `--export-json` option. This folder contains scripts that can be used in combination with hyperfines `--export-json` option.
### Example: ### Example:
```bash ```bash
hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
./plot_whisker.py sleep.json
```
### Pre-requisites
To make these scripts work, you will need `numpy`, `matplotlib` and `scipy`.
If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/)
inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx),
you can directly run the scripts using
```bash
uv run plot_whisker.py sleep.json uv run plot_whisker.py sleep.json
``` ```
### Prerequisites Otherwise, install the dependencies via your system package manager or using `pip`:
Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the ```bash
commands above from this directory. pip install numpy matplotlib scipy # pip3, if you are using python3
```
The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
inline script metadata. `uv run` automatically installs these dependencies in an
isolated environment before running the script.
+4 -7
View File
@@ -7,7 +7,8 @@
# /// # ///
"""This script performs Welch's t-test on a JSON export file with two """This script performs Welch's t-test on a JSON export file with two
benchmark results to test for a difference in their population mean runtimes.""" benchmark results to test whether or not the two distributions are
the same."""
import argparse import argparse
import json import json
@@ -39,10 +40,6 @@ print(f"t = {t:.3}, p = {p:.3}")
print() print()
if dispose: if dispose:
print( print(f"There is a difference between the two benchmarks (p < {th}).")
f"A statistically significant difference in mean runtime was detected (p < {th})."
)
else: else:
print( print(f"The two benchmarks are almost the same (p >= {th}).")
f"No statistically significant difference in mean runtime was detected (p >= {th})."
)
+40 -21
View File
@@ -7,14 +7,14 @@ use crate::options::{
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell, CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
}; };
use crate::output::progress_bar::get_progress_bar; use crate::output::progress_bar::get_progress_bar;
use crate::timer::execute_and_measure; use crate::timer::{execute_and_measure, TimerResult};
use crate::util::randomized_environment_offset; use crate::util::randomized_environment_offset;
use crate::util::statistics::mean;
use crate::util::units::Second; use crate::util::units::Second;
use super::measurement::Measurement; use super::timing_result::TimingResult;
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use statistical::mean;
pub enum BenchmarkIteration { pub enum BenchmarkIteration {
NonBenchmarkRun, NonBenchmarkRun,
@@ -40,7 +40,7 @@ pub trait Executor {
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement>; ) -> Result<(TimingResult, ExitStatus)>;
/// Perform a calibration of this executor. For example, /// Perform a calibration of this executor. For example,
/// when running commands through a shell, we need to /// when running commands through a shell, we need to
@@ -62,7 +62,7 @@ fn run_command_and_measure_common(
command_input_policy: &CommandInputPolicy, command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy, command_output_policy: &CommandOutputPolicy,
command_name: &str, command_name: &str,
) -> Result<Measurement> { ) -> Result<TimerResult> {
let stdin = command_input_policy.get_stdin()?; let stdin = command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?; let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
command.stdin(stdin).stdout(stdout).stderr(stderr); command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -136,15 +136,25 @@ impl Executor for RawExecutor<'_> {
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<(TimingResult, ExitStatus)> {
run_command_and_measure_common( let result = run_command_and_measure_common(
command.get_command()?, command.get_command()?,
iteration, iteration,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy, &self.options.command_input_policy,
output_policy, output_policy,
&command.get_command_line(), &command.get_command_line(),
) )?;
Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
} }
fn calibrate(&mut self) -> Result<()> { fn calibrate(&mut self) -> Result<()> {
@@ -159,7 +169,7 @@ impl Executor for RawExecutor<'_> {
pub struct ShellExecutor<'a> { pub struct ShellExecutor<'a> {
options: &'a Options, options: &'a Options,
shell: &'a Shell, shell: &'a Shell,
shell_spawning_time: Option<Measurement>, shell_spawning_time: Option<TimingResult>,
} }
impl<'a> ShellExecutor<'a> { impl<'a> ShellExecutor<'a> {
@@ -179,7 +189,7 @@ impl Executor for ShellExecutor<'_> {
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<(TimingResult, ExitStatus)> {
let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
let mut command_builder = self.shell.command(); let mut command_builder = self.shell.command();
command_builder.arg(if on_windows_cmd { "/C" } else { "-c" }); command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
@@ -208,7 +218,15 @@ impl Executor for ShellExecutor<'_> {
result.time_system = (result.time_system - spawning_time.time_system).max(0.0); result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
} }
Ok(result) Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
} }
/// Measure the average shell spawning time /// Measure the average shell spawning time
@@ -250,7 +268,7 @@ impl Executor for ShellExecutor<'_> {
shell_cmd shell_cmd
); );
} }
Ok(r) => { Ok((r, _)) => {
times_real.push(r.time_real); times_real.push(r.time_real);
times_user.push(r.time_user); times_user.push(r.time_user);
times_system.push(r.time_system); times_system.push(r.time_system);
@@ -266,12 +284,11 @@ impl Executor for ShellExecutor<'_> {
bar.finish_and_clear() bar.finish_and_clear()
} }
self.shell_spawning_time = Some(Measurement { self.shell_spawning_time = Some(TimingResult {
time_real: mean(&times_real), time_real: mean(&times_real),
time_user: mean(&times_user), time_user: mean(&times_user),
time_system: mean(&times_system), time_system: mean(&times_system),
memory_usage_byte: 0, memory_usage_byte: 0,
status: ExitStatus::default(),
}); });
Ok(()) Ok(())
@@ -309,7 +326,7 @@ impl Executor for MockExecutor {
_iteration: BenchmarkIteration, _iteration: BenchmarkIteration,
_command_failure_action: Option<CmdFailureAction>, _command_failure_action: Option<CmdFailureAction>,
_output_policy: &CommandOutputPolicy, _output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<(TimingResult, ExitStatus)> {
#[cfg(unix)] #[cfg(unix)]
let status = { let status = {
use std::os::unix::process::ExitStatusExt; use std::os::unix::process::ExitStatusExt;
@@ -322,13 +339,15 @@ impl Executor for MockExecutor {
ExitStatus::from_raw(0) ExitStatus::from_raw(0)
}; };
Ok(Measurement { Ok((
time_real: Self::extract_time(command.get_command_line()), TimingResult {
time_user: 0.0, time_real: Self::extract_time(command.get_command_line()),
time_system: 0.0, time_user: 0.0,
memory_usage_byte: 0, time_system: 0.0,
memory_usage_byte: 0,
},
status, status,
}) ))
} }
fn calibrate(&mut self) -> Result<()> { fn calibrate(&mut self) -> Result<()> {
+19 -17
View File
@@ -1,8 +1,8 @@
pub mod benchmark_result; pub mod benchmark_result;
pub mod executor; pub mod executor;
pub mod measurement;
pub mod relative_speed; pub mod relative_speed;
pub mod scheduler; pub mod scheduler;
pub mod timing_result;
use std::cmp; use std::cmp;
use std::io::{self, Write}; use std::io::{self, Write};
@@ -19,13 +19,14 @@ use crate::output::progress_bar::get_progress_bar;
use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::output::warnings::{OutlierWarningOptions, Warnings};
use crate::parameter::ParameterNameAndValue; use crate::parameter::ParameterNameAndValue;
use crate::util::exit_code::extract_exit_code; use crate::util::exit_code::extract_exit_code;
use crate::util::statistics::{max, mean, median, min, standard_deviation}; use crate::util::min_max::{max, min};
use crate::util::units::Second; use crate::util::units::Second;
use benchmark_result::BenchmarkResult; use benchmark_result::BenchmarkResult;
use measurement::Measurement; use timing_result::TimingResult;
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use colored::*; use colored::*;
use statistical::{mean, median, standard_deviation};
use self::executor::Executor; use self::executor::Executor;
@@ -60,7 +61,7 @@ impl<'a> Benchmark<'a> {
command: &Command<'_>, command: &Command<'_>,
error_output: &'static str, error_output: &'static str,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
self.executor self.executor
.run_command_and_measure( .run_command_and_measure(
command, command,
@@ -68,6 +69,7 @@ impl<'a> Benchmark<'a> {
Some(CmdFailureAction::RaiseError), Some(CmdFailureAction::RaiseError),
output_policy, output_policy,
) )
.map(|r| r.0)
.map_err(|_| anyhow!(error_output)) .map_err(|_| anyhow!(error_output))
} }
@@ -76,7 +78,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>, parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let command = self let command = self
.options .options
.setup_command .setup_command
@@ -97,7 +99,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>, parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let command = self let command = self
.options .options
.cleanup_command .cleanup_command
@@ -118,7 +120,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
command: &Command<'_>, command: &Command<'_>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let error_output = "The preparation command terminated with a non-zero exit code. \ let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored."; Append ' || true' to the command if you are sure that this can be ignored.";
@@ -130,7 +132,7 @@ impl<'a> Benchmark<'a> {
&self, &self,
command: &Command<'_>, command: &Command<'_>,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement> { ) -> Result<TimingResult> {
let error_output = "The conclusion command terminated with a non-zero exit code. \ let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored."; Append ' || true' to the command if you are sure that this can be ignored.";
@@ -245,13 +247,13 @@ impl<'a> Benchmark<'a> {
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
// Initial timing run // Initial timing run
let res = self.executor.run_command_and_measure( let (res, status) = self.executor.run_command_and_measure(
self.command, self.command,
BenchmarkIteration::Benchmark(0), BenchmarkIteration::Benchmark(0),
None, None,
output_policy, output_policy,
)?; )?;
let success = res.status.success(); let success = status.success();
let conclusion_result = run_conclusion_command()?; let conclusion_result = run_conclusion_command()?;
let conclusion_overhead = let conclusion_overhead =
@@ -282,7 +284,7 @@ impl<'a> Benchmark<'a> {
times_user.push(res.time_user); times_user.push(res.time_user);
times_system.push(res.time_system); times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte); memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(res.status)); exit_codes.push(extract_exit_code(status));
all_succeeded = all_succeeded && success; all_succeeded = all_succeeded && success;
@@ -307,19 +309,19 @@ impl<'a> Benchmark<'a> {
bar.set_message(msg.to_owned()) bar.set_message(msg.to_owned())
} }
let res = self.executor.run_command_and_measure( let (res, status) = self.executor.run_command_and_measure(
self.command, self.command,
BenchmarkIteration::Benchmark(i + 1), BenchmarkIteration::Benchmark(i + 1),
None, None,
output_policy, output_policy,
)?; )?;
let success = res.status.success(); let success = status.success();
times_real.push(res.time_real); times_real.push(res.time_real);
times_user.push(res.time_user); times_user.push(res.time_user);
times_system.push(res.time_system); times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte); memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(res.status)); exit_codes.push(extract_exit_code(status));
all_succeeded = all_succeeded && success; all_succeeded = all_succeeded && success;
@@ -338,13 +340,13 @@ impl<'a> Benchmark<'a> {
let t_num = times_real.len(); let t_num = times_real.len();
let t_mean = mean(&times_real); let t_mean = mean(&times_real);
let t_stddev = if times_real.len() > 1 { let t_stddev = if times_real.len() > 1 {
Some(standard_deviation(&times_real, t_mean)) Some(standard_deviation(&times_real, Some(t_mean)))
} else { } else {
None None
}; };
let t_median = median(&times_real); let t_median = median(&times_real);
let t_min = min(times_real.iter().copied()); let t_min = min(&times_real);
let t_max = max(times_real.iter().copied()); let t_max = max(&times_real);
let user_mean = mean(&times_user); let user_mean = mean(&times_user);
let system_mean = mean(&times_system); let system_mean = mean(&times_system);
@@ -1,10 +1,8 @@
use std::process::ExitStatus;
use crate::util::units::Second; use crate::util::units::Second;
/// Performance measurements and exit status from running a single command /// Results from timing a single command
#[derive(Debug, Default, Copy, Clone)] #[derive(Debug, Default, Copy, Clone)]
pub struct Measurement { pub struct TimingResult {
/// Wall clock time /// Wall clock time
pub time_real: Second, pub time_real: Second,
@@ -16,7 +14,4 @@ pub struct Measurement {
/// Maximum amount of memory used, in bytes /// Maximum amount of memory used, in bytes
pub memory_usage_byte: u64, pub memory_usage_byte: u64,
/// The exit status of the process
pub status: ExitStatus,
} }
+1 -5
View File
@@ -41,7 +41,6 @@ fn build_command() -> Command {
.short('w') .short('w')
.value_name("NUM") .value_name("NUM")
.action(ArgAction::Set) .action(ArgAction::Set)
.value_parser(clap::value_parser!(u64))
.help( .help(
"Perform NUM warmup runs before the actual benchmark. This can be used \ "Perform NUM warmup runs before the actual benchmark. This can be used \
to fill (disk) caches for I/O-heavy programs.", to fill (disk) caches for I/O-heavy programs.",
@@ -53,7 +52,6 @@ fn build_command() -> Command {
.short('m') .short('m')
.action(ArgAction::Set) .action(ArgAction::Set)
.value_name("NUM") .value_name("NUM")
.value_parser(clap::value_parser!(u64).range(1..))
.help("Perform at least NUM runs for each command (default: 10)."), .help("Perform at least NUM runs for each command (default: 10)."),
) )
.arg( .arg(
@@ -62,7 +60,6 @@ fn build_command() -> Command {
.short('M') .short('M')
.action(ArgAction::Set) .action(ArgAction::Set)
.value_name("NUM") .value_name("NUM")
.value_parser(clap::value_parser!(u64).range(1..))
.help("Perform at most NUM runs for each command. By default, there is no limit."), .help("Perform at most NUM runs for each command. By default, there is no limit."),
) )
.arg( .arg(
@@ -72,7 +69,6 @@ fn build_command() -> Command {
.short('r') .short('r')
.action(ArgAction::Set) .action(ArgAction::Set)
.value_name("NUM") .value_name("NUM")
.value_parser(clap::value_parser!(u64).range(1..))
.help("Perform exactly NUM runs for each command. If this option is not specified, \ .help("Perform exactly NUM runs for each command. If this option is not specified, \
hyperfine automatically determines the number of runs."), hyperfine automatically determines the number of runs."),
) )
@@ -165,7 +161,7 @@ fn build_command() -> Command {
.help( .help(
"Perform benchmark runs for each value in the range MIN..MAX. Replaces the \ "Perform benchmark runs for each value in the range MIN..MAX. Replaces the \
string '{VAR}' in each command by the current parameter value.\n\n \ string '{VAR}' in each command by the current parameter value.\n\n \
Example: hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\ Example: hyperfine -P threads 1 8 'make -j {threads}'\n\n\
This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\ This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
To have the value increase following different patterns, use shell arithmetics.\n\n \ To have the value increase following different patterns, use shell arithmetics.\n\n \
Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\ Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\
+14 -7
View File
@@ -275,15 +275,22 @@ impl Default for Options {
impl Options { impl Options {
pub fn from_cli_arguments<'a>(matches: &ArgMatches) -> Result<Self, OptionsError<'a>> { pub fn from_cli_arguments<'a>(matches: &ArgMatches) -> Result<Self, OptionsError<'a>> {
let mut options = Self::default(); let mut options = Self::default();
options.warmup_count = matches let param_to_u64 = |param| {
.get_one::<u64>("warmup") matches
.copied() .get_one::<String>(param)
.unwrap_or(options.warmup_count); .map(|n| {
n.parse::<u64>()
.map_err(|e| OptionsError::IntParsingError(param, e))
})
.transpose()
};
let mut min_runs = matches.get_one::<u64>("min-runs").copied(); options.warmup_count = param_to_u64("warmup")?.unwrap_or(options.warmup_count);
let mut max_runs = matches.get_one::<u64>("max-runs").copied();
if let Some(&runs) = matches.get_one::<u64>("runs") { let mut min_runs = param_to_u64("min-runs")?;
let mut max_runs = param_to_u64("max-runs")?;
if let Some(runs) = param_to_u64("runs")? {
min_runs = Some(runs); min_runs = Some(runs);
max_runs = Some(runs); max_runs = Some(runs);
} }
+1 -1
View File
@@ -5,7 +5,7 @@
//! The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka, //! The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
//! Ph.D., Editor. //! Ph.D., Editor.
use crate::util::statistics::median; use statistical::median;
/// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that /// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
/// converts the MAD to an estimator for the standard deviation. The second factor is the number /// converts the MAD to an estimator for the standard deviation. The second factor is the number
+15 -4
View File
@@ -16,11 +16,11 @@ use std::os::fd::AsFd;
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
use windows_sys::Win32::System::Threading::CREATE_SUSPENDED; use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
use crate::benchmark::measurement::Measurement; use crate::util::units::Second;
use wall_clock_timer::WallClockTimer; use wall_clock_timer::WallClockTimer;
use std::io::Read; use std::io::Read;
use std::process::{ChildStdout, Command}; use std::process::{ChildStdout, Command, ExitStatus};
use anyhow::Result; use anyhow::Result;
@@ -37,6 +37,17 @@ struct CPUTimes {
pub memory_usage_byte: u64, pub memory_usage_byte: u64,
} }
/// Used to indicate the result of running a command
#[derive(Debug, Copy, Clone)]
pub struct TimerResult {
pub time_real: Second,
pub time_user: Second,
pub time_system: Second,
pub memory_usage_byte: u64,
/// The exit status of the process
pub status: ExitStatus,
}
/// Discard the output of a child process. /// Discard the output of a child process.
fn discard(output: ChildStdout) { fn discard(output: ChildStdout) {
const CHUNK_SIZE: usize = 64 << 10; const CHUNK_SIZE: usize = 64 << 10;
@@ -69,7 +80,7 @@ fn discard(output: ChildStdout) {
} }
/// Execute the given command and return a timing summary /// Execute the given command and return a timing summary
pub fn execute_and_measure(mut command: Command) -> Result<Measurement> { pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
#[cfg(not(windows))] #[cfg(not(windows))]
let cpu_timer = self::unix_timer::CPUTimer::start(); let cpu_timer = self::unix_timer::CPUTimer::start();
@@ -100,7 +111,7 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
let time_real = wallclock_timer.stop(); let time_real = wallclock_timer.stop();
let (time_user, time_system, memory_usage_byte) = cpu_timer.stop(); let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
Ok(Measurement { Ok(TimerResult {
time_real, time_real,
time_user, time_user,
time_system, time_system,
+28
View File
@@ -0,0 +1,28 @@
/// A max function for f64's without NaNs
pub fn max(vals: &[f64]) -> f64 {
*vals
.iter()
.max_by(|a, b| a.partial_cmp(b).unwrap())
.unwrap()
}
/// A min function for f64's without NaNs
pub fn min(vals: &[f64]) -> f64 {
*vals
.iter()
.min_by(|a, b| a.partial_cmp(b).unwrap())
.unwrap()
}
#[test]
fn test_max() {
let assert_float_eq = |a: f64, b: f64| {
assert!((a - b).abs() < f64::EPSILON);
};
assert_float_eq(1.0, max(&[1.0]));
assert_float_eq(-1.0, max(&[-1.0]));
assert_float_eq(-1.0, max(&[-2.0, -1.0]));
assert_float_eq(1.0, max(&[-1.0, 1.0]));
assert_float_eq(1.0, max(&[-1.0, 1.0, 0.0]));
}
+1 -1
View File
@@ -1,5 +1,5 @@
pub mod exit_code; pub mod exit_code;
pub mod min_max;
pub mod number; pub mod number;
pub mod randomized_environment_offset; pub mod randomized_environment_offset;
pub mod statistics;
pub mod units; pub mod units;
-113
View File
@@ -1,113 +0,0 @@
//! Statistics for non-empty samples of finite measurements.
/// A min function that assumes no NaNs and at least one element
pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
values
.into_iter()
.min_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
.expect("'min' requires at least one element")
}
/// A max function that assumes no NaNs and at least one element
pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
values
.into_iter()
.max_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
.expect("'max' requires at least one element")
}
/// Arithmetic mean, summing measurements in their original order.
pub fn mean(values: &[f64]) -> f64 {
values.iter().fold(0.0, |sum, value| sum + value) / values.len() as f64
}
/// Median without modifying the input sample.
pub fn median(values: &[f64]) -> f64 {
assert!(
!values.is_empty(),
"median requires at least one measurement"
);
let mut sorted = values.to_vec();
sorted.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
let mid = sorted.len() / 2;
if sorted.len() % 2 == 1 {
sorted[mid]
} else {
(sorted[mid] + sorted[mid - 1]) / 2.0
}
}
/// Sample standard deviation, using an already computed mean.
pub fn standard_deviation(values: &[f64], mean: f64) -> f64 {
assert!(
values.len() > 1,
"standard deviation requires at least two measurements"
);
let sum = values
.iter()
.map(|value| (value - mean) * (value - mean))
.fold(0.0, |sum, deviation| sum + deviation);
assert!(sum >= 0.0, "invalid sum of squared deviations");
// Preserve the operation order of statistical 1.0: dividing by n - 1
// before taking the square root also preserves exported floating-point values.
(sum / (values.len() as f64 - 1.0)).sqrt()
}
#[cfg(test)]
mod tests {
use super::*;
use approx::assert_relative_eq;
#[test]
fn test_min() {
assert_eq!(1.0, min([1.0]));
assert_eq!(-1.0, min([-1.0]));
assert_eq!(-2.0, min([-2.0, -1.0]));
assert_eq!(-1.0, min([-1.0, 1.0]));
assert_eq!(-1.0, min([1.0, -1.0, 0.0]));
let values = vec![1024.0, 2.0, 3.0];
assert_eq!(*min(&values), 2.0);
}
#[test]
fn test_max() {
assert_eq!(1.0, max([1.0]));
assert_eq!(-1.0, max([-1.0]));
assert_eq!(-1.0, max([-2.0, -1.0]));
assert_eq!(1.0, max([-1.0, 1.0]));
assert_eq!(1.0, max([-1.0, 1.0, 0.0]));
let values = vec![1.0, 2048.0, 3.0];
assert_eq!(*max(&values), 2048.0);
}
#[test]
fn test_mean() {
assert_eq!(1.0, mean(&[1.0]));
assert_relative_eq!(2.0, mean(&[1.0, 3.0]));
let values = [0.1, 0.2, 0.6];
assert_relative_eq!(mean(&values), 0.3);
}
#[test]
fn test_median() {
assert_eq!(1.0, median(&[1.0]));
assert_relative_eq!(2.0, median(&[1.0, 3.0]));
let values = [0.1, 0.2, 0.6];
assert_relative_eq!(median(&values), 0.2);
let values = [0.1, 0.2, 0.3, 0.6];
assert_relative_eq!(median(&values), 0.25);
}
#[test]
fn test_standard_deviation() {
let values = [0.1, 0.2, 0.3];
assert_relative_eq!(standard_deviation(&values, mean(&values)), 0.1);
}
}
-26
View File
@@ -74,32 +74,6 @@ fn exits_quietly_when_stdout_is_closed() {
} }
} }
#[test]
fn fails_with_zero_runs() {
for option in ["--runs", "--min-runs", "--max-runs"] {
hyperfine()
.arg(option)
.arg("0")
.arg("echo dummy benchmark")
.assert()
.code(2)
.stderr(predicate::str::contains(format!(
"invalid value '0' for '{option} <NUM>'"
)));
}
}
#[test]
fn min_runs_of_one_still_performs_one_run() {
hyperfine_debug()
.arg("--min-runs=1")
.arg("--warmup=0")
.arg("sleep 4")
.assert()
.success()
.stdout(predicate::str::contains("Time (abs ≡)"));
}
#[test] #[test]
fn can_run_commands_without_a_shell() { fn can_run_commands_without_a_shell() {
hyperfine() hyperfine()