mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-17 18:01:30 +00:00
Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0bccf959b9 | ||
|
|
16e23a1b42 | ||
|
|
a24f76c0ce | ||
|
|
db3a3dedc7 | ||
|
|
35c94a0e36 | ||
|
|
750fe11b85 | ||
|
|
c8f8c4ea5b | ||
|
|
305e89d536 | ||
|
|
694057829b | ||
|
|
1e2e0be25c | ||
|
|
1fb03c3807 | ||
|
|
258af076cb | ||
|
|
fe75d1cb1b | ||
|
|
926b89c770 | ||
|
|
7720bfaba8 |
No files matched your search
Generated
+30
-242
@@ -97,27 +97,12 @@ dependencies = [
|
||||
"wait-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
@@ -231,15 +216,6 @@ version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
||||
|
||||
[[package]]
|
||||
name = "cloudabi"
|
||||
version = "0.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
@@ -352,12 +328,6 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-cprng"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -412,6 +382,7 @@ dependencies = [
|
||||
"csv",
|
||||
"indicatif",
|
||||
"insta",
|
||||
"insta-cmd",
|
||||
"libc",
|
||||
"nix",
|
||||
"once_cell",
|
||||
@@ -421,7 +392,6 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"shell-words",
|
||||
"statistical",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -458,11 +428,24 @@ checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
|
||||
dependencies = [
|
||||
"console 0.16.6",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"similar",
|
||||
"strip-ansi-escapes",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "insta-cmd"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bffdf4af1db390cf0401535d7c1303cd079a074d28d8473b026fdb6559c41403"
|
||||
dependencies = [
|
||||
"insta",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.13"
|
||||
@@ -508,7 +491,7 @@ version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
@@ -520,79 +503,13 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
|
||||
|
||||
[[package]]
|
||||
name = "num"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-iter",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-iter"
|
||||
version = "0.1.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg 1.5.1",
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -697,25 +614,6 @@ version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
|
||||
dependencies = [
|
||||
"autocfg 0.1.8",
|
||||
"libc",
|
||||
"rand_chacha 0.1.1",
|
||||
"rand_core 0.4.3",
|
||||
"rand_hc",
|
||||
"rand_isaac",
|
||||
"rand_jitter",
|
||||
"rand_os",
|
||||
"rand_pcg",
|
||||
"rand_xorshift",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.8"
|
||||
@@ -748,16 +646,6 @@ dependencies = [
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
|
||||
dependencies = [
|
||||
"autocfg 0.1.8",
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
@@ -778,21 +666,6 @@ dependencies = [
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96f815e01bbd9678b50d927f79aa1cf3ffdfdb1b9787317c1284dadb894ad0e8"
|
||||
dependencies = [
|
||||
"rand_core 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e5937858e6fd18cd595d558f90bb5de3b72ae23f9e3763af0e805949b04ef60"
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
@@ -817,77 +690,6 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_isaac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_jitter"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_core 0.4.3",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_os"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
|
||||
dependencies = [
|
||||
"cloudabi",
|
||||
"fuchsia-cprng",
|
||||
"libc",
|
||||
"rand_core 0.4.3",
|
||||
"rdrand",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_pcg"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
|
||||
dependencies = [
|
||||
"autocfg 0.1.8",
|
||||
"rand_core 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_xorshift"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rdrand"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
|
||||
dependencies = [
|
||||
"rand_core 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
@@ -940,7 +742,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
@@ -1015,13 +817,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
|
||||
|
||||
[[package]]
|
||||
name = "statistical"
|
||||
version = "1.0.0"
|
||||
name = "strip-ansi-escapes"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49d57902bb128e5e38b5218d3681215ae3e322d99f65d5420e9849730d2ea372"
|
||||
checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
|
||||
dependencies = [
|
||||
"num",
|
||||
"rand 0.6.5",
|
||||
"vte",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1155,6 +956,15 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "vte"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wait-timeout"
|
||||
version = "0.2.1"
|
||||
@@ -1225,28 +1035,6 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
|
||||
+2
-2
@@ -20,7 +20,6 @@ windows_process_extensions_main_thread_handle = []
|
||||
colored = "3.1"
|
||||
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
|
||||
indicatif = "=0.17.4"
|
||||
statistical = "1.0"
|
||||
csv = "1.4"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
@@ -64,7 +63,8 @@ features = [
|
||||
[dev-dependencies]
|
||||
approx = "0.5"
|
||||
assert_cmd = "2.2"
|
||||
insta = { version = "1.48", features = ["yaml"] }
|
||||
insta = { version = "1.48", features = ["yaml", "filters"] }
|
||||
insta-cmd = "0.7"
|
||||
predicates = "3.1"
|
||||
tempfile = "3.27"
|
||||
|
||||
|
||||
+19
-23
@@ -11,10 +11,9 @@ use crate::timer::execute_and_measure;
|
||||
use crate::util::randomized_environment_offset;
|
||||
use crate::util::units::Second;
|
||||
|
||||
use super::measurement::Measurement;
|
||||
use super::measurement::{Measurement, Measurements};
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use statistical::mean;
|
||||
|
||||
pub enum BenchmarkIteration {
|
||||
NonBenchmarkRun,
|
||||
@@ -79,7 +78,7 @@ fn run_command_and_measure_common(
|
||||
let result = execute_and_measure(command)
|
||||
.with_context(|| format!("Failed to run command '{command_name}'"))?;
|
||||
|
||||
if !result.status.success() {
|
||||
if !result.exit_status.success() {
|
||||
use crate::util::exit_code::extract_exit_code;
|
||||
|
||||
let should_fail = match command_failure_action {
|
||||
@@ -87,7 +86,7 @@ fn run_command_and_measure_common(
|
||||
CmdFailureAction::IgnoreAllFailures => false,
|
||||
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
|
||||
// Only fail if the exit code is not in the list of codes to ignore
|
||||
if let Some(exit_code) = extract_exit_code(result.status) {
|
||||
if let Some(exit_code) = extract_exit_code(result.exit_status) {
|
||||
!codes.contains(&exit_code)
|
||||
} else {
|
||||
// If we can't extract an exit code, treat it as a failure
|
||||
@@ -107,7 +106,7 @@ fn run_command_and_measure_common(
|
||||
bail!(
|
||||
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
|
||||
Alternatively, use the '--show-output' option to debug what went wrong.",
|
||||
cause=result.status.code().map_or(
|
||||
cause=result.exit_status.code().map_or(
|
||||
"The process has been terminated by a signal".into(),
|
||||
|c| format!("Command terminated with non-zero exit code {c}")
|
||||
|
||||
@@ -203,7 +202,8 @@ impl Executor for ShellExecutor<'_> {
|
||||
|
||||
// Subtract shell spawning time
|
||||
if let Some(spawning_time) = self.shell_spawning_time {
|
||||
result.time_real = (result.time_real - spawning_time.time_real).max(0.0);
|
||||
result.time_wall_clock =
|
||||
(result.time_wall_clock - spawning_time.time_wall_clock).max(0.0);
|
||||
result.time_user = (result.time_user - spawning_time.time_user).max(0.0);
|
||||
result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
|
||||
}
|
||||
@@ -224,9 +224,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
None
|
||||
};
|
||||
|
||||
let mut times_real: Vec<Second> = vec![];
|
||||
let mut times_user: Vec<Second> = vec![];
|
||||
let mut times_system: Vec<Second> = vec![];
|
||||
let mut measurements = Measurements::default();
|
||||
|
||||
for _ in 0..COUNT {
|
||||
// Just run the shell without any command
|
||||
@@ -251,9 +249,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
);
|
||||
}
|
||||
Ok(r) => {
|
||||
times_real.push(r.time_real);
|
||||
times_user.push(r.time_user);
|
||||
times_system.push(r.time_system);
|
||||
measurements.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,18 +263,18 @@ impl Executor for ShellExecutor<'_> {
|
||||
}
|
||||
|
||||
self.shell_spawning_time = Some(Measurement {
|
||||
time_real: mean(×_real),
|
||||
time_user: mean(×_user),
|
||||
time_system: mean(×_system),
|
||||
memory_usage_byte: 0,
|
||||
status: ExitStatus::default(),
|
||||
time_wall_clock: measurements.time_wall_clock_mean(),
|
||||
time_user: measurements.time_user_mean(),
|
||||
time_system: measurements.time_system_mean(),
|
||||
peak_memory_usage: 0,
|
||||
exit_status: ExitStatus::default(),
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn time_overhead(&self) -> Second {
|
||||
self.shell_spawning_time.unwrap().time_real
|
||||
self.shell_spawning_time.unwrap().time_wall_clock
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,23 +307,23 @@ impl Executor for MockExecutor {
|
||||
_output_policy: &CommandOutputPolicy,
|
||||
) -> Result<Measurement> {
|
||||
#[cfg(unix)]
|
||||
let status = {
|
||||
let exit_status = {
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
ExitStatus::from_raw(0)
|
||||
};
|
||||
|
||||
#[cfg(windows)]
|
||||
let status = {
|
||||
let exit_status = {
|
||||
use std::os::windows::process::ExitStatusExt;
|
||||
ExitStatus::from_raw(0)
|
||||
};
|
||||
|
||||
Ok(Measurement {
|
||||
time_real: Self::extract_time(command.get_command_line()),
|
||||
time_wall_clock: Self::extract_time(command.get_command_line()),
|
||||
time_user: 0.0,
|
||||
time_system: 0.0,
|
||||
memory_usage_byte: 0,
|
||||
status,
|
||||
peak_memory_usage: 0,
|
||||
exit_status,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
use std::process::ExitStatus;
|
||||
|
||||
use crate::outlier_detection::modified_zscores;
|
||||
use crate::util::statistics::{max, mean, median, min, standard_deviation};
|
||||
use crate::util::units::Second;
|
||||
|
||||
/// Performance measurements and exit status from running a single command
|
||||
#[derive(Debug, Default, Copy, Clone)]
|
||||
pub struct Measurement {
|
||||
/// Wall clock time
|
||||
pub time_real: Second,
|
||||
pub time_wall_clock: Second,
|
||||
|
||||
/// Time spent in user mode
|
||||
pub time_user: Second,
|
||||
@@ -15,8 +17,77 @@ pub struct Measurement {
|
||||
pub time_system: Second,
|
||||
|
||||
/// Maximum amount of memory used, in bytes
|
||||
pub memory_usage_byte: u64,
|
||||
pub peak_memory_usage: u64,
|
||||
|
||||
/// The exit status of the process
|
||||
pub status: ExitStatus,
|
||||
pub exit_status: ExitStatus,
|
||||
}
|
||||
|
||||
/// Performance measurements and exit statuses for all runs of a command.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct Measurements {
|
||||
pub measurements: Vec<Measurement>,
|
||||
}
|
||||
|
||||
impl Measurements {
|
||||
pub fn len(&self) -> usize {
|
||||
self.measurements.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.measurements.is_empty()
|
||||
}
|
||||
|
||||
pub fn push(&mut self, measurement: Measurement) {
|
||||
self.measurements.push(measurement);
|
||||
}
|
||||
|
||||
pub fn wall_clock_times(&self) -> impl Iterator<Item = Second> + '_ {
|
||||
self.measurements.iter().map(|m| m.time_wall_clock)
|
||||
}
|
||||
|
||||
/// The average wall clock time.
|
||||
pub fn time_wall_clock_mean(&self) -> Second {
|
||||
mean(self.wall_clock_times())
|
||||
}
|
||||
|
||||
/// The standard deviation of wall clock times, if at least two runs were measured.
|
||||
pub fn stddev(&self) -> Option<Second> {
|
||||
if self.len() < 2 {
|
||||
None
|
||||
} else {
|
||||
let times: Vec<_> = self.wall_clock_times().collect();
|
||||
Some(standard_deviation(×, self.time_wall_clock_mean()))
|
||||
}
|
||||
}
|
||||
|
||||
/// The median wall clock time.
|
||||
pub fn median(&self) -> Second {
|
||||
median(&self.wall_clock_times().collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
/// The minimum wall clock time.
|
||||
pub fn min(&self) -> Second {
|
||||
min(self.wall_clock_times())
|
||||
}
|
||||
|
||||
/// The maximum wall clock time.
|
||||
pub fn max(&self) -> Second {
|
||||
max(self.wall_clock_times())
|
||||
}
|
||||
|
||||
/// Compute modified Z-scores for the wall clock times.
|
||||
pub fn modified_zscores(&self) -> Vec<f64> {
|
||||
modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
/// The average user time.
|
||||
pub fn time_user_mean(&self) -> Second {
|
||||
mean(self.measurements.iter().map(|m| m.time_user))
|
||||
}
|
||||
|
||||
/// The average system time.
|
||||
pub fn time_system_mean(&self) -> Second {
|
||||
mean(self.measurements.iter().map(|m| m.time_system))
|
||||
}
|
||||
}
|
||||
+43
-46
@@ -12,21 +12,19 @@ use crate::command::Command;
|
||||
use crate::options::{
|
||||
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
|
||||
};
|
||||
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
|
||||
use crate::outlier_detection::OUTLIER_THRESHOLD;
|
||||
use crate::output::console_writeln;
|
||||
use crate::output::format::{format_duration, format_duration_unit};
|
||||
use crate::output::progress_bar::get_progress_bar;
|
||||
use crate::output::warnings::{OutlierWarningOptions, Warnings};
|
||||
use crate::parameter::ParameterNameAndValue;
|
||||
use crate::util::exit_code::extract_exit_code;
|
||||
use crate::util::min_max::{max, min};
|
||||
use crate::util::units::Second;
|
||||
use benchmark_result::BenchmarkResult;
|
||||
use measurement::Measurement;
|
||||
use measurement::{Measurement, Measurements};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use colored::*;
|
||||
use statistical::{mean, median, standard_deviation};
|
||||
|
||||
use self::executor::Executor;
|
||||
|
||||
@@ -150,11 +148,7 @@ impl<'a> Benchmark<'a> {
|
||||
)?;
|
||||
}
|
||||
|
||||
let mut times_real: Vec<Second> = vec![];
|
||||
let mut times_user: Vec<Second> = vec![];
|
||||
let mut times_system: Vec<Second> = vec![];
|
||||
let mut memory_usage_byte: Vec<u64> = vec![];
|
||||
let mut exit_codes: Vec<Option<i32>> = vec![];
|
||||
let mut measurements = Measurements::default();
|
||||
let mut all_succeeded = true;
|
||||
|
||||
let output_policy = &self.options.command_output_policies[self.number];
|
||||
@@ -242,8 +236,9 @@ impl<'a> Benchmark<'a> {
|
||||
};
|
||||
|
||||
let preparation_result = run_preparation_command()?;
|
||||
let preparation_overhead =
|
||||
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
|
||||
let preparation_overhead = preparation_result.map_or(0.0, |res| {
|
||||
res.time_wall_clock + self.executor.time_overhead()
|
||||
});
|
||||
|
||||
// Initial timing run
|
||||
let res = self.executor.run_command_and_measure(
|
||||
@@ -252,15 +247,16 @@ impl<'a> Benchmark<'a> {
|
||||
None,
|
||||
output_policy,
|
||||
)?;
|
||||
let success = res.status.success();
|
||||
let success = res.exit_status.success();
|
||||
|
||||
let conclusion_result = run_conclusion_command()?;
|
||||
let conclusion_overhead =
|
||||
conclusion_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
|
||||
let conclusion_overhead = conclusion_result.map_or(0.0, |res| {
|
||||
res.time_wall_clock + self.executor.time_overhead()
|
||||
});
|
||||
|
||||
// Determine number of benchmark runs
|
||||
let runs_in_min_time = (self.options.min_benchmarking_time
|
||||
/ (res.time_real
|
||||
/ (res.time_wall_clock
|
||||
+ self.executor.time_overhead()
|
||||
+ preparation_overhead
|
||||
+ conclusion_overhead)) as u64;
|
||||
@@ -279,11 +275,7 @@ impl<'a> Benchmark<'a> {
|
||||
let count_remaining = count - 1;
|
||||
|
||||
// Save the first result
|
||||
times_real.push(res.time_real);
|
||||
times_user.push(res.time_user);
|
||||
times_system.push(res.time_system);
|
||||
memory_usage_byte.push(res.memory_usage_byte);
|
||||
exit_codes.push(extract_exit_code(res.status));
|
||||
measurements.push(res);
|
||||
|
||||
all_succeeded = all_succeeded && success;
|
||||
|
||||
@@ -300,7 +292,8 @@ impl<'a> Benchmark<'a> {
|
||||
run_preparation_command()?;
|
||||
|
||||
let msg = {
|
||||
let mean = format_duration(mean(×_real), self.options.time_unit);
|
||||
let mean =
|
||||
format_duration(measurements.time_wall_clock_mean(), self.options.time_unit);
|
||||
format!("Current estimate: {}", mean.to_string().green())
|
||||
};
|
||||
|
||||
@@ -314,13 +307,9 @@ impl<'a> Benchmark<'a> {
|
||||
None,
|
||||
output_policy,
|
||||
)?;
|
||||
let success = res.status.success();
|
||||
let success = res.exit_status.success();
|
||||
|
||||
times_real.push(res.time_real);
|
||||
times_user.push(res.time_user);
|
||||
times_system.push(res.time_system);
|
||||
memory_usage_byte.push(res.memory_usage_byte);
|
||||
exit_codes.push(extract_exit_code(res.status));
|
||||
measurements.push(res);
|
||||
|
||||
all_succeeded = all_succeeded && success;
|
||||
|
||||
@@ -336,19 +325,15 @@ impl<'a> Benchmark<'a> {
|
||||
}
|
||||
|
||||
// Compute statistical quantities
|
||||
let t_num = times_real.len();
|
||||
let t_mean = mean(×_real);
|
||||
let t_stddev = if times_real.len() > 1 {
|
||||
Some(standard_deviation(×_real, Some(t_mean)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let t_median = median(×_real);
|
||||
let t_min = min(×_real);
|
||||
let t_max = max(×_real);
|
||||
let t_num = measurements.len();
|
||||
let t_mean = measurements.time_wall_clock_mean();
|
||||
let t_stddev = measurements.stddev();
|
||||
let t_median = measurements.median();
|
||||
let t_min = measurements.min();
|
||||
let t_max = measurements.max();
|
||||
|
||||
let user_mean = mean(×_user);
|
||||
let system_mean = mean(×_system);
|
||||
let user_mean = measurements.time_user_mean();
|
||||
let system_mean = measurements.time_system_mean();
|
||||
|
||||
// Formatting and console output
|
||||
let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);
|
||||
@@ -361,7 +346,7 @@ impl<'a> Benchmark<'a> {
|
||||
|
||||
if self.options.output_style != OutputStyleOption::Disabled {
|
||||
let mut stdout = io::stdout().lock();
|
||||
if times_real.len() == 1 {
|
||||
if measurements.len() == 1 {
|
||||
console_writeln!(
|
||||
stdout,
|
||||
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
|
||||
@@ -402,7 +387,9 @@ impl<'a> Benchmark<'a> {
|
||||
|
||||
// Check execution time
|
||||
if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
|
||||
&& times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
|
||||
&& measurements
|
||||
.wall_clock_times()
|
||||
.any(|t| t < MIN_EXECUTION_TIME)
|
||||
{
|
||||
warnings.push(Warnings::FastExecutionTime);
|
||||
}
|
||||
@@ -413,7 +400,7 @@ impl<'a> Benchmark<'a> {
|
||||
}
|
||||
|
||||
// Run outlier detection
|
||||
let scores = modified_zscores(×_real);
|
||||
let scores = measurements.modified_zscores();
|
||||
|
||||
let outlier_warning_options = OutlierWarningOptions {
|
||||
warmup_in_use: self.options.warmup_count > 0,
|
||||
@@ -428,7 +415,7 @@ impl<'a> Benchmark<'a> {
|
||||
|
||||
if scores[0] > OUTLIER_THRESHOLD {
|
||||
warnings.push(Warnings::SlowInitialRun(
|
||||
times_real[0],
|
||||
measurements.wall_clock_times().next().unwrap(),
|
||||
outlier_warning_options,
|
||||
));
|
||||
} else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
|
||||
@@ -460,9 +447,19 @@ impl<'a> Benchmark<'a> {
|
||||
system: system_mean,
|
||||
min: t_min,
|
||||
max: t_max,
|
||||
times: Some(times_real),
|
||||
memory_usage_byte: Some(memory_usage_byte),
|
||||
exit_codes,
|
||||
times: Some(measurements.wall_clock_times().collect()),
|
||||
memory_usage_byte: Some(
|
||||
measurements
|
||||
.measurements
|
||||
.iter()
|
||||
.map(|m| m.peak_memory_usage)
|
||||
.collect(),
|
||||
),
|
||||
exit_codes: measurements
|
||||
.measurements
|
||||
.iter()
|
||||
.map(|m| extract_exit_code(m.exit_status))
|
||||
.collect(),
|
||||
parameters: self
|
||||
.command
|
||||
.get_parameters()
|
||||
|
||||
@@ -52,7 +52,7 @@ fn compute_relative_speeds<'a>(
|
||||
};
|
||||
|
||||
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
|
||||
// Covariance asssumed to be 0, i.e. variables are assumed to be independent
|
||||
// Covariance assumed to be 0, i.e. variables are assumed to be independent
|
||||
let ratio_stddev = match (result.stddev, reference.stddev) {
|
||||
(Some(result_stddev), Some(fastest_stddev)) => Some(
|
||||
ratio
|
||||
|
||||
+1
-1
@@ -167,7 +167,7 @@ fn build_command() -> Command {
|
||||
string '{VAR}' in each command by the current parameter value.\n\n \
|
||||
Example: hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\
|
||||
This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
|
||||
To have the value increase following different patterns, use shell arithmetics.\n\n \
|
||||
To have the value increase following different patterns, use shell arithmetic.\n\n \
|
||||
Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\
|
||||
This performs benchmarks with power of 2 increases: 'sleep 1', 'sleep 2', 'sleep 4', …\n\
|
||||
The exact syntax may vary depending on your shell and OS."
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
|
||||
//! Ph.D., Editor.
|
||||
|
||||
use statistical::median;
|
||||
use crate::util::statistics::median;
|
||||
|
||||
/// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
|
||||
/// converts the MAD to an estimator for the standard deviation. The second factor is the number
|
||||
|
||||
+6
-6
@@ -95,16 +95,16 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
|
||||
discard(output);
|
||||
}
|
||||
|
||||
let status = child.wait()?;
|
||||
let exit_status = child.wait()?;
|
||||
|
||||
let time_real = wallclock_timer.stop();
|
||||
let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
|
||||
let time_wall_clock = wallclock_timer.stop();
|
||||
let (time_user, time_system, peak_memory_usage) = cpu_timer.stop();
|
||||
|
||||
Ok(Measurement {
|
||||
time_real,
|
||||
time_wall_clock,
|
||||
time_user,
|
||||
time_system,
|
||||
memory_usage_byte,
|
||||
status,
|
||||
peak_memory_usage,
|
||||
exit_status,
|
||||
})
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
/// A max function for f64's without NaNs
|
||||
pub fn max(vals: &[f64]) -> f64 {
|
||||
*vals
|
||||
.iter()
|
||||
.max_by(|a, b| a.partial_cmp(b).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// A min function for f64's without NaNs
|
||||
pub fn min(vals: &[f64]) -> f64 {
|
||||
*vals
|
||||
.iter()
|
||||
.min_by(|a, b| a.partial_cmp(b).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max() {
|
||||
let assert_float_eq = |a: f64, b: f64| {
|
||||
assert!((a - b).abs() < f64::EPSILON);
|
||||
};
|
||||
|
||||
assert_float_eq(1.0, max(&[1.0]));
|
||||
assert_float_eq(-1.0, max(&[-1.0]));
|
||||
assert_float_eq(-1.0, max(&[-2.0, -1.0]));
|
||||
assert_float_eq(1.0, max(&[-1.0, 1.0]));
|
||||
assert_float_eq(1.0, max(&[-1.0, 1.0, 0.0]));
|
||||
}
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
pub mod exit_code;
|
||||
pub mod min_max;
|
||||
pub mod number;
|
||||
pub mod randomized_environment_offset;
|
||||
pub mod statistics;
|
||||
pub mod units;
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Statistics for non-empty samples of finite measurements.
|
||||
|
||||
/// A min function that assumes no NaNs and at least one element
|
||||
pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
|
||||
values
|
||||
.into_iter()
|
||||
.min_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
|
||||
.expect("'min' requires at least one element")
|
||||
}
|
||||
|
||||
/// A max function that assumes no NaNs and at least one element
|
||||
pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
|
||||
values
|
||||
.into_iter()
|
||||
.max_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
|
||||
.expect("'max' requires at least one element")
|
||||
}
|
||||
|
||||
/// Arithmetic mean, summing measurements in their original order.
|
||||
pub fn mean(values: impl IntoIterator<Item = f64>) -> f64 {
|
||||
let mut sum = 0.0;
|
||||
let mut count = 0usize;
|
||||
for value in values {
|
||||
sum += value;
|
||||
count += 1;
|
||||
}
|
||||
sum / count as f64
|
||||
}
|
||||
|
||||
/// Median without modifying the input sample.
|
||||
pub fn median(values: &[f64]) -> f64 {
|
||||
assert!(
|
||||
!values.is_empty(),
|
||||
"median requires at least one measurement"
|
||||
);
|
||||
let mut sorted = values.to_vec();
|
||||
sorted.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
|
||||
|
||||
let mid = sorted.len() / 2;
|
||||
if sorted.len() % 2 == 1 {
|
||||
sorted[mid]
|
||||
} else {
|
||||
(sorted[mid] + sorted[mid - 1]) / 2.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Sample standard deviation, using an already computed mean.
|
||||
pub fn standard_deviation(values: &[f64], mean: f64) -> f64 {
|
||||
assert!(
|
||||
values.len() > 1,
|
||||
"standard deviation requires at least two measurements"
|
||||
);
|
||||
let sum = values
|
||||
.iter()
|
||||
.map(|value| (value - mean) * (value - mean))
|
||||
.fold(0.0, |sum, deviation| sum + deviation);
|
||||
assert!(sum >= 0.0, "invalid sum of squared deviations");
|
||||
|
||||
// Preserve the operation order of statistical 1.0: dividing by n - 1
|
||||
// before taking the square root also preserves exported floating-point values.
|
||||
(sum / (values.len() as f64 - 1.0)).sqrt()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use approx::assert_relative_eq;
|
||||
|
||||
#[test]
|
||||
fn test_min() {
|
||||
assert_eq!(1.0, min([1.0]));
|
||||
assert_eq!(-1.0, min([-1.0]));
|
||||
assert_eq!(-2.0, min([-2.0, -1.0]));
|
||||
assert_eq!(-1.0, min([-1.0, 1.0]));
|
||||
assert_eq!(-1.0, min([1.0, -1.0, 0.0]));
|
||||
|
||||
let values = vec![1024.0, 2.0, 3.0];
|
||||
assert_eq!(*min(&values), 2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max() {
|
||||
assert_eq!(1.0, max([1.0]));
|
||||
assert_eq!(-1.0, max([-1.0]));
|
||||
assert_eq!(-1.0, max([-2.0, -1.0]));
|
||||
assert_eq!(1.0, max([-1.0, 1.0]));
|
||||
assert_eq!(1.0, max([-1.0, 1.0, 0.0]));
|
||||
|
||||
let values = vec![1.0, 2048.0, 3.0];
|
||||
assert_eq!(*max(&values), 2048.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mean() {
|
||||
assert_eq!(1.0, mean([1.0]));
|
||||
assert_relative_eq!(2.0, mean([1.0, 3.0]));
|
||||
|
||||
let values = [0.1, 0.2, 0.6];
|
||||
assert_relative_eq!(mean(values), 0.3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_median() {
|
||||
assert_eq!(1.0, median(&[1.0]));
|
||||
assert_relative_eq!(2.0, median(&[1.0, 3.0]));
|
||||
|
||||
let values = [0.1, 0.2, 0.6];
|
||||
assert_relative_eq!(median(&values), 0.2);
|
||||
|
||||
let values = [0.1, 0.2, 0.3, 0.6];
|
||||
assert_relative_eq!(median(&values), 0.25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_standard_deviation() {
|
||||
let values = [0.1, 0.2, 0.3];
|
||||
assert_relative_eq!(standard_deviation(&values, mean(values)), 0.1);
|
||||
}
|
||||
}
|
||||
+248
-80
@@ -1,6 +1,8 @@
|
||||
mod common;
|
||||
use common::{hyperfine, hyperfine_raw_command};
|
||||
|
||||
use assert_cmd::assert::OutputAssertExt;
|
||||
use insta_cmd::assert_cmd_snapshot;
|
||||
use predicates::prelude::*;
|
||||
|
||||
/// Platform-specific I/O utility.
|
||||
@@ -9,12 +11,19 @@ use predicates::prelude::*;
|
||||
/// See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
|
||||
const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
|
||||
|
||||
pub fn hyperfine_debug() -> assert_cmd::Command {
|
||||
let mut cmd = hyperfine();
|
||||
pub fn hyperfine_debug() -> std::process::Command {
|
||||
let mut cmd = hyperfine_raw_command();
|
||||
cmd.arg("--debug-mode");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn snapshot_settings() -> insta::Settings {
|
||||
let mut settings = insta::Settings::clone_current();
|
||||
// Ignore trailing report padding while preserving indentation and internal spacing.
|
||||
settings.add_filter(r"(?m)[ \t]+$", "");
|
||||
settings
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runs_successfully() {
|
||||
hyperfine()
|
||||
@@ -594,86 +603,187 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_relative_times() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("2.00 ± 0.00 times faster")
|
||||
.and(predicate::str::contains("3.00 ± 0.00 times faster")),
|
||||
);
|
||||
.arg("sleep 3.0"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1.0 ran
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
3.00 ± 0.00 times faster than sleep 3.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_same_time() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--command-name=A")
|
||||
.arg("--command-name=B")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 1000.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("As fast (1.00 ± 0.00) as")
|
||||
.and(predicate::str::contains("2.00 ± 0.00 times faster"))
|
||||
.and(predicate::str::contains("1000.00 ± 0.00 times faster")),
|
||||
);
|
||||
.arg("sleep 1000.0"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: A
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 2: B
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 1000.0
|
||||
Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1000.000 s … 1000.000 s 10 runs
|
||||
|
||||
Summary
|
||||
A ran
|
||||
As fast (1.00 ± 0.00) as B
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
1000.00 ± 0.00 times faster than sleep 1000.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_relative_to_reference() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("2.00 ± 0.00 times slower")
|
||||
.and(predicate::str::contains("1.50 ± 0.00 times faster")),
|
||||
);
|
||||
.arg("sleep 3.0"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 2.0 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_reference_name() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("--reference-name=refabc123")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
|
||||
.arg("sleep 3.0"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: refabc123
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
refabc123 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
.arg("30")
|
||||
.arg("45")
|
||||
.arg("--parameter-step-size")
|
||||
.arg("5")
|
||||
.arg("sleep {time}")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Benchmark 1: sleep 30")
|
||||
.and(predicate::str::contains("Benchmark 2: sleep 35"))
|
||||
.and(predicate::str::contains("Benchmark 3: sleep 40"))
|
||||
.and(predicate::str::contains("Benchmark 4: sleep 45"))
|
||||
.and(predicate::str::contains("Benchmark 5: sleep 50").not()),
|
||||
);
|
||||
.arg("sleep {time}"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 30 ran
|
||||
1.17 ± 0.00 times faster than sleep 35
|
||||
1.33 ± 0.00 times faster than sleep 40
|
||||
1.50 ± 0.00 times faster than sleep 45
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 25")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
@@ -681,33 +791,62 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||
.arg("45")
|
||||
.arg("--parameter-step-size")
|
||||
.arg("5")
|
||||
.arg("sleep {time}")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Benchmark 1: sleep 25")
|
||||
.and(predicate::str::contains("Benchmark 2: sleep 30"))
|
||||
.and(predicate::str::contains("Benchmark 3: sleep 35"))
|
||||
.and(predicate::str::contains("Benchmark 4: sleep 40"))
|
||||
.and(predicate::str::contains("Benchmark 5: sleep 45"))
|
||||
.and(predicate::str::contains("Benchmark 6: sleep 50").not()),
|
||||
);
|
||||
.arg("sleep {time}"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 25
|
||||
Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 25.000 s … 25.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
|
||||
Benchmark 5: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 25 ran
|
||||
1.20 ± 0.00 times faster than sleep 30
|
||||
1.40 ± 0.00 times faster than sleep 35
|
||||
1.60 ± 0.00 times faster than sleep 40
|
||||
1.80 ± 0.00 times faster than sleep 45
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intermediate_results_are_not_exported_to_stdout() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=none") // To only see the Markdown export on stdout
|
||||
.arg("--export-markdown")
|
||||
.arg("-")
|
||||
.arg("sleep 1")
|
||||
.arg("sleep 2")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
(predicate::str::contains("sleep 1").count(1))
|
||||
.and(predicate::str::contains("sleep 2").count(1)),
|
||||
);
|
||||
.arg("sleep 2"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||||
| `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
|
||||
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -749,27 +888,56 @@ fn unused_parameters_are_shown_in_benchmark_name() {
|
||||
|
||||
#[test]
|
||||
fn speed_comparison_sort_order() {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
hyperfine_debug()
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}"))
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"sleep 1 ran\n 2.00 ± 0.00 times faster than sleep 2",
|
||||
));
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
insta::allow_duplicates! {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}")), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1 ran
|
||||
2.00 ± 0.00 times faster than sleep 2
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
}
|
||||
|
||||
hyperfine_debug()
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg("--sort=command")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"2.00 ± 0.00 sleep 2\n 1.00 sleep 1",
|
||||
));
|
||||
.arg("--sort=command"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Relative speed comparison
|
||||
2.00 ± 0.00 sleep 2
|
||||
1.00 sleep 1
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
|
||||
Reference in new issue
Block a user