mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-20 03:11:28 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f326201882 |
No files matched your search
Generated
+25
-35
@@ -23,7 +23,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -53,15 +53,7 @@ dependencies = [
|
||||
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colored"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -76,7 +68,7 @@ dependencies = [
|
||||
"regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"termios 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -95,12 +87,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "hyperfine"
|
||||
version = "0.4.0"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"approx 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"clap 2.29.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"colored 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"indicatif 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@@ -158,8 +149,8 @@ dependencies = [
|
||||
"num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-rational 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -168,7 +159,7 @@ version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
@@ -178,7 +169,7 @@ name = "num-complex"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@@ -187,7 +178,7 @@ name = "num-integer"
|
||||
version = "0.1.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -196,23 +187,23 @@ version = "0.1.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.1.41"
|
||||
version = "0.1.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.1.42"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
@@ -240,7 +231,7 @@ dependencies = [
|
||||
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"smallvec 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -387,11 +378,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.4"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -401,12 +392,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[metadata]
|
||||
@@ -417,7 +408,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf"
|
||||
"checksum clap 2.29.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4151c5790817c7d21bbdc6c3530811f798172915f93258244948b93ba19604a6"
|
||||
"checksum clicolors-control 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1f84dec9bc083ce2503908cd305af98bd363da6f54bf8d4bf0ac14ee749ad5d1"
|
||||
"checksum colored 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b0aa3473e85a3161b59845d6096b289bb577874cafeaf75ea1b1beaa6572c7fc"
|
||||
"checksum console 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7649ca90478264b9686aac8d269fcb014f14c2bed7c79a7e51b9f6afd4d783eb"
|
||||
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
||||
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
|
||||
@@ -432,8 +422,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
"checksum num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "58de7b4bf7cf5dbecb635a5797d489864eadd03b107930cbccf9e0fd7428b47c"
|
||||
"checksum num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "d1452e8b06e448a07f0e6ebb0bb1d92b8890eea63288c0b627331d53514d0fba"
|
||||
"checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01"
|
||||
"checksum num-rational 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "0b950f75e042fdd710460084d19c8efdcd72d65183ead8ecd04b90483f5a55d2"
|
||||
"checksum num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "9936036cc70fe4a8b2d338ab665900323290efb03983c86cbe235ae800ad8017"
|
||||
"checksum num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "0c7cb72a95250d8a370105c828f388932373e0e94414919891a0f945222310fe"
|
||||
"checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070"
|
||||
"checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37"
|
||||
"checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412"
|
||||
"checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8"
|
||||
@@ -457,7 +447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
|
||||
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
|
||||
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
|
||||
"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3"
|
||||
"checksum winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b09fb3b6f248ea4cd42c9a65113a847d612e17505d6ebd1f7357ad68a8bf8693"
|
||||
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
|
||||
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
"checksum winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ec6667f60c23eca65c561e63a13d81b44234c2e38a6b6c959025ee907ec614cc"
|
||||
"checksum winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98f12c52b2630cd05d2c3ffd8e008f7f48252c042b4871c72aed9dc733b96668"
|
||||
+2
-3
@@ -7,14 +7,13 @@ license = "MIT/Apache-2.0"
|
||||
name = "hyperfine"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/sharkdp/hyperfine"
|
||||
version = "0.4.0"
|
||||
version = "0.3.0"
|
||||
|
||||
[dependencies]
|
||||
colored = "1.6"
|
||||
ansi_term = "0.10"
|
||||
indicatif = "0.9"
|
||||
statistical = "0.1"
|
||||
libc = "0.2"
|
||||
atty = "0.2.2"
|
||||
|
||||
[dependencies.clap]
|
||||
version = "2"
|
||||
|
||||
@@ -11,12 +11,10 @@ A command-line benchmarking tool (*inspired by [bench](https://github.com/Gabrie
|
||||
|
||||
## Features
|
||||
|
||||
* Statistical analysis across multiple runs.
|
||||
* Support for arbitrary shell commands.
|
||||
* Constant feedback about the benchmark progress and current estimates.
|
||||
* Warmup runs can be executed before the actual benchmark.
|
||||
* Cache-clearing commands can be set up before each timing run.
|
||||
* Statisical outlier detection.
|
||||
* Statistical analysis across multiple runs
|
||||
* Support for arbitrary shell commands
|
||||
* Constant feedback about the benchmark progress and current estimates
|
||||
* Warmup runs can be executed before the actual benchmark
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -25,14 +23,14 @@ A command-line benchmarking tool (*inspired by [bench](https://github.com/Gabrie
|
||||
To run a benchmark, you can simply call `hyperfine <command>...`. The argument(s) can be any
|
||||
shell command. For example:
|
||||
``` bash
|
||||
hyperfine 'sleep 0.3'
|
||||
> hyperfine 'sleep 0.3'
|
||||
```
|
||||
|
||||
Hyperfine will automatically determine the number of runs to perform for each command. By default,
|
||||
it will perform *at least* 10 benchmarking runs. To change this, you can use the `-m`/`--min-runs`
|
||||
option:
|
||||
``` bash
|
||||
hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
|
||||
> hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
|
||||
```
|
||||
|
||||
### I/O-heavy programs
|
||||
@@ -43,7 +41,7 @@ influence by disk caches and whether they are cold or warm.
|
||||
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform
|
||||
a certain number of program executions before the actual benchmark:
|
||||
``` bash
|
||||
hyperfine --warmup 3 'grep -R TODO *'
|
||||
> hyperfine --warmup 3 'grep -R TODO *'
|
||||
```
|
||||
|
||||
Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`
|
||||
@@ -52,7 +50,7 @@ on Linux, you can run
|
||||
``` bash
|
||||
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
|
||||
```
|
||||
To use this specific command with Hyperfine, call `sudo -v` to temporarily gain sudo permissions
|
||||
To use this specific command with Hyperfine, call `sudo echo` to temporarily gain sudo permissions
|
||||
and then call:
|
||||
``` bash
|
||||
hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *'
|
||||
@@ -72,16 +70,6 @@ On Arch Linux, hyperfine can be installed [from the AUR](https://aur.archlinux.o
|
||||
yaourt -S hyperfine
|
||||
```
|
||||
|
||||
### Ubuntu
|
||||
|
||||
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
|
||||
and install it via `dpkg`:
|
||||
```
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v0.4.0/hyperfine_0.4.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_0.4.0_amd64.deb
|
||||
```
|
||||
|
||||
|
||||
## Origin of the name
|
||||
|
||||
The name *hyperfine* was chosen in reference to the hyperfine levels of caesium 133 which play a crucial role in the
|
||||
|
||||
+30
-59
@@ -2,15 +2,13 @@ use std::io;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
use colored::*;
|
||||
use ansi_term::Colour::{Blue, Cyan, Green, Purple, White, Yellow};
|
||||
use statistical::{mean, standard_deviation};
|
||||
|
||||
use hyperfine::internal::{get_progress_bar, max, min, CmdFailureAction, HyperfineOptions,
|
||||
OutputStyleOption, Second, MIN_EXECUTION_TIME};
|
||||
use hyperfine::warnings::Warnings;
|
||||
use hyperfine::internal::{get_progress_bar, max, min, CmdFailureAction, HyperfineOptions, Second,
|
||||
Warnings, MIN_EXECUTION_TIME};
|
||||
use hyperfine::format::{format_duration, format_duration_unit};
|
||||
use hyperfine::cputime::{cpu_time_interval, get_cpu_times};
|
||||
use hyperfine::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
|
||||
|
||||
/// Results from timing a single shell command
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
@@ -88,9 +86,9 @@ pub fn time_shell_command(
|
||||
}
|
||||
|
||||
/// Measure the average shell spawning time
|
||||
pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result<TimingResult> {
|
||||
pub fn mean_shell_spawning_time() -> io::Result<TimingResult> {
|
||||
const COUNT: u64 = 200;
|
||||
let progress_bar = get_progress_bar(COUNT, "Measuring shell spawning time", style);
|
||||
let progress_bar = get_progress_bar(COUNT, "Measuring shell spawning time");
|
||||
|
||||
let mut times_real: Vec<Second> = vec![];
|
||||
let mut times_user: Vec<Second> = vec![];
|
||||
@@ -125,21 +123,6 @@ pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result<TimingR
|
||||
})
|
||||
}
|
||||
|
||||
/// Run the command specified by `--prepare`.
|
||||
fn run_preparation_command(command: &Option<String>) -> io::Result<()> {
|
||||
if let &Some(ref preparation_command) = command {
|
||||
let res = time_shell_command(preparation_command, CmdFailureAction::RaiseError, None);
|
||||
if res.is_err() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"The preparation command terminated with a non-zero exit code. \
|
||||
Append ' || true' to the command if you are sure that this can be ignored.",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the benchmark for a single shell command
|
||||
pub fn run_benchmark(
|
||||
num: usize,
|
||||
@@ -149,8 +132,8 @@ pub fn run_benchmark(
|
||||
) -> io::Result<()> {
|
||||
println!(
|
||||
"{}{}: {}",
|
||||
"Benchmark #".bold(),
|
||||
(num + 1).to_string().bold(),
|
||||
White.bold().paint("Benchmark #"),
|
||||
White.bold().paint((num + 1).to_string()),
|
||||
cmd
|
||||
);
|
||||
println!();
|
||||
@@ -162,11 +145,7 @@ pub fn run_benchmark(
|
||||
|
||||
// Warmup phase
|
||||
if options.warmup_count > 0 {
|
||||
let progress_bar = get_progress_bar(
|
||||
options.warmup_count,
|
||||
"Performing warmup runs",
|
||||
&options.output_style,
|
||||
);
|
||||
let progress_bar = get_progress_bar(options.warmup_count, "Performing warmup runs");
|
||||
|
||||
for _ in 0..options.warmup_count {
|
||||
let _ = time_shell_command(cmd, options.failure_action, None)?;
|
||||
@@ -176,14 +155,15 @@ pub fn run_benchmark(
|
||||
}
|
||||
|
||||
// Set up progress bar (and spinner for initial measurement)
|
||||
let progress_bar = get_progress_bar(
|
||||
options.min_runs,
|
||||
"Initial time measurement",
|
||||
&options.output_style,
|
||||
);
|
||||
let progress_bar = get_progress_bar(options.min_runs, "Initial time measurement");
|
||||
|
||||
// Run init / cleanup command
|
||||
run_preparation_command(&options.preparation_command)?;
|
||||
let run_preparation_command = || {
|
||||
if let Some(ref preparation_command) = options.preparation_command {
|
||||
let _ = time_shell_command(preparation_command, options.failure_action, None);
|
||||
}
|
||||
};
|
||||
run_preparation_command();
|
||||
|
||||
// Initial timing run
|
||||
let (res, success) =
|
||||
@@ -213,11 +193,11 @@ pub fn run_benchmark(
|
||||
|
||||
// Gather statistics
|
||||
for _ in 0..count_remaining {
|
||||
run_preparation_command(&options.preparation_command)?;
|
||||
run_preparation_command();
|
||||
|
||||
let msg = {
|
||||
let mean = format_duration(mean(×_real), None);
|
||||
format!("Current estimate: {}", mean.to_string().green())
|
||||
format!("Current estimate: {}", Green.paint(mean))
|
||||
};
|
||||
progress_bar.set_message(&msg);
|
||||
|
||||
@@ -253,22 +233,22 @@ pub fn run_benchmark(
|
||||
let system_str = format_duration(system_mean, Some(user_unit));
|
||||
|
||||
println!(
|
||||
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
|
||||
"mean".green().bold(),
|
||||
"σ".green(),
|
||||
mean_str.green().bold(),
|
||||
stddev_str.green(),
|
||||
user_str.blue(),
|
||||
system_str.blue()
|
||||
" Time ({} ± {}): {} ± {} [User: {}, System: {}]",
|
||||
Green.bold().paint("mean"),
|
||||
Green.paint("σ"),
|
||||
Green.bold().paint(mean_str),
|
||||
Green.paint(stddev_str),
|
||||
Blue.paint(user_str),
|
||||
Blue.paint(system_str)
|
||||
);
|
||||
println!(" ");
|
||||
|
||||
println!(
|
||||
" Range ({} … {}): {:>8} … {:>8}",
|
||||
"min".cyan(),
|
||||
"max".purple(),
|
||||
min_str.cyan(),
|
||||
max_str.purple()
|
||||
" Range ({} … {}): {} … {}",
|
||||
Cyan.paint("min"),
|
||||
Purple.paint("max"),
|
||||
Cyan.paint(min_str),
|
||||
Purple.paint(max_str)
|
||||
);
|
||||
|
||||
// Warnings
|
||||
@@ -284,19 +264,10 @@ pub fn run_benchmark(
|
||||
warnings.push(Warnings::NonZeroExitCode);
|
||||
}
|
||||
|
||||
// Run outlier detection
|
||||
let scores = modified_zscores(×_real);
|
||||
if scores[0] > OUTLIER_THRESHOLD {
|
||||
warnings.push(Warnings::SlowInitialRun(times_real[0]));
|
||||
} else if scores.iter().any(|&s| s > OUTLIER_THRESHOLD) {
|
||||
warnings.push(Warnings::OutliersDetected);
|
||||
}
|
||||
|
||||
if !warnings.is_empty() {
|
||||
eprintln!(" ");
|
||||
|
||||
for warning in &warnings {
|
||||
eprintln!(" {}: {}", "Warning".yellow(), warning);
|
||||
eprintln!(" {}: {}", Yellow.paint("Warning"), warning);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ pub fn format_duration(duration: Second, unit: Option<Unit>) -> String {
|
||||
/// Like `format_duration`, but returns the target unit as well.
|
||||
pub fn format_duration_unit(duration: Second, unit: Option<Unit>) -> (String, Unit) {
|
||||
if (duration < 1.0 && unit.is_none()) || unit == Some(Unit::MilliSecond) {
|
||||
(format!("{:.1} ms", duration * 1e3), Unit::MilliSecond)
|
||||
(format!("{:5.1} ms", duration * 1e3), Unit::MilliSecond)
|
||||
} else {
|
||||
(format!("{:.3} s", duration), Unit::Second)
|
||||
(format!("{:6.3} s", duration), Unit::Second)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,12 +27,12 @@ pub fn format_duration_unit(duration: Second, unit: Option<Unit>) -> (String, Un
|
||||
fn test_format_duration_unit_basic() {
|
||||
let (out_str, out_unit) = format_duration_unit(1.3, None);
|
||||
|
||||
assert_eq!("1.300 s", out_str);
|
||||
assert_eq!(" 1.300 s", out_str);
|
||||
assert_eq!(Unit::Second, out_unit);
|
||||
|
||||
let (out_str, out_unit) = format_duration_unit(1.0, None);
|
||||
|
||||
assert_eq!("1.000 s", out_str);
|
||||
assert_eq!(" 1.000 s", out_str);
|
||||
assert_eq!(Unit::Second, out_unit);
|
||||
|
||||
let (out_str, out_unit) = format_duration_unit(0.999, None);
|
||||
@@ -42,7 +42,7 @@ fn test_format_duration_unit_basic() {
|
||||
|
||||
let (out_str, out_unit) = format_duration_unit(0.0, None);
|
||||
|
||||
assert_eq!("0.0 ms", out_str);
|
||||
assert_eq!(" 0.0 ms", out_str);
|
||||
assert_eq!(Unit::MilliSecond, out_unit);
|
||||
|
||||
let (out_str, out_unit) = format_duration_unit(1000.0, None);
|
||||
@@ -55,7 +55,7 @@ fn test_format_duration_unit_basic() {
|
||||
fn test_format_duration_unit_with_unit() {
|
||||
let (out_str, out_unit) = format_duration_unit(1.3, Some(Unit::Second));
|
||||
|
||||
assert_eq!("1.300 s", out_str);
|
||||
assert_eq!(" 1.300 s", out_str);
|
||||
assert_eq!(Unit::Second, out_unit);
|
||||
|
||||
let (out_str, out_unit) = format_duration_unit(1.3, Some(Unit::MilliSecond));
|
||||
|
||||
+26
-25
@@ -1,3 +1,5 @@
|
||||
use std::fmt;
|
||||
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
|
||||
/// Type alias for unit of time
|
||||
@@ -16,16 +18,6 @@ pub enum CmdFailureAction {
|
||||
Ignore,
|
||||
}
|
||||
|
||||
/// Output style type option
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub enum OutputStyleOption {
|
||||
/// Do not output with colors or any special formatting
|
||||
Basic,
|
||||
|
||||
/// Output with full color and formatting
|
||||
Full,
|
||||
}
|
||||
|
||||
/// A set of options for hyperfine
|
||||
pub struct HyperfineOptions {
|
||||
/// Number of warmup runs
|
||||
@@ -42,9 +34,6 @@ pub struct HyperfineOptions {
|
||||
|
||||
/// Command to run before each timing run
|
||||
pub preparation_command: Option<String>,
|
||||
|
||||
/// What color mode to use for output
|
||||
pub output_style: OutputStyleOption,
|
||||
}
|
||||
|
||||
impl Default for HyperfineOptions {
|
||||
@@ -55,24 +44,17 @@ impl Default for HyperfineOptions {
|
||||
min_time_sec: 3.0,
|
||||
failure_action: CmdFailureAction::RaiseError,
|
||||
preparation_command: None,
|
||||
output_style: OutputStyleOption::Full,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a pre-configured progress bar
|
||||
pub fn get_progress_bar(length: u64, msg: &str, option: &OutputStyleOption) -> ProgressBar {
|
||||
let progressbar_style = match *option {
|
||||
OutputStyleOption::Basic => ProgressStyle::default_bar(),
|
||||
OutputStyleOption::Full => ProgressStyle::default_spinner()
|
||||
.tick_chars("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏")
|
||||
.template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise}"),
|
||||
};
|
||||
pub fn get_progress_bar(length: u64, msg: &str) -> ProgressBar {
|
||||
let progressbar_style = ProgressStyle::default_spinner()
|
||||
.tick_chars("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏")
|
||||
.template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise}");
|
||||
|
||||
let progress_bar = match *option {
|
||||
OutputStyleOption::Basic => ProgressBar::hidden(),
|
||||
OutputStyleOption::Full => ProgressBar::new(length),
|
||||
};
|
||||
let progress_bar = ProgressBar::new(length);
|
||||
progress_bar.set_style(progressbar_style.clone());
|
||||
progress_bar.enable_steady_tick(80);
|
||||
progress_bar.set_message(msg);
|
||||
@@ -80,6 +62,25 @@ pub fn get_progress_bar(length: u64, msg: &str, option: &OutputStyleOption) -> P
|
||||
progress_bar
|
||||
}
|
||||
|
||||
/// Possible benchmark warnings
|
||||
pub enum Warnings {
|
||||
FastExecutionTime,
|
||||
NonZeroExitCode,
|
||||
}
|
||||
|
||||
impl fmt::Display for Warnings {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match *self {
|
||||
Warnings::FastExecutionTime => write!(
|
||||
f,
|
||||
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
|
||||
MIN_EXECUTION_TIME * 1e3
|
||||
),
|
||||
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A max function for f64's without NaNs
|
||||
pub fn max(vals: &[f64]) -> f64 {
|
||||
*vals.iter()
|
||||
|
||||
@@ -2,5 +2,3 @@ pub mod benchmark;
|
||||
pub mod format;
|
||||
pub mod internal;
|
||||
pub mod cputime;
|
||||
pub mod outlier_detection;
|
||||
pub mod warnings;
|
||||
@@ -1,102 +0,0 @@
|
||||
/// A module for statistical outlier detection.
|
||||
///
|
||||
/// References:
|
||||
/// - Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and Handle Outliers",
|
||||
/// The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
|
||||
/// Ph.D., Editor.
|
||||
|
||||
use statistical::median;
|
||||
|
||||
/// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
|
||||
/// converts the MAD to an estimator for the standard deviation. The second factor is the number
|
||||
/// of standard deviations.
|
||||
pub const OUTLIER_THRESHOLD: f64 = 1.4826 * 10.0;
|
||||
|
||||
/// Compute modifized Z-scores for a given sample. A (unmodified) Z-score is defined by
|
||||
/// `(x_i - x_mean)/x_stddev` whereas the modified Z-score is defined by `|x_i - x_median|/MAD`
|
||||
/// where MAD is the median average deviation.
|
||||
///
|
||||
/// References:
|
||||
/// - <https://en.wikipedia.org/wiki/Median_absolute_deviation>
|
||||
pub fn modified_zscores(xs: &[f64]) -> Vec<f64> {
|
||||
assert!(!xs.is_empty());
|
||||
|
||||
// Compute sample median:
|
||||
let x_median = median(xs);
|
||||
|
||||
// Compute the absolute deviations from the median:
|
||||
let deviations: Vec<f64> = xs.iter().map(|x| (x - x_median).abs()).collect();
|
||||
|
||||
// Compute median absolute deviation:
|
||||
let mad = median(&deviations);
|
||||
|
||||
// Compute modified Z-scores (x_i - x_median) / MAD
|
||||
xs.iter().map(|&x| (x - x_median) / mad).collect()
|
||||
}
|
||||
|
||||
/// Return the number of outliers in a given sample. Outliers are defined as data points with a
|
||||
/// modified Z-score that is larger than `OUTLIER_THRESHOLD`.
|
||||
#[cfg(test)]
|
||||
pub fn num_outliers(xs: &[f64]) -> usize {
|
||||
if xs.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let scores = modified_zscores(xs);
|
||||
scores
|
||||
.iter()
|
||||
.filter(|&&s| s.abs() > OUTLIER_THRESHOLD)
|
||||
.count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_outliers() {
|
||||
// Should not detect outliers in small samples
|
||||
assert_eq!(0, num_outliers(&[]));
|
||||
assert_eq!(0, num_outliers(&[50.0]));
|
||||
assert_eq!(0, num_outliers(&[1000.0, 0.0]));
|
||||
|
||||
// Should not detect outliers in low-variance samples
|
||||
let xs = [-0.2, 0.0, 0.2];
|
||||
assert_eq!(0, num_outliers(&xs));
|
||||
|
||||
// Should detect a single outlier
|
||||
let xs = [-0.2, 0.0, 0.2, 4.0];
|
||||
assert_eq!(1, num_outliers(&xs));
|
||||
|
||||
// Should detect a single outlier
|
||||
let xs = [0.5, 0.30, 0.29, 0.31, 0.30];
|
||||
assert_eq!(1, num_outliers(&xs));
|
||||
|
||||
// Should detect no outliers in sample drawn from normal distribution
|
||||
let xs = [
|
||||
2.33269488,
|
||||
1.42195907,
|
||||
-0.57527698,
|
||||
-0.31293437,
|
||||
2.2948158,
|
||||
0.75813273,
|
||||
-1.0712388,
|
||||
-0.96394741,
|
||||
-1.15897446,
|
||||
1.10976285,
|
||||
];
|
||||
assert_eq!(0, num_outliers(&xs));
|
||||
|
||||
// Should detect two outliers that were manually added
|
||||
let xs = [
|
||||
2.33269488,
|
||||
1.42195907,
|
||||
-0.57527698,
|
||||
-0.31293437,
|
||||
2.2948158,
|
||||
0.75813273,
|
||||
-1.0712388,
|
||||
-0.96394741,
|
||||
-1.15897446,
|
||||
1.10976285,
|
||||
20.0,
|
||||
-500.0,
|
||||
];
|
||||
assert_eq!(2, num_outliers(&xs));
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
use std::fmt;
|
||||
|
||||
use hyperfine::internal::{Second, MIN_EXECUTION_TIME};
|
||||
use hyperfine::format::format_duration;
|
||||
|
||||
/// A list of all possible warnings
|
||||
pub enum Warnings {
|
||||
FastExecutionTime,
|
||||
NonZeroExitCode,
|
||||
SlowInitialRun(Second),
|
||||
OutliersDetected,
|
||||
}
|
||||
|
||||
impl fmt::Display for Warnings {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match *self {
|
||||
Warnings::FastExecutionTime => write!(
|
||||
f,
|
||||
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
|
||||
MIN_EXECUTION_TIME * 1e3
|
||||
),
|
||||
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
|
||||
Warnings::SlowInitialRun(t_first) => write!(
|
||||
f,
|
||||
"The first benchmarking run for this command was significantly slower than the \
|
||||
rest ({}). This could be caused by (filesystem) caches that were not filled until \
|
||||
after the first run. You should consider using the '--warmup' option to fill \
|
||||
those caches before the actual benchmark. Alternatively, use the '--prepare' \
|
||||
option to clear the caches before each timing run.",
|
||||
format_duration(t_first, None)
|
||||
),
|
||||
Warnings::OutliersDetected => write!(
|
||||
f,
|
||||
"Statistical outliers were detected. Consider re-running this benchmark on a quiet \
|
||||
PC without any interferences from other programs. It might help to use the \
|
||||
'--warmup' or '--prepare' options."
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-41
@@ -1,8 +1,6 @@
|
||||
extern crate atty;
|
||||
|
||||
extern crate ansi_term;
|
||||
#[macro_use]
|
||||
extern crate clap;
|
||||
extern crate colored;
|
||||
extern crate indicatif;
|
||||
extern crate libc;
|
||||
extern crate statistical;
|
||||
@@ -15,24 +13,23 @@ use std::cmp;
|
||||
use std::error::Error;
|
||||
use std::io;
|
||||
|
||||
use atty::Stream;
|
||||
use colored::*;
|
||||
use ansi_term::Colour::Red;
|
||||
use clap::{App, AppSettings, Arg};
|
||||
|
||||
mod hyperfine;
|
||||
|
||||
use hyperfine::internal::{CmdFailureAction, HyperfineOptions, OutputStyleOption};
|
||||
use hyperfine::internal::{CmdFailureAction, HyperfineOptions};
|
||||
use hyperfine::benchmark::{mean_shell_spawning_time, run_benchmark};
|
||||
|
||||
/// Print error message to stderr and terminate
|
||||
pub fn error(message: &str) -> ! {
|
||||
eprintln!("{} {}", "Error:".red(), message);
|
||||
eprintln!("{}: {}", Red.paint("Error"), message);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
/// Runs the benchmark for the given commands
|
||||
fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result<()> {
|
||||
let shell_spawning_time = mean_shell_spawning_time(&options.output_style)?;
|
||||
let shell_spawning_time = mean_shell_spawning_time()?;
|
||||
|
||||
// Run the benchmarks
|
||||
for (num, cmd) in commands.iter().enumerate() {
|
||||
@@ -46,15 +43,9 @@ fn main() {
|
||||
// Process command line options
|
||||
let mut options = HyperfineOptions::default();
|
||||
|
||||
let clap_color_setting = if atty::is(Stream::Stdout) {
|
||||
AppSettings::ColoredHelp
|
||||
} else {
|
||||
AppSettings::ColorNever
|
||||
};
|
||||
|
||||
let matches = App::new("hyperfine")
|
||||
.version(crate_version!())
|
||||
.setting(clap_color_setting)
|
||||
.setting(AppSettings::ColoredHelp)
|
||||
.setting(AppSettings::DeriveDisplayOrder)
|
||||
.setting(AppSettings::UnifiedHelpMessage)
|
||||
.max_term_width(90)
|
||||
@@ -99,19 +90,6 @@ fn main() {
|
||||
clearing disk caches, for example.",
|
||||
),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("style")
|
||||
.long("style")
|
||||
.short("s")
|
||||
.takes_value(true)
|
||||
.value_name("TYPE")
|
||||
.possible_values(&["auto", "basic", "full"])
|
||||
.help(
|
||||
"Set output style type (default: auto). Set this to 'basic' to disable output \
|
||||
coloring and interactive elements. Set it to 'full' to enable all effects \
|
||||
even if no interactive terminal was detected.",
|
||||
),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("ignore-failure")
|
||||
.long("ignore-failure")
|
||||
@@ -136,19 +114,6 @@ fn main() {
|
||||
|
||||
options.preparation_command = matches.value_of("prepare").map(String::from);
|
||||
|
||||
options.output_style = match matches.value_of("style") {
|
||||
Some("full") => OutputStyleOption::Full,
|
||||
Some("basic") => OutputStyleOption::Basic,
|
||||
_ => if atty::is(Stream::Stdout) {
|
||||
OutputStyleOption::Full
|
||||
} else {
|
||||
OutputStyleOption::Basic
|
||||
},
|
||||
};
|
||||
|
||||
if options.output_style == OutputStyleOption::Basic {
|
||||
colored::control::set_override(false);
|
||||
}
|
||||
if matches.is_present("ignore-failure") {
|
||||
options.failure_action = CmdFailureAction::Ignore;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user