Compare commits

..
9 Commits
Author SHA1 Message Date
sharkdp 2859150219 Updates for 0.4.0 2018-01-28 18:43:04 +01:00
Steve Pentland 2e24ff012a Add new style option to disable coloring and progress bars (#24) 2018-01-28 14:35:52 +01:00
sharkdp d5c21ec272 Update warnings, switch to colored crate 2018-01-24 20:30:40 +01:00
sharkdp 2ddc21c7d4 Add outlier detection 2018-01-24 20:30:40 +01:00
David Peter cda5816dd3 Remove prompt characters 2018-01-24 08:32:11 +01:00
Anthony Som e4e937715e Recommend use of sudo -v over sudo echo
`sudo -v` (or `sudo --validate`) updates the sudo credentials cache without running any commands.

It's slightly safer than `sudo echo` under the unlikely chance that `~/.local/bin/echo` (or elsewhere on the PATH) could be set to something malicious.
2018-01-23 07:50:06 +01:00
sharkdp 2784b7fc1b Detect failure of preparation command 2018-01-21 18:11:27 +01:00
sharkdp 1fc78cd59b Add correct secure key 2018-01-21 18:11:27 +01:00
David Peter a08581e33a Add Ubuntu install instructions 2018-01-20 14:22:26 +01:00
10 changed files with 333 additions and 103 deletions

No files matched your search

Generated
+35 -25
View File
@@ -23,7 +23,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
"termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -53,7 +53,15 @@ dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "colored"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -68,7 +76,7 @@ dependencies = [
"regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"termios 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -87,11 +95,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "hyperfine"
version = "0.3.0"
version = "0.4.0"
dependencies = [
"ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
"approx 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.29.2 (registry+https://github.com/rust-lang/crates.io-index)",
"colored 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"indicatif 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
"statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -149,8 +158,8 @@ dependencies = [
"num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)",
"num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-rational 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -159,7 +168,7 @@ version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -169,7 +178,7 @@ name = "num-complex"
version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -178,7 +187,7 @@ name = "num-integer"
version = "0.1.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -187,23 +196,23 @@ version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-rational"
version = "0.1.40"
version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-traits"
version = "0.1.41"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@@ -231,7 +240,7 @@ dependencies = [
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -378,11 +387,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -392,12 +401,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.3.2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.3.2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
@@ -408,6 +417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf"
"checksum clap 2.29.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4151c5790817c7d21bbdc6c3530811f798172915f93258244948b93ba19604a6"
"checksum clicolors-control 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1f84dec9bc083ce2503908cd305af98bd363da6f54bf8d4bf0ac14ee749ad5d1"
"checksum colored 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b0aa3473e85a3161b59845d6096b289bb577874cafeaf75ea1b1beaa6572c7fc"
"checksum console 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7649ca90478264b9686aac8d269fcb014f14c2bed7c79a7e51b9f6afd4d783eb"
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
@@ -422,8 +432,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "58de7b4bf7cf5dbecb635a5797d489864eadd03b107930cbccf9e0fd7428b47c"
"checksum num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "d1452e8b06e448a07f0e6ebb0bb1d92b8890eea63288c0b627331d53514d0fba"
"checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01"
"checksum num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "0c7cb72a95250d8a370105c828f388932373e0e94414919891a0f945222310fe"
"checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070"
"checksum num-rational 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "0b950f75e042fdd710460084d19c8efdcd72d65183ead8ecd04b90483f5a55d2"
"checksum num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "9936036cc70fe4a8b2d338ab665900323290efb03983c86cbe235ae800ad8017"
"checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37"
"checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412"
"checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8"
@@ -447,7 +457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b09fb3b6f248ea4cd42c9a65113a847d612e17505d6ebd1f7357ad68a8bf8693"
"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ec6667f60c23eca65c561e63a13d81b44234c2e38a6b6c959025ee907ec614cc"
"checksum winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98f12c52b2630cd05d2c3ffd8e008f7f48252c042b4871c72aed9dc733b96668"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+3 -2
View File
@@ -7,13 +7,14 @@ license = "MIT/Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "0.3.0"
version = "0.4.0"
[dependencies]
ansi_term = "0.10"
colored = "1.6"
indicatif = "0.9"
statistical = "0.1"
libc = "0.2"
atty = "0.2.2"
[dependencies.clap]
version = "2"
+20 -8
View File
@@ -11,10 +11,12 @@ A command-line benchmarking tool (*inspired by [bench](https://github.com/Gabrie
## Features
* Statistical analysis across multiple runs
* Support for arbitrary shell commands
* Constant feedback about the benchmark progress and current estimates
* Warmup runs can be executed before the actual benchmark
* Statistical analysis across multiple runs.
* Support for arbitrary shell commands.
* Constant feedback about the benchmark progress and current estimates.
* Warmup runs can be executed before the actual benchmark.
* Cache-clearing commands can be set up before each timing run.
* Statisical outlier detection.
## Usage
@@ -23,14 +25,14 @@ A command-line benchmarking tool (*inspired by [bench](https://github.com/Gabrie
To run a benchmark, you can simply call `hyperfine <command>...`. The argument(s) can be any
shell command. For example:
``` bash
> hyperfine 'sleep 0.3'
hyperfine 'sleep 0.3'
```
Hyperfine will automatically determine the number of runs to perform for each command. By default,
it will perform *at least* 10 benchmarking runs. To change this, you can use the `-m`/`--min-runs`
option:
``` bash
> hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
hyperfine --min-runs 5 'sleep 0.2' 'sleep 3.2'
```
### I/O-heavy programs
@@ -41,7 +43,7 @@ influence by disk caches and whether they are cold or warm.
If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform
a certain number of program executions before the actual benchmark:
``` bash
> hyperfine --warmup 3 'grep -R TODO *'
hyperfine --warmup 3 'grep -R TODO *'
```
Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare`
@@ -50,7 +52,7 @@ on Linux, you can run
``` bash
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
```
To use this specific command with Hyperfine, call `sudo echo` to temporarily gain sudo permissions
To use this specific command with Hyperfine, call `sudo -v` to temporarily gain sudo permissions
and then call:
``` bash
hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *'
@@ -70,6 +72,16 @@ On Arch Linux, hyperfine can be installed [from the AUR](https://aur.archlinux.o
yaourt -S hyperfine
```
### Ubuntu
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v0.4.0/hyperfine_0.4.0_amd64.deb
sudo dpkg -i hyperfine_0.4.0_amd64.deb
```
## Origin of the name
The name *hyperfine* was chosen in reference to the hyperfine levels of caesium 133 which play a crucial role in the
+59 -30
View File
@@ -2,13 +2,15 @@ use std::io;
use std::process::{Command, Stdio};
use std::time::Instant;
use ansi_term::Colour::{Blue, Cyan, Green, Purple, White, Yellow};
use colored::*;
use statistical::{mean, standard_deviation};
use hyperfine::internal::{get_progress_bar, max, min, CmdFailureAction, HyperfineOptions, Second,
Warnings, MIN_EXECUTION_TIME};
use hyperfine::internal::{get_progress_bar, max, min, CmdFailureAction, HyperfineOptions,
OutputStyleOption, Second, MIN_EXECUTION_TIME};
use hyperfine::warnings::Warnings;
use hyperfine::format::{format_duration, format_duration_unit};
use hyperfine::cputime::{cpu_time_interval, get_cpu_times};
use hyperfine::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
/// Results from timing a single shell command
#[derive(Debug, Copy, Clone)]
@@ -86,9 +88,9 @@ pub fn time_shell_command(
}
/// Measure the average shell spawning time
pub fn mean_shell_spawning_time() -> io::Result<TimingResult> {
pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result<TimingResult> {
const COUNT: u64 = 200;
let progress_bar = get_progress_bar(COUNT, "Measuring shell spawning time");
let progress_bar = get_progress_bar(COUNT, "Measuring shell spawning time", style);
let mut times_real: Vec<Second> = vec![];
let mut times_user: Vec<Second> = vec![];
@@ -123,6 +125,21 @@ pub fn mean_shell_spawning_time() -> io::Result<TimingResult> {
})
}
/// Run the command specified by `--prepare`.
fn run_preparation_command(command: &Option<String>) -> io::Result<()> {
if let &Some(ref preparation_command) = command {
let res = time_shell_command(preparation_command, CmdFailureAction::RaiseError, None);
if res.is_err() {
return Err(io::Error::new(
io::ErrorKind::Other,
"The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.",
));
}
}
Ok(())
}
/// Run the benchmark for a single shell command
pub fn run_benchmark(
num: usize,
@@ -132,8 +149,8 @@ pub fn run_benchmark(
) -> io::Result<()> {
println!(
"{}{}: {}",
White.bold().paint("Benchmark #"),
White.bold().paint((num + 1).to_string()),
"Benchmark #".bold(),
(num + 1).to_string().bold(),
cmd
);
println!();
@@ -145,7 +162,11 @@ pub fn run_benchmark(
// Warmup phase
if options.warmup_count > 0 {
let progress_bar = get_progress_bar(options.warmup_count, "Performing warmup runs");
let progress_bar = get_progress_bar(
options.warmup_count,
"Performing warmup runs",
&options.output_style,
);
for _ in 0..options.warmup_count {
let _ = time_shell_command(cmd, options.failure_action, None)?;
@@ -155,15 +176,14 @@ pub fn run_benchmark(
}
// Set up progress bar (and spinner for initial measurement)
let progress_bar = get_progress_bar(options.min_runs, "Initial time measurement");
let progress_bar = get_progress_bar(
options.min_runs,
"Initial time measurement",
&options.output_style,
);
// Run init / cleanup command
let run_preparation_command = || {
if let Some(ref preparation_command) = options.preparation_command {
let _ = time_shell_command(preparation_command, options.failure_action, None);
}
};
run_preparation_command();
run_preparation_command(&options.preparation_command)?;
// Initial timing run
let (res, success) =
@@ -193,11 +213,11 @@ pub fn run_benchmark(
// Gather statistics
for _ in 0..count_remaining {
run_preparation_command();
run_preparation_command(&options.preparation_command)?;
let msg = {
let mean = format_duration(mean(&times_real), None);
format!("Current estimate: {}", Green.paint(mean))
format!("Current estimate: {}", mean.to_string().green())
};
progress_bar.set_message(&msg);
@@ -233,22 +253,22 @@ pub fn run_benchmark(
let system_str = format_duration(system_mean, Some(user_unit));
println!(
" Time ({} ± {}): {} ± {} [User: {}, System: {}]",
Green.bold().paint("mean"),
Green.paint("σ"),
Green.bold().paint(mean_str),
Green.paint(stddev_str),
Blue.paint(user_str),
Blue.paint(system_str)
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
"mean".green().bold(),
"σ".green(),
mean_str.green().bold(),
stddev_str.green(),
user_str.blue(),
system_str.blue()
);
println!(" ");
println!(
" Range ({} … {}): {} … {}",
Cyan.paint("min"),
Purple.paint("max"),
Cyan.paint(min_str),
Purple.paint(max_str)
" Range ({} … {}): {:>8} … {:>8}",
"min".cyan(),
"max".purple(),
min_str.cyan(),
max_str.purple()
);
// Warnings
@@ -264,10 +284,19 @@ pub fn run_benchmark(
warnings.push(Warnings::NonZeroExitCode);
}
// Run outlier detection
let scores = modified_zscores(&times_real);
if scores[0] > OUTLIER_THRESHOLD {
warnings.push(Warnings::SlowInitialRun(times_real[0]));
} else if scores.iter().any(|&s| s > OUTLIER_THRESHOLD) {
warnings.push(Warnings::OutliersDetected);
}
if !warnings.is_empty() {
eprintln!(" ");
for warning in &warnings {
eprintln!(" {}: {}", Yellow.paint("Warning"), warning);
eprintln!(" {}: {}", "Warning".yellow(), warning);
}
}
+6 -6
View File
@@ -17,9 +17,9 @@ pub fn format_duration(duration: Second, unit: Option<Unit>) -> String {
/// Like `format_duration`, but returns the target unit as well.
pub fn format_duration_unit(duration: Second, unit: Option<Unit>) -> (String, Unit) {
if (duration < 1.0 && unit.is_none()) || unit == Some(Unit::MilliSecond) {
(format!("{:5.1} ms", duration * 1e3), Unit::MilliSecond)
(format!("{:.1} ms", duration * 1e3), Unit::MilliSecond)
} else {
(format!("{:6.3} s", duration), Unit::Second)
(format!("{:.3} s", duration), Unit::Second)
}
}
@@ -27,12 +27,12 @@ pub fn format_duration_unit(duration: Second, unit: Option<Unit>) -> (String, Un
fn test_format_duration_unit_basic() {
let (out_str, out_unit) = format_duration_unit(1.3, None);
assert_eq!(" 1.300 s", out_str);
assert_eq!("1.300 s", out_str);
assert_eq!(Unit::Second, out_unit);
let (out_str, out_unit) = format_duration_unit(1.0, None);
assert_eq!(" 1.000 s", out_str);
assert_eq!("1.000 s", out_str);
assert_eq!(Unit::Second, out_unit);
let (out_str, out_unit) = format_duration_unit(0.999, None);
@@ -42,7 +42,7 @@ fn test_format_duration_unit_basic() {
let (out_str, out_unit) = format_duration_unit(0.0, None);
assert_eq!(" 0.0 ms", out_str);
assert_eq!("0.0 ms", out_str);
assert_eq!(Unit::MilliSecond, out_unit);
let (out_str, out_unit) = format_duration_unit(1000.0, None);
@@ -55,7 +55,7 @@ fn test_format_duration_unit_basic() {
fn test_format_duration_unit_with_unit() {
let (out_str, out_unit) = format_duration_unit(1.3, Some(Unit::Second));
assert_eq!(" 1.300 s", out_str);
assert_eq!("1.300 s", out_str);
assert_eq!(Unit::Second, out_unit);
let (out_str, out_unit) = format_duration_unit(1.3, Some(Unit::MilliSecond));
+25 -26
View File
@@ -1,5 +1,3 @@
use std::fmt;
use indicatif::{ProgressBar, ProgressStyle};
/// Type alias for unit of time
@@ -18,6 +16,16 @@ pub enum CmdFailureAction {
Ignore,
}
/// Output style type option
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum OutputStyleOption {
/// Do not output with colors or any special formatting
Basic,
/// Output with full color and formatting
Full,
}
/// A set of options for hyperfine
pub struct HyperfineOptions {
/// Number of warmup runs
@@ -34,6 +42,9 @@ pub struct HyperfineOptions {
/// Command to run before each timing run
pub preparation_command: Option<String>,
/// What color mode to use for output
pub output_style: OutputStyleOption,
}
impl Default for HyperfineOptions {
@@ -44,17 +55,24 @@ impl Default for HyperfineOptions {
min_time_sec: 3.0,
failure_action: CmdFailureAction::RaiseError,
preparation_command: None,
output_style: OutputStyleOption::Full,
}
}
}
/// Return a pre-configured progress bar
pub fn get_progress_bar(length: u64, msg: &str) -> ProgressBar {
let progressbar_style = ProgressStyle::default_spinner()
.tick_chars("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏")
.template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise}");
pub fn get_progress_bar(length: u64, msg: &str, option: &OutputStyleOption) -> ProgressBar {
let progressbar_style = match *option {
OutputStyleOption::Basic => ProgressStyle::default_bar(),
OutputStyleOption::Full => ProgressStyle::default_spinner()
.tick_chars("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏")
.template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise}"),
};
let progress_bar = ProgressBar::new(length);
let progress_bar = match *option {
OutputStyleOption::Basic => ProgressBar::hidden(),
OutputStyleOption::Full => ProgressBar::new(length),
};
progress_bar.set_style(progressbar_style.clone());
progress_bar.enable_steady_tick(80);
progress_bar.set_message(msg);
@@ -62,25 +80,6 @@ pub fn get_progress_bar(length: u64, msg: &str) -> ProgressBar {
progress_bar
}
/// Possible benchmark warnings
pub enum Warnings {
FastExecutionTime,
NonZeroExitCode,
}
impl fmt::Display for Warnings {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Warnings::FastExecutionTime => write!(
f,
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
MIN_EXECUTION_TIME * 1e3
),
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
}
}
}
/// A max function for f64's without NaNs
pub fn max(vals: &[f64]) -> f64 {
*vals.iter()
+2
View File
@@ -2,3 +2,5 @@ pub mod benchmark;
pub mod format;
pub mod internal;
pub mod cputime;
pub mod outlier_detection;
pub mod warnings;
+102
View File
@@ -0,0 +1,102 @@
/// A module for statistical outlier detection.
///
/// References:
/// - Boris Iglewicz and David Hoaglin (1993), "Volume 16: How to Detect and Handle Outliers",
/// The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
/// Ph.D., Editor.
use statistical::median;
/// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
/// converts the MAD to an estimator for the standard deviation. The second factor is the number
/// of standard deviations.
pub const OUTLIER_THRESHOLD: f64 = 1.4826 * 10.0;
/// Compute modifized Z-scores for a given sample. A (unmodified) Z-score is defined by
/// `(x_i - x_mean)/x_stddev` whereas the modified Z-score is defined by `|x_i - x_median|/MAD`
/// where MAD is the median average deviation.
///
/// References:
/// - <https://en.wikipedia.org/wiki/Median_absolute_deviation>
pub fn modified_zscores(xs: &[f64]) -> Vec<f64> {
assert!(!xs.is_empty());
// Compute sample median:
let x_median = median(xs);
// Compute the absolute deviations from the median:
let deviations: Vec<f64> = xs.iter().map(|x| (x - x_median).abs()).collect();
// Compute median absolute deviation:
let mad = median(&deviations);
// Compute modified Z-scores (x_i - x_median) / MAD
xs.iter().map(|&x| (x - x_median) / mad).collect()
}
/// Return the number of outliers in a given sample. Outliers are defined as data points with a
/// modified Z-score that is larger than `OUTLIER_THRESHOLD`.
#[cfg(test)]
pub fn num_outliers(xs: &[f64]) -> usize {
if xs.is_empty() {
return 0;
}
let scores = modified_zscores(xs);
scores
.iter()
.filter(|&&s| s.abs() > OUTLIER_THRESHOLD)
.count()
}
#[test]
fn test_detect_outliers() {
// Should not detect outliers in small samples
assert_eq!(0, num_outliers(&[]));
assert_eq!(0, num_outliers(&[50.0]));
assert_eq!(0, num_outliers(&[1000.0, 0.0]));
// Should not detect outliers in low-variance samples
let xs = [-0.2, 0.0, 0.2];
assert_eq!(0, num_outliers(&xs));
// Should detect a single outlier
let xs = [-0.2, 0.0, 0.2, 4.0];
assert_eq!(1, num_outliers(&xs));
// Should detect a single outlier
let xs = [0.5, 0.30, 0.29, 0.31, 0.30];
assert_eq!(1, num_outliers(&xs));
// Should detect no outliers in sample drawn from normal distribution
let xs = [
2.33269488,
1.42195907,
-0.57527698,
-0.31293437,
2.2948158,
0.75813273,
-1.0712388,
-0.96394741,
-1.15897446,
1.10976285,
];
assert_eq!(0, num_outliers(&xs));
// Should detect two outliers that were manually added
let xs = [
2.33269488,
1.42195907,
-0.57527698,
-0.31293437,
2.2948158,
0.75813273,
-1.0712388,
-0.96394741,
-1.15897446,
1.10976285,
20.0,
-500.0,
];
assert_eq!(2, num_outliers(&xs));
}
+40
View File
@@ -0,0 +1,40 @@
use std::fmt;
use hyperfine::internal::{Second, MIN_EXECUTION_TIME};
use hyperfine::format::format_duration;
/// A list of all possible warnings
pub enum Warnings {
FastExecutionTime,
NonZeroExitCode,
SlowInitialRun(Second),
OutliersDetected,
}
impl fmt::Display for Warnings {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Warnings::FastExecutionTime => write!(
f,
"Command took less than {:.0} ms to complete. Results might be inaccurate.",
MIN_EXECUTION_TIME * 1e3
),
Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
Warnings::SlowInitialRun(t_first) => write!(
f,
"The first benchmarking run for this command was significantly slower than the \
rest ({}). This could be caused by (filesystem) caches that were not filled until \
after the first run. You should consider using the '--warmup' option to fill \
those caches before the actual benchmark. Alternatively, use the '--prepare' \
option to clear the caches before each timing run.",
format_duration(t_first, None)
),
Warnings::OutliersDetected => write!(
f,
"Statistical outliers were detected. Consider re-running this benchmark on a quiet \
PC without any interferences from other programs. It might help to use the \
'--warmup' or '--prepare' options."
),
}
}
}
+41 -6
View File
@@ -1,6 +1,8 @@
extern crate ansi_term;
extern crate atty;
#[macro_use]
extern crate clap;
extern crate colored;
extern crate indicatif;
extern crate libc;
extern crate statistical;
@@ -13,23 +15,24 @@ use std::cmp;
use std::error::Error;
use std::io;
use ansi_term::Colour::Red;
use atty::Stream;
use colored::*;
use clap::{App, AppSettings, Arg};
mod hyperfine;
use hyperfine::internal::{CmdFailureAction, HyperfineOptions};
use hyperfine::internal::{CmdFailureAction, HyperfineOptions, OutputStyleOption};
use hyperfine::benchmark::{mean_shell_spawning_time, run_benchmark};
/// Print error message to stderr and terminate
pub fn error(message: &str) -> ! {
eprintln!("{}: {}", Red.paint("Error"), message);
eprintln!("{} {}", "Error:".red(), message);
std::process::exit(1);
}
/// Runs the benchmark for the given commands
fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result<()> {
let shell_spawning_time = mean_shell_spawning_time()?;
let shell_spawning_time = mean_shell_spawning_time(&options.output_style)?;
// Run the benchmarks
for (num, cmd) in commands.iter().enumerate() {
@@ -43,9 +46,15 @@ fn main() {
// Process command line options
let mut options = HyperfineOptions::default();
let clap_color_setting = if atty::is(Stream::Stdout) {
AppSettings::ColoredHelp
} else {
AppSettings::ColorNever
};
let matches = App::new("hyperfine")
.version(crate_version!())
.setting(AppSettings::ColoredHelp)
.setting(clap_color_setting)
.setting(AppSettings::DeriveDisplayOrder)
.setting(AppSettings::UnifiedHelpMessage)
.max_term_width(90)
@@ -90,6 +99,19 @@ fn main() {
clearing disk caches, for example.",
),
)
.arg(
Arg::with_name("style")
.long("style")
.short("s")
.takes_value(true)
.value_name("TYPE")
.possible_values(&["auto", "basic", "full"])
.help(
"Set output style type (default: auto). Set this to 'basic' to disable output \
coloring and interactive elements. Set it to 'full' to enable all effects \
even if no interactive terminal was detected.",
),
)
.arg(
Arg::with_name("ignore-failure")
.long("ignore-failure")
@@ -114,6 +136,19 @@ fn main() {
options.preparation_command = matches.value_of("prepare").map(String::from);
options.output_style = match matches.value_of("style") {
Some("full") => OutputStyleOption::Full,
Some("basic") => OutputStyleOption::Basic,
_ => if atty::is(Stream::Stdout) {
OutputStyleOption::Full
} else {
OutputStyleOption::Basic
},
};
if options.output_style == OutputStyleOption::Basic {
colored::control::set_override(false);
}
if matches.is_present("ignore-failure") {
options.failure_action = CmdFailureAction::Ignore;
}