Compare commits

...
Author SHA1 Message Date
sharkdp 148460b950 Fix nan/inf output for very fast commands
closes #319
2020-10-16 10:50:52 +02:00
4 changed files with 74 additions and 32 deletions

No files matched your search

+2
View File
@@ -24,6 +24,8 @@
## Bugfixes ## Bugfixes
- Fix a bug in the outlier detection which would only detect "slow outliers" but not the fast ones (runs that are much faster than the rest of the benchmarking runs), see #329 - Fix a bug in the outlier detection which would only detect "slow outliers" but not the fast ones (runs that are much faster than the rest of the benchmarking runs), see #329
- Better error messages for very fast commands that would lead to inf/nan results in the relative
speed comparison, see #319
- Keep output colorized when the output is not interactive and `--style=full` or `--style=color` is used. - Keep output colorized when the output is not interactive and `--style=full` or `--style=color` is used.
## Other ## Other
+8 -3
View File
@@ -5,7 +5,7 @@ use crate::hyperfine::internal::{compute_relative_speed, BenchmarkResultWithRela
use crate::hyperfine::types::BenchmarkResult; use crate::hyperfine::types::BenchmarkResult;
use crate::hyperfine::units::Unit; use crate::hyperfine::units::Unit;
use std::io::Result; use std::io::{Error, ErrorKind, Result};
#[derive(Default)] #[derive(Default)]
pub struct MarkdownExporter {} pub struct MarkdownExporter {}
@@ -23,8 +23,7 @@ impl Exporter for MarkdownExporter {
Unit::Second Unit::Second
}; };
let annotated_results = compute_relative_speed(results); if let Some(annotated_results) = compute_relative_speed(results) {
let mut destination = start_table(unit); let mut destination = start_table(unit);
for result in annotated_results { for result in annotated_results {
@@ -32,6 +31,12 @@ impl Exporter for MarkdownExporter {
} }
Ok(destination) Ok(destination)
} else {
Err(Error::new(
ErrorKind::Other,
"Relative speed comparison is not available for Markdown export.",
))
}
} }
} }
+41 -10
View File
@@ -52,6 +52,7 @@ pub fn min(vals: &[f64]) -> f64 {
.unwrap() .unwrap()
} }
#[derive(Debug)]
pub struct BenchmarkResultWithRelativeSpeed<'a> { pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub result: &'a BenchmarkResult, pub result: &'a BenchmarkResult,
pub relative_speed: Scalar, pub relative_speed: Scalar,
@@ -65,12 +66,17 @@ fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
pub fn compute_relative_speed<'a>( pub fn compute_relative_speed<'a>(
results: &'a [BenchmarkResult], results: &'a [BenchmarkResult],
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> { ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
let fastest: &BenchmarkResult = results let fastest: &BenchmarkResult = results
.iter() .iter()
.min_by(|&l, &r| compare_mean_time(l, r)) .min_by(|&l, &r| compare_mean_time(l, r))
.expect("at least one benchmark result"); .expect("at least one benchmark result");
if fastest.mean == 0.0 {
return None;
}
Some(
results results
.iter() .iter()
.map(|result| { .map(|result| {
@@ -79,7 +85,8 @@ pub fn compute_relative_speed<'a>(
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance asssumed to be 0, i.e. variables are assumed to be independent // Covariance asssumed to be 0, i.e. variables are assumed to be independent
let ratio_stddev = ratio let ratio_stddev = ratio
* ((result.stddev / result.mean).powi(2) + (fastest.stddev / fastest.mean).powi(2)) * ((result.stddev / result.mean).powi(2)
+ (fastest.stddev / fastest.mean).powi(2))
.sqrt(); .sqrt();
BenchmarkResultWithRelativeSpeed { BenchmarkResultWithRelativeSpeed {
@@ -89,7 +96,8 @@ pub fn compute_relative_speed<'a>(
is_fastest: result == fastest, is_fastest: result == fastest,
} }
}) })
.collect() .collect(),
)
} }
pub fn write_benchmark_comparison(results: &[BenchmarkResult]) { pub fn write_benchmark_comparison(results: &[BenchmarkResult]) {
@@ -97,7 +105,7 @@ pub fn write_benchmark_comparison(results: &[BenchmarkResult]) {
return; return;
} }
let mut annotated_results = compute_relative_speed(&results); if let Some(mut annotated_results) = compute_relative_speed(&results) {
annotated_results.sort_by(|l, r| compare_mean_time(l.result, r.result)); annotated_results.sort_by(|l, r| compare_mean_time(l.result, r.result));
let fastest = &annotated_results[0]; let fastest = &annotated_results[0];
@@ -114,6 +122,16 @@ pub fn write_benchmark_comparison(results: &[BenchmarkResult]) {
&item.result.command.magenta() &item.result.command.magenta()
); );
} }
} else {
eprintln!(
"{}: The benchmark comparison could not be computed as some benchmark times are zero. \
This could be caused by background interference during the initial calibration phase \
of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
Try to re-run the benchmark on a quiet system. If it does not help, you command is \
most likely too fast to be accurately benchmarked by hyperfine.",
"Note".bold().red()
);
}
} }
#[test] #[test]
@@ -125,12 +143,11 @@ fn test_max() {
assert_eq!(1.0, max(&[-1.0, 1.0, 0.0])); assert_eq!(1.0, max(&[-1.0, 1.0, 0.0]));
} }
#[test] #[cfg(test)]
fn test_compute_relative_speed() { fn create_result(name: &str, mean: Scalar) -> BenchmarkResult {
use approx::assert_relative_eq;
use std::collections::BTreeMap; use std::collections::BTreeMap;
let create_result = |name: &str, mean| BenchmarkResult { BenchmarkResult {
command: name.into(), command: name.into(),
mean, mean,
stddev: 1.0, stddev: 1.0,
@@ -141,7 +158,12 @@ fn test_compute_relative_speed() {
max: mean, max: mean,
times: None, times: None,
parameters: BTreeMap::new(), parameters: BTreeMap::new(),
}; }
}
#[test]
fn test_compute_relative_speed() {
use approx::assert_relative_eq;
let results = vec![ let results = vec![
create_result("cmd1", 3.0), create_result("cmd1", 3.0),
@@ -149,13 +171,22 @@ fn test_compute_relative_speed() {
create_result("cmd3", 5.0), create_result("cmd3", 5.0),
]; ];
let annotated_results = compute_relative_speed(&results); let annotated_results = compute_relative_speed(&results).unwrap();
assert_relative_eq!(1.5, annotated_results[0].relative_speed); assert_relative_eq!(1.5, annotated_results[0].relative_speed);
assert_relative_eq!(1.0, annotated_results[1].relative_speed); assert_relative_eq!(1.0, annotated_results[1].relative_speed);
assert_relative_eq!(2.5, annotated_results[2].relative_speed); assert_relative_eq!(2.5, annotated_results[2].relative_speed);
} }
#[test]
fn test_compute_relative_speed_for_zero_times() {
let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
let annotated_results = compute_relative_speed(&results);
assert!(annotated_results.is_none());
}
pub fn tokenize<'a>(values: &'a str) -> Vec<String> { pub fn tokenize<'a>(values: &'a str) -> Vec<String> {
let mut tokens = vec![]; let mut tokens = vec![];
let mut buf = String::new(); let mut buf = String::new();
+23 -19
View File
@@ -16,7 +16,7 @@ use crate::hyperfine::export::{ExportManager, ExportType};
use crate::hyperfine::internal::{tokenize, write_benchmark_comparison}; use crate::hyperfine::internal::{tokenize, write_benchmark_comparison};
use crate::hyperfine::parameter_range::get_parameterized_commands; use crate::hyperfine::parameter_range::get_parameterized_commands;
use crate::hyperfine::types::{ use crate::hyperfine::types::{
BenchmarkResult, CmdFailureAction, Command, HyperfineOptions, OutputStyleOption, ParameterValue, CmdFailureAction, Command, HyperfineOptions, OutputStyleOption, ParameterValue,
}; };
use crate::hyperfine::units::Unit; use crate::hyperfine::units::Unit;
@@ -27,7 +27,11 @@ pub fn error(message: &str) -> ! {
} }
/// Runs the benchmark for the given commands /// Runs the benchmark for the given commands
fn run(commands: &[Command<'_>], options: &HyperfineOptions) -> io::Result<Vec<BenchmarkResult>> { fn run(
commands: &[Command<'_>],
options: &HyperfineOptions,
export_manager: &ExportManager,
) -> io::Result<()> {
let shell_spawning_time = let shell_spawning_time =
mean_shell_spawning_time(&options.shell, options.output_style, options.show_output)?; mean_shell_spawning_time(&options.shell, options.output_style, options.show_output)?;
@@ -47,7 +51,21 @@ fn run(commands: &[Command<'_>], options: &HyperfineOptions) -> io::Result<Vec<B
timing_results.push(run_benchmark(num, cmd, shell_spawning_time, options)?); timing_results.push(run_benchmark(num, cmd, shell_spawning_time, options)?);
} }
Ok(timing_results) // Print relative speed comparison
if options.output_style != OutputStyleOption::Disabled {
write_benchmark_comparison(&timing_results);
}
// Export results
let ans = export_manager.write_results(timing_results, options.time_unit);
if let Err(e) = ans {
error(&format!(
"The following error occurred while exporting: {}",
e
));
}
Ok(())
} }
fn main() { fn main() {
@@ -57,26 +75,12 @@ fn main() {
let commands = build_commands(&matches); let commands = build_commands(&matches);
let res = match options { let res = match options {
Ok(ref opts) => run(&commands, &opts), Ok(ref opts) => run(&commands, &opts, &export_manager),
Err(ref e) => error(&e.to_string()), Err(ref e) => error(&e.to_string()),
}; };
match res { match res {
Ok(timing_results) => { Ok(_) => {}
let options = options.unwrap();
if options.output_style != OutputStyleOption::Disabled {
write_benchmark_comparison(&timing_results);
}
let ans = export_manager.write_results(timing_results, options.time_unit);
if let Err(e) = ans {
error(&format!(
"The following error occurred while exporting: {}",
e
));
}
}
Err(e) => error(&e.to_string()), Err(e) => error(&e.to_string()),
} }
} }