Compare commits

..
13 changed files with 463 additions and 302 deletions

No files matched your search

+6 -4
View File
@@ -116,15 +116,17 @@ not every time as would happen with the \fB\-\-prepare\fR option.
.HP
\fB\-\-reference\fR \fICMD\fP
.IP
Benchmark \fICMD\fP as the reference for the relative comparison of results.
If this option is not specified, the fastest command is used as the reference.
In the command-sorted comparison and exported markup tables, values below 1
indicate commands faster than the reference.
Use \fICMD\fP as the reference for the relative comparison of results. Without
parameters, benchmark \fICMD\fP as a separate reference command. If this option
is not specified, the fastest command is used as the reference.
With \fB\-\-parameter\-scan\fR or \fB\-\-parameter\-list\fR, select an existing
benchmark by its exact printed name. The name must match exactly one benchmark.
.HP
\fB\-\-reference\-name\fR \fINAME\fP
.IP
Give a meaningful \fINAME\fP to the reference command.
Requires \fB\-\-reference\fR.
Cannot be used with parameterized benchmarks; use \fB\-\-command\-name\fR instead.
.HP
\fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP
.IP
+28 -7
View File
@@ -6,6 +6,7 @@ pub mod scheduler;
use std::cmp;
use std::io::{self, Write};
use std::time::Instant;
use crate::benchmark::executor::BenchmarkIteration;
use crate::command::Command;
@@ -14,7 +15,9 @@ use crate::options::{
};
use crate::outlier_detection::OUTLIER_THRESHOLD;
use crate::output::console_writeln;
use crate::output::progress_bar::get_progress_bar;
use crate::output::progress_bar::{
finish_initial_measurement, get_progress_bar, start_initial_measurement,
};
use crate::output::warnings::{OutlierWarningOptions, Warnings};
use crate::parameter::ParameterNameAndValue;
use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero};
@@ -244,7 +247,11 @@ impl<'a> Benchmark<'a> {
let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
Some(get_progress_bar(
self.options.run_bounds.min,
"Initial time measurement",
if preparation_command.is_some() {
"Running preparation command"
} else {
"Initial run"
},
self.options.output_style,
))
} else {
@@ -258,6 +265,9 @@ impl<'a> Benchmark<'a> {
});
// Initial timing run
if let Some(bar) = progress_bar.as_ref() {
start_initial_measurement(bar, Instant::now());
}
let res = self.executor.run_command_and_measure(
self.command,
benchmark_iteration,
@@ -266,6 +276,18 @@ impl<'a> Benchmark<'a> {
)?;
let success = res.exit_status.success();
if let Some(bar) = progress_bar.as_ref() {
let time_unit = self
.options
.time_unit
.unwrap_or(res.time_wall_clock.suitable_unit());
let estimate = res.time_wall_clock.format(time_unit);
finish_initial_measurement(
bar,
format!("Current estimate: {}", estimate.to_string().green()),
);
}
let conclusion_result = run_conclusion_command(benchmark_iteration)?;
let conclusion_overhead = conclusion_result.map_or(Time::zero(), |res| {
res.time_wall_clock + self.executor.time_overhead()
@@ -299,16 +321,13 @@ impl<'a> Benchmark<'a> {
// Re-configure the progress bar
if let Some(bar) = progress_bar.as_ref() {
bar.set_length(count)
}
if let Some(bar) = progress_bar.as_ref() {
bar.inc(1)
bar.set_length(count);
bar.inc(1);
}
// Gather statistics (perform the actual benchmark)
for i in 0..count_remaining {
let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
run_preparation_command(benchmark_iteration)?;
let msg = {
let t_wall_clock_mean = measurements.time_wall_clock_mean();
@@ -324,6 +343,8 @@ impl<'a> Benchmark<'a> {
bar.set_message(msg.to_owned())
}
run_preparation_command(benchmark_iteration)?;
let res = self.executor.run_command_and_measure(
self.command,
benchmark_iteration,
+57 -82
View File
@@ -3,13 +3,12 @@ use std::cmp::Ordering;
use super::benchmark_result::BenchmarkResult;
use crate::{
options::SortOrder,
quantity::{self, Time, Zero},
quantity::{self, Ratio, Time, Zero},
};
#[derive(Debug)]
pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub result: &'a BenchmarkResult,
/// Runtime divided by the reference runtime; values below one are faster.
pub relative_speed: f64,
pub relative_speed_stddev: Option<f64>,
pub is_reference: bool,
@@ -17,6 +16,20 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub relative_ordering: Ordering,
}
impl BenchmarkResultWithRelativeSpeed<'_> {
pub fn reference_label(&self) -> &'static str {
if self.is_reference {
" (reference)"
} else {
match self.relative_ordering {
Ordering::Less => " (faster)",
Ordering::Equal => " (same speed)",
Ordering::Greater => " (slower)",
}
}
}
}
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
l.mean_wall_clock_time()
.partial_cmp(&r.mean_wall_clock_time())
@@ -38,11 +51,13 @@ fn compute_relative_speeds<'a>(
let mut results: Vec<_> = results
.iter()
.map(|result| {
let is_reference = result == reference;
// Separate benchmarks can have identical names and measurements.
let is_reference = std::ptr::eq(result, reference);
let relative_ordering = compare_mean_time(result, reference);
let reference_mean = reference.mean_wall_clock_time();
if reference_mean == Time::zero() {
if result.mean_wall_clock_time() == Time::zero()
|| reference.mean_wall_clock_time() == Time::zero()
{
return BenchmarkResultWithRelativeSpeed {
result,
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -52,7 +67,13 @@ fn compute_relative_speeds<'a>(
};
}
let ratio = result.mean_wall_clock_time() / reference_mean;
let ratio = match relative_ordering {
Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(),
Ordering::Equal => Ratio::new::<quantity::ratio>(1.0),
Ordering::Greater => {
result.mean_wall_clock_time() / reference.mean_wall_clock_time()
}
};
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance assumed to be 0, i.e. variables are assumed to be independent
@@ -61,10 +82,12 @@ fn compute_relative_speeds<'a>(
reference.measurements.stddev(),
) {
(Some(result_stddev), Some(reference_stddev)) => Some(
((result_stddev / reference_mean).powi(uom::typenum::P2::new())
+ (ratio * (reference_stddev / reference_mean))
.powi(uom::typenum::P2::new()))
.sqrt(),
ratio
* ((result_stddev / result.mean_wall_clock_time())
.powi(uom::typenum::P2::new())
+ (reference_stddev / reference.mean_wall_clock_time())
.powi(uom::typenum::P2::new()))
.sqrt(),
),
_ => None,
};
@@ -116,25 +139,18 @@ pub fn compute_with_check(
Some(compute_relative_speeds(results, fastest, sort_order))
}
/// Compute ratios relative to the given reference, or the fastest result if omitted.
/// A zero reference runtime can produce infinite ratios.
/// Compute relative speeds against the given reference, or the fastest result.
pub fn compute<'a>(
results: &'a [BenchmarkResult],
sort_order: SortOrder,
reference: Option<&'a BenchmarkResult>,
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
let reference = reference.unwrap_or_else(|| fastest_of(results));
compute_relative_speeds(results, reference, sort_order)
}
#[cfg(test)]
fn create_result(name: &str, mean: f64) -> BenchmarkResult {
create_result_from_times(name, &[mean])
}
#[cfg(test)]
fn create_result_from_times(name: &str, times: &[f64]) -> BenchmarkResult {
use std::collections::BTreeMap;
use crate::benchmark::measurement::{Measurement, Measurements};
@@ -144,14 +160,11 @@ fn create_result_from_times(name: &str, times: &[f64]) -> BenchmarkResult {
command: name.into(),
command_with_unused_parameters: name.into(),
measurements: Measurements {
measurements: times
.iter()
.map(|&time| Measurement {
time_wall_clock: Time::new::<second>(time),
time_user: Time::new::<second>(time),
..Default::default()
})
.collect(),
measurements: vec![Measurement {
time_wall_clock: Time::new::<second>(mean),
time_user: Time::new::<second>(mean),
..Default::default()
}],
},
parameters: BTreeMap::new(),
}
@@ -178,20 +191,13 @@ fn test_compute_relative_speed() {
fn test_compute_relative_speed_with_reference() {
use approx::assert_relative_eq;
let results = vec![
create_result_from_times("cmd2", &[1.0, 2.0, 3.0]),
create_result_from_times("cmd3", &[4.0, 5.0, 6.0]),
];
let reference = create_result_from_times("reference", &[3.0, 4.0, 5.0]);
let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
let reference = create_result("cmd2", 4.0);
let annotated_results =
compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
assert_relative_eq!(0.5, annotated_results[0].relative_speed);
assert_relative_eq!(
0.2795084971874737,
annotated_results[0].relative_speed_stddev.unwrap()
);
assert_relative_eq!(2.0, annotated_results[0].relative_speed);
assert_relative_eq!(1.25, annotated_results[1].relative_speed);
}
@@ -205,57 +211,26 @@ fn test_compute_relative_speed_for_zero_times() {
}
#[test]
fn test_reference_ratios_do_not_depend_on_sort_order() {
use approx::assert_relative_eq;
fn reference_identity_distinguishes_equal_results() {
let results = vec![create_result("same", 1.0), create_result("same", 1.0)];
let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
let results = vec![
create_result("reference", 2.0),
create_result("slower", 3.0),
create_result("faster", 1.0),
create_result("equal", 2.0),
];
assert!(entries[0].is_reference);
assert!(!entries[1].is_reference);
}
for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
let entries = compute(&results, sort_order, Some(&results[0]));
for entry in entries {
assert_relative_eq!(
entry
.result
.mean_wall_clock_time()
.get::<quantity::second>()
/ 2.0,
entry.relative_speed
);
assert_eq!(entry.is_reference, entry.result.command == "reference");
}
#[test]
fn reference_ratios_handle_zero_times() {
let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)];
for result in &mut results {
let measurement = result.measurements.measurements[0];
result.measurements.measurements.push(measurement);
}
}
let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
assert_eq!(entries[1].relative_speed, f64::INFINITY);
#[test]
fn test_reference_ratios_without_stddev() {
let results = vec![
create_result("reference", 2.0),
create_result("faster", 1.0),
];
let entries = compute(&results, SortOrder::Command, Some(&results[0]));
assert_eq!(entries[1].relative_speed, 0.5);
assert_eq!(entries[1].relative_speed_stddev, None);
}
#[test]
fn test_reference_ratios_with_zero_times() {
let results = vec![
create_result_from_times("reference", &[1.0, 2.0, 3.0]),
create_result_from_times("zero", &[0.0, 0.0, 0.0]),
];
let entries = compute(&results, SortOrder::Command, Some(&results[0]));
assert_eq!(entries[1].relative_speed, 0.0);
assert_eq!(entries[1].relative_speed_stddev, Some(0.0));
let entries = compute(&results, SortOrder::Command, Some(&results[1]));
let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
assert_eq!(entries[0].relative_speed, f64::INFINITY);
assert_eq!(entries[0].relative_speed_stddev, None);
assert_eq!(entries[1].relative_speed, 1.0);
assert!(compute_with_check_from_reference(&results, &results[0], SortOrder::Command).is_none());
}
+31 -32
View File
@@ -54,7 +54,8 @@ impl<'a> Scheduler<'a> {
// We export results after each individual benchmark, because
// we would risk losing them if a later benchmark fails.
self.export_manager.write_results(&self.results, true)?;
self.export_manager
.write_results(&self.results, true, self.options.reference_index)?;
}
Ok(())
@@ -71,9 +72,8 @@ impl<'a> Scheduler<'a> {
let reference = self
.options
.reference_command
.as_ref()
.map(|_| &self.results[0])
.reference_index
.map(|index| &self.results[index])
.unwrap_or_else(|| relative_speed::fastest_of(&self.results));
if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
@@ -96,36 +96,28 @@ impl<'a> Scheduler<'a> {
)?;
for item in others {
// The prose summary expresses both directions as factors above one.
let (relative_speed, relative_speed_stddev, prefix, suffix) =
match item.relative_ordering {
Ordering::Less => (
format!("{:8.2}", 1.0 / item.relative_speed),
item.relative_speed_stddev
.map(|stddev| stddev / item.relative_speed.powi(2)),
"",
" times slower than",
),
Ordering::Greater => (
format!("{:8.2}", item.relative_speed),
item.relative_speed_stddev,
"",
" times faster than",
),
Ordering::Equal => (
format!("{:.2}", item.relative_speed),
item.relative_speed_stddev,
" As fast (",
") as",
),
};
let stddev = if let Some(stddev) = relative_speed_stddev {
let stddev = if let Some(stddev) = item.relative_speed_stddev {
format!(" ± {}", format!("{:.2}", stddev).green())
} else {
"".into()
};
let comparator =
format!("{prefix}{}{stddev}{suffix}", relative_speed.bold().green());
let comparator = match item.relative_ordering {
Ordering::Less => format!(
"{}{} times slower than",
format!("{:8.2}", item.relative_speed).bold().green(),
stddev
),
Ordering::Greater => format!(
"{}{} times faster than",
format!("{:8.2}", item.relative_speed).bold().green(),
stddev
),
Ordering::Equal => format!(
" As fast ({}{}) as",
format!("{:.2}", item.relative_speed).bold().green(),
stddev
),
};
console_writeln!(
stdout,
"{} {}",
@@ -138,9 +130,14 @@ impl<'a> Scheduler<'a> {
console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
for item in annotated_results {
let relationship = if self.options.reference_index.is_some() {
item.reference_label()
} else {
""
};
console_writeln!(
stdout,
" {}{} {}",
" {}{} {}{}",
format!("{:10.2}", item.relative_speed).bold().green(),
if item.is_reference {
" ".into()
@@ -150,6 +147,7 @@ impl<'a> Scheduler<'a> {
" ".into()
},
item.result.command_with_unused_parameters,
relationship,
)?;
}
}
@@ -171,7 +169,8 @@ impl<'a> Scheduler<'a> {
}
pub fn final_export(&self) -> Result<()> {
self.export_manager.write_results(&self.results, false)
self.export_manager
.write_results(&self.results, false, self.options.reference_index)
}
}
+7 -5
View File
@@ -105,10 +105,11 @@ fn build_command() -> Command {
.action(ArgAction::Set)
.value_name("CMD")
.help(
"The reference command for the relative comparison of results. \
If this is unset, results are compared with the fastest command as reference. \
In the command-sorted comparison and exported markup tables, values below 1 \
indicate commands faster than the reference."
"The reference for the relative comparison of results. Without parameters, \
CMD is run as a separate reference command. With --parameter-scan or \
--parameter-list, CMD must exactly match one unique benchmark name as \
shown in the output. If this is unset, results are compared with the \
fastest command as reference."
)
)
.arg(
@@ -116,7 +117,8 @@ fn build_command() -> Command {
.long("reference-name")
.action(ArgAction::Set)
.value_name("CMD")
.help("Give a meaningful name to the reference command.")
.help("Give a meaningful name to the reference command. This cannot be used \
with parameterized benchmarks; use --command-name instead.")
.requires("reference")
)
.arg(
+1 -1
View File
@@ -18,7 +18,7 @@ impl Exporter for CsvExporter {
results: &[BenchmarkResult],
_unit: Option<TimeUnit>,
_sort_order: SortOrder,
_reference: Option<&BenchmarkResult>,
_reference_index: Option<usize>,
) -> Result<Vec<u8>> {
const CSV_UNIT: TimeUnit = TimeUnit::Second;
const CSV_PRECISION: usize = 6;
+1 -1
View File
@@ -22,7 +22,7 @@ impl Exporter for JsonExporter {
results: &[BenchmarkResult],
_unit: Option<TimeUnit>,
_sort_order: SortOrder,
_reference: Option<&BenchmarkResult>,
_reference_index: Option<usize>,
) -> Result<Vec<u8>> {
let mut output = to_vec_pretty(&HyperfineSummary { results });
if let Ok(ref mut content) = output {
+34 -11
View File
@@ -1,10 +1,11 @@
use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed;
use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed};
use crate::options::SortOrder;
use crate::quantity::{FormatQuantity, IsUnit, TimeUnit};
use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero};
use super::Exporter;
use anyhow::Result;
use std::cmp::Ordering;
pub enum Alignment {
Left,
@@ -16,6 +17,8 @@ pub trait MarkupExporter {
&self,
entries: &[BenchmarkResultWithRelativeSpeed],
unit: TimeUnit,
reference_pending: bool,
explicit_reference: bool,
) -> String {
// prepare table header strings
let notation = format!("[{}]", unit.short_name());
@@ -58,13 +61,25 @@ pub trait MarkupExporter {
};
let min_str = measurement.measurements.min().format_value(unit);
let max_str = measurement.measurements.max().format_value(unit);
let rel_str = format!("{:.2}", entry.relative_speed);
let rel_stddev_str = if entry.is_reference {
"".into()
} else if let Some(stddev) = entry.relative_speed_stddev {
format!(" ± {stddev:.2}")
// The ratio of two zero times is undefined, even if they compare equal.
let relative_unavailable = reference_pending
|| (explicit_reference
&& !entry.is_reference
&& entry.relative_ordering == Ordering::Equal
&& measurement.mean_wall_clock_time() == Time::zero());
let relative = if relative_unavailable {
"N/A".to_string()
} else {
"".into()
let stddev = match (entry.is_reference, entry.relative_speed_stddev) {
(false, Some(stddev)) => format!(" ± {stddev:.2}"),
_ => String::new(),
};
let label = if explicit_reference {
entry.reference_label()
} else {
""
};
format!("{:.2}{stddev}{label}", entry.relative_speed)
};
// prepare table row entries
@@ -73,7 +88,7 @@ pub trait MarkupExporter {
&format!("{mean_str}{stddev_str}"),
&min_str,
&max_str,
&format!("{rel_str}{rel_stddev_str}"),
&relative,
]))
}
@@ -114,12 +129,20 @@ impl<T: MarkupExporter> Exporter for T {
results: &[BenchmarkResult],
unit: Option<TimeUnit>,
sort_order: SortOrder,
reference: Option<&BenchmarkResult>,
reference_index: Option<usize>,
) -> Result<Vec<u8>> {
let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
let entries = relative_speed::compute(results, sort_order, reference);
// Do not report ratios against another benchmark while the selected
// reference is still pending in an intermediate export.
let reference_pending = reference_index.is_some_and(|i| i >= results.len());
let entries = relative_speed::compute(
results,
sort_order,
reference_index.and_then(|i| results.get(i)),
);
let table = self.table_results(&entries, unit);
let table =
self.table_results(&entries, unit, reference_pending, reference_index.is_some());
Ok(table.as_bytes().to_vec())
}
}
+10 -11
View File
@@ -51,7 +51,7 @@ trait Exporter {
results: &[BenchmarkResult],
unit: Option<TimeUnit>,
sort_order: SortOrder,
reference: Option<&BenchmarkResult>,
reference_index: Option<usize>,
) -> Result<Vec<u8>>;
}
@@ -70,7 +70,6 @@ pub struct ExportManager {
exporters: Vec<ExporterWithTarget>,
time_unit: Option<TimeUnit>,
sort_order: SortOrder,
has_reference: bool,
}
impl ExportManager {
@@ -85,7 +84,6 @@ impl ExportManager {
exporters: vec![],
time_unit,
sort_order,
has_reference: matches.contains_id("reference"),
};
{
let mut add_exporter = |flag, exporttype| -> Result<()> {
@@ -133,17 +131,18 @@ impl ExportManager {
/// results are written to all file targets (to always have them up to date, even
/// if a benchmark fails). In the latter case, we only print to stdout targets (in
/// order not to clutter the output of hyperfine with intermediate results).
pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
// The scheduler benchmarks an explicit reference before the other commands.
let reference = if self.has_reference {
results.first()
} else {
None
};
/// `reference_index` refers to the complete benchmark sequence, so it can
/// point past `results` while that benchmark is still pending.
pub fn write_results(
&self,
results: &[BenchmarkResult],
intermediate: bool,
reference_index: Option<usize>,
) -> Result<()> {
for e in &self.exporters {
let content = || {
e.exporter
.serialize(results, self.time_unit, self.sort_order, reference)
.serialize(results, self.time_unit, self.sort_order, reference_index)
};
match e.target {
+35 -2
View File
@@ -4,7 +4,7 @@ use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::{cmp, env, fmt, io};
use anyhow::ensure;
use anyhow::{bail, ensure};
use clap::ArgMatches;
use crate::command::Commands;
@@ -208,12 +208,15 @@ pub struct Options {
/// Whether or not to ignore non-zero exit codes
pub command_failure_action: CmdFailureAction,
// Command to use as a reference for relative speed comparison
// Standalone reference command, cleared when an existing benchmark is selected.
pub reference_command: Option<String>,
// Name of the reference command
pub reference_name: Option<String>,
// Index of the reference in the benchmark sequence, if any.
pub reference_index: Option<usize>,
/// Command(s) to run before each timing run
pub preparation_command: Option<Vec<String>>,
@@ -257,6 +260,7 @@ impl Default for Options {
command_failure_action: CmdFailureAction::RaiseError,
reference_command: None,
reference_name: None,
reference_index: None,
preparation_command: None,
conclusion_command: None,
setup_command: None,
@@ -464,6 +468,35 @@ impl Options {
}
pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> {
if let Some(reference) = &self.reference_command {
if commands
.iter()
.next()
.is_some_and(|command| !command.get_parameters().is_empty())
{
ensure!(
self.reference_name.is_none(),
"--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark"
);
let mut matches = commands
.iter()
.enumerate()
.filter(|(_, command)| command.get_name_with_unused_parameters() == *reference);
let Some((index, _)) = matches.next() else {
bail!(
"Reference '{reference}' does not match any parameterized benchmark. Use its full displayed benchmark name."
);
};
ensure!(
matches.next().is_none(),
"Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names."
);
self.reference_index = Some(index);
self.reference_command = None;
} else {
self.reference_index = Some(0);
}
}
let has_reference_command = self.reference_command.is_some();
let num_commands = commands.num_commands(has_reference_command);
+35 -3
View File
@@ -1,5 +1,6 @@
use indicatif::{ProgressBar, ProgressStyle};
use std::time::Duration;
use indicatif::{FormattedDuration, ProgressBar, ProgressState, ProgressStyle};
use std::fmt;
use std::time::{Duration, Instant};
use crate::options::OutputStyleOption;
@@ -9,13 +10,44 @@ const TICK_SETTINGS: (&str, u64) = ("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏ ", 80);
#[cfg(windows)]
const TICK_SETTINGS: (&str, u64) = (r"+-x| ", 200);
// Keep the message area 30 columns wide across preparation, the initial run,
// and estimates so that switching between them does not move the bar.
const DEFAULT_TEMPLATE: &str = " {spinner} {msg:<30} {wide_bar} ETA {eta_precise} ";
const INITIAL_TEMPLATE: &str =
" {spinner} Initial run: {initial_elapsed:<17} {wide_bar} ETA {eta_precise} ";
/// Show the time elapsed since the initial benchmark command started.
pub fn start_initial_measurement(bar: &ProgressBar, started: Instant) {
let style = bar
.style()
.template(INITIAL_TEMPLATE)
.expect("no template error")
.with_key(
"initial_elapsed",
move |_: &ProgressState, w: &mut dyn fmt::Write| {
write!(w, "{}", FormattedDuration(started.elapsed())).unwrap();
},
);
bar.set_style(style);
}
/// Stop showing the initial timer and display the next phase's message.
pub fn finish_initial_measurement(bar: &ProgressBar, message: String) {
bar.set_message(message);
bar.set_style(
bar.style()
.template(DEFAULT_TEMPLATE)
.expect("no template error"),
);
}
/// Return a pre-configured progress bar
pub fn get_progress_bar(length: u64, msg: &str, option: OutputStyleOption) -> ProgressBar {
let progressbar_style = match option {
OutputStyleOption::Basic | OutputStyleOption::Color => ProgressStyle::default_bar(),
_ => ProgressStyle::default_spinner()
.tick_chars(TICK_SETTINGS.0)
.template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise} ")
.template(DEFAULT_TEMPLATE)
.expect("no template error"),
};
-20
View File
@@ -383,26 +383,6 @@ fn reference_is_executed_first() {
.run();
}
#[test]
fn reference_is_executed_first_parameter_value() {
ExecutionOrderTest::new()
.arg("--runs=2")
.reference("reference")
.arg("--parameter-list")
.arg("number")
.arg("1,2,3")
.command("command {number}")
.expect_output("reference")
.expect_output("reference")
.expect_output("command 1")
.expect_output("command 1")
.expect_output("command 2")
.expect_output("command 2")
.expect_output("command 3")
.expect_output("command 3")
.run();
}
#[test]
fn reference_is_executed_separately_from_commands() {
ExecutionOrderTest::new()
+218 -123
View File
@@ -706,102 +706,65 @@ fn shows_benchmark_comparison_relative_to_reference() {
}
#[test]
fn command_sorted_comparison_relative_to_reference() {
fn command_sorted_comparison_and_markup_identify_reference() {
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
let directory = tempfile::tempdir().unwrap();
let export_path = directory.path().join("results.md");
let output = hyperfine_debug()
.args([
"--style=basic",
"--runs=1",
"--sort=command",
"--reference=sleep 2",
"--reference-name=baseline",
"sleep 1",
"sleep 3",
"--reference",
"sleep 2",
]), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: baseline
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 2: sleep 1
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 3: sleep 3
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 3.000 s … 3.000 s 10 runs
Benchmark 4: sleep 2
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Relative speed comparison
1.00 baseline
0.50 ± 0.00 sleep 1
1.50 ± 0.00 sleep 3
1.00 ± 0.00 sleep 2
----- stderr -----
"--reference-name",
"baseline",
"--export-markdown",
])
.arg(&export_path)
.args(["sleep 1", "sleep 2", "sleep 3"])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1;
insta::assert_snapshot!(comparison, @r"
1.00 baseline (reference)
2.00 sleep 1 (faster)
1.00 sleep 2 (same speed)
1.50 sleep 3 (slower)
");
let markdown = std::fs::read_to_string(export_path).unwrap();
insta::assert_snapshot!(markdown, @r"
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `baseline` | 2.000 | 2.000 | 2.000 | 1.00 (reference) |
| `sleep 1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) |
| `sleep 2` | 2.000 | 2.000 | 2.000 | 1.00 (same speed) |
| `sleep 3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) |
");
}
#[test]
fn markdown_export_relative_to_reference() {
let tempdir = tempfile::tempdir().unwrap();
let export_path = tempdir.path().join("results.md");
insta::allow_duplicates! {
for sort_order in ["auto", "command", "mean-time"] {
for to_stdout in [false, true] {
let output = hyperfine_debug()
.args([
"--style=none",
"--time-unit=second",
"--reference=sleep 2",
"--reference-name=baseline",
"sleep 1",
"sleep 3",
])
.arg(format!("--sort={sort_order}"))
.arg("--export-markdown")
.arg(if to_stdout {
std::path::Path::new("-")
} else {
&export_path
})
.assert()
.success()
.get_output()
.stdout
.clone();
let table = if to_stdout {
String::from_utf8(output).unwrap()
} else {
std::fs::read_to_string(&export_path).unwrap()
};
if sort_order == "mean-time" {
insta::assert_snapshot!(table.trim(), @r"
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 |
| `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 |
| `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 |
");
} else {
insta::assert_snapshot!(table.trim(), @r"
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 |
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 |
| `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 |
");
}
}
}
}
// Equal zero times have no meaningful relative factor.
let output = hyperfine_debug()
.args([
"--style=none",
"--runs=1",
"--export-markdown=-",
"--reference",
"sleep 0",
"--reference-name",
"baseline",
"sleep 0",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let markdown = String::from_utf8(output.stdout).unwrap();
let row = markdown
.lines()
.find(|line| line.contains("sleep 0"))
.unwrap();
assert!(row.ends_with("| N/A |"), "{}", row);
}
#[test]
@@ -894,48 +857,180 @@ fn rejects_negative_parameter_steps() {
}
}
#[cfg(unix)]
#[test]
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
fn selects_reference_with_parameterized_prepare() {
let directory = tempfile::tempdir().unwrap();
let csv_path = directory.path().join("results.csv");
let json_path = directory.path().join("results.json");
let output = hyperfine_raw_command()
.args([
"--style=basic",
"--shell=none",
"--runs=1",
"-L",
"delay",
"0.2,0.4,0.6",
"--prepare",
"sleep {delay}",
"--reference",
"echo a (delay = 0.4)",
"--export-csv",
])
.arg(&csv_path)
.arg("--export-json")
.arg(&json_path)
.arg("echo a")
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(stdout.matches("Benchmark ").count(), 3);
for (number, delay) in ["0.2", "0.4", "0.6"].iter().enumerate() {
assert!(
stdout.contains(&format!(
"Benchmark {}: echo a (delay = {})",
number + 1,
delay
)),
"{}",
stdout
);
}
assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout);
let mut csv = csv::Reader::from_path(csv_path).unwrap();
assert!(csv
.headers()
.unwrap()
.iter()
.any(|header| header == "parameter_delay"));
assert_eq!(
csv.records().collect::<Result<Vec<_>, _>>().unwrap().len(),
3
);
let json: serde_json::Value =
serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap();
let results = json["results"].as_array().unwrap();
assert_eq!(results.len(), 3);
assert_eq!(
results
.iter()
.map(|result| result["parameters"]["delay"].as_str().unwrap())
.collect::<Vec<_>>(),
["0.2", "0.4", "0.6"]
);
}
#[test]
fn rejects_unmatched_reference_before_export_or_setup() {
let directory = tempfile::tempdir().unwrap();
let export_path = directory.path().join("results.csv");
let marker_path = directory.path().join("setup-ran");
std::fs::write(&export_path, "previous contents").unwrap();
hyperfine()
.args(["--reference", "sleep 1", "-P", "secs", "2", "3", "--setup"])
.arg(format!("echo touched > \"{}\"", marker_path.display()))
.arg("--export-csv")
.arg(&export_path)
.arg("sleep {secs}")
.assert()
.failure()
.stdout(predicate::str::is_empty())
.stderr(predicate::str::contains(
"does not match any parameterized benchmark",
));
assert_eq!(
std::fs::read_to_string(export_path).unwrap(),
"previous contents"
);
assert!(!marker_path.exists());
}
#[test]
fn rejects_ambiguous_parameterized_reference() {
hyperfine_debug()
.args([
"-L",
"x",
"1,2",
"--command-name",
"case",
"--reference",
"case",
"sleep {x}",
])
.assert()
.failure()
.stderr(predicate::str::contains(
"matches multiple parameterized benchmarks",
));
}
#[cfg(unix)]
#[test]
fn intermediate_markdown_waits_for_selected_reference() {
let directory = tempfile::tempdir().unwrap();
let export_path = directory.path().join("results.md");
hyperfine()
.args([
"--style=none",
"--shell=none",
"--runs=1",
"-L",
"delay",
"0.01,0.02,0.03",
"--reference",
"sleep 0.02",
"--prepare",
"true",
"--prepare",
"false",
"--prepare",
"true",
"--export-markdown",
])
.arg(&export_path)
.arg("sleep {delay}")
.assert()
.failure();
let markdown = std::fs::read_to_string(export_path).unwrap();
let row = markdown
.lines()
.find(|line| line.contains("sleep 0.01"))
.unwrap();
assert!(row.ends_with("| N/A |"), "{}", row);
}
#[test]
fn markdown_export_uses_selected_parameterized_reference() {
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("--reference=sleep 25")
.arg("--parameter-scan")
.arg("time")
.arg("30")
.arg("45")
.arg("--parameter-step-size")
.arg("5")
.arg("sleep {time}"), @r"
.args([
"--style=none",
"--runs=1",
"--export-markdown=-",
"-P",
"x",
"1",
"3",
"--command-name",
"case-{x}",
"--reference",
"case-2",
"sleep {x}",
]), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 25
Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 25.000 s … 25.000 s 10 runs
Benchmark 2: sleep 30
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 30.000 s … 30.000 s 10 runs
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `case-1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) |
| `case-2` | 2.000 | 2.000 | 2.000 | 1.00 (reference) |
| `case-3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) |
Benchmark 3: sleep 35
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 35.000 s … 35.000 s 10 runs
Benchmark 4: sleep 40
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 40.000 s … 40.000 s 10 runs
Benchmark 5: sleep 45
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 45.000 s … 45.000 s 10 runs
Summary
sleep 25 ran
1.20 ± 0.00 times faster than sleep 30
1.40 ± 0.00 times faster than sleep 35
1.60 ± 0.00 times faster than sleep 40
1.80 ± 0.00 times faster than sleep 45
----- stderr -----
");