Compare commits

..
8 changed files with 292 additions and 122 deletions

No files matched your search

Generated
+32
View File
@@ -382,6 +382,7 @@ dependencies = [
"csv",
"indicatif",
"insta",
"insta-cmd",
"libc",
"nix",
"once_cell",
@@ -427,11 +428,24 @@ checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
dependencies = [
"console 0.16.6",
"once_cell",
"regex",
"serde",
"similar",
"strip-ansi-escapes",
"tempfile",
]
[[package]]
name = "insta-cmd"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffdf4af1db390cf0401535d7c1303cd079a074d28d8473b026fdb6559c41403"
dependencies = [
"insta",
"serde",
"serde_json",
]
[[package]]
name = "instant"
version = "0.1.13"
@@ -802,6 +816,15 @@ version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]]
name = "strip-ansi-escapes"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
dependencies = [
"vte",
]
[[package]]
name = "strsim"
version = "0.11.1"
@@ -933,6 +956,15 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "vte"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
dependencies = [
"memchr",
]
[[package]]
name = "wait-timeout"
version = "0.2.1"
+2 -1
View File
@@ -63,7 +63,8 @@ features = [
[dev-dependencies]
approx = "0.5"
assert_cmd = "2.2"
insta = { version = "1.48", features = ["yaml"] }
insta = { version = "1.48", features = ["yaml", "filters"] }
insta-cmd = "0.7"
predicates = "3.1"
tempfile = "3.27"
+2 -15
View File
@@ -255,21 +255,8 @@ order benchmarks by mean runtime
.HP
\fB\-u\fR, \fB\-\-time\-unit\fR \fIUNIT\fP
.IP
Set the time unit to be used. If the option is not given, the time unit is
determined automatically. Possible values:
.RS
.IP "microsecond"
Aliases: µs, us, microseconds.
.IP "millisecond"
Aliases: ms, milliseconds.
.IP "second"
Aliases: s, seconds.
.IP "minute"
Aliases: min, minutes.
.IP "hour"
Aliases: h, hours.
.RE
.IP
Set the time unit to be used. Possible values: microsecond, millisecond, second. If
the option is not given, the time unit is determined automatically.
This option affects the standard output as well as all export formats
except for CSV and JSON.
.HP
+4 -9
View File
@@ -278,15 +278,10 @@ fn build_command() -> Command {
.short('u')
.action(ArgAction::Set)
.value_name("UNIT")
.value_parser(["µs", "us", "microsecond", "microseconds", "ms", "millisecond", "milliseconds", "s", "second", "seconds", "min", "minute", "minutes", "h", "hour", "hours"])
.help("Set the time unit to be used. If the option is not given, the time unit is determined automatically. \
This option affects the standard output as well as all export formats except for CSV and JSON.\n\
Possible values:\n \
* 'µs', 'us', 'microsecond', 'microseconds'\n \
* 'ms', 'millisecond', 'milliseconds'\n \
* 's', 'second', 'seconds'\n \
* 'min', 'minute', 'minutes'\n \
* 'h', 'hour', 'hours'"),
.value_parser(["microsecond", "millisecond", "second"])
.help("Set the time unit to be used. Possible values: microsecond, millisecond, second. \
If the option is not given, the time unit is determined automatically. \
This option affects the standard output as well as all export formats except for CSV and JSON."),
)
.arg(
Arg::new("export-asciidoc")
+3 -5
View File
@@ -429,11 +429,9 @@ impl Options {
}
options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
Some("µs" | "us" | "microsecond" | "microseconds") => Some(Unit::MicroSecond),
Some("ms" | "millisecond" | "milliseconds") => Some(Unit::MilliSecond),
Some("s" | "second" | "seconds") => Some(Unit::Second),
Some("min" | "minute" | "minutes") => Some(Unit::Minute),
Some("h" | "hour" | "hours") => Some(Unit::Hour),
Some("microsecond") => Some(Unit::MicroSecond),
Some("millisecond") => Some(Unit::MilliSecond),
Some("second") => Some(Unit::Second),
_ => None,
};
+1 -2
View File
@@ -21,8 +21,7 @@ pub fn format_duration_value(duration: Second, unit: Option<Unit>) -> (String, U
} else if (duration < 1.0 && unit.is_none()) || unit == Some(Unit::MilliSecond) {
(Unit::MilliSecond.format(duration), Unit::MilliSecond)
} else {
let unit = unit.unwrap_or(Unit::Second);
(unit.format(duration), unit)
(Unit::Second.format(duration), Unit::Second)
}
}
-10
View File
@@ -11,8 +11,6 @@ pub enum Unit {
Second,
MilliSecond,
MicroSecond,
Minute,
Hour,
}
impl Unit {
@@ -22,8 +20,6 @@ impl Unit {
Unit::Second => String::from("s"),
Unit::MilliSecond => String::from("ms"),
Unit::MicroSecond => String::from("µs"),
Unit::Minute => String::from("min"),
Unit::Hour => String::from("h"),
}
}
@@ -33,8 +29,6 @@ impl Unit {
Unit::Second => format!("{value:.3}"),
Unit::MilliSecond => format!("{:.1}", value * 1e3),
Unit::MicroSecond => format!("{:.1}", value * 1e6),
Unit::Minute => format!("{:.1}", value / 60.0),
Unit::Hour => format!("{:.1}", value / 3600.0),
}
}
}
@@ -44,8 +38,6 @@ fn test_unit_short_name() {
assert_eq!("s", Unit::Second.short_name());
assert_eq!("ms", Unit::MilliSecond.short_name());
assert_eq!("µs", Unit::MicroSecond.short_name());
assert_eq!("min", Unit::Minute.short_name());
assert_eq!("h", Unit::Hour.short_name());
}
// Note - the values are rounded when formatted.
@@ -56,6 +48,4 @@ fn test_unit_format() {
assert_eq!("123456.8", Unit::MilliSecond.format(value));
assert_eq!("1234.6", Unit::MicroSecond.format(0.00123456));
assert_eq!("2.1", Unit::Minute.format(value));
assert_eq!("0.0", Unit::Hour.format(value));
}
+248 -80
View File
@@ -1,6 +1,8 @@
mod common;
use common::{hyperfine, hyperfine_raw_command};
use assert_cmd::assert::OutputAssertExt;
use insta_cmd::assert_cmd_snapshot;
use predicates::prelude::*;
/// Platform-specific I/O utility.
@@ -9,12 +11,19 @@ use predicates::prelude::*;
/// See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
pub fn hyperfine_debug() -> assert_cmd::Command {
let mut cmd = hyperfine();
pub fn hyperfine_debug() -> std::process::Command {
let mut cmd = hyperfine_raw_command();
cmd.arg("--debug-mode");
cmd
}
fn snapshot_settings() -> insta::Settings {
let mut settings = insta::Settings::clone_current();
// Ignore trailing report padding while preserving indentation and internal spacing.
settings.add_filter(r"(?m)[ \t]+$", "");
settings
}
#[test]
fn runs_successfully() {
hyperfine()
@@ -594,86 +603,187 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
#[test]
fn shows_benchmark_comparison_with_relative_times() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("sleep 1.0")
.arg("sleep 2.0")
.arg("sleep 3.0")
.assert()
.success()
.stdout(
predicate::str::contains("2.00 ± 0.00 times faster")
.and(predicate::str::contains("3.00 ± 0.00 times faster")),
);
.arg("sleep 3.0"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 1.0
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 2: sleep 2.0
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 3: sleep 3.0
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 3.000 s … 3.000 s 10 runs
Summary
sleep 1.0 ran
2.00 ± 0.00 times faster than sleep 2.0
3.00 ± 0.00 times faster than sleep 3.0
----- stderr -----
");
}
#[test]
fn shows_benchmark_comparison_with_same_time() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("--command-name=A")
.arg("--command-name=B")
.arg("sleep 1.0")
.arg("sleep 1.0")
.arg("sleep 2.0")
.arg("sleep 1000.0")
.assert()
.success()
.stdout(
predicate::str::contains("As fast (1.00 ± 0.00) as")
.and(predicate::str::contains("2.00 ± 0.00 times faster"))
.and(predicate::str::contains("1000.00 ± 0.00 times faster")),
);
.arg("sleep 1000.0"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: A
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 2: B
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 3: sleep 2.0
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 4: sleep 1000.0
Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1000.000 s … 1000.000 s 10 runs
Summary
A ran
As fast (1.00 ± 0.00) as B
2.00 ± 0.00 times faster than sleep 2.0
1000.00 ± 0.00 times faster than sleep 1000.0
----- stderr -----
");
}
#[test]
fn shows_benchmark_comparison_relative_to_reference() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("--reference=sleep 2.0")
.arg("sleep 1.0")
.arg("sleep 3.0")
.assert()
.success()
.stdout(
predicate::str::contains("2.00 ± 0.00 times slower")
.and(predicate::str::contains("1.50 ± 0.00 times faster")),
);
.arg("sleep 3.0"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 2.0
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 2: sleep 1.0
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 3: sleep 3.0
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 3.000 s … 3.000 s 10 runs
Summary
sleep 2.0 ran
2.00 ± 0.00 times slower than sleep 1.0
1.50 ± 0.00 times faster than sleep 3.0
----- stderr -----
");
}
#[test]
fn shows_reference_name() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("--reference=sleep 2.0")
.arg("--reference-name=refabc123")
.arg("sleep 1.0")
.arg("sleep 3.0")
.assert()
.success()
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
.arg("sleep 3.0"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: refabc123
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 2: sleep 1.0
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 3: sleep 3.0
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 3.000 s … 3.000 s 10 runs
Summary
refabc123 ran
2.00 ± 0.00 times slower than sleep 1.0
1.50 ± 0.00 times faster than sleep 3.0
----- stderr -----
");
}
#[test]
fn performs_all_benchmarks_in_parameter_scan() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("--parameter-scan")
.arg("time")
.arg("30")
.arg("45")
.arg("--parameter-step-size")
.arg("5")
.arg("sleep {time}")
.assert()
.success()
.stdout(
predicate::str::contains("Benchmark 1: sleep 30")
.and(predicate::str::contains("Benchmark 2: sleep 35"))
.and(predicate::str::contains("Benchmark 3: sleep 40"))
.and(predicate::str::contains("Benchmark 4: sleep 45"))
.and(predicate::str::contains("Benchmark 5: sleep 50").not()),
);
.arg("sleep {time}"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 30
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 30.000 s … 30.000 s 10 runs
Benchmark 2: sleep 35
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 35.000 s … 35.000 s 10 runs
Benchmark 3: sleep 40
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 40.000 s … 40.000 s 10 runs
Benchmark 4: sleep 45
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 45.000 s … 45.000 s 10 runs
Summary
sleep 30 ran
1.17 ± 0.00 times faster than sleep 35
1.33 ± 0.00 times faster than sleep 40
1.50 ± 0.00 times faster than sleep 45
----- stderr -----
");
}
#[test]
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("--reference=sleep 25")
.arg("--parameter-scan")
.arg("time")
@@ -681,33 +791,62 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
.arg("45")
.arg("--parameter-step-size")
.arg("5")
.arg("sleep {time}")
.assert()
.success()
.stdout(
predicate::str::contains("Benchmark 1: sleep 25")
.and(predicate::str::contains("Benchmark 2: sleep 30"))
.and(predicate::str::contains("Benchmark 3: sleep 35"))
.and(predicate::str::contains("Benchmark 4: sleep 40"))
.and(predicate::str::contains("Benchmark 5: sleep 45"))
.and(predicate::str::contains("Benchmark 6: sleep 50").not()),
);
.arg("sleep {time}"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 25
Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 25.000 s … 25.000 s 10 runs
Benchmark 2: sleep 30
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 30.000 s … 30.000 s 10 runs
Benchmark 3: sleep 35
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 35.000 s … 35.000 s 10 runs
Benchmark 4: sleep 40
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 40.000 s … 40.000 s 10 runs
Benchmark 5: sleep 45
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 45.000 s … 45.000 s 10 runs
Summary
sleep 25 ran
1.20 ± 0.00 times faster than sleep 30
1.40 ± 0.00 times faster than sleep 35
1.60 ± 0.00 times faster than sleep 40
1.80 ± 0.00 times faster than sleep 45
----- stderr -----
");
}
#[test]
fn intermediate_results_are_not_exported_to_stdout() {
hyperfine_debug()
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=none") // To only see the Markdown export on stdout
.arg("--export-markdown")
.arg("-")
.arg("sleep 1")
.arg("sleep 2")
.assert()
.success()
.stdout(
(predicate::str::contains("sleep 1").count(1))
.and(predicate::str::contains("sleep 2").count(1)),
);
.arg("sleep 2"), @r"
success: true
exit_code: 0
----- stdout -----
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
| `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
----- stderr -----
");
}
#[test]
@@ -749,27 +888,56 @@ fn unused_parameters_are_shown_in_benchmark_name() {
#[test]
fn speed_comparison_sort_order() {
for sort_order in ["auto", "mean-time"] {
hyperfine_debug()
.arg("sleep 2")
.arg("sleep 1")
.arg(format!("--sort={sort_order}"))
.assert()
.success()
.stdout(predicate::str::contains(
"sleep 1 ran\n 2.00 ± 0.00 times faster than sleep 2",
));
let _settings = snapshot_settings().bind_to_scope();
insta::allow_duplicates! {
for sort_order in ["auto", "mean-time"] {
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("sleep 2")
.arg("sleep 1")
.arg(format!("--sort={sort_order}")), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 2
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 2: sleep 1
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Summary
sleep 1 ran
2.00 ± 0.00 times faster than sleep 2
----- stderr -----
");
}
}
hyperfine_debug()
assert_cmd_snapshot!(hyperfine_debug()
.arg("--style=basic")
.arg("sleep 2")
.arg("sleep 1")
.arg("--sort=command")
.assert()
.success()
.stdout(predicate::str::contains(
"2.00 ± 0.00 sleep 2\n 1.00 sleep 1",
));
.arg("--sort=command"), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: sleep 2
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 2: sleep 1
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Relative speed comparison
2.00 ± 0.00 sleep 2
1.00 sleep 1
----- stderr -----
");
}
#[cfg(windows)]