Compare commits

..
Author SHA1 Message Date
David Peter 06167908cb Choose prose comparison and ratio in one match 2026-10-05 14:46:46 +02:00
David Peter 7167df6951 Fix relative runtime ratios with an explicit reference 2026-10-05 14:20:04 +02:00
David Peter 79aea8872a Merge pull request #977 from sharkdp/codex/fix-clippy-warnings
Fix Windows Clippy warnings
2026-10-05 14:13:44 +02:00
David Peter f48f44b336 Merge pull request #976 from sharkdp/codex/remaining-statistics
Compute standard deviation directly from samples
2026-10-05 14:10:05 +02:00
David Peter ebb3182a6e Fix Windows Clippy warnings 2026-10-05 14:07:39 +02:00
David Peter 0fe93f9d0b Keep the nonempty-sample assertion in median 2026-10-05 14:02:52 +02:00
David Peter 8084a9eb7f Adopt the remaining statistics changes from #790 2026-10-05 13:57:31 +02:00
David Peter efd57b3a89 Merge pull request #975 from sharkdp/codex/unified-windows-timer
Unify the Windows CPU timer stop interface
2026-10-05 13:55:36 +02:00
David Peter 580b500052 Merge pull request #973 from sharkdp/dependabot/github_actions/softprops/action-gh-release-3
Bump softprops/action-gh-release from 2 to 3
2026-10-05 13:54:02 +02:00
David Peter ee96552dae Merge pull request #974 from sharkdp/dependabot/github_actions/actions/checkout-7
Bump actions/checkout from 6 to 7
2026-10-05 13:53:45 +02:00
David Peter 6af23bb885 Unify the Windows CPU timer stop interface 2026-10-05 13:51:06 +02:00
David Peter 683885a6c8 Merge pull request #972 from sharkdp/codex/csv-quantity-formatting
Format CSV measurements through FormatQuantity
2026-10-05 13:45:52 +02:00
David Peter 33aafdca07 Merge pull request #971 from sharkdp/codex/typed-relative-speed
Use typed quantities for relative-speed calculations
2026-10-05 13:44:35 +02:00
dependabot[bot] 0e2acb67e5 Bump actions/checkout from 6 to 7
Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v6...v7)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-05 11:42:47 +00:00
dependabot[bot] de0afb81c5 Bump softprops/action-gh-release from 2 to 3
Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 2 to 3.
- [Release notes](https://github.com/softprops/action-gh-release/releases)
- [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md)
- [Commits](https://github.com/softprops/action-gh-release/compare/v2...v3)

---
updated-dependencies:
- dependency-name: softprops/action-gh-release
  dependency-version: '3'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-10-05 11:42:43 +00:00
David Peter df49a04a66 Use typed quantities for relative-speed calculations 2026-10-05 13:41:20 +02:00
15 changed files with 307 additions and 124 deletions

No files matched your search

+5 -5
View File
@@ -18,7 +18,7 @@ jobs:
name: Extract crate metadata
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- name: Extract crate information
id: crate_metadata
run: |
@@ -49,7 +49,7 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- run: cargo fmt -- --check
min_version:
@@ -58,7 +58,7 @@ jobs:
needs: crate_metadata
steps:
- name: Checkout source code
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }})
uses: dtolnay/rust-toolchain@master
@@ -96,7 +96,7 @@ jobs:
BUILD_CMD: cargo
steps:
- name: Checkout source code
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Install prerequisites
shell: bash
@@ -337,7 +337,7 @@ jobs:
echo "IS_RELEASE=${IS_RELEASE}" >> $GITHUB_OUTPUT
- name: Publish archives and packages
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@v3
if: steps.is-release.outputs.IS_RELEASE
with:
files: |
+2
View File
@@ -118,6 +118,8 @@ not every time as would happen with the \fB\-\-prepare\fR option.
.IP
Benchmark \fICMD\fP as the reference for the relative comparison of results.
If this option is not specified, the fastest command is used as the reference.
In the command-sorted comparison and exported markup tables, values below 1
indicate commands faster than the reference.
.HP
\fB\-\-reference\-name\fR \fINAME\fP
.IP
+5 -4
View File
@@ -54,13 +54,14 @@ impl Measurements {
mean(self.wall_clock_times())
}
/// The standard deviation of wall clock times, if at least two runs were measured.
/// The standard deviation of all wall clock times. Not available if only one run has been performed
pub fn stddev(&self) -> Option<Time> {
if self.len() < 2 {
let times: Vec<_> = self.wall_clock_times().collect(); // TODO: Avoid collecting
if times.len() < 2 {
None
} else {
let times: Vec<_> = self.wall_clock_times().collect();
Some(standard_deviation(&times, self.time_wall_clock_mean()))
Some(standard_deviation(times))
}
}
+108 -49
View File
@@ -1,12 +1,15 @@
use std::cmp::Ordering;
use super::benchmark_result::BenchmarkResult;
use crate::options::SortOrder;
use crate::quantity::second;
use crate::{
options::SortOrder,
quantity::{self, Time, Zero},
};
#[derive(Debug)]
pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub result: &'a BenchmarkResult,
/// Runtime divided by the reference runtime; values below one are faster.
pub relative_speed: f64,
pub relative_speed_stddev: Option<f64>,
pub is_reference: bool,
@@ -16,8 +19,7 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
l.mean_wall_clock_time()
.get::<second>()
.partial_cmp(&r.mean_wall_clock_time().get::<second>())
.partial_cmp(&r.mean_wall_clock_time())
.unwrap_or(Ordering::Equal)
}
@@ -39,7 +41,8 @@ fn compute_relative_speeds<'a>(
let is_reference = result == reference;
let relative_ordering = compare_mean_time(result, reference);
if result.mean_wall_clock_time().get::<second>() == 0.0 {
let reference_mean = reference.mean_wall_clock_time();
if reference_mean == Time::zero() {
return BenchmarkResultWithRelativeSpeed {
result,
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -49,44 +52,27 @@ fn compute_relative_speeds<'a>(
};
}
let ratio = match relative_ordering {
Ordering::Less => {
reference.mean_wall_clock_time().get::<second>()
/ result.mean_wall_clock_time().get::<second>()
}
Ordering::Equal => 1.0,
Ordering::Greater => {
result.mean_wall_clock_time().get::<second>()
/ reference.mean_wall_clock_time().get::<second>()
}
};
let ratio = result.mean_wall_clock_time() / reference_mean;
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance assumed to be 0, i.e. variables are assumed to be independent
let ratio_stddev = match (
result
.measurements
.stddev()
.map(|time| time.get::<second>()),
reference
.measurements
.stddev()
.map(|time| time.get::<second>()),
result.measurements.stddev(),
reference.measurements.stddev(),
) {
(Some(result_stddev), Some(fastest_stddev)) => Some(
ratio
* ((result_stddev / result.mean_wall_clock_time().get::<second>()).powi(2)
+ (fastest_stddev / reference.mean_wall_clock_time().get::<second>())
.powi(2))
.sqrt(),
(Some(result_stddev), Some(reference_stddev)) => Some(
((result_stddev / reference_mean).powi(uom::typenum::P2::new())
+ (ratio * (reference_stddev / reference_mean))
.powi(uom::typenum::P2::new()))
.sqrt(),
),
_ => None,
};
BenchmarkResultWithRelativeSpeed {
result,
relative_speed: ratio,
relative_speed_stddev: ratio_stddev,
relative_speed: ratio.get::<quantity::ratio>(),
relative_speed_stddev: ratio_stddev.map(|r| r.get::<quantity::ratio>()),
is_reference,
relative_ordering,
}
@@ -108,8 +94,8 @@ pub fn compute_with_check_from_reference<'a>(
reference: &'a BenchmarkResult,
sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
if fastest_of(results).mean_wall_clock_time().get::<second>() == 0.0
|| reference.mean_wall_clock_time().get::<second>() == 0.0
if fastest_of(results).mean_wall_clock_time() == Time::zero()
|| reference.mean_wall_clock_time() == Time::zero()
{
return None;
}
@@ -123,25 +109,32 @@ pub fn compute_with_check(
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
let fastest = fastest_of(results);
if fastest.mean_wall_clock_time().get::<second>() == 0.0 {
if fastest.mean_wall_clock_time() == Time::zero() {
return None;
}
Some(compute_relative_speeds(results, fastest, sort_order))
}
/// Same as compute_with_check, potentially resulting in relative speeds of infinity
pub fn compute(
results: &[BenchmarkResult],
/// Compute ratios relative to the given reference, or the fastest result if omitted.
/// A zero reference runtime can produce infinite ratios.
pub fn compute<'a>(
results: &'a [BenchmarkResult],
sort_order: SortOrder,
) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
let fastest = fastest_of(results);
reference: Option<&'a BenchmarkResult>,
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
let reference = reference.unwrap_or_else(|| fastest_of(results));
compute_relative_speeds(results, fastest, sort_order)
compute_relative_speeds(results, reference, sort_order)
}
#[cfg(test)]
fn create_result(name: &str, mean: f64) -> BenchmarkResult {
create_result_from_times(name, &[mean])
}
#[cfg(test)]
fn create_result_from_times(name: &str, times: &[f64]) -> BenchmarkResult {
use std::collections::BTreeMap;
use crate::benchmark::measurement::{Measurement, Measurements};
@@ -151,11 +144,14 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult {
command: name.into(),
command_with_unused_parameters: name.into(),
measurements: Measurements {
measurements: vec![Measurement {
time_wall_clock: Time::new::<second>(mean),
time_user: Time::new::<second>(mean),
..Default::default()
}],
measurements: times
.iter()
.map(|&time| Measurement {
time_wall_clock: Time::new::<second>(time),
time_user: Time::new::<second>(time),
..Default::default()
})
.collect(),
},
parameters: BTreeMap::new(),
}
@@ -182,13 +178,20 @@ fn test_compute_relative_speed() {
fn test_compute_relative_speed_with_reference() {
use approx::assert_relative_eq;
let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
let reference = create_result("cmd2", 4.0);
let results = vec![
create_result_from_times("cmd2", &[1.0, 2.0, 3.0]),
create_result_from_times("cmd3", &[4.0, 5.0, 6.0]),
];
let reference = create_result_from_times("reference", &[3.0, 4.0, 5.0]);
let annotated_results =
compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
assert_relative_eq!(2.0, annotated_results[0].relative_speed);
assert_relative_eq!(0.5, annotated_results[0].relative_speed);
assert_relative_eq!(
0.2795084971874737,
annotated_results[0].relative_speed_stddev.unwrap()
);
assert_relative_eq!(1.25, annotated_results[1].relative_speed);
}
@@ -200,3 +203,59 @@ fn test_compute_relative_speed_for_zero_times() {
assert!(annotated_results.is_none());
}
#[test]
fn test_reference_ratios_do_not_depend_on_sort_order() {
use approx::assert_relative_eq;
let results = vec![
create_result("reference", 2.0),
create_result("slower", 3.0),
create_result("faster", 1.0),
create_result("equal", 2.0),
];
for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
let entries = compute(&results, sort_order, Some(&results[0]));
for entry in entries {
assert_relative_eq!(
entry
.result
.mean_wall_clock_time()
.get::<quantity::second>()
/ 2.0,
entry.relative_speed
);
assert_eq!(entry.is_reference, entry.result.command == "reference");
}
}
}
#[test]
fn test_reference_ratios_without_stddev() {
let results = vec![
create_result("reference", 2.0),
create_result("faster", 1.0),
];
let entries = compute(&results, SortOrder::Command, Some(&results[0]));
assert_eq!(entries[1].relative_speed, 0.5);
assert_eq!(entries[1].relative_speed_stddev, None);
}
#[test]
fn test_reference_ratios_with_zero_times() {
let results = vec![
create_result_from_times("reference", &[1.0, 2.0, 3.0]),
create_result_from_times("zero", &[0.0, 0.0, 0.0]),
];
let entries = compute(&results, SortOrder::Command, Some(&results[0]));
assert_eq!(entries[1].relative_speed, 0.0);
assert_eq!(entries[1].relative_speed_stddev, Some(0.0));
let entries = compute(&results, SortOrder::Command, Some(&results[1]));
assert_eq!(entries[0].relative_speed, f64::INFINITY);
assert_eq!(entries[0].relative_speed_stddev, None);
assert_eq!(entries[1].relative_speed, 1.0);
assert!(compute_with_check_from_reference(&results, &results[0], SortOrder::Command).is_none());
}
+26 -18
View File
@@ -96,28 +96,36 @@ impl<'a> Scheduler<'a> {
)?;
for item in others {
let stddev = if let Some(stddev) = item.relative_speed_stddev {
// The prose summary expresses both directions as factors above one.
let (relative_speed, relative_speed_stddev, prefix, suffix) =
match item.relative_ordering {
Ordering::Less => (
format!("{:8.2}", 1.0 / item.relative_speed),
item.relative_speed_stddev
.map(|stddev| stddev / item.relative_speed.powi(2)),
"",
" times slower than",
),
Ordering::Greater => (
format!("{:8.2}", item.relative_speed),
item.relative_speed_stddev,
"",
" times faster than",
),
Ordering::Equal => (
format!("{:.2}", item.relative_speed),
item.relative_speed_stddev,
" As fast (",
") as",
),
};
let stddev = if let Some(stddev) = relative_speed_stddev {
format!(" ± {}", format!("{:.2}", stddev).green())
} else {
"".into()
};
let comparator = match item.relative_ordering {
Ordering::Less => format!(
"{}{} times slower than",
format!("{:8.2}", item.relative_speed).bold().green(),
stddev
),
Ordering::Greater => format!(
"{}{} times faster than",
format!("{:8.2}", item.relative_speed).bold().green(),
stddev
),
Ordering::Equal => format!(
" As fast ({}{}) as",
format!("{:.2}", item.relative_speed).bold().green(),
stddev
),
};
let comparator =
format!("{prefix}{}{stddev}{suffix}", relative_speed.bold().green());
console_writeln!(
stdout,
"{} {}",
+3 -1
View File
@@ -106,7 +106,9 @@ fn build_command() -> Command {
.value_name("CMD")
.help(
"The reference command for the relative comparison of results. \
If this is unset, results are compared with the fastest command as reference."
If this is unset, results are compared with the fastest command as reference. \
In the command-sorted comparison and exported markup tables, values below 1 \
indicate commands faster than the reference."
)
)
.arg(
+2 -1
View File
@@ -18,6 +18,7 @@ impl Exporter for CsvExporter {
results: &[BenchmarkResult],
_unit: Option<TimeUnit>,
_sort_order: SortOrder,
_reference: Option<&BenchmarkResult>,
) -> Result<Vec<u8>> {
const CSV_UNIT: TimeUnit = TimeUnit::Second;
const CSV_PRECISION: usize = 6;
@@ -172,7 +173,7 @@ fn test_csv() {
let actual = String::from_utf8(
exporter
.serialize(&results, Some(TimeUnit::Second), SortOrder::Command)
.serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None)
.unwrap(),
)
.unwrap();
+1
View File
@@ -22,6 +22,7 @@ impl Exporter for JsonExporter {
results: &[BenchmarkResult],
_unit: Option<TimeUnit>,
_sort_order: SortOrder,
_reference: Option<&BenchmarkResult>,
) -> Result<Vec<u8>> {
let mut output = to_vec_pretty(&HyperfineSummary { results });
if let Ok(ref mut content) = output {
+2 -1
View File
@@ -114,9 +114,10 @@ impl<T: MarkupExporter> Exporter for T {
results: &[BenchmarkResult],
unit: Option<TimeUnit>,
sort_order: SortOrder,
reference: Option<&BenchmarkResult>,
) -> Result<Vec<u8>> {
let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
let entries = relative_speed::compute(results, sort_order);
let entries = relative_speed::compute(results, sort_order, reference);
let table = self.table_results(&entries, unit);
Ok(table.as_bytes().to_vec())
+10 -1
View File
@@ -51,6 +51,7 @@ trait Exporter {
results: &[BenchmarkResult],
unit: Option<TimeUnit>,
sort_order: SortOrder,
reference: Option<&BenchmarkResult>,
) -> Result<Vec<u8>>;
}
@@ -69,6 +70,7 @@ pub struct ExportManager {
exporters: Vec<ExporterWithTarget>,
time_unit: Option<TimeUnit>,
sort_order: SortOrder,
has_reference: bool,
}
impl ExportManager {
@@ -83,6 +85,7 @@ impl ExportManager {
exporters: vec![],
time_unit,
sort_order,
has_reference: matches.contains_id("reference"),
};
{
let mut add_exporter = |flag, exporttype| -> Result<()> {
@@ -131,10 +134,16 @@ impl ExportManager {
/// if a benchmark fails). In the latter case, we only print to stdout targets (in
/// order not to clutter the output of hyperfine with intermediate results).
pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
// The scheduler benchmarks an explicit reference before the other commands.
let reference = if self.has_reference {
results.first()
} else {
None
};
for e in &self.exporters {
let content = || {
e.exporter
.serialize(results, self.time_unit, self.sort_order)
.serialize(results, self.time_unit, self.sort_order, reference)
};
match e.target {
+1 -1
View File
@@ -14,7 +14,7 @@ fn get_output<E: Exporter + Default>(
sort_order: SortOrder,
) -> String {
let exporter = E::default();
String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap()
String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap()
}
/// Ensure the makrup output includes the table header and the multiple
+30 -25
View File
@@ -1,8 +1,12 @@
//! Statistics for non-empty samples of finite measurements.
use std::ops::Add;
use std::ops::AddAssign;
use std::ops::Div;
use std::ops::{Add, AddAssign, Div};
use uom::num_traits;
use uom::si::f64::Ratio;
use uom::si::ratio::ratio;
use crate::quantity::{byte, ratio, second, Information, Ratio, Time, Zero};
use super::{byte, second, Information, Time};
/// A min function that assumes no NaNs and at least one element
pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
@@ -20,14 +24,14 @@ pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
.expect("'max' requires at least one element")
}
/// Arithmetic mean, summing measurements in their original order.
/// A mean function that assumes at least one element
pub fn mean<Q, P>(values: impl IntoIterator<Item = Q>) -> Q
where
Q: AddAssign + Zero + Div<Ratio, Output = P>,
Q: AddAssign + num_traits::Zero + Div<Ratio, Output = P>,
P: Into<Q>,
{
let mut sum = Q::zero();
let mut count = 0usize;
let mut count = 0;
for value in values {
sum += value;
count += 1;
@@ -37,7 +41,6 @@ where
(sum / count).into()
}
/// Median without modifying the input sample.
pub fn median<Q, P>(values: impl IntoIterator<Item = Q>) -> Q
where
Q: Copy + PartialOrd + Add<Output = Q> + Div<Ratio, Output = P>,
@@ -48,14 +51,14 @@ where
!values.is_empty(),
"median requires at least one measurement"
);
values.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
values.sort_by(|a, b| a.partial_cmp(b).expect("No NaN values"));
let len = values.len();
if len % 2 == 0 {
let mid = len / 2;
let a = &values[mid - 1];
let b = &values[mid];
((*b + *a) / Ratio::new::<ratio>(2.)).into()
((*a + *b) / Ratio::new::<ratio>(2.)).into()
} else {
values[len / 2]
}
@@ -88,22 +91,24 @@ impl UnsafeRawValue for Information {
}
}
/// Sample standard deviation, using an already computed mean.
pub fn standard_deviation<Q: UnsafeRawValue>(values: &[Q], mean: Q) -> Q {
let mean = mean.unsafe_raw_value();
assert!(
values.len() > 1,
"standard deviation requires at least two measurements"
);
let sum = values
.iter()
.map(|value| (value.unsafe_raw_value() - mean) * (value.unsafe_raw_value() - mean))
.fold(0.0, |sum, deviation| sum + deviation);
assert!(sum >= 0.0, "invalid sum of squared deviations");
fn standard_deviation_f64(values: &[f64]) -> f64 {
let mean_value = mean(values.iter().copied());
// Preserve the operation order of statistical 1.0: dividing by n - 1
// before taking the square root also preserves exported floating-point values.
Q::unsafe_from_raw_value((sum / (values.len() as f64 - 1.0)).sqrt())
let mut squared_deviations = 0.;
let mut n = 0;
for value in values {
let deviation = value - mean_value;
squared_deviations += deviation * deviation;
n += 1;
}
(1. / ((n - 1) as f64) * squared_deviations).sqrt()
}
pub fn standard_deviation<Q: UnsafeRawValue>(values: impl IntoIterator<Item = Q> + Clone) -> Q {
let values: Vec<_> = values.into_iter().map(|q| q.unsafe_raw_value()).collect();
let result = standard_deviation_f64(&values);
Q::unsafe_from_raw_value(result)
}
/// Compute modified Z-scores for a given sample. A (unmodified) Z-score is defined by
@@ -219,7 +224,7 @@ mod tests {
Time::new::<millisecond>(200.0),
Time::new::<microsecond>(300_000.0),
];
let result = standard_deviation(&values, mean(values));
let result = standard_deviation(values);
assert_relative_eq!(result.get::<millisecond>(), 100.0);
}
}
-8
View File
@@ -82,17 +82,9 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
discard(output);
}
#[cfg(not(windows))]
let (time_user, time_system, peak_memory_usage, exit_status) = cpu_timer.stop(child)?;
#[cfg(windows)]
let exit_status = child.wait()?;
let time_wall_clock = wallclock_timer.stop();
#[cfg(windows)]
let (time_user, time_system, peak_memory_usage) = cpu_timer.stop();
Ok(Measurement {
time_wall_clock,
time_user,
+13 -10
View File
@@ -1,7 +1,10 @@
#![cfg(windows)]
#![warn(unsafe_op_in_unsafe_fn)]
use std::{mem, os::windows::io::AsRawHandle, process, ptr};
use std::process::{self, Child, ExitStatus};
use std::{mem, os::windows::io::AsRawHandle, ptr};
use anyhow::Result;
use windows_sys::Win32::{
Foundation::{CloseHandle, HANDLE},
@@ -31,11 +34,11 @@ use crate::quantity::{nanosecond, Information, Time, Zero};
#[cfg(not(feature = "windows_process_extensions_main_thread_handle"))]
#[allow(non_upper_case_globals)]
static NtResumeProcess: Lazy<unsafe extern "system" fn(ProcessHandle: HANDLE) -> NTSTATUS> =
static NtResumeProcess: Lazy<unsafe extern "system" fn(process_handle: HANDLE) -> NTSTATUS> =
Lazy::new(|| {
// SAFETY: Getting the module handle for ntdll.dll is safe
let ntdll = unsafe { GetModuleHandleW(w!("ntdll.dll")) };
assert!(ntdll != std::ptr::null_mut(), "GetModuleHandleW failed");
assert!(!ntdll.is_null(), "GetModuleHandleW failed");
// SAFETY: The ntdll handle is valid
let nt_resume_process = unsafe { GetProcAddress(ntdll, s!("NtResumeProcess")) };
@@ -54,10 +57,7 @@ impl CPUTimer {
// SAFETY: Creating a new job object is safe
let job_object = unsafe { CreateJobObjectW(ptr::null_mut(), ptr::null_mut()) };
assert!(
job_object != std::ptr::null_mut(),
"CreateJobObjectW failed"
);
assert!(!job_object.is_null(), "CreateJobObjectW failed");
// SAFETY: The job object handle is valid
let ret = unsafe { AssignProcessToJobObject(job_object, child_handle) };
@@ -84,7 +84,9 @@ impl CPUTimer {
Self { job_object }
}
pub fn stop(&self) -> (Time, Time, Information) {
pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Information, ExitStatus)> {
let status = child.wait()?;
let mut job_object_info =
mem::MaybeUninit::<JOBOBJECT_BASIC_ACCOUNTING_INFORMATION>::uninit();
@@ -113,9 +115,10 @@ impl CPUTimer {
// processes no longer associated with the job, in 100-nanosecond ticks."
let system_time =
Time::new::<nanosecond>((job_object_info.TotalKernelTime as f64) * 100.0);
(user_time, system_time, Information::zero())
Ok((user_time, system_time, Information::zero(), status))
} else {
(Time::zero(), Time::zero(), Information::zero())
Ok((Time::zero(), Time::zero(), Information::zero(), status))
}
}
}
+99
View File
@@ -705,6 +705,105 @@ fn shows_benchmark_comparison_relative_to_reference() {
");
}
#[test]
fn command_sorted_comparison_relative_to_reference() {
let _settings = snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug()
.args([
"--style=basic",
"--sort=command",
"--reference=sleep 2",
"--reference-name=baseline",
"sleep 1",
"sleep 3",
"sleep 2",
]), @r"
success: true
exit_code: 0
----- stdout -----
Benchmark 1: baseline
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Benchmark 2: sleep 1
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 1.000 s … 1.000 s 10 runs
Benchmark 3: sleep 3
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 3.000 s … 3.000 s 10 runs
Benchmark 4: sleep 2
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
Range (min … max): 2.000 s … 2.000 s 10 runs
Relative speed comparison
1.00 baseline
0.50 ± 0.00 sleep 1
1.50 ± 0.00 sleep 3
1.00 ± 0.00 sleep 2
----- stderr -----
");
}
#[test]
fn markdown_export_relative_to_reference() {
let tempdir = tempfile::tempdir().unwrap();
let export_path = tempdir.path().join("results.md");
insta::allow_duplicates! {
for sort_order in ["auto", "command", "mean-time"] {
for to_stdout in [false, true] {
let output = hyperfine_debug()
.args([
"--style=none",
"--time-unit=second",
"--reference=sleep 2",
"--reference-name=baseline",
"sleep 1",
"sleep 3",
])
.arg(format!("--sort={sort_order}"))
.arg("--export-markdown")
.arg(if to_stdout {
std::path::Path::new("-")
} else {
&export_path
})
.assert()
.success()
.get_output()
.stdout
.clone();
let table = if to_stdout {
String::from_utf8(output).unwrap()
} else {
std::fs::read_to_string(&export_path).unwrap()
};
if sort_order == "mean-time" {
insta::assert_snapshot!(table.trim(), @r"
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 |
| `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 |
| `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 |
");
} else {
insta::assert_snapshot!(table.trim(), @r"
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 |
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 |
| `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 |
");
}
}
}
}
}
#[test]
fn shows_reference_name() {
let _settings = snapshot_settings().bind_to_scope();