Compare commits

...
Author SHA1 Message Date
David Peter f0f2dccea6 Update dependencies 2024-11-10 22:51:56 +01:00
David Peter 83f04cdd31 Add iteration information to failure error message 2024-11-10 22:26:13 +01:00
David Carlier f096c266f0 fix build warning proposal
```rust
warning: elided lifetime has a name
   --> src/command.rs:137:66
    |
136 | impl<'a> Commands<'a> {
    |      -- lifetime `'a` declared here
137 |     pub fn from_cli_arguments(matches: &'a ArgMatches) -> Result<Commands> {
    |                                                                  ^^^^^^^^ this elided lifetime gets resolved as `'
```
2024-11-10 20:45:03 +01:00
David Peter e3e86174d9 Minor rewording 2024-08-28 17:26:40 +02:00
9 changed files with 337 additions and 341 deletions

No files matched your search

Generated
+243 -314
View File
File diff suppressed because it is too large. Load diff
+16 -8
View File
@@ -10,7 +10,7 @@ repository = "https://github.com/sharkdp/hyperfine"
version = "1.18.0" version = "1.18.0"
edition = "2018" edition = "2018"
build = "build.rs" build = "build.rs"
rust-version = "1.70.0" rust-version = "1.76.0"
[features] [features]
# Use the nightly feature windows_process_extensions_main_thread_handle # Use the nightly feature windows_process_extensions_main_thread_handle
@@ -20,20 +20,20 @@ windows_process_extensions_main_thread_handle = []
colored = "2.1" colored = "2.1"
indicatif = "=0.17.4" indicatif = "=0.17.4"
statistical = "1.0" statistical = "1.0"
csv = "1.1" csv = "1.3"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
rust_decimal = "1.35" rust_decimal = "1.36"
rand = "0.8" rand = "0.8"
shell-words = "1.0" shell-words = "1.0"
thiserror = "1.0" thiserror = "2.0"
anyhow = "1.0" anyhow = "1.0"
[target.'cfg(not(windows))'.dependencies] [target.'cfg(not(windows))'.dependencies]
libc = "0.2" libc = "0.2"
[target.'cfg(windows)'.dependencies] [target.'cfg(windows)'.dependencies]
windows-sys = { version = "0.48", features = [ windows-sys = { version = "0.59", features = [
"Win32_Foundation", "Win32_Foundation",
"Win32_Security", "Win32_Security",
"Win32_System_JobObjects", "Win32_System_JobObjects",
@@ -45,18 +45,26 @@ windows-sys = { version = "0.48", features = [
once_cell = "1.19" once_cell = "1.19"
[target.'cfg(target_os="linux")'.dependencies] [target.'cfg(target_os="linux")'.dependencies]
nix = { version = "0.27.1", features = ["zerocopy"] } nix = { version = "0.29", features = ["zerocopy"] }
[dependencies.clap] [dependencies.clap]
version = "4" version = "4"
default-features = false default-features = false
features = ["suggestions", "color", "wrap_help", "cargo", "help", "usage", "error-context"] features = [
"suggestions",
"color",
"wrap_help",
"cargo",
"help",
"usage",
"error-context",
]
[dev-dependencies] [dev-dependencies]
approx = "0.5" approx = "0.5"
assert_cmd = "2.0" assert_cmd = "2.0"
predicates = "3.0" predicates = "3.0"
tempfile = "3.9" tempfile = "3.14"
[build-dependencies] [build-dependencies]
clap = "4.4.12" clap = "4.4.12"
+1 -1
View File
@@ -316,7 +316,7 @@ Hyperfine can be installed from source via [cargo](https://doc.rust-lang.org/car
cargo install --locked hyperfine cargo install --locked hyperfine
``` ```
Make sure that you use Rust 1.70 or newer. Make sure that you use Rust 1.76 or newer.
### From binaries (Linux, macOS, Windows) ### From binaries (Linux, macOS, Windows)
+1 -1
View File
@@ -9,7 +9,7 @@ hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
### Pre-requisites ### Pre-requisites
To make these scripts work, you will need to install `numpy`, `matplotlib` and `scipy`. To make these scripts work, you will need `numpy`, `matplotlib` and `scipy`.
If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/) If you have a Python package manager that understands [PEP-723](https://peps.python.org/pep-0723/)
inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx), inline script requirements like [`uv`](https://github.com/astral-sh/uv) or [`pipx`](https://github.com/pypa/pipx),
+44 -5
View File
@@ -16,11 +16,28 @@ use super::timing_result::TimingResult;
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use statistical::mean; use statistical::mean;
pub enum BenchmarkIteration {
NonBenchmarkRun,
Warmup(u64),
Benchmark(u64),
}
impl BenchmarkIteration {
pub fn to_env_var_value(&self) -> Option<String> {
match self {
BenchmarkIteration::NonBenchmarkRun => None,
BenchmarkIteration::Warmup(i) => Some(format!("warmup-{}", i)),
BenchmarkIteration::Benchmark(i) => Some(format!("{}", i)),
}
}
}
pub trait Executor { pub trait Executor {
/// Run the given command and measure the execution time /// Run the given command and measure the execution time
fn run_command_and_measure( fn run_command_and_measure(
&self, &self,
command: &Command<'_>, command: &Command<'_>,
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
) -> Result<(TimingResult, ExitStatus)>; ) -> Result<(TimingResult, ExitStatus)>;
@@ -39,6 +56,7 @@ pub trait Executor {
fn run_command_and_measure_common( fn run_command_and_measure_common(
mut command: std::process::Command, mut command: std::process::Command,
iteration: BenchmarkIteration,
command_failure_action: CmdFailureAction, command_failure_action: CmdFailureAction,
command_input_policy: &CommandInputPolicy, command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy, command_output_policy: &CommandOutputPolicy,
@@ -53,17 +71,29 @@ fn run_command_and_measure_common(
randomized_environment_offset::value(), randomized_environment_offset::value(),
); );
if let Some(value) = iteration.to_env_var_value() {
command.env("HYPERFINE_ITERATION", value);
}
let result = execute_and_measure(command) let result = execute_and_measure(command)
.with_context(|| format!("Failed to run command '{command_name}'"))?; .with_context(|| format!("Failed to run command '{command_name}'"))?;
if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() { if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
let when = match iteration {
BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(),
BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(),
BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"),
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
};
bail!( bail!(
"{}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.", Alternatively, use the '--show-output' option to debug what went wrong.",
result.status.code().map_or( cause=result.status.code().map_or(
"The process has been terminated by a signal".into(), "The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code: {c}") |c| format!("Command terminated with non-zero exit code {c}")
)
),
); );
} }
@@ -84,10 +114,12 @@ impl<'a> Executor for RawExecutor<'a> {
fn run_command_and_measure( fn run_command_and_measure(
&self, &self,
command: &Command<'_>, command: &Command<'_>,
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
) -> Result<(TimingResult, ExitStatus)> { ) -> Result<(TimingResult, ExitStatus)> {
let result = run_command_and_measure_common( let result = run_command_and_measure_common(
command.get_command()?, command.get_command()?,
iteration,
command_failure_action.unwrap_or(self.options.command_failure_action), command_failure_action.unwrap_or(self.options.command_failure_action),
&self.options.command_input_policy, &self.options.command_input_policy,
&self.options.command_output_policy, &self.options.command_output_policy,
@@ -133,6 +165,7 @@ impl<'a> Executor for ShellExecutor<'a> {
fn run_command_and_measure( fn run_command_and_measure(
&self, &self,
command: &Command<'_>, command: &Command<'_>,
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>, command_failure_action: Option<CmdFailureAction>,
) -> Result<(TimingResult, ExitStatus)> { ) -> Result<(TimingResult, ExitStatus)> {
let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
@@ -149,6 +182,7 @@ impl<'a> Executor for ShellExecutor<'a> {
let mut result = run_command_and_measure_common( let mut result = run_command_and_measure_common(
command_builder, command_builder,
iteration,
command_failure_action.unwrap_or(self.options.command_failure_action), command_failure_action.unwrap_or(self.options.command_failure_action),
&self.options.command_input_policy, &self.options.command_input_policy,
&self.options.command_output_policy, &self.options.command_output_policy,
@@ -191,7 +225,11 @@ impl<'a> Executor for ShellExecutor<'a> {
for _ in 0..COUNT { for _ in 0..COUNT {
// Just run the shell without any command // Just run the shell without any command
let res = self.run_command_and_measure(&Command::new(None, ""), None); let res = self.run_command_and_measure(
&Command::new(None, ""),
BenchmarkIteration::NonBenchmarkRun,
None,
);
match res { match res {
Err(_) => { Err(_) => {
@@ -260,6 +298,7 @@ impl Executor for MockExecutor {
fn run_command_and_measure( fn run_command_and_measure(
&self, &self,
command: &Command<'_>, command: &Command<'_>,
_iteration: BenchmarkIteration,
_command_failure_action: Option<CmdFailureAction>, _command_failure_action: Option<CmdFailureAction>,
) -> Result<(TimingResult, ExitStatus)> { ) -> Result<(TimingResult, ExitStatus)> {
#[cfg(unix)] #[cfg(unix)]
+23 -6
View File
@@ -6,6 +6,7 @@ pub mod timing_result;
use std::cmp; use std::cmp;
use crate::benchmark::executor::BenchmarkIteration;
use crate::command::Command; use crate::command::Command;
use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption}; use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption};
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
@@ -57,7 +58,11 @@ impl<'a> Benchmark<'a> {
error_output: &'static str, error_output: &'static str,
) -> Result<TimingResult> { ) -> Result<TimingResult> {
self.executor self.executor
.run_command_and_measure(command, Some(CmdFailureAction::RaiseError)) .run_command_and_measure(
command,
executor::BenchmarkIteration::NonBenchmarkRun,
Some(CmdFailureAction::RaiseError),
)
.map(|r| r.0) .map(|r| r.0)
.map_err(|_| anyhow!(error_output)) .map_err(|_| anyhow!(error_output))
} }
@@ -187,9 +192,13 @@ impl<'a> Benchmark<'a> {
None None
}; };
for _ in 0..self.options.warmup_count { for i in 0..self.options.warmup_count {
let _ = run_preparation_command()?; let _ = run_preparation_command()?;
let _ = self.executor.run_command_and_measure(self.command, None)?; let _ = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Warmup(i),
None,
)?;
let _ = run_conclusion_command()?; let _ = run_conclusion_command()?;
if let Some(bar) = progress_bar.as_ref() { if let Some(bar) = progress_bar.as_ref() {
bar.inc(1) bar.inc(1)
@@ -216,7 +225,11 @@ impl<'a> Benchmark<'a> {
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
// Initial timing run // Initial timing run
let (res, status) = self.executor.run_command_and_measure(self.command, None)?; let (res, status) = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(0),
None,
)?;
let success = status.success(); let success = status.success();
let conclusion_result = run_conclusion_command()?; let conclusion_result = run_conclusion_command()?;
@@ -260,7 +273,7 @@ impl<'a> Benchmark<'a> {
} }
// Gather statistics (perform the actual benchmark) // Gather statistics (perform the actual benchmark)
for _ in 0..count_remaining { for i in 0..count_remaining {
run_preparation_command()?; run_preparation_command()?;
let msg = { let msg = {
@@ -272,7 +285,11 @@ impl<'a> Benchmark<'a> {
bar.set_message(msg.to_owned()) bar.set_message(msg.to_owned())
} }
let (res, status) = self.executor.run_command_and_measure(self.command, None)?; let (res, status) = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(i + 1),
None,
)?;
let success = status.success(); let success = status.success();
times_real.push(res.time_real); times_real.push(res.time_real);
+1 -1
View File
@@ -134,7 +134,7 @@ impl<'a> Command<'a> {
pub struct Commands<'a>(Vec<Command<'a>>); pub struct Commands<'a>(Vec<Command<'a>>);
impl<'a> Commands<'a> { impl<'a> Commands<'a> {
pub fn from_cli_arguments(matches: &'a ArgMatches) -> Result<Commands> { pub fn from_cli_arguments(matches: &'a ArgMatches) -> Result<Commands<'a>> {
let command_names = matches.get_many::<String>("command-name"); let command_names = matches.get_many::<String>("command-name");
let command_strings = matches let command_strings = matches
.get_many::<String>("command") .get_many::<String>("command")
+3 -3
View File
@@ -11,7 +11,7 @@ use nix::fcntl::{splice, SpliceFFlags};
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
use std::fs::File; use std::fs::File;
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
use std::os::unix::io::AsRawFd; use std::os::fd::AsFd;
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
use windows_sys::Win32::System::Threading::CREATE_SUSPENDED; use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
@@ -53,9 +53,9 @@ fn discard(output: ChildStdout) {
{ {
if let Ok(file) = File::create("/dev/null") { if let Ok(file) = File::create("/dev/null") {
while let Ok(bytes) = splice( while let Ok(bytes) = splice(
output.as_raw_fd(), output.as_fd(),
None, None,
file.as_raw_fd(), file.as_fd(),
None, None,
CHUNK_SIZE, CHUNK_SIZE,
SpliceFFlags::empty(), SpliceFFlags::empty(),
+5 -2
View File
@@ -37,7 +37,7 @@ static NtResumeProcess: Lazy<unsafe extern "system" fn(ProcessHandle: HANDLE) ->
Lazy::new(|| { Lazy::new(|| {
// SAFETY: Getting the module handle for ntdll.dll is safe // SAFETY: Getting the module handle for ntdll.dll is safe
let ntdll = unsafe { GetModuleHandleW(w!("ntdll.dll")) }; let ntdll = unsafe { GetModuleHandleW(w!("ntdll.dll")) };
assert!(ntdll != 0, "GetModuleHandleW failed"); assert!(ntdll != std::ptr::null_mut(), "GetModuleHandleW failed");
// SAFETY: The ntdll handle is valid // SAFETY: The ntdll handle is valid
let nt_resume_process = unsafe { GetProcAddress(ntdll, s!("NtResumeProcess")) }; let nt_resume_process = unsafe { GetProcAddress(ntdll, s!("NtResumeProcess")) };
@@ -56,7 +56,10 @@ impl CPUTimer {
// SAFETY: Creating a new job object is safe // SAFETY: Creating a new job object is safe
let job_object = unsafe { CreateJobObjectW(ptr::null_mut(), ptr::null_mut()) }; let job_object = unsafe { CreateJobObjectW(ptr::null_mut(), ptr::null_mut()) };
assert!(job_object != 0, "CreateJobObjectW failed"); assert!(
job_object != std::ptr::null_mut(),
"CreateJobObjectW failed"
);
// SAFETY: The job object handle is valid // SAFETY: The job object handle is valid
let ret = unsafe { AssignProcessToJobObject(job_object, child_handle) }; let ret = unsafe { AssignProcessToJobObject(job_object, child_handle) };