Compare commits

...
6 changed files with 103 additions and 80 deletions

No files matched your search

+5 -4
View File
@@ -108,7 +108,7 @@ hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log'
### Intermediate shell ### Intermediate shell
By default, commands are executed directly, without an intermediate shell (`--shell=none`). By default, benchmarked commands, including warmup runs, are executed directly, without an intermediate shell (`--shell=none`).
Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported. Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported.
Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted. Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted.
This avoids shell startup overhead and the noise from correcting for it, especially for fast commands This avoids shell startup overhead and the noise from correcting for it, especially for fast commands
@@ -125,10 +125,11 @@ You can also select a specific shell with `--shell <SHELL>`:
hyperfine --shell zsh 'for i in {1..10000}; do echo test; done' hyperfine --shell zsh 'for i in {1..10000}; do echo test; done'
``` ```
The shell setting applies to all commands, including `--setup`, `--prepare`, `--conclude`, and `--cleanup`. The `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands always run through a shell.
If any of these commands need shell syntax, enable a shell explicitly. They use the shell selected by `--shell`, or the platform shell (`sh` on Unix, `cmd.exe` on Windows)
when `--shell` is omitted or set to `none`.
When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an When benchmarked commands use a shell, hyperfine *corrects for the shell spawning time*. It runs the shell with an
empty command multiple times to measure its startup time, then subtracts this time from each empty command multiple times to measure its startup time, then subtracts this time from each
time measurement. time measurement.
+6 -4
View File
@@ -194,14 +194,16 @@ possible parameter combinations.
.HP .HP
\fB\-\-shell\fR \fISHELL\fP \fB\-\-shell\fR \fISHELL\fP
.IP .IP
Set the shell to use for executing commands (default: none). This can be Set the shell to use for executing benchmarked commands, including warmup runs
the name or the path to the shell executable, or a full command line (default: none). This can be the name or the path to the shell executable,
or a full command line
like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly
select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be
set to "none" to disable the shell. In this case, commands will be set to "none" to disable the shell. In this case, benchmarked commands will be
executed directly. They can still have arguments, but more complex executed directly. They can still have arguments, but more complex
things like "sleep 0.1; sleep 0.2" are not possible without a shell. things like "sleep 0.1; sleep 0.2" are not possible without a shell.
This option also applies to setup, prepare, conclude, and cleanup commands. Setup, prepare, conclude, and cleanup commands always use a shell: the selected
shell, or the platform shell when this option is omitted or set to "none".
.HP .HP
\fB\-S\fR \fB\-S\fR
.IP .IP
+51 -20
View File
@@ -5,9 +5,7 @@ use std::process::ExitStatus;
use crate::benchmark::measurement::Measurement; use crate::benchmark::measurement::Measurement;
use crate::benchmark::measurement::Measurements; use crate::benchmark::measurement::Measurements;
use crate::command::Command; use crate::command::Command;
use crate::options::{ use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell};
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
};
use crate::output::progress_bar::get_progress_bar; use crate::output::progress_bar::get_progress_bar;
use crate::quantity::{second, Information, Time, Zero}; use crate::quantity::{second, Information, Time, Zero};
use crate::timer::execute_and_measure; use crate::timer::execute_and_measure;
@@ -42,6 +40,21 @@ pub trait Executor {
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
) -> Result<Measurement>; ) -> Result<Measurement>;
/// Run a lifecycle command through a shell, always treating failures as errors.
fn run_intermediate_command_and_measure(
&self,
command: &Command<'_>,
iteration: BenchmarkIteration,
output_policy: &CommandOutputPolicy,
) -> Result<Measurement> {
self.run_command_and_measure(
command,
iteration,
Some(CmdFailureAction::RaiseError),
output_policy,
)
}
/// Perform a calibration of this executor. For example, /// Perform a calibration of this executor. For example,
/// when running commands through a shell, we need to /// when running commands through a shell, we need to
/// measure the shell spawning time separately in order /// measure the shell spawning time separately in order
@@ -58,13 +71,13 @@ pub trait Executor {
fn run_command_and_measure_common( fn run_command_and_measure_common(
mut command: std::process::Command, mut command: std::process::Command,
iteration: BenchmarkIteration, iteration: BenchmarkIteration,
command_failure_action: CmdFailureAction, command_failure_action: Option<CmdFailureAction>,
command_input_policy: &CommandInputPolicy, options: &Options,
command_output_policy: &CommandOutputPolicy, command_output_policy: &CommandOutputPolicy,
command_name: &str, command_name: &str,
collect_hardware_counters: bool, collect_hardware_counters: bool,
) -> Result<Measurement> { ) -> Result<Measurement> {
let stdin = command_input_policy.get_stdin()?; let stdin = options.command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?; let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
command.stdin(stdin).stdout(stdout).stderr(stderr); command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -83,10 +96,14 @@ fn run_command_and_measure_common(
if !measurement.exit_status.success() { if !measurement.exit_status.success() {
use crate::util::exit_code::extract_exit_code; use crate::util::exit_code::extract_exit_code;
let should_fail = match command_failure_action { let show_ignore_failure_hint = command_failure_action.is_none();
let should_fail = match command_failure_action
.as_ref()
.unwrap_or(&options.command_failure_action)
{
CmdFailureAction::RaiseError => true, CmdFailureAction::RaiseError => true,
CmdFailureAction::IgnoreAllFailures => false, CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(ref codes) => { CmdFailureAction::IgnoreSpecificFailures(codes) => {
// Only fail if the exit code is not in the list of codes to ignore // Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(measurement.exit_status) { if let Some(exit_code) = extract_exit_code(measurement.exit_status) {
!codes.contains(&exit_code) !codes.contains(&exit_code)
@@ -105,15 +122,17 @@ fn run_command_and_measure_common(
BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(), BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(),
BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"), BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"),
}; };
bail!( let cause = measurement.exit_status.code().map_or_else(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ || "was terminated by a signal".to_owned(),
Alternatively, use the '--show-output' option to debug what went wrong.", |code| format!("terminated with non-zero exit code {code}"),
cause=measurement.exit_status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
),
); );
let hint = if show_ignore_failure_hint {
" Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong."
} else {
" Use the '--show-output' option to debug what went wrong."
};
bail!("Command '{command_name}' {cause} in {when}.{hint}");
} }
} }
@@ -141,14 +160,26 @@ impl Executor for RawExecutor<'_> {
run_command_and_measure_common( run_command_and_measure_common(
command.get_command()?, command.get_command()?,
iteration, iteration,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), command_failure_action,
&self.options.command_input_policy, self.options,
output_policy, output_policy,
&command.get_command_line(), &command.get_command_line(),
matches!(iteration, BenchmarkIteration::Benchmark(_)), matches!(iteration, BenchmarkIteration::Benchmark(_)),
) )
} }
fn run_intermediate_command_and_measure(
&self,
command: &Command<'_>,
iteration: BenchmarkIteration,
output_policy: &CommandOutputPolicy,
) -> Result<Measurement> {
// Keep shell startup in the hook's duration: raw benchmarks have no
// calibrated shell overhead to add back when estimating the run count.
ShellExecutor::new(&Shell::platform_default(), self.options)
.run_intermediate_command_and_measure(command, iteration, output_policy)
}
fn calibrate(&mut self) -> Result<()> { fn calibrate(&mut self) -> Result<()> {
Ok(()) Ok(())
} }
@@ -197,8 +228,8 @@ impl Executor for ShellExecutor<'_> {
let mut measurement = run_command_and_measure_common( let mut measurement = run_command_and_measure_common(
command_builder, command_builder,
iteration, iteration,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), command_failure_action,
&self.options.command_input_policy, self.options,
output_policy, output_policy,
&command.get_command_line(), &command.get_command_line(),
false, false,
+16 -28
View File
@@ -11,9 +11,7 @@ use crate::benchmark::benchmark_result::Parameter;
use crate::benchmark::executor::BenchmarkIteration; use crate::benchmark::executor::BenchmarkIteration;
use crate::benchmark::measurement::{Measurement, Measurements}; use crate::benchmark::measurement::{Measurement, Measurements};
use crate::command::Command; use crate::command::Command;
use crate::options::{ use crate::options::{CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption};
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
};
use crate::outlier_detection::OUTLIER_THRESHOLD; use crate::outlier_detection::OUTLIER_THRESHOLD;
use crate::output::console_writeln; use crate::output::console_writeln;
use crate::output::progress_bar::{ use crate::output::progress_bar::{
@@ -55,22 +53,24 @@ impl<'a> Benchmark<'a> {
} }
} }
/// Run setup, cleanup, or preparation commands /// Run setup, cleanup, preparation, or conclusion commands
fn run_intermediate_command( fn run_intermediate_command(
&self, &self,
command: &Command<'_>, command: &Command<'_>,
error_output: &'static str, kind: &'static str,
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration, iteration: executor::BenchmarkIteration,
) -> Result<Measurement> { ) -> Result<Measurement> {
self.executor self.executor
.run_command_and_measure( .run_intermediate_command_and_measure(command, iteration, output_policy)
command, .map_err(|error| {
iteration, let hint = if cfg!(windows) {
Some(CmdFailureAction::RaiseError), "Append ' || exit /b 0' to the command if this failure can be ignored."
output_policy, } else {
) "Append ' || true' to the command if this failure can be ignored."
.map_err(|_| anyhow!(error_output)) };
anyhow!("The {kind} command failed: {error:#} {hint}")
})
} }
/// Run the command specified by `--setup`. /// Run the command specified by `--setup`.
@@ -85,14 +85,11 @@ impl<'a> Benchmark<'a> {
.as_ref() .as_ref()
.map(|setup_command| Command::new_parametrized(None, setup_command, parameters)); .map(|setup_command| Command::new_parametrized(None, setup_command, parameters));
let error_output = "The setup command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command Ok(command
.map(|cmd| { .map(|cmd| {
self.run_intermediate_command( self.run_intermediate_command(
&cmd, &cmd,
error_output, "setup",
output_policy, output_policy,
BenchmarkIteration::NonBenchmarkRun, BenchmarkIteration::NonBenchmarkRun,
) )
@@ -113,14 +110,11 @@ impl<'a> Benchmark<'a> {
.as_ref() .as_ref()
.map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters)); .map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters));
let error_output = "The cleanup command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
Ok(command Ok(command
.map(|cmd| { .map(|cmd| {
self.run_intermediate_command( self.run_intermediate_command(
&cmd, &cmd,
error_output, "cleanup",
output_policy, output_policy,
BenchmarkIteration::NonBenchmarkRun, BenchmarkIteration::NonBenchmarkRun,
) )
@@ -136,10 +130,7 @@ impl<'a> Benchmark<'a> {
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration, iteration: executor::BenchmarkIteration,
) -> Result<Measurement> { ) -> Result<Measurement> {
let error_output = "The preparation command terminated with a non-zero exit code. \ self.run_intermediate_command(command, "preparation", output_policy, iteration)
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy, iteration)
} }
/// Run the command specified by `--conclude`. /// Run the command specified by `--conclude`.
@@ -149,10 +140,7 @@ impl<'a> Benchmark<'a> {
output_policy: &CommandOutputPolicy, output_policy: &CommandOutputPolicy,
iteration: executor::BenchmarkIteration, iteration: executor::BenchmarkIteration,
) -> Result<Measurement> { ) -> Result<Measurement> {
let error_output = "The conclusion command terminated with a non-zero exit code. \ self.run_intermediate_command(command, "conclusion", output_policy, iteration)
Append ' || true' to the command if you are sure that this can be ignored.";
self.run_intermediate_command(command, error_output, output_policy, iteration)
} }
fn validate_measurement(&self, measurement: &Measurement) -> Result<()> { fn validate_measurement(&self, measurement: &Measurement) -> Result<()> {
+6 -5
View File
@@ -236,14 +236,15 @@ fn build_command() -> Command {
.value_name("SHELL") .value_name("SHELL")
.overrides_with("shell") .overrides_with("shell")
.value_hint(ValueHint::CommandString) .value_hint(ValueHint::CommandString)
.help("Set the shell to use for executing commands (default: none). This can be the \ .help("Set the shell to use for executing benchmarked commands, including warmup runs \
name or the path to the shell executable, or a full command line \ (default: none). This can be the name or the path to the shell executable, or a full command line \
like \"bash --norc\". It can also be set to \"default\" to explicitly select \ like \"bash --norc\". It can also be set to \"default\" to explicitly select \
the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \ the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \
\"none\" to disable the shell. In this case, commands will be executed \ \"none\" to disable the shell. In this case, benchmarked commands will be executed \
directly. They can still have arguments, but more complex things like \ directly. They can still have arguments, but more complex things like \
\"sleep 0.1; sleep 0.2\" are not possible without a shell. This option also \ \"sleep 0.1; sleep 0.2\" are not possible without a shell. Setup, prepare, conclude, \
applies to setup, prepare, conclude, and cleanup commands.") and cleanup commands always use a shell: the selected shell, or the platform \
shell when this option is omitted or set to \"none\".")
) )
.arg( .arg(
Arg::new("default-shell") Arg::new("default-shell")
+19 -19
View File
@@ -275,7 +275,7 @@ fn fails_for_unknown_command() {
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains(
"Command terminated with non-zero exit code", "Command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577' terminated with non-zero exit code",
)); ));
} }
@@ -298,12 +298,15 @@ fn fails_for_failing_command_without_shell() {
hyperfine() hyperfine()
.arg("--shell=none") .arg("--shell=none")
.arg("--runs=1") .arg("--runs=1")
.arg("false") .arg("--command-name=named benchmark")
.args(["--parameter-list", "status", "0,1"])
.arg("test {status} -eq 0")
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains(
"Command terminated with non-zero exit code", "Command 'test 1 -eq 0' terminated with non-zero exit code 1 in the first benchmark run.",
)); ))
.stderr(predicate::str::contains("--ignore-failure"));
} }
#[test] #[test]
@@ -315,9 +318,7 @@ fn fails_for_unknown_setup_command() {
.arg("echo test") .arg("echo test")
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains("The setup command failed:"));
"The setup command terminated with a non-zero exit code.",
));
} }
#[test] #[test]
@@ -329,9 +330,7 @@ fn fails_for_unknown_cleanup_command() {
.arg("echo test") .arg("echo test")
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains("The cleanup command failed:"));
"The cleanup command terminated with a non-zero exit code.",
));
} }
#[test] #[test]
@@ -342,9 +341,7 @@ fn fails_for_unknown_prepare_command() {
.arg("echo test") .arg("echo test")
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains("The preparation command failed:"));
"The preparation command terminated with a non-zero exit code.",
));
} }
#[test] #[test]
@@ -355,9 +352,7 @@ fn fails_for_unknown_conclude_command() {
.arg("echo test") .arg("echo test")
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains("The conclusion command failed:"));
"The conclusion command terminated with a non-zero exit code.",
));
} }
#[cfg(unix)] #[cfg(unix)]
@@ -368,7 +363,7 @@ fn can_run_failing_commands_with_ignore_failure_option() {
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains(
"Command terminated with non-zero exit code", "Command 'false' terminated with non-zero exit code",
)); ));
hyperfine() hyperfine()
@@ -400,7 +395,7 @@ fn can_ignore_specific_exit_codes() {
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains(
"Command terminated with non-zero exit code 2", "Command 'exit 2' terminated with non-zero exit code 2",
)); ));
} }
@@ -441,7 +436,7 @@ fn can_ignore_multiple_exit_codes() {
.assert() .assert()
.failure() .failure()
.stderr(predicate::str::contains( .stderr(predicate::str::contains(
"Command terminated with non-zero exit code 4", "Command 'exit 4' terminated with non-zero exit code 4",
)); ));
} }
@@ -477,9 +472,14 @@ fn runs_commands_using_user_defined_shell() {
.arg("--show-output") .arg("--show-output")
.arg("--shell") .arg("--shell")
.arg("echo 'custom_shell' '--shell-arg'") .arg("echo 'custom_shell' '--shell-arg'")
.arg("--setup=echo setup")
.arg("--prepare=echo prepare")
.arg("--conclude=echo conclude")
.arg("--cleanup=echo cleanup")
.arg("echo benchmark") .arg("echo benchmark")
.assert() .assert()
.success() .success()
.stdout(predicate::str::contains("custom_shell --shell-arg").count(5))
.stdout( .stdout(
predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or( predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or(
predicate::str::contains("custom_shell --shell-arg /C echo benchmark"), predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),