Proper shell overhead computation

src/app.rs

@@ -249,6 +249,12 @@ fn build_app() -> App<'static> {
                 .help("Give a meaningful name to a command. This can be specified multiple times \
                        if several commands are benchmarked."),
         )
+        .arg(
+            Arg::new("debug-mode")
+            .long("debug-mode")
+            .hide(true)
+            .help("Enable debug mode which does not actually run commands, but returns fake times when the command is 'sleep <time>'")
+        )
 }
 
 #[test]

src/benchmark/benchmark_result.rs

@@ -9,10 +9,10 @@ use crate::util::units::Second;
 // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
 pub struct BenchmarkResult {
-    /// The command that was run
+    /// The full command line of the program that is being benchmarked
     pub command: String,
 
-    /// The mean run time
+    /// The average run time
     pub mean: Second,
 
     /// The standard deviation of all run times. Not available if only one run has been performed
@@ -21,26 +21,26 @@ pub struct BenchmarkResult {
     /// The median run time
     pub median: Second,
 
-    /// Time spend in user space
+    /// Time spent in user mode
     pub user: Second,
 
-    /// Time spent in system space
+    /// Time spent in kernel mode
     pub system: Second,
 
-    /// Min time measured
+    /// Minimum of all measured times
     pub min: Second,
 
-    /// Max time measured
+    /// Maximum of all measured times
     pub max: Second,
 
     /// All run time measurements
     #[serde(skip_serializing_if = "Option::is_none")]
     pub times: Option<Vec<Second>>,
 
-    /// All run exit codes
+    /// Exit codes of all command invocations
     pub exit_codes: Vec<Option<i32>>,
 
-    /// Any parameter values used
+    /// Parameter values for this benchmark
     #[serde(skip_serializing_if = "BTreeMap::is_empty")]
     pub parameters: BTreeMap<String, String>,
 }

src/benchmark/executor.rs

@@ -0,0 +1,230 @@
+use std::process::{ExitStatus, Stdio};
+
+use crate::command::Command;
+use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell};
+use crate::output::progress_bar::get_progress_bar;
+use crate::shell::execute_and_time;
+use crate::timer::wallclocktimer::WallClockTimer;
+use crate::timer::{TimerStart, TimerStop};
+use crate::util::units::Second;
+
+use super::timing_result::TimingResult;
+
+use anyhow::{bail, Result};
+use statistical::mean;
+
+pub trait Executor {
+    fn time_command(
+        &self,
+        command: &Command<'_>,
+        command_failure_action: Option<CmdFailureAction>,
+    ) -> Result<(TimingResult, ExitStatus)>;
+
+    fn calibrate(&mut self) -> Result<()>;
+
+    fn time_overhead(&self) -> Second;
+}
+
+pub struct ShellExecutor<'a> {
+    options: &'a Options,
+    shell: &'a Shell,
+    shell_spawning_time: Option<TimingResult>,
+}
+
+impl<'a> ShellExecutor<'a> {
+    /// Correct for shell spawning time
+    fn subtract_shell_spawning_time(&self, time: Second, shell_spawning_time: Second) -> Second {
+        if time < shell_spawning_time {
+            0.0
+        } else {
+            time - shell_spawning_time
+        }
+    }
+
+    pub fn new(shell: &'a Shell, options: &'a Options) -> Self {
+        ShellExecutor {
+            shell,
+            options,
+            shell_spawning_time: None,
+        }
+    }
+}
+
+impl<'a> Executor for ShellExecutor<'a> {
+    /// Run the given shell command and measure the execution time
+    fn time_command(
+        &self,
+        command: &Command<'_>,
+        command_failure_action: Option<CmdFailureAction>,
+    ) -> Result<(TimingResult, ExitStatus)> {
+        let (stdout, stderr) = match self.options.command_output_policy {
+            CommandOutputPolicy::Discard => (Stdio::null(), Stdio::null()),
+            CommandOutputPolicy::Forward => (Stdio::inherit(), Stdio::inherit()),
+        };
+
+        let wallclock_timer = WallClockTimer::start();
+        let result = execute_and_time(stdout, stderr, &command.get_shell_command(), &self.shell)?;
+        let mut time_real = wallclock_timer.stop();
+
+        let mut time_user = result.user_time;
+        let mut time_system = result.system_time;
+
+        if command_failure_action.unwrap_or(self.options.command_failure_action)
+            == CmdFailureAction::RaiseError
+            && !result.status.success()
+        {
+            bail!(
+                "{}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
+                Alternatively, use the '--show-output' option to debug what went wrong.",
+                result.status.code().map_or(
+                    "The process has been terminated by a signal".into(),
+                    |c| format!("Command terminated with non-zero exit code: {}", c)
+                )
+            );
+        }
+
+        // Subtract shell spawning time
+        if let Some(spawning_time) = self.shell_spawning_time {
+            time_real = self.subtract_shell_spawning_time(time_real, spawning_time.time_real);
+            time_user = self.subtract_shell_spawning_time(time_user, spawning_time.time_user);
+            time_system = self.subtract_shell_spawning_time(time_system, spawning_time.time_system);
+        }
+
+        Ok((
+            TimingResult {
+                time_real,
+                time_user,
+                time_system,
+            },
+            result.status,
+        ))
+    }
+
+    /// Measure the average shell spawning time
+    fn calibrate(&mut self) -> Result<()> {
+        const COUNT: u64 = 50;
+        let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
+            Some(get_progress_bar(
+                COUNT,
+                "Measuring shell spawning time",
+                self.options.output_style,
+            ))
+        } else {
+            None
+        };
+
+        let mut times_real: Vec<Second> = vec![];
+        let mut times_user: Vec<Second> = vec![];
+        let mut times_system: Vec<Second> = vec![];
+
+        for _ in 0..COUNT {
+            // Just run the shell without any command
+            let res = self.time_command(&Command::new(None, ""), None);
+
+            match res {
+                Err(_) => {
+                    let shell_cmd = if cfg!(windows) {
+                        format!("{} /C \"\"", self.shell)
+                    } else {
+                        format!("{} -c \"\"", self.shell)
+                    };
+
+                    bail!(
+                        "Could not measure shell execution time. Make sure you can run '{}'.",
+                        shell_cmd
+                    );
+                }
+                Ok((r, _)) => {
+                    times_real.push(r.time_real);
+                    times_user.push(r.time_user);
+                    times_system.push(r.time_system);
+                }
+            }
+
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.inc(1)
+            }
+        }
+
+        if let Some(bar) = progress_bar.as_ref() {
+            bar.finish_and_clear()
+        }
+
+        self.shell_spawning_time = Some(TimingResult {
+            time_real: mean(&times_real),
+            time_user: mean(&times_user),
+            time_system: mean(&times_system),
+        });
+
+        Ok(())
+    }
+
+    fn time_overhead(&self) -> Second {
+        self.shell_spawning_time.unwrap().time_real
+    }
+}
+
+#[derive(Clone)]
+pub struct MockExecutor {
+    shell: Option<String>,
+}
+
+impl MockExecutor {
+    pub fn new(shell: Option<String>) -> Self {
+        MockExecutor { shell }
+    }
+
+    fn extract_time<S: AsRef<str>>(sleep_command: S) -> Second {
+        assert!(sleep_command.as_ref().starts_with("sleep "));
+        sleep_command
+            .as_ref()
+            .trim_start_matches("sleep ")
+            .parse::<Second>()
+            .unwrap()
+    }
+}
+
+impl Executor for MockExecutor {
+    fn time_command(
+        &self,
+        command: &Command<'_>,
+        _command_failure_action: Option<CmdFailureAction>,
+    ) -> Result<(TimingResult, ExitStatus)> {
+        #[cfg(unix)]
+        let status = {
+            use std::os::unix::process::ExitStatusExt;
+            ExitStatus::from_raw(0)
+        };
+
+        #[cfg(windows)]
+        let status = {
+            use std::os::windows::process::ExitStatusExt;
+            ExitStatus::from_raw(0)
+        };
+
+        Ok((
+            TimingResult {
+                time_real: Self::extract_time(command.get_shell_command()),
+                time_user: 0.0,
+                time_system: 0.0,
+            },
+            status,
+        ))
+    }
+
+    fn calibrate(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn time_overhead(&self) -> Second {
+        match &self.shell {
+            None => 0.0,
+            Some(shell) => Self::extract_time(shell),
+        }
+    }
+}
+
+#[test]
+fn test_mock_executor_extract_time() {
+    assert_eq!(MockExecutor::extract_time("sleep 0.1"), 0.1);
+}

src/benchmark/mod.rs

@@ -1,392 +1,216 @@
+pub mod benchmark_result;
+pub mod executor;
 pub mod relative_speed;
-pub mod result;
 pub mod scheduler;
+pub mod timing_result;
 
 use std::cmp;
-use std::process::{ExitStatus, Stdio};
 
-use colored::*;
-use statistical::{mean, median, standard_deviation};
-
-use crate::benchmark::result::BenchmarkResult;
 use crate::command::Command;
-use crate::options::{CmdFailureAction, Options, OutputStyleOption, Shell};
+use crate::options::{CmdFailureAction, Options, OutputStyleOption};
 use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
 use crate::output::format::{format_duration, format_duration_unit};
 use crate::output::progress_bar::get_progress_bar;
 use crate::output::warnings::Warnings;
-use crate::shell::execute_and_time;
-use crate::timer::wallclocktimer::WallClockTimer;
-use crate::timer::{TimerStart, TimerStop};
+use crate::parameter::ParameterNameAndValue;
+use crate::util::exit_code::extract_exit_code;
 use crate::util::min_max::{max, min};
 use crate::util::units::Second;
+use benchmark_result::BenchmarkResult;
+use timing_result::TimingResult;
 
-use anyhow::{bail, Result};
+use anyhow::{anyhow, Result};
+use colored::*;
+use statistical::{mean, median, standard_deviation};
+
+use self::executor::Executor;
 
 /// Threshold for warning about fast execution time
 pub const MIN_EXECUTION_TIME: Second = 5e-3;
 
-/// Results from timing a single command
-#[derive(Debug, Default, Copy, Clone)]
-pub struct TimingResult {
-    /// Wall clock time
-    pub time_real: Second,
-
-    /// Time spent in user mode
-    pub time_user: Second,
-
-    /// Time spent in kernel mode
-    pub time_system: Second,
+pub struct Benchmark<'a> {
+    number: usize,
+    command: &'a Command<'a>,
+    options: &'a Options,
+    executor: &'a dyn Executor,
 }
 
-/// Correct for shell spawning time
-fn subtract_shell_spawning_time(time: Second, shell_spawning_time: Second) -> Second {
-    if time < shell_spawning_time {
-        0.0
-    } else {
-        time - shell_spawning_time
+impl<'a> Benchmark<'a> {
+    pub fn new(
+        number: usize,
+        command: &'a Command<'a>,
+        options: &'a Options,
+        executor: &'a dyn Executor,
+    ) -> Self {
+        Benchmark {
+            number,
+            command,
+            options,
+            executor,
+        }
     }
-}
 
-/// Run the given shell command and measure the execution time
-pub fn time_shell_command(
-    shell: &Shell,
-    command: &Command<'_>,
-    show_output: bool,
-    failure_action: CmdFailureAction,
-    shell_spawning_time: Option<TimingResult>,
-) -> Result<(TimingResult, ExitStatus)> {
-    let (stdout, stderr) = if show_output {
-        (Stdio::inherit(), Stdio::inherit())
-    } else {
-        (Stdio::null(), Stdio::null())
-    };
+    /// Run setup, cleanup, or preparation commands
+    fn run_intermediate_command(
+        &self,
+        command: &Command<'_>,
+        error_output: &'static str,
+    ) -> Result<TimingResult> {
+        self.executor
+            .time_command(command, Some(CmdFailureAction::RaiseError))
+            .map(|r| r.0)
+            .map_err(|_| anyhow!(error_output))
+    }
 
-    let wallclock_timer = WallClockTimer::start();
-    let result = execute_and_time(stdout, stderr, &command.get_shell_command(), shell)?;
-    let mut time_real = wallclock_timer.stop();
+    /// Run the command specified by `--setup`.
+    fn run_setup_command(
+        &self,
+        parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
+    ) -> Result<TimingResult> {
+        let command = self
+            .options
+            .setup_command
+            .as_ref()
+            .map(|setup_command| Command::new_parametrized(None, setup_command, parameters));
 
-    let mut time_user = result.user_time;
-    let mut time_system = result.system_time;
+        let error_output = "The setup command terminated with a non-zero exit code. \
+                            Append ' || true' to the command if you are sure that this can be ignored.";
 
-    if failure_action == CmdFailureAction::RaiseError && !result.status.success() {
-        bail!(
-            "{}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
-                Alternatively, use the '--show-output' option to debug what went wrong.",
-            result.status.code().map_or(
-                "The process has been terminated by a signal".into(),
-                |c| format!("Command terminated with non-zero exit code: {}", c)
+        Ok(command
+            .map(|cmd| self.run_intermediate_command(&cmd, error_output))
+            .transpose()?
+            .unwrap_or_default())
+    }
+
+    /// Run the command specified by `--cleanup`.
+    fn run_cleanup_command(
+        &self,
+        parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
+    ) -> Result<TimingResult> {
+        let command = self
+            .options
+            .cleanup_command
+            .as_ref()
+            .map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters));
+
+        let error_output = "The cleanup command terminated with a non-zero exit code. \
+                            Append ' || true' to the command if you are sure that this can be ignored.";
+
+        Ok(command
+            .map(|cmd| self.run_intermediate_command(&cmd, error_output))
+            .transpose()?
+            .unwrap_or_default())
+    }
+
+    /// Run the command specified by `--prepare`.
+    fn run_preparation_command(&self, command: &Command<'_>) -> Result<TimingResult> {
+        let error_output = "The preparation command terminated with a non-zero exit code. \
+                            Append ' || true' to the command if you are sure that this can be ignored.";
+
+        self.run_intermediate_command(command, error_output)
+    }
+
+    /// Run the benchmark for a single command
+    pub fn run(&self) -> Result<BenchmarkResult> {
+        let command_name = self.command.get_name();
+        if self.options.output_style != OutputStyleOption::Disabled {
+            println!(
+                "{}{}: {}",
+                "Benchmark ".bold(),
+                (self.number + 1).to_string().bold(),
+                command_name,
+            );
+        }
+
+        let mut times_real: Vec<Second> = vec![];
+        let mut times_user: Vec<Second> = vec![];
+        let mut times_system: Vec<Second> = vec![];
+        let mut exit_codes: Vec<Option<i32>> = vec![];
+        let mut all_succeeded = true;
+
+        let preparation_command = self.options.preparation_command.as_ref().map(|values| {
+            let preparation_command = if values.len() == 1 {
+                &values[0]
+            } else {
+                &values[self.number]
+            };
+            Command::new_parametrized(
+                None,
+                preparation_command,
+                self.command.get_parameters().iter().cloned(),
             )
-        );
-    }
-
-    // Correct for shell spawning time
-    if let Some(spawning_time) = shell_spawning_time {
-        time_real = subtract_shell_spawning_time(time_real, spawning_time.time_real);
-        time_user = subtract_shell_spawning_time(time_user, spawning_time.time_user);
-        time_system = subtract_shell_spawning_time(time_system, spawning_time.time_system);
-    }
-
-    Ok((
-        TimingResult {
-            time_real,
-            time_user,
-            time_system,
-        },
-        result.status,
-    ))
-}
-
-/// Measure the average shell spawning time
-pub fn mean_shell_spawning_time(
-    shell: &Shell,
-    style: OutputStyleOption,
-    show_output: bool,
-) -> Result<TimingResult> {
-    const COUNT: u64 = 50;
-    let progress_bar = if style != OutputStyleOption::Disabled {
-        Some(get_progress_bar(
-            COUNT,
-            "Measuring shell spawning time",
-            style,
-        ))
-    } else {
-        None
-    };
-
-    let mut times_real: Vec<Second> = vec![];
-    let mut times_user: Vec<Second> = vec![];
-    let mut times_system: Vec<Second> = vec![];
-
-    for _ in 0..COUNT {
-        // Just run the shell without any command
-        let res = time_shell_command(
-            shell,
-            &Command::new(None, ""),
-            show_output,
-            CmdFailureAction::RaiseError,
-            None,
-        );
-
-        match res {
-            Err(_) => {
-                let shell_cmd = if cfg!(windows) {
-                    format!("{} /C \"\"", shell)
-                } else {
-                    format!("{} -c \"\"", shell)
-                };
-
-                bail!(
-                    "Could not measure shell execution time. Make sure you can run '{}'.",
-                    shell_cmd
-                );
-            }
-            Ok((r, _)) => {
-                times_real.push(r.time_real);
-                times_user.push(r.time_user);
-                times_system.push(r.time_system);
-            }
-        }
-
-        if let Some(bar) = progress_bar.as_ref() {
-            bar.inc(1)
-        }
-    }
-
-    if let Some(bar) = progress_bar.as_ref() {
-        bar.finish_and_clear()
-    }
-
-    Ok(TimingResult {
-        time_real: mean(&times_real),
-        time_user: mean(&times_user),
-        time_system: mean(&times_system),
-    })
-}
-
-fn run_intermediate_command(
-    shell: &Shell,
-    command: &Option<Command<'_>>,
-    show_output: bool,
-    error_output: &'static str,
-) -> Result<TimingResult> {
-    if let Some(ref cmd) = command {
-        let res = time_shell_command(shell, cmd, show_output, CmdFailureAction::RaiseError, None);
-        if res.is_err() {
-            bail!(error_output);
-        }
-        return res.map(|r| r.0);
-    }
-    Ok(TimingResult {
-        ..Default::default()
-    })
-}
-
-/// Run the command specified by `--setup`.
-fn run_setup_command(
-    shell: &Shell,
-    command: &Option<Command<'_>>,
-    show_output: bool,
-) -> Result<TimingResult> {
-    let error_output = "The setup command terminated with a non-zero exit code. \
-                        Append ' || true' to the command if you are sure that this can be ignored.";
-
-    run_intermediate_command(shell, command, show_output, error_output)
-}
-
-/// Run the command specified by `--prepare`.
-fn run_preparation_command(
-    shell: &Shell,
-    command: &Option<Command<'_>>,
-    show_output: bool,
-) -> Result<TimingResult> {
-    let error_output = "The preparation command terminated with a non-zero exit code. \
-                        Append ' || true' to the command if you are sure that this can be ignored.";
-
-    run_intermediate_command(shell, command, show_output, error_output)
-}
-
-/// Run the command specified by `--cleanup`.
-fn run_cleanup_command(
-    shell: &Shell,
-    command: &Option<Command<'_>>,
-    show_output: bool,
-) -> Result<TimingResult> {
-    let error_output = "The cleanup command terminated with a non-zero exit code. \
-                        Append ' || true' to the command if you are sure that this can be ignored.";
-
-    run_intermediate_command(shell, command, show_output, error_output)
-}
-
-#[cfg(unix)]
-fn extract_exit_code(status: ExitStatus) -> Option<i32> {
-    use std::os::unix::process::ExitStatusExt;
-
-    /* From the ExitStatus::code documentation:
-       "On Unix, this will return None if the process was terminated by a signal."
-       In that case, ExitStatusExt::signal should never return None.
-    */
-    status.code().or_else(||
-        /* To differentiate between "normal" exit codes and signals, we are using
-           something similar to bash exit codes (https://tldp.org/LDP/abs/html/exitcodes.html)
-           by adding 128 to a signal integer value.
-         */
-        status.signal().map(|s| 128 + s))
-}
-
-#[cfg(not(unix))]
-fn extract_exit_code(status: ExitStatus) -> Option<i32> {
-    status.code()
-}
-
-/// Run the benchmark for a single shell command
-pub fn run_benchmark(
-    num: usize,
-    cmd: &Command<'_>,
-    shell_spawning_time: TimingResult,
-    options: &Options,
-) -> Result<BenchmarkResult> {
-    let command_name = cmd.get_name();
-    if options.output_style != OutputStyleOption::Disabled {
-        println!(
-            "{}{}: {}",
-            "Benchmark ".bold(),
-            (num + 1).to_string().bold(),
-            command_name,
-        );
-    }
-
-    let mut times_real: Vec<Second> = vec![];
-    let mut times_user: Vec<Second> = vec![];
-    let mut times_system: Vec<Second> = vec![];
-    let mut exit_codes: Vec<Option<i32>> = vec![];
-    let mut all_succeeded = true;
-
-    // Run init command
-    let prepare_cmd = options.preparation_command.as_ref().map(|values| {
-        let preparation_command = if values.len() == 1 {
-            &values[0]
-        } else {
-            &values[num]
+        });
+        let run_preparation_command = || {
+            preparation_command
+                .as_ref()
+                .map(|cmd| self.run_preparation_command(cmd))
+                .transpose()
         };
-        Command::new_parametrized(None, preparation_command, cmd.get_parameters().clone())
-    });
 
-    // Run setup command
-    let setup_cmd = options.setup_command.as_ref().map(|setup_command| {
-        Command::new_parametrized(None, setup_command, cmd.get_parameters().clone())
-    });
-    run_setup_command(&options.shell, &setup_cmd, options.show_output)?;
+        self.run_setup_command(self.command.get_parameters().iter().cloned())?;
 
-    // Warmup phase
-    if options.warmup_count > 0 {
-        let progress_bar = if options.output_style != OutputStyleOption::Disabled {
+        // Warmup phase
+        if self.options.warmup_count > 0 {
+            let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
+                Some(get_progress_bar(
+                    self.options.warmup_count,
+                    "Performing warmup runs",
+                    self.options.output_style,
+                ))
+            } else {
+                None
+            };
+
+            for _ in 0..self.options.warmup_count {
+                let _ = run_preparation_command()?;
+                let _ = self.executor.time_command(self.command, None)?;
+                if let Some(bar) = progress_bar.as_ref() {
+                    bar.inc(1)
+                }
+            }
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.finish_and_clear()
+            }
+        }
+
+        // Set up progress bar (and spinner for initial measurement)
+        let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
             Some(get_progress_bar(
-                options.warmup_count,
-                "Performing warmup runs",
-                options.output_style,
+                self.options.run_bounds.min,
+                "Initial time measurement",
+                self.options.output_style,
             ))
         } else {
             None
         };
 
-        for _ in 0..options.warmup_count {
-            let _ = run_preparation_command(&options.shell, &prepare_cmd, options.show_output)?;
-            let _ = time_shell_command(
-                &options.shell,
-                cmd,
-                options.show_output,
-                options.failure_action,
-                None,
-            )?;
-            if let Some(bar) = progress_bar.as_ref() {
-                bar.inc(1)
-            }
-        }
-        if let Some(bar) = progress_bar.as_ref() {
-            bar.finish_and_clear()
-        }
-    }
+        let preparation_result = run_preparation_command()?;
+        let preparation_overhead =
+            preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
 
-    // Set up progress bar (and spinner for initial measurement)
-    let progress_bar = if options.output_style != OutputStyleOption::Disabled {
-        Some(get_progress_bar(
-            options.runs.min,
-            "Initial time measurement",
-            options.output_style,
-        ))
-    } else {
-        None
-    };
-
-    let prepare_res = run_preparation_command(&options.shell, &prepare_cmd, options.show_output)?;
-
-    // Initial timing run
-    let (res, status) = time_shell_command(
-        &options.shell,
-        cmd,
-        options.show_output,
-        options.failure_action,
-        Some(shell_spawning_time),
-    )?;
-    let success = status.success();
-
-    // Determine number of benchmark runs
-    let runs_in_min_time = (options.min_time_sec
-        / (res.time_real + prepare_res.time_real + shell_spawning_time.time_real))
-        as u64;
-
-    let count = {
-        let min = cmp::max(runs_in_min_time, options.runs.min);
-
-        options
-            .runs
-            .max
-            .as_ref()
-            .map(|max| cmp::min(min, *max))
-            .unwrap_or(min)
-    };
-
-    let count_remaining = count - 1;
-
-    // Save the first result
-    times_real.push(res.time_real);
-    times_user.push(res.time_user);
-    times_system.push(res.time_system);
-    exit_codes.push(extract_exit_code(status));
-
-    all_succeeded = all_succeeded && success;
-
-    // Re-configure the progress bar
-    if let Some(bar) = progress_bar.as_ref() {
-        bar.set_length(count)
-    }
-    if let Some(bar) = progress_bar.as_ref() {
-        bar.inc(1)
-    }
-
-    // Gather statistics
-    for _ in 0..count_remaining {
-        run_preparation_command(&options.shell, &prepare_cmd, options.show_output)?;
-
-        let msg = {
-            let mean = format_duration(mean(&times_real), options.time_unit);
-            format!("Current estimate: {}", mean.to_string().green())
-        };
-
-        if let Some(bar) = progress_bar.as_ref() {
-            bar.set_message(msg.to_owned())
-        }
-
-        let (res, status) = time_shell_command(
-            &options.shell,
-            cmd,
-            options.show_output,
-            options.failure_action,
-            Some(shell_spawning_time),
-        )?;
+        // Initial timing run
+        let (res, status) = self.executor.time_command(self.command, None)?;
         let success = status.success();
 
+        // Determine number of benchmark runs
+        let runs_in_min_time = (self.options.min_benchmarking_time
+            / (res.time_real + self.executor.time_overhead() + preparation_overhead))
+            as u64;
+
+        let count = {
+            let min = cmp::max(runs_in_min_time, self.options.run_bounds.min);
+
+            self.options
+                .run_bounds
+                .max
+                .as_ref()
+                .map(|max| cmp::min(min, *max))
+                .unwrap_or(min)
+        };
+
+        let count_remaining = count - 1;
+
+        // Save the first result
         times_real.push(res.time_real);
         times_user.push(res.time_user);
         times_system.push(res.time_system);
@@ -394,127 +218,156 @@ pub fn run_benchmark(
 
         all_succeeded = all_succeeded && success;
 
+        // Re-configure the progress bar
+        if let Some(bar) = progress_bar.as_ref() {
+            bar.set_length(count)
+        }
         if let Some(bar) = progress_bar.as_ref() {
             bar.inc(1)
         }
-    }
 
-    if let Some(bar) = progress_bar.as_ref() {
-        bar.finish_and_clear()
-    }
+        // Gather statistics (perform the actual benchmark)
+        for _ in 0..count_remaining {
+            run_preparation_command()?;
 
-    // Compute statistical quantities
-    let t_num = times_real.len();
-    let t_mean = mean(&times_real);
-    let t_stddev = if times_real.len() > 1 {
-        Some(standard_deviation(&times_real, Some(t_mean)))
-    } else {
-        None
-    };
-    let t_median = median(&times_real);
-    let t_min = min(&times_real);
-    let t_max = max(&times_real);
+            let msg = {
+                let mean = format_duration(mean(&times_real), self.options.time_unit);
+                format!("Current estimate: {}", mean.to_string().green())
+            };
 
-    let user_mean = mean(&times_user);
-    let system_mean = mean(&times_system);
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.set_message(msg.to_owned())
+            }
 
-    // Formatting and console output
-    let (mean_str, time_unit) = format_duration_unit(t_mean, options.time_unit);
-    let min_str = format_duration(t_min, Some(time_unit));
-    let max_str = format_duration(t_max, Some(time_unit));
-    let num_str = format!("{} runs", t_num);
+            let (res, status) = self.executor.time_command(self.command, None)?;
+            let success = status.success();
 
-    let user_str = format_duration(user_mean, Some(time_unit));
-    let system_str = format_duration(system_mean, Some(time_unit));
+            times_real.push(res.time_real);
+            times_user.push(res.time_user);
+            times_system.push(res.time_system);
+            exit_codes.push(extract_exit_code(status));
 
-    if options.output_style != OutputStyleOption::Disabled {
-        if times_real.len() == 1 {
-            println!(
-                "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
-                "abs".green().bold(),
-                mean_str.green().bold(),
-                "        ".to_string(), // alignment
-                user_str.blue(),
-                system_str.blue()
-            );
+            all_succeeded = all_succeeded && success;
+
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.inc(1)
+            }
+        }
+
+        if let Some(bar) = progress_bar.as_ref() {
+            bar.finish_and_clear()
+        }
+
+        // Compute statistical quantities
+        let t_num = times_real.len();
+        let t_mean = mean(&times_real);
+        let t_stddev = if times_real.len() > 1 {
+            Some(standard_deviation(&times_real, Some(t_mean)))
         } else {
-            let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
+            None
+        };
+        let t_median = median(&times_real);
+        let t_min = min(&times_real);
+        let t_max = max(&times_real);
 
-            println!(
-                "  Time ({} ± {}):     {:>8} ± {:>8}    [User: {}, System: {}]",
-                "mean".green().bold(),
-                "σ".green(),
-                mean_str.green().bold(),
-                stddev_str.green(),
-                user_str.blue(),
-                system_str.blue()
-            );
+        let user_mean = mean(&times_user);
+        let system_mean = mean(&times_system);
 
-            println!(
-                "  Range ({} … {}):   {:>8} … {:>8}    {}",
-                "min".cyan(),
-                "max".purple(),
-                min_str.cyan(),
-                max_str.purple(),
-                num_str.dimmed()
-            );
+        // Formatting and console output
+        let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);
+        let min_str = format_duration(t_min, Some(time_unit));
+        let max_str = format_duration(t_max, Some(time_unit));
+        let num_str = format!("{} runs", t_num);
+
+        let user_str = format_duration(user_mean, Some(time_unit));
+        let system_str = format_duration(system_mean, Some(time_unit));
+
+        if self.options.output_style != OutputStyleOption::Disabled {
+            if times_real.len() == 1 {
+                println!(
+                    "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
+                    "abs".green().bold(),
+                    mean_str.green().bold(),
+                    "        ".to_string(), // alignment
+                    user_str.blue(),
+                    system_str.blue()
+                );
+            } else {
+                let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
+
+                println!(
+                    "  Time ({} ± {}):     {:>8} ± {:>8}    [User: {}, System: {}]",
+                    "mean".green().bold(),
+                    "σ".green(),
+                    mean_str.green().bold(),
+                    stddev_str.green(),
+                    user_str.blue(),
+                    system_str.blue()
+                );
+
+                println!(
+                    "  Range ({} … {}):   {:>8} … {:>8}    {}",
+                    "min".cyan(),
+                    "max".purple(),
+                    min_str.cyan(),
+                    max_str.purple(),
+                    num_str.dimmed()
+                );
+            }
         }
-    }
 
-    // Warnings
-    let mut warnings = vec![];
+        // Warnings
+        let mut warnings = vec![];
 
-    // Check execution time
-    if times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) {
-        warnings.push(Warnings::FastExecutionTime);
-    }
-
-    // Check programm exit codes
-    if !all_succeeded {
-        warnings.push(Warnings::NonZeroExitCode);
-    }
-
-    // Run outlier detection
-    let scores = modified_zscores(&times_real);
-    if scores[0] > OUTLIER_THRESHOLD {
-        warnings.push(Warnings::SlowInitialRun(times_real[0]));
-    } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
-        warnings.push(Warnings::OutliersDetected);
-    }
-
-    if !warnings.is_empty() {
-        eprintln!(" ");
-
-        for warning in &warnings {
-            eprintln!("  {}: {}", "Warning".yellow(), warning);
+        // Check execution time
+        if times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) {
+            warnings.push(Warnings::FastExecutionTime);
         }
+
+        // Check programm exit codes
+        if !all_succeeded {
+            warnings.push(Warnings::NonZeroExitCode);
+        }
+
+        // Run outlier detection
+        let scores = modified_zscores(&times_real);
+        if scores[0] > OUTLIER_THRESHOLD {
+            warnings.push(Warnings::SlowInitialRun(times_real[0]));
+        } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
+            warnings.push(Warnings::OutliersDetected);
+        }
+
+        if !warnings.is_empty() {
+            eprintln!(" ");
+
+            for warning in &warnings {
+                eprintln!("  {}: {}", "Warning".yellow(), warning);
+            }
+        }
+
+        if self.options.output_style != OutputStyleOption::Disabled {
+            println!(" ");
+        }
+
+        self.run_cleanup_command(self.command.get_parameters().iter().cloned())?;
+
+        Ok(BenchmarkResult {
+            command: command_name,
+            mean: t_mean,
+            stddev: t_stddev,
+            median: t_median,
+            user: user_mean,
+            system: system_mean,
+            min: t_min,
+            max: t_max,
+            times: Some(times_real),
+            exit_codes,
+            parameters: self
+                .command
+                .get_parameters()
+                .iter()
+                .map(|(name, value)| (name.to_string(), value.to_string()))
+                .collect(),
+        })
     }
-
-    if options.output_style != OutputStyleOption::Disabled {
-        println!(" ");
-    }
-
-    // Run cleanup command
-    let cleanup_cmd = options.cleanup_command.as_ref().map(|cleanup_command| {
-        Command::new_parametrized(None, cleanup_command, cmd.get_parameters().clone())
-    });
-    run_cleanup_command(&options.shell, &cleanup_cmd, options.show_output)?;
-
-    Ok(BenchmarkResult {
-        command: command_name,
-        mean: t_mean,
-        stddev: t_stddev,
-        median: t_median,
-        user: user_mean,
-        system: system_mean,
-        min: t_min,
-        max: t_max,
-        times: Some(times_real),
-        exit_codes,
-        parameters: cmd
-            .get_parameters()
-            .iter()
-            .map(|(name, value)| ((*name).to_string(), value.to_string()))
-            .collect(),
-    })
 }

src/benchmark/relative_speed.rs

@@ -1,6 +1,6 @@
 use std::cmp::Ordering;
 
-use super::result::BenchmarkResult;
+use super::benchmark_result::BenchmarkResult;
 use crate::util::units::Scalar;
 
 #[derive(Debug)]

src/benchmark/scheduler.rs

@@ -1,11 +1,14 @@
+use colored::*;
+
+use super::benchmark_result::BenchmarkResult;
+use super::executor::{Executor, MockExecutor, ShellExecutor};
+use super::{relative_speed, Benchmark};
+
 use crate::command::Commands;
 use crate::export::ExportManager;
-use crate::options::{Options, OutputStyleOption};
-
-use super::{mean_shell_spawning_time, relative_speed, result::BenchmarkResult, run_benchmark};
+use crate::options::{ExecutorKind, Options, OutputStyleOption};
 
 use anyhow::Result;
-use colored::*;
 
 pub struct Scheduler<'a> {
     commands: &'a Commands<'a>,
@@ -29,15 +32,16 @@ impl<'a> Scheduler<'a> {
     }
 
     pub fn run_benchmarks(&mut self) -> Result<()> {
-        let shell_spawning_time = mean_shell_spawning_time(
-            &self.options.shell,
-            self.options.output_style,
-            self.options.show_output,
-        )?;
+        let mut executor: Box<dyn Executor> = match self.options.executor_kind {
+            ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),
+            ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, &self.options)),
+        };
 
-        for (num, cmd) in self.commands.iter().enumerate() {
+        executor.calibrate()?;
+
+        for (number, cmd) in self.commands.iter().enumerate() {
             self.results
-                .push(run_benchmark(num, cmd, shell_spawning_time, self.options)?);
+                .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
 
             // We export (all results so far) after each individual benchmark, because
             // we would risk losing all results if a later benchmark fails.

src/benchmark/timing_result.rs

@@ -0,0 +1,14 @@
+use crate::util::units::Second;
+
+/// Results from timing a single command
+#[derive(Debug, Default, Copy, Clone)]
+pub struct TimingResult {
+    /// Wall clock time
+    pub time_real: Second,
+
+    /// Time spent in user mode
+    pub time_user: Second,
+
+    /// Time spent in kernel mode
+    pub time_system: Second,
+}

src/command.rs

@@ -4,7 +4,10 @@ use std::str::FromStr;
 
 use crate::{
     error::{OptionsError, ParameterScanError},
-    parameter::range_step::{Numeric, RangeStep},
+    parameter::{
+        range_step::{Numeric, RangeStep},
+        ParameterNameAndValue,
+    },
 };
 
 use clap::ArgMatches;
@@ -26,7 +29,7 @@ pub struct Command<'a> {
     expression: &'a str,
 
     /// Zero or more parameter values.
-    parameters: Vec<(&'a str, ParameterValue)>,
+    parameters: Vec<ParameterNameAndValue<'a>>,
 }
 
 impl<'a> Command<'a> {
@@ -41,12 +44,12 @@ impl<'a> Command<'a> {
     pub fn new_parametrized(
         name: Option<&'a str>,
         expression: &'a str,
-        parameters: Vec<(&'a str, ParameterValue)>,
+        parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
     ) -> Command<'a> {
         Command {
             name,
             expression,
-            parameters,
+            parameters: parameters.into_iter().collect(),
         }
     }
 
@@ -61,7 +64,7 @@ impl<'a> Command<'a> {
         self.replace_parameters_in(self.expression)
     }
 
-    pub fn get_parameters(&self) -> &Vec<(&'a str, ParameterValue)> {
+    pub fn get_parameters(&self) -> &[(&'a str, ParameterValue)] {
         &self.parameters
     }
 
@@ -166,7 +169,7 @@ impl<'a> Commands<'a> {
                 i += 1;
 
                 let (command_index, params_indices) = index.split_first().unwrap();
-                let parameters = param_names_and_values
+                let parameters: Vec<_> = param_names_and_values
                     .iter()
                     .zip(params_indices)
                     .map(|((name, values), i)| (*name, ParameterValue::Text(values[*i].clone())))
@@ -209,7 +212,7 @@ impl<'a> Commands<'a> {
         self.0.iter()
     }
 
-    pub fn len(&self) -> usize {
+    pub fn num_commands(&self) -> usize {
         self.0.len()
     }
 

src/error.rs

@@ -15,7 +15,7 @@ pub enum ParameterScanError {
     TooLarge,
     #[error("Zero is not a valid parameter step")]
     ZeroStep,
-    #[error("A step size is required when the range bounds are floating point numbers. The step size can be specified with the '--parameter-step-size' parameter")]
+    #[error("A step size is required when the range bounds are floating point numbers. The step size can be specified with the '-D/--parameter-step-size <DELTA>' parameter")]
     StepRequired,
     #[error("'--command-name' has been specified {0} times. It has to appear exactly once, or exactly {1} times (number of benchmarks)")]
     UnexpectedCommandNameCount(usize, usize),

src/export/asciidoc.rs

@@ -1,5 +1,5 @@
+use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::benchmark::relative_speed::{self, BenchmarkResultWithRelativeSpeed};
-use crate::benchmark::result::BenchmarkResult;
 use crate::output::format::format_duration_value;
 use crate::util::units::Unit;
 

src/export/csv.rs

@@ -3,7 +3,7 @@ use std::borrow::Cow;
 use csv::WriterBuilder;
 
 use super::Exporter;
-use crate::benchmark::result::BenchmarkResult;
+use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::util::units::Unit;
 
 use anyhow::Result;

src/export/json.rs

@@ -2,7 +2,7 @@ use serde::*;
 use serde_json::to_vec_pretty;
 
 use super::Exporter;
-use crate::benchmark::result::BenchmarkResult;
+use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::util::units::Unit;
 
 use anyhow::Result;

src/export/markdown.rs

@@ -1,6 +1,6 @@
 use super::Exporter;
+use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::benchmark::relative_speed::{self, BenchmarkResultWithRelativeSpeed};
-use crate::benchmark::result::BenchmarkResult;
 use crate::output::format::format_duration_value;
 use crate::util::units::Unit;
 

src/export/mod.rs

@@ -11,7 +11,7 @@ use self::csv::CsvExporter;
 use self::json::JsonExporter;
 use self::markdown::MarkdownExporter;
 
-use crate::benchmark::result::BenchmarkResult;
+use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::util::units::Unit;
 
 use anyhow::{Context, Result};

src/options.rs

@@ -22,6 +22,7 @@ pub const DEFAULT_SHELL: &str = "cmd.exe";
 pub enum Shell {
     /// Default shell command
     Default(&'static str),
+
     /// Custom shell command specified via --shell
     Custom(Vec<String>),
 }
@@ -43,7 +44,7 @@ impl fmt::Display for Shell {
 
 impl Shell {
     /// Parse given string as shell command line
-    pub fn parse<'a>(s: &str) -> Result<Self, OptionsError<'a>> {
+    pub fn from_str<'a>(s: &str) -> Result<Self, OptionsError<'a>> {
         let v = shell_words::split(s).map_err(OptionsError::ShellParseError)?;
         if v.is_empty() || v[0].is_empty() {
             return Err(OptionsError::EmptyShell);
@@ -92,8 +93,8 @@ pub enum OutputStyleOption {
     Disabled,
 }
 
-/// Number of runs for a benchmark
-pub struct Runs {
+/// Bounds for the number of benchmark runs
+pub struct RunBounds {
     /// Minimum number of benchmark runs
     pub min: u64,
 
@@ -101,67 +102,88 @@ pub struct Runs {
     pub max: Option<u64>,
 }
 
-impl Default for Runs {
-    fn default() -> Runs {
-        Runs { min: 10, max: None }
+impl Default for RunBounds {
+    fn default() -> Self {
+        RunBounds { min: 10, max: None }
     }
 }
 
-/// A set of options for hyperfine
+/// How to handle the output of benchmarked commands
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum CommandOutputPolicy {
+    /// Discard all output
+    Discard,
+
+    /// Show command output on the terminal
+    Forward,
+}
+
+impl Default for CommandOutputPolicy {
+    fn default() -> Self {
+        CommandOutputPolicy::Discard
+    }
+}
+
+pub enum ExecutorKind {
+    Shell(Shell),
+    Mock(Option<String>),
+}
+
+impl Default for ExecutorKind {
+    fn default() -> Self {
+        ExecutorKind::Shell(Shell::default())
+    }
+}
+
+/// The main settings for a hyperfine benchmark session
 pub struct Options {
+    /// Upper and lower bound for the number of benchmark runs
+    pub run_bounds: RunBounds,
+
     /// Number of warmup runs
     pub warmup_count: u64,
 
-    /// Number of benchmark runs
-    pub runs: Runs,
-
     /// Minimum benchmarking time
-    pub min_time_sec: Second,
+    pub min_benchmarking_time: Second,
 
     /// Whether or not to ignore non-zero exit codes
-    pub failure_action: CmdFailureAction,
+    pub command_failure_action: CmdFailureAction,
 
-    /// Command to run before each batch of timing runs
-    pub setup_command: Option<String>,
-
-    /// Command to run before each timing run
+    /// Command(s) to run before each timing run
     pub preparation_command: Option<Vec<String>>,
 
-    /// Command to run after each benchmark
+    /// Command to run before each *batch* of timing runs, i.e. before each individual benchmark
+    pub setup_command: Option<String>,
+
+    /// Command to run after each *batch* of timing runs, i.e. after each individual benchmark
     pub cleanup_command: Option<String>,
 
-    /// What color mode to use for output
+    /// What color mode to use for the terminal output
     pub output_style: OutputStyleOption,
 
-    /// The shell to use for executing commands.
-    pub shell: Shell,
+    /// Determines how we run commands
+    pub executor_kind: ExecutorKind,
 
-    /// Forward benchmark's stdout to hyperfine's stdout
-    pub show_output: bool,
+    /// What to do with the output of the benchmarked command
+    pub command_output_policy: CommandOutputPolicy,
 
-    /// Which time unit to use for CLI & Markdown output
+    /// Which time unit to use when displaying resuls
     pub time_unit: Option<Unit>,
-
-    /// A list of custom command names that, if defined,
-    /// will be used instead of the command itself in
-    /// benchmark outputs.
-    pub names: Option<Vec<String>>,
 }
 
 impl Default for Options {
     fn default() -> Options {
         Options {
-            names: None,
+            run_bounds: RunBounds::default(),
             warmup_count: 0,
-            runs: Runs::default(),
-            min_time_sec: 3.0,
-            failure_action: CmdFailureAction::RaiseError,
-            setup_command: None,
+            min_benchmarking_time: 3.0,
+            command_failure_action: CmdFailureAction::RaiseError,
             preparation_command: None,
+            setup_command: None,
             cleanup_command: None,
             output_style: OutputStyleOption::Full,
-            shell: Shell::default(),
-            show_output: false,
+            executor_kind: ExecutorKind::default(),
+            command_output_policy: CommandOutputPolicy::Discard,
             time_unit: None,
         }
     }
@@ -192,19 +214,19 @@ impl Options {
 
         match (min_runs, max_runs) {
             (Some(min), None) => {
-                options.runs.min = min;
+                options.run_bounds.min = min;
             }
             (None, Some(max)) => {
                 // Since the minimum was not explicit we lower it if max is below the default min.
-                options.runs.min = cmp::min(options.runs.min, max);
-                options.runs.max = Some(max);
+                options.run_bounds.min = cmp::min(options.run_bounds.min, max);
+                options.run_bounds.max = Some(max);
             }
             (Some(min), Some(max)) if min > max => {
                 return Err(OptionsError::EmptyRunsRange);
             }
             (Some(min), Some(max)) => {
-                options.runs.min = min;
-                options.runs.max = Some(max);
+                options.run_bounds.min = min;
+                options.run_bounds.max = Some(max);
             }
             (None, None) => {}
         };
@@ -217,7 +239,11 @@ impl Options {
 
         options.cleanup_command = matches.value_of("cleanup").map(String::from);
 
-        options.show_output = matches.is_present("show-output");
+        options.command_output_policy = if matches.is_present("show-output") {
+            CommandOutputPolicy::Forward
+        } else {
+            CommandOutputPolicy::Discard
+        };
 
         options.output_style = match matches.value_of("style") {
             Some("full") => OutputStyleOption::Full,
@@ -226,7 +252,9 @@ impl Options {
             Some("color") => OutputStyleOption::Color,
             Some("none") => OutputStyleOption::Disabled,
             _ => {
-                if !options.show_output && atty::is(Stream::Stdout) {
+                if options.command_output_policy == CommandOutputPolicy::Discard
+                    && atty::is(Stream::Stdout)
+                {
                     OutputStyleOption::Full
                 } else {
                     OutputStyleOption::Basic
@@ -244,12 +272,16 @@ impl Options {
             OutputStyleOption::Disabled => {}
         };
 
-        if let Some(shell) = matches.value_of("shell") {
-            options.shell = Shell::parse(shell)?;
-        }
+        options.executor_kind = match (matches.is_present("debug-mode"), matches.value_of("shell"))
+        {
+            (false, Some(shell)) => ExecutorKind::Shell(Shell::from_str(shell)?),
+            (false, None) => ExecutorKind::Shell(Shell::default()),
+            (true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())),
+            (true, None) => ExecutorKind::Mock(None),
+        };
 
         if matches.is_present("ignore-failure") {
-            options.failure_action = CmdFailureAction::Ignore;
+            options.command_failure_action = CmdFailureAction::Ignore;
         }
 
         options.time_unit = match matches.value_of("time-unit") {
@@ -264,7 +296,8 @@ impl Options {
     pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
         if let Some(preparation_command) = &self.preparation_command {
             ensure!(
-                preparation_command.len() <= 1 || commands.len() == preparation_command.len(),
+                preparation_command.len() <= 1
+                    || commands.num_commands() == preparation_command.len(),
                 "The '--prepare' option has to be provided just once or N times, where N is the \
              number of benchmark commands."
             );
@@ -275,22 +308,19 @@ impl Options {
 }
 
 #[test]
-fn test_shell_default_command() {
+fn test_default_shell() {
     let shell = Shell::default();
 
     let s = format!("{}", shell);
     assert_eq!(&s, DEFAULT_SHELL);
 
     let cmd = shell.command();
-    // Command::get_program is not yet available in stable channel.
-    // https://doc.rust-lang.org/std/process/struct.Command.html#method.get_program
-    let s = format!("{:?}", cmd);
-    assert_eq!(s, format!("\"{}\"", DEFAULT_SHELL));
+    assert_eq!(cmd.get_program(), DEFAULT_SHELL);
 }
 
 #[test]
-fn test_shell_parse_command() {
-    let shell = Shell::parse("shell -x 'aaa bbb'").unwrap();
+fn test_can_parse_shell_command_line_from_str() {
+    let shell = Shell::from_str("shell -x 'aaa bbb'").unwrap();
 
     let s = format!("{}", shell);
     assert_eq!(&s, "shell -x 'aaa bbb'");
@@ -306,17 +336,17 @@ fn test_shell_parse_command() {
 
     // Error cases
     assert!(matches!(
-        Shell::parse("shell 'foo").unwrap_err(),
+        Shell::from_str("shell 'foo").unwrap_err(),
         OptionsError::ShellParseError(_)
     ));
 
     assert!(matches!(
-        Shell::parse("").unwrap_err(),
+        Shell::from_str("").unwrap_err(),
         OptionsError::EmptyShell
     ));
 
     assert!(matches!(
-        Shell::parse("''").unwrap_err(),
+        Shell::from_str("''").unwrap_err(),
         OptionsError::EmptyShell
     ));
 }

src/parameter/mod.rs

@@ -17,3 +17,5 @@ impl<'a> ToString for ParameterValue {
         }
     }
 }
+
+pub type ParameterNameAndValue<'a> = (&'a str, ParameterValue);

src/util/exit_code.rs

@@ -0,0 +1,22 @@
+use std::process::ExitStatus;
+
+#[cfg(unix)]
+pub fn extract_exit_code(status: ExitStatus) -> Option<i32> {
+    use std::os::unix::process::ExitStatusExt;
+
+    // From the ExitStatus::code documentation:
+    //
+    //   "On Unix, this will return None if the process was terminated by a signal."
+    //
+    // In that case, ExitStatusExt::signal should never return None.
+    //
+    // To differentiate between "normal" exit codes and signals, we are using a technique
+    // similar to bash (https://tldp.org/LDP/abs/html/exitcodes.html) and add 128 to the
+    // signal value.
+    status.code().or_else(|| status.signal().map(|s| s + 128))
+}
+
+#[cfg(not(unix))]
+pub fn extract_exit_code(status: ExitStatus) -> Option<i32> {
+    status.code()
+}

src/util/mod.rs

@@ -1,3 +1,4 @@
+pub mod exit_code;
 pub mod min_max;
 pub mod number;
 pub mod units;

tests/integration_tests.rs

@@ -3,6 +3,12 @@ use common::hyperfine;
 
 use predicates::prelude::*;
 
+pub fn hyperfine_debug() -> assert_cmd::Command {
+    let mut cmd = hyperfine();
+    cmd.arg("--debug-mode");
+    cmd
+}
+
 #[test]
 fn hyperfine_runs_successfully() {
     hyperfine()
@@ -75,6 +81,7 @@ fn fails_with_duplicate_parameter_names() {
 #[test]
 fn fails_for_unknown_command() {
     hyperfine()
+        .arg("--runs=1")
         .arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
         .assert()
         .failure()
@@ -83,6 +90,44 @@ fn fails_for_unknown_command() {
         ));
 }
 
+#[test]
+fn fails_for_unknown_setup_command() {
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--setup=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
+        .arg("echo test")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The setup command terminated with a non-zero exit code.",
+        ));
+}
+
+#[test]
+fn fails_for_unknown_cleanup_command() {
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--cleanup=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
+        .arg("echo test")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The cleanup command terminated with a non-zero exit code.",
+        ));
+}
+
+#[test]
+fn fails_for_unknown_prepare_command() {
+    hyperfine()
+        .arg("--prepare=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
+        .arg("echo test")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The preparation command terminated with a non-zero exit code.",
+        ));
+}
+
 #[cfg(unix)]
 #[test]
 fn can_run_failing_commands_with_ignore_failure_option() {
@@ -101,3 +146,125 @@ fn can_run_failing_commands_with_ignore_failure_option() {
         .assert()
         .success();
 }
+
+#[test]
+fn runs_commands_using_user_defined_shell() {
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--show-output")
+        .arg("--shell")
+        .arg("echo 'custom_shell' '--shell-arg'")
+        .arg("echo benchmark")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or(
+                predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),
+            ),
+        );
+}
+
+#[test]
+fn returns_mean_time_in_correct_unit() {
+    hyperfine_debug()
+        .arg("sleep 1.234")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Time (mean ± σ):      1.234 s ±"));
+
+    hyperfine_debug()
+        .arg("sleep 0.123")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Time (mean ± σ):     123.0 ms ±"));
+
+    hyperfine_debug()
+        .arg("--time-unit=millisecond")
+        .arg("sleep 1.234")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Time (mean ± σ):     1234.0 ms ±"));
+}
+
+#[test]
+fn performs_ten_runs_for_slow_commands() {
+    hyperfine_debug()
+        .arg("sleep 0.5")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("10 runs"));
+}
+
+#[test]
+fn performs_three_seconds_of_benchmarking_for_fast_commands() {
+    hyperfine_debug()
+        .arg("sleep 0.01")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("300 runs"));
+}
+
+#[test]
+fn takes_shell_spawning_time_into_account_for_computing_number_of_runs() {
+    hyperfine_debug()
+        .arg("--shell=sleep 0.02")
+        .arg("sleep 0.01")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("100 runs"));
+}
+
+#[test]
+fn takes_preparation_command_into_account_for_computing_number_of_runs() {
+    hyperfine_debug()
+        .arg("--prepare=sleep 0.02")
+        .arg("sleep 0.01")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("100 runs"));
+
+    // Shell overhead needs to be added to both the prepare command and the actual command,
+    // leading to a total benchmark time of (prepare + shell + cmd + shell = 0.1 s)
+    hyperfine_debug()
+        .arg("--shell=sleep 0.01")
+        .arg("--prepare=sleep 0.03")
+        .arg("sleep 0.05")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("30 runs"));
+}
+
+#[test]
+fn shows_benchmark_comparison_with_relative_times() {
+    hyperfine_debug()
+        .arg("sleep 1.0")
+        .arg("sleep 2.0")
+        .arg("sleep 3.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("2.00 ± 0.00 times faster")
+                .and(predicate::str::contains("3.00 ± 0.00 times faster")),
+        );
+}
+
+#[test]
+fn performs_all_benchmarks_in_parameter_scan() {
+    hyperfine_debug()
+        .arg("--parameter-scan")
+        .arg("time")
+        .arg("30")
+        .arg("45")
+        .arg("--parameter-step-size")
+        .arg("5")
+        .arg("sleep {time}")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Benchmark 1: sleep 30")
+                .and(predicate::str::contains("Benchmark 2: sleep 35"))
+                .and(predicate::str::contains("Benchmark 3: sleep 40"))
+                .and(predicate::str::contains("Benchmark 4: sleep 45"))
+                .and(predicate::str::contains("Benchmark 5: sleep 50").not()),
+        );
+}