Store parameter usage metadata in benchmark results

src/benchmark/benchmark_result.rs

@@ -2,6 +2,15 @@ use std::collections::BTreeMap;
 
 use serde::Serialize;
 
+/// Parameter value and whether it was used in the command line template
+#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+#[serde(transparent)]
+pub struct Parameter {
+    pub value: String,
+    #[serde(skip)]
+    pub is_unused: bool,
+}
+
 /// Set of values that will be exported.
 // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
 // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
@@ -49,5 +58,5 @@ pub struct BenchmarkResult {
 
     /// Parameter values for this benchmark
     #[serde(skip_serializing_if = "BTreeMap::is_empty")]
-    pub parameters: BTreeMap<String, String>,
+    pub parameters: BTreeMap<String, Parameter>,
 }

src/benchmark/executor.rs

@@ -15,7 +15,6 @@ use super::measurement::{Measurement, Measurements};
 
 use anyhow::{bail, Context, Result};
 
-#[derive(Clone, Copy)]
 pub enum BenchmarkIteration {
     NonBenchmarkRun,
     Warmup(u64),

src/benchmark/mod.rs

@@ -19,7 +19,7 @@ use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
 use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero};
 use crate::util::exit_code::extract_exit_code;
-use benchmark_result::BenchmarkResult;
+use benchmark_result::{BenchmarkResult, Parameter};
 use measurement::{Measurement, Measurements};
 
 use anyhow::{anyhow, Result};
@@ -58,12 +58,11 @@ impl<'a> Benchmark<'a> {
         command: &Command<'_>,
         error_output: &'static str,
         output_policy: &CommandOutputPolicy,
-        iteration: executor::BenchmarkIteration,
     ) -> Result<Measurement> {
         self.executor
             .run_command_and_measure(
                 command,
-                iteration,
+                executor::BenchmarkIteration::NonBenchmarkRun,
                 Some(CmdFailureAction::RaiseError),
                 output_policy,
             )
@@ -86,14 +85,7 @@ impl<'a> Benchmark<'a> {
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
         Ok(command
-            .map(|cmd| {
-                self.run_intermediate_command(
-                    &cmd,
-                    error_output,
-                    output_policy,
-                    BenchmarkIteration::NonBenchmarkRun,
-                )
-            })
+            .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
             .transpose()?
             .unwrap_or_default())
     }
@@ -114,14 +106,7 @@ impl<'a> Benchmark<'a> {
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
         Ok(command
-            .map(|cmd| {
-                self.run_intermediate_command(
-                    &cmd,
-                    error_output,
-                    output_policy,
-                    BenchmarkIteration::NonBenchmarkRun,
-                )
-            })
+            .map(|cmd| self.run_intermediate_command(&cmd, error_output, output_policy))
             .transpose()?
             .unwrap_or_default())
     }
@@ -131,12 +116,11 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         output_policy: &CommandOutputPolicy,
-        iteration: executor::BenchmarkIteration,
     ) -> Result<Measurement> {
         let error_output = "The preparation command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
-        self.run_intermediate_command(command, error_output, output_policy, iteration)
+        self.run_intermediate_command(command, error_output, output_policy)
     }
 
     /// Run the command specified by `--conclude`.
@@ -144,12 +128,11 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         output_policy: &CommandOutputPolicy,
-        iteration: executor::BenchmarkIteration,
     ) -> Result<Measurement> {
         let error_output = "The conclusion command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
-        self.run_intermediate_command(command, error_output, output_policy, iteration)
+        self.run_intermediate_command(command, error_output, output_policy)
     }
 
     /// Run the benchmark for a single command
@@ -182,10 +165,10 @@ impl<'a> Benchmark<'a> {
             )
         });
 
-        let run_preparation_command = |iteration: executor::BenchmarkIteration| {
+        let run_preparation_command = || {
             preparation_command
                 .as_ref()
-                .map(|cmd| self.run_preparation_command(cmd, output_policy, iteration))
+                .map(|cmd| self.run_preparation_command(cmd, output_policy))
                 .transpose()
         };
 
@@ -201,10 +184,10 @@ impl<'a> Benchmark<'a> {
                 self.command.get_parameters().iter().cloned(),
             )
         });
-        let run_conclusion_command = |iteration: executor::BenchmarkIteration| {
+        let run_conclusion_command = || {
             conclusion_command
                 .as_ref()
-                .map(|cmd| self.run_conclusion_command(cmd, output_policy, iteration))
+                .map(|cmd| self.run_conclusion_command(cmd, output_policy))
                 .transpose()
         };
 
@@ -223,15 +206,14 @@ impl<'a> Benchmark<'a> {
             };
 
             for i in 0..self.options.warmup_count {
-                let warmup_iteration = BenchmarkIteration::Warmup(i);
-                let _ = run_preparation_command(warmup_iteration)?;
+                let _ = run_preparation_command()?;
                 let _ = self.executor.run_command_and_measure(
                     self.command,
-                    warmup_iteration,
+                    BenchmarkIteration::Warmup(i),
                     None,
                     output_policy,
                 )?;
-                let _ = run_conclusion_command(warmup_iteration)?;
+                let _ = run_conclusion_command()?;
                 if let Some(bar) = progress_bar.as_ref() {
                     bar.inc(1)
                 }
@@ -252,8 +234,7 @@ impl<'a> Benchmark<'a> {
             None
         };
 
-        let benchmark_iteration = BenchmarkIteration::Benchmark(0);
-        let preparation_result = run_preparation_command(benchmark_iteration)?;
+        let preparation_result = run_preparation_command()?;
         let preparation_overhead = preparation_result.map_or(Time::zero(), |res| {
             res.time_wall_clock + self.executor.time_overhead()
         });
@@ -261,13 +242,13 @@ impl<'a> Benchmark<'a> {
         // Initial timing run
         let res = self.executor.run_command_and_measure(
             self.command,
-            benchmark_iteration,
+            BenchmarkIteration::Benchmark(0),
             None,
             output_policy,
         )?;
         let success = res.exit_status.success();
 
-        let conclusion_result = run_conclusion_command(benchmark_iteration)?;
+        let conclusion_result = run_conclusion_command()?;
         let conclusion_overhead = conclusion_result.map_or(Time::zero(), |res| {
             res.time_wall_clock + self.executor.time_overhead()
         });
@@ -308,8 +289,7 @@ impl<'a> Benchmark<'a> {
 
         // Gather statistics (perform the actual benchmark)
         for i in 0..count_remaining {
-            let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
-            run_preparation_command(benchmark_iteration)?;
+            run_preparation_command()?;
 
             let msg = {
                 let t_wall_clock_mean = measurements.time_wall_clock_mean();
@@ -327,7 +307,7 @@ impl<'a> Benchmark<'a> {
 
             let res = self.executor.run_command_and_measure(
                 self.command,
-                benchmark_iteration,
+                BenchmarkIteration::Benchmark(i + 1),
                 None,
                 output_policy,
             )?;
@@ -341,7 +321,7 @@ impl<'a> Benchmark<'a> {
                 bar.inc(1)
             }
 
-            run_conclusion_command(benchmark_iteration)?;
+            run_conclusion_command()?;
         }
 
         if let Some(bar) = progress_bar.as_ref() {
@@ -494,7 +474,15 @@ impl<'a> Benchmark<'a> {
                 .command
                 .get_parameters()
                 .iter()
-                .map(|(name, value)| (name.to_string(), value.to_string()))
+                .map(|(name, value)| {
+                    (
+                        name.to_string(),
+                        Parameter {
+                            value: value.to_string(),
+                            is_unused: self.command.is_parameter_unused(name),
+                        },
+                    )
+                })
                 .collect(),
         })
     }

src/export/csv.rs

@@ -53,7 +53,7 @@ impl Exporter for CsvExporter {
                 fields.push(Cow::Owned(format!("{f:.CSV_PRECISION$}").into_bytes()))
             }
             for v in res.parameters.values() {
-                fields.push(Cow::Borrowed(v.as_bytes()))
+                fields.push(Cow::Borrowed(v.value.as_bytes()))
             }
             writer.write_record(fields)?;
         }
@@ -64,6 +64,7 @@ impl Exporter for CsvExporter {
 
 #[test]
 fn test_csv() {
+    use crate::benchmark::benchmark_result::Parameter;
     use std::collections::BTreeMap;
     let exporter = CsvExporter::default();
 
@@ -83,8 +84,20 @@ fn test_csv() {
             exit_codes: vec![Some(0), Some(0), Some(0)],
             parameters: {
                 let mut params = BTreeMap::new();
-                params.insert("foo".into(), "one".into());
-                params.insert("bar".into(), "two".into());
+                params.insert(
+                    "foo".into(),
+                    Parameter {
+                        value: "one".into(),
+                        is_unused: false,
+                    },
+                );
+                params.insert(
+                    "bar".into(),
+                    Parameter {
+                        value: "two".into(),
+                        is_unused: false,
+                    },
+                );
                 params
             },
         },
@@ -103,8 +116,20 @@ fn test_csv() {
             exit_codes: vec![Some(0), Some(0), Some(0)],
             parameters: {
                 let mut params = BTreeMap::new();
-                params.insert("foo".into(), "one".into());
-                params.insert("bar".into(), "seven".into());
+                params.insert(
+                    "foo".into(),
+                    Parameter {
+                        value: "one".into(),
+                        is_unused: false,
+                    },
+                );
+                params.insert(
+                    "bar".into(),
+                    Parameter {
+                        value: "seven".into(),
+                        is_unused: false,
+                    },
+                );
                 params
             },
         },

tests/integration_tests.rs

@@ -1037,45 +1037,3 @@ fn windows_quote_before_quote_args() {
         .assert()
         .success();
 }
-
-#[test]
-fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() {
-    let tempdir = tempfile::tempdir().unwrap();
-    let output_path = tempdir.path().join("iteration output.txt");
-    let iteration = if cfg!(windows) {
-        "%HYPERFINE_ITERATION%"
-    } else {
-        "$HYPERFINE_ITERATION"
-    };
-    let command = |phase| format!(r#"echo {phase}:{iteration} >> "iteration output.txt""#);
-
-    hyperfine()
-        .current_dir(tempdir.path())
-        .arg("--runs=2")
-        .arg("--warmup=1")
-        .arg("--prepare")
-        .arg(command("prepare"))
-        .arg(command("benchmark"))
-        .arg("--conclude")
-        .arg(command("conclude"))
-        .assert()
-        .success();
-
-    let contents = std::fs::read_to_string(output_path).unwrap();
-    // cmd.exe's echo includes the space before the redirection operator.
-    let lines: Vec<&str> = contents.lines().map(str::trim_end).collect();
-    assert_eq!(
-        lines,
-        [
-            "prepare:warmup-0",
-            "benchmark:warmup-0",
-            "conclude:warmup-0",
-            "prepare:0",
-            "benchmark:0",
-            "conclude:0",
-            "prepare:1",
-            "benchmark:1",
-            "conclude:1",
-        ]
-    );
-}