Add --env for benchmark environment overrides

README.md

@@ -2,8 +2,7 @@
 
 A command-line benchmarking tool.
 
-<img width="800" height="195" alt="hyperfine-demo-800" src="https://github.com/user-attachments/assets/9a5a45d3-df3f-4e0e-8cc9-5d8153f39e48" />
-
+<img width="839" height="205" alt="hyperfine demo" src="https://github.com/user-attachments/assets/13c20b1d-39c1-4da1-8fcb-0c775dd032f5" />
 
 *(hyperfine in action, benchmarking [`mypy`](https://mypy-lang.org/) and [`ty`](https://ty.dev/))*
 
@@ -265,9 +264,8 @@ You can also use a preset to select a group of metrics:
 
 ### Exporting results
 
-Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. CSV exports all
-selected metrics, while Markdown, AsciiDoc, and org-mode contain only the primary metric,
-which is the first metric selected by `--metrics`.
+Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. Non-JSON formats
+contain only the primary metric, which is the first metric selected by `--metrics`.
 
 #### Markdown
 
@@ -303,14 +301,6 @@ uvx --from 'git+https://github.com/sharkdp/hyperfine' \
 |---:|---:|
 
 
-#### CSV
-
-Use `--export-csv <file>` to export one row per command. Each metric selected by `--metrics`
-contributes a group of columns, in selection order: `<metric>_unit`, `<metric>_mean`,
-`<metric>_stddev`, `<metric>_median`, `<metric>_min`, and `<metric>_max`.
-
-CSV uses explicitly selected units, or base units (seconds, bytes, or counts) when omitted.
-
 ### Detailed benchmark flowchart
 
 The following chart explains the execution order of various benchmark runs when using options

doc/hyperfine.1

@@ -243,7 +243,7 @@ of the tool.
 .HP
 \fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|time|all\fP
 .IP
-Select the metrics to measure, in order, with an optional unit for each.
+Select the metrics to display, in order, with an optional unit for each.
 Default: \fBtime_wall_clock,memory_peak_resident\fR, or \fBtime_wall_clock\fR
 on Windows, where peak resident memory is not yet supported.
 .IP
@@ -277,11 +277,9 @@ Export the primary metric's summary statistics as an AsciiDoc table to the given
 .HP
 \fB\-\-export\-csv\fR \fIFILE\fP
 .IP
-Export summary statistics for all selected metrics as CSV to the given \fIFILE\fP,
-one row per command. Each metric contributes columns for unit, mean, stddev, median,
-min, and max, prefixed with the metric name, in selection order. The command column
-comes first and parameter columns come last. Use the explicitly selected units,
-or base units (seconds, bytes, or unscaled counts) when omitted.
+Export the primary metric's summary statistics as CSV to the given \fIFILE\fP,
+identifying the metric and unit explicitly. Use the explicitly selected unit,
+or the base unit (seconds, bytes, or unscaled counts) when omitted.
 .HP
 \fB\-\-export\-json\fR \fIFILE\fP
 .IP

src/benchmark/scheduler.rs

@@ -69,7 +69,7 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
     assert_eq!(options.executor_kind, ExecutorKind::Mock(None));
 
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
-    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
 
     options.validate_against_command_list(&commands)?;
 

src/cli.rs

@@ -50,19 +50,6 @@ fn build_command() -> Command {
                 .value_hint(ValueHint::CommandString)
                 .value_parser(NonEmptyStringValueParser::new()),
         )
-        // Retain the removed option to provide migration guidance for old invocations.
-        .arg(
-            Arg::new("reference")
-                .long("reference")
-                .hide(true)
-                .value_name("CMD")
-                .num_args(0..=1)
-                .default_missing_value("")
-                .value_parser(|_: &str| -> Result<String, String> {
-                    Err("The --reference option has been removed. Put your reference command first instead.".into())
-                })
-                .help("Removed: put your reference command first instead."),
-        )
         .arg(
             Arg::new("metrics")
                 .long("metrics")
@@ -71,7 +58,7 @@ fn build_command() -> Command {
                 .hide_default_value(true)
                 .help("Performance metrics to measure, in order.\n\n\
                        This is either a preset name, or a comma-separated list of metrics with optional units. \
-                       CSV exports all selected metrics; markup exports use the first metric. \
+                       The first metric is used for non-JSON exports, \
                        JSON always includes all collected metrics.\n\n\
                        Presets (use one on its own):\n  \
                          default  Wall-clock time and peak RSS\n  \
@@ -338,10 +325,9 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export summary statistics for all selected metrics as CSV to the given FILE. If you need \
+                .help("Export the primary metric summary statistics as CSV to the given FILE. If you need \
                        all metrics for each individual run, use the JSON export format. \
-                       Column names are prefixed with the metric name. \
-                       Use the explicitly selected units, or base units when unspecified."),
+                       Use the explicitly selected unit, or the base unit when unspecified."),
         )
         .arg(
             Arg::new("export-json")

src/export/csv.rs

@@ -4,34 +4,23 @@ use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::metric::MetricSelection;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 
 #[derive(Default)]
 pub struct CsvExporter {}
 
 impl Exporter for CsvExporter {
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        metrics: &[MetricSelection],
-    ) -> Result<Vec<u8>> {
-        // Omit metrics unavailable for all commands (possible with `--metrics all`).
-        let metrics: Vec<_> = metrics
-            .iter()
-            .filter(|selection| {
-                results
-                    .iter()
-                    .any(|result| selection.metric.summarize(&result.measurements).is_some())
-            })
-            .collect();
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
+        // CSV never chooses units automatically: an omitted suffix means base units.
+        let unit = primary.csv_unit();
         let mut writer = WriterBuilder::new().from_writer(vec![]);
-        let mut headers = vec!["command".to_owned()];
-        for selection in &metrics {
-            headers.extend(
-                ["unit", "mean", "stddev", "median", "min", "max"]
-                    .map(|field| format!("{}_{field}", selection.metric.name())),
-            );
-        }
+        let mut headers: Vec<String> = [
+            "command", "metric", "unit", "mean", "stddev", "median", "min", "max",
+        ]
+        .iter()
+        .copied()
+        .map(str::to_owned)
+        .collect();
         if let Some(result) = results.first() {
             headers.extend(
                 result
@@ -43,28 +32,36 @@ impl Exporter for CsvExporter {
         writer.write_record(headers)?;
 
         for result in results {
-            let mut fields = vec![result.get_name().to_owned()];
-            for selection in &metrics {
-                // CSV never chooses units automatically: an omitted suffix means base units.
-                let unit = selection.csv_unit();
-                let stats = selection.metric.summarize(&result.measurements);
-                fields.push(unit.symbol.to_owned());
-                // Preserve numeric precision rather than using rounded display values.
-                fields.extend(
-                    [
-                        stats.map(|stats| stats.mean),
-                        stats.and_then(|stats| stats.stddev),
-                        stats.map(|stats| stats.median),
-                        stats.map(|stats| stats.min),
-                        stats.map(|stats| stats.max),
-                    ]
-                    .map(|value| {
-                        value
-                            .map(|value| (value / unit.scale).to_string())
-                            .unwrap_or_default()
-                    }),
-                );
-            }
+            let stats = primary
+                .metric
+                .summarize(&result.measurements)
+                .with_context(|| {
+                    format!(
+                        "Metric '{}' is unavailable for '{}'",
+                        primary.metric.name(),
+                        result.get_name()
+                    )
+                })?;
+            let mut fields = vec![
+                result.get_name().to_owned(),
+                primary.metric.name().to_owned(),
+                unit.symbol.to_owned(),
+            ];
+            // Preserve numeric precision rather than using rounded display values.
+            fields.extend(
+                [
+                    Some(stats.mean),
+                    stats.stddev,
+                    Some(stats.median),
+                    Some(stats.min),
+                    Some(stats.max),
+                ]
+                .map(|value| {
+                    value
+                        .map(|value| (value / unit.scale).to_string())
+                        .unwrap_or_default()
+                }),
+            );
             fields.extend(
                 result
                     .parameters
@@ -196,18 +193,15 @@ fn test_csv() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(
-                &results,
-                &MetricSelection::parse_list("time_wall_clock,memory_peak_resident").unwrap(),
-            )
+            .serialize(&results, MetricSelection::default())
             .unwrap(),
     )
     .unwrap();
 
     insta::assert_snapshot!(actual, @r#"
-    command,time_wall_clock_unit,time_wall_clock_mean,time_wall_clock_stddev,time_wall_clock_median,time_wall_clock_min,time_wall_clock_max,memory_peak_resident_unit,memory_peak_resident_mean,memory_peak_resident_stddev,memory_peak_resident_median,memory_peak_resident_min,memory_peak_resident_max,parameter_bar,parameter_foo
-    command_a,s,9,2.6457513110645907,8,7,12,B,1024,0,1024,1024,1024,two,one
-    command_b,s,18,1,18,17,19,B,1024,0,1024,1024,1024,seven,one
+    command,metric,unit,mean,stddev,median,min,max,parameter_bar,parameter_foo
+    command_a,time_wall_clock,s,9,2.6457513110645907,8,7,12,two,one
+    command_b,time_wall_clock,s,18,1,18,17,19,seven,one
     "#);
 
     for (selection, unit, mean) in [
@@ -215,11 +209,12 @@ fn test_csv() {
         ("memory_peak_resident:KiB", "KiB", "1"),
     ] {
         let output = exporter
-            .serialize(&results, &MetricSelection::parse_list(selection).unwrap())
+            .serialize(&results, MetricSelection::parse_list(selection).unwrap()[0])
             .unwrap();
         let mut reader = csv::Reader::from_reader(output.as_slice());
         let row = reader.records().next().unwrap().unwrap();
-        assert_eq!(&row[1], unit);
-        assert_eq!(&row[2], mean);
+        assert_eq!(&row[1], "memory_peak_resident");
+        assert_eq!(&row[2], unit);
+        assert_eq!(&row[3], mean);
     }
 }

src/export/json.rs

@@ -65,12 +65,7 @@ pub struct JsonExporter {
 }
 
 impl Exporter for JsonExporter {
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        metrics: &[MetricSelection],
-    ) -> Result<Vec<u8>> {
-        let primary = metrics[0];
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary {
             schema_version: JSON_SCHEMA_VERSION,
             metadata: &self.metadata,

src/export/markup.rs

@@ -77,12 +77,7 @@ pub trait MarkupExporter {
 }
 
 impl<T: MarkupExporter> Exporter for T {
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        metrics: &[MetricSelection],
-    ) -> Result<Vec<u8>> {
-        let primary = metrics[0];
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
         let stats: Vec<_> = results
             .iter()
             .map(|result| {

src/export/mod.rs

@@ -44,12 +44,8 @@ pub enum ExportType {
 
 /// Interface for different exporters.
 trait Exporter {
-    /// Export the given entries using a nonempty list of selected metrics, primary first.
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        metrics: &[MetricSelection],
-    ) -> Result<Vec<u8>>;
+    /// Export the given entries in the serialized form.
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>>;
 }
 
 pub enum ExportTarget {
@@ -65,16 +61,16 @@ struct ExporterWithTarget {
 /// Handles the management of multiple file exporters.
 pub struct ExportManager {
     exporters: Vec<ExporterWithTarget>,
-    metrics: Vec<MetricSelection>,
+    primary: MetricSelection,
 }
 
 impl ExportManager {
     /// Build the ExportManager that will export the results specified
     /// in the given ArgMatches
-    pub fn from_cli_arguments(matches: &ArgMatches, metrics: &[MetricSelection]) -> Result<Self> {
+    pub fn from_cli_arguments(matches: &ArgMatches, primary: MetricSelection) -> Result<Self> {
         let mut export_manager = Self {
             exporters: vec![],
-            metrics: metrics.to_vec(),
+            primary,
         };
         {
             let mut add_exporter = |flag, exporttype| -> Result<()> {
@@ -124,7 +120,7 @@ impl ExportManager {
     /// order not to clutter the output of hyperfine with intermediate results).
     pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
         for e in &self.exporters {
-            let content = || e.exporter.serialize(results, &self.metrics);
+            let content = || e.exporter.serialize(results, self.primary);
 
             match e.target {
                 ExportTarget::File(ref filename) => {

src/export/tests.rs

@@ -15,10 +15,10 @@ fn get_output<E: Exporter + Default>(results: &[BenchmarkResult], unit: Option<U
         exporter
             .serialize(
                 results,
-                &[MetricSelection {
+                MetricSelection {
                     unit,
                     ..MetricSelection::default()
-                }],
+                },
             )
             .unwrap(),
     )

src/main.rs

@@ -70,7 +70,7 @@ fn run() -> Result<()> {
         }
     }
 
-    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
 
     let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
     scheduler.run_benchmarks()?;

tests/integration_tests.rs

@@ -44,25 +44,6 @@ fn runs_successfully() {
         .success();
 }
 
-#[test]
-fn rejects_removed_reference_option_with_migration_guidance() {
-    for args in [
-        vec!["--reference", "echo reference", "echo comparison"],
-        vec!["echo comparison", "--reference=echo reference"],
-        vec!["--reference", "echo reference"],
-        vec!["--reference"],
-    ] {
-        hyperfine()
-            .args(args)
-            .assert()
-            .code(2)
-            .stdout("")
-            .stderr(predicate::str::contains(
-            "The --reference option has been removed. Put your reference command first instead.",
-        ));
-    }
-}
-
 #[test]
 fn one_run_is_supported() {
     hyperfine()