mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-13 16:07:49 +00:00
Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6645f62b39 | ||
|
|
c516152498 | ||
|
|
b33755fe1f | ||
|
|
26760e8365 | ||
|
|
6dddb02d73 | ||
|
|
067ee70f20 | ||
|
|
25b0b54a5d | ||
|
|
9361ee7fb9 |
No files matched your search
+23
-22
@@ -1,48 +1,49 @@
|
||||
# v2.0.0-alpha.2
|
||||
# v2.0.0
|
||||
|
||||
## Changes
|
||||
|
||||
- Always run `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands in a shell, even when benchmarked commands run directly. Improve error messages for these commands, see #990 (@sharkdp).
|
||||
- Require an explicit shell choice for commands containing unquoted, standalone shell operators such as `&&` or `>`: use `-S` to interpret shell syntax or `-N` to pass operators as literal arguments (@sharkdp).
|
||||
- Use a compact output layout on narrow terminals (@sharkdp).
|
||||
- Make plotting and analysis scripts runnable directly from GitHub with `uvx --from 'git+https://github.com/sharkdp/hyperfine' <script>` (@sharkdp).
|
||||
|
||||
# v2.0.0-alpha.1
|
||||
|
||||
This is the first alpha release of hyperfine 2.0. Command-line options and export formats may change before the stable release. See the breaking changes below when migrating from hyperfine 1.x.
|
||||
See the breaking changes below when migrating from hyperfine 1.x.
|
||||
|
||||
## Features
|
||||
|
||||
- Add support for alternative performance metrics (peak memory usage, CPU cycles, instructions, cache misses, branch misses, ...). By default, hyperfine will now display wall-clock time and peak memory usage (wall-clock time only on Windows),
|
||||
but users can use the new `--metrics` option to select different performance metrics.
|
||||
- The terminal output now shows an overview of all selected performance metrics, including relative changes.
|
||||
- Python plotting and analysis scripts support the new JSON format and can select performance metrics with `--metric`, see #981 and #984 (@sharkdp).
|
||||
- Add support for alternative performance metrics (peak memory usage, instructions, CPU cycles, cache misses, branch misses, ...). By default, hyperfine will now display wall-clock time and peak memory usage, but users can use the [new `--metrics` option](README.md#choosing-metrics-and-units) to select different performance metrics. The terminal output now shows an overview of all selected performance metrics, including relative changes.
|
||||
- Add `--env` to set environment variables even without an intermediate shell (which is the new default behavior, see below). For example:
|
||||
|
||||
```sh
|
||||
# Parameterized benchmark, comparing different thread counts:
|
||||
hyperfine -P threads 1 8 --env 'OMP_NUM_THREADS={threads}' 'my_command'
|
||||
```
|
||||
|
||||
- The plotting and analysis scripts now support different metrics as well, see #981 and #984 (@sharkdp).
|
||||
|
||||
## Breaking changes
|
||||
|
||||
- Benchmarked commands are now executed directly by default, without an intermediate shell (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell <SHELL>`.
|
||||
- The JSON format for `--export-json` has changed (schema version 2). The new format now includes metadata, per-run measurements, and statistical summaries for all measured quantities, see #790 (@sharkdp). Code that reads those exported JSON files must be updated. For example, `results[i].mean` is now `results[i].summary.time_wall_clock.mean`, and `results[i].times` is replaced by `results[i].measurements[j].time_wall_clock.value`. The new structure looks like this:
|
||||
- Benchmarked commands are now executed directly by default, without an intermediate shell (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell <SHELL>`. See [the rationale for this change](https://github.com/sharkdp/hyperfine/issues/787#issuecomment-6038898252).
|
||||
- Always run `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands in a shell, even when using `--shell=none` (the new default behavior), see #990 (@sharkdp).
|
||||
- The JSON format for `--export-json` has changed (schema version 2). The [new format](README.md#json) now includes metadata, per-run measurements, and statistical summaries for all measured quantities, see #790 (@sharkdp). Code that reads those exported JSON files must be updated. For example, `results[i].mean` is now `results[i].summary.time_wall_clock.mean`, and `results[i].times` is replaced by `results[i].measurements[j].time_wall_clock.value`. The new structure looks like this:
|
||||
|
||||
```python
|
||||
{
|
||||
"schema_version": 2,
|
||||
"primary_metric": "time_wall_clock",
|
||||
"metadata": {
|
||||
"hyperfine_version": "2.0.0-alpha.1",
|
||||
"hyperfine_version": "2.0.0",
|
||||
"start_time": "2026-10-06T12:00:00Z",
|
||||
"platform": {"os": "Linux", "architecture": "x86_64"}
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"command": "sleep 1", # Actual command after parameter substitution
|
||||
"name": "wait 1s", # Optional custom name, separate from the command
|
||||
"name": "wait 1s", # Optional custom or automatically generated name
|
||||
"parameters": {"duration": {"value": "1"}}, # Values are now objects
|
||||
"environment": {"OMP_NUM_THREADS": "8"}, # Only present with --env overrides
|
||||
"measurements": [ # One entry per run, excluding warmups
|
||||
{
|
||||
"time_wall_clock": {"value": 1.0, "unit": "second"},
|
||||
"time_cpu": {"value": 0.0, "unit": "second"},
|
||||
"time_user": {"value": 0.0, "unit": "second"},
|
||||
"time_system": {"value": 0.0, "unit": "second"},
|
||||
"memory_peak_resident": {"value": 1048576.0, "unit": "byte"},
|
||||
"cpu_cycles": {"value": 5000000}, # Hardware counters, when available
|
||||
"instructions": {"value": 10000000},
|
||||
"exit_code": 0
|
||||
},
|
||||
... # Further runs omitted
|
||||
@@ -64,11 +65,11 @@ This is the first alpha release of hyperfine 2.0. Command-line options and expor
|
||||
}
|
||||
```
|
||||
|
||||
Times are always exported in seconds and memory in bytes.
|
||||
Times are always exported in seconds and memory in bytes. Hardware counters and their summaries have no `unit` field. Unavailable memory measurements and hardware counters are omitted; other available counters may also be included. `stddev` is `null` for a single run.
|
||||
- `--time-unit`/`-u` has been removed. Units can now be selected using the `--metrics METRIC[:UNIT],…` option, e.g. `--metrics time_wall_clock:ms,memory_peak_resident:MiB`.
|
||||
- `--sort` has been removed. It complicated hyperfine significantly and doesn't make too much sense with the new output format.
|
||||
- `--reference` and `--reference-name` have been removed (for now). The first command is always considered as the reference command.
|
||||
- The format of Markdown, AsciiDoc, org-mode and CSV exports has also been changed. These formats contain only the primary metric (the first metric selected by `--metrics`). CSV now includes columns with the name and the unit of the primary metric.
|
||||
- `--reference` and `--reference-name` have been removed (for now). The first command is always considered as the reference command. Invocations using `--reference` now show migration guidance to put the reference command first.
|
||||
- The format of [Markdown](README.md#markdown), AsciiDoc and org-mode, and [CSV](README.md#csv) exports has also been changed. Markdown, AsciiDoc, and org-mode contain only the primary metric (the first metric selected by `--metrics`). CSV exports all selected metrics side by side.
|
||||
|
||||
# v1.21.0
|
||||
|
||||
|
||||
+1
-1
@@ -13,5 +13,5 @@ authors:
|
||||
repository-code: 'https://github.com/sharkdp/hyperfine'
|
||||
abstract: A command-line benchmarking tool.
|
||||
license: MIT
|
||||
version: 2.0.0-alpha.2
|
||||
version: 2.0.0
|
||||
date-released: '2026-10-07'
|
||||
Generated
+3
-3
@@ -377,7 +377,7 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
|
||||
|
||||
[[package]]
|
||||
name = "hyperfine"
|
||||
version = "2.0.0-alpha.2"
|
||||
version = "2.0.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"approx",
|
||||
@@ -480,9 +480,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
version = "0.2.191"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0"
|
||||
name = "hyperfine"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/sharkdp/hyperfine"
|
||||
version = "2.0.0-alpha.2"
|
||||
version = "2.0.0"
|
||||
edition = "2018"
|
||||
build = "build.rs"
|
||||
rust-version = "1.97.0"
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
A command-line benchmarking tool.
|
||||
|
||||
<img width="839" height="205" alt="hyperfine demo" src="https://github.com/user-attachments/assets/13c20b1d-39c1-4da1-8fcb-0c775dd032f5" />
|
||||
<img width="800" height="195" alt="hyperfine-demo-800" src="https://github.com/user-attachments/assets/9a5a45d3-df3f-4e0e-8cc9-5d8153f39e48" />
|
||||
|
||||
|
||||
*(hyperfine in action, benchmarking [`mypy`](https://mypy-lang.org/) and [`ty`](https://ty.dev/))*
|
||||
|
||||
@@ -154,6 +155,31 @@ Otherwise, inline the function into the benchmarked command:
|
||||
hyperfine -S 'my_function() { sleep 1; }; my_function'
|
||||
```
|
||||
|
||||
### Environment variables
|
||||
|
||||
Use `--env` to set environment variables for benchmark commands and their respective setup, prepare, conclude,
|
||||
and cleanup commands. Assignments are space-separated and support shell-style quoting,
|
||||
but no shell expansion:
|
||||
|
||||
```sh
|
||||
hyperfine --env 'OMP_NUM_THREADS=8 CFLAGS="-O3 -march=native"' 'my_command'
|
||||
```
|
||||
|
||||
Specify once for all commands or once per benchmark command:
|
||||
|
||||
```sh
|
||||
hyperfine \
|
||||
--env 'OMP_NUM_THREADS=8' './benchmark-cpp' \
|
||||
--env 'RAYON_NUM_THREADS=8' './benchmark-rust'
|
||||
```
|
||||
|
||||
Use an empty `--env=''` for no overrides. Values can contain parameter placeholders.
|
||||
For example, the following benchmarks `my_command` with `OMP_NUM_THREADS` ranging from 1 to 8:
|
||||
|
||||
```sh
|
||||
hyperfine -P threads 1 8 --env 'OMP_NUM_THREADS={threads}' 'my_command'
|
||||
```
|
||||
|
||||
### Choosing metrics and units
|
||||
|
||||
By default, hyperfine displays wall-clock time and memory usage
|
||||
@@ -239,8 +265,9 @@ You can also use a preset to select a group of metrics:
|
||||
|
||||
### Exporting results
|
||||
|
||||
Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. Non-JSON formats
|
||||
contain only the primary metric, which is the first metric selected by `--metrics`.
|
||||
Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. CSV exports all
|
||||
selected metrics, while Markdown, AsciiDoc, and org-mode contain only the primary metric,
|
||||
which is the first metric selected by `--metrics`.
|
||||
|
||||
#### Markdown
|
||||
|
||||
@@ -276,6 +303,14 @@ uvx --from 'git+https://github.com/sharkdp/hyperfine' \
|
||||
|---:|---:|
|
||||
|
||||
|
||||
#### CSV
|
||||
|
||||
Use `--export-csv <file>` to export one row per command. Each metric selected by `--metrics`
|
||||
contributes a group of columns, in selection order: `<metric>_unit`, `<metric>_mean`,
|
||||
`<metric>_stddev`, `<metric>_median`, `<metric>_min`, and `<metric>_max`.
|
||||
|
||||
CSV uses explicitly selected units, or base units (seconds, bytes, or counts) when omitted.
|
||||
|
||||
### Detailed benchmark flowchart
|
||||
|
||||
The following chart explains the execution order of various benchmark runs when using options
|
||||
@@ -296,8 +331,8 @@ apt install hyperfine
|
||||
|
||||
Alternatively, for the latest version, you can download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases) and install it via `dpkg`:
|
||||
```
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v1.21.0/hyperfine_1.21.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_1.21.0_amd64.deb
|
||||
wget https://github.com/sharkdp/hyperfine/releases/download/v2.0.0/hyperfine_2.0.0_amd64.deb
|
||||
sudo dpkg -i hyperfine_2.0.0_amd64.deb
|
||||
```
|
||||
|
||||
### On Fedora
|
||||
|
||||
+17
-4
@@ -192,6 +192,17 @@ This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'.
|
||||
The option can be specified multiple times to run benchmarks for all
|
||||
possible parameter combinations.
|
||||
.HP
|
||||
\fB\-\-env\fR \fIASSIGNMENTS\fP
|
||||
.IP
|
||||
Set environment variables for benchmark commands and their respective setup, prepare, conclude,
|
||||
and cleanup commands. Assignments are space-separated and support shell-style quoting,
|
||||
but no shell expansion, e.g. \fB\-\-env\fR 'OMP_NUM_THREADS=8 CFLAGS="\-O3 \-march=native"'.
|
||||
Specify once for all commands or once per benchmark command, like \fB\-\-prepare\fR.
|
||||
Use an empty string for no overrides. Values can contain parameter placeholders.
|
||||
.IP
|
||||
For example, \fBhyperfine \-P threads 1 8 \-\-env\fR 'OMP_NUM_THREADS={threads}' 'my_command'
|
||||
benchmarks 'my_command' with OMP_NUM_THREADS ranging from 1 to 8.
|
||||
.HP
|
||||
\fB\-\-shell\fR \fISHELL\fP
|
||||
.IP
|
||||
Set the shell to use for executing benchmarked commands, including warmup runs
|
||||
@@ -232,7 +243,7 @@ of the tool.
|
||||
.HP
|
||||
\fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|time|all\fP
|
||||
.IP
|
||||
Select the metrics to display, in order, with an optional unit for each.
|
||||
Select the metrics to measure, in order, with an optional unit for each.
|
||||
Default: \fBtime_wall_clock,memory_peak_resident\fR, or \fBtime_wall_clock\fR
|
||||
on Windows, where peak resident memory is not yet supported.
|
||||
.IP
|
||||
@@ -266,9 +277,11 @@ Export the primary metric's summary statistics as an AsciiDoc table to the given
|
||||
.HP
|
||||
\fB\-\-export\-csv\fR \fIFILE\fP
|
||||
.IP
|
||||
Export the primary metric's summary statistics as CSV to the given \fIFILE\fP,
|
||||
identifying the metric and unit explicitly. Use the explicitly selected unit,
|
||||
or the base unit (seconds, bytes, or unscaled counts) when omitted.
|
||||
Export summary statistics for all selected metrics as CSV to the given \fIFILE\fP,
|
||||
one row per command. Each metric contributes columns for unit, mean, stddev, median,
|
||||
min, and max, prefixed with the metric name, in selection order. The command column
|
||||
comes first and parameter columns come last. Use the explicitly selected units,
|
||||
or base units (seconds, bytes, or unscaled counts) when omitted.
|
||||
.HP
|
||||
\fB\-\-export\-json\fR \fIFILE\fP
|
||||
.IP
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "hyperfine-scripts"
|
||||
version = "0.1.0"
|
||||
version = "2.0.0"
|
||||
description = "Plot and analyze hyperfine JSON benchmark results"
|
||||
requires-python = ">=3.10"
|
||||
license-files = ["LICENSE-APACHE", "LICENSE-MIT"]
|
||||
|
||||
@@ -18,15 +18,13 @@ pub struct BenchmarkResult {
|
||||
/// regardless of any custom name.
|
||||
pub command: String,
|
||||
|
||||
/// The custom name after parameter substitution, if it differs from `command`.
|
||||
/// For example, `--command-name="wait {duration}s"` with `duration=1`
|
||||
/// produces `Some("wait 1s")`. Without a custom name, or when the expanded
|
||||
/// name equals `command`, this is `None`.
|
||||
/// The custom or automatically generated name after parameter substitution,
|
||||
/// if it differs from `command`. Automatic names include environment overrides.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
|
||||
/// The custom name (or `command`), followed by parameters not used in the
|
||||
/// command template, in command-line order. For example, `sleep {duration}`
|
||||
/// The name, followed by parameters not used in the command or environment
|
||||
/// templates, in command-line order. For example, `sleep {duration}`
|
||||
/// with `duration=1`, `branch=main`, and the custom name `wait {duration}s`
|
||||
/// produces `wait 1s (branch = main)`. Without the custom name, this is
|
||||
/// `sleep 1 (branch = main)`. Used in summaries and table exports.
|
||||
@@ -40,6 +38,10 @@ pub struct BenchmarkResult {
|
||||
/// Parameter values for this benchmark
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub parameters: BTreeMap<String, Parameter>,
|
||||
|
||||
/// Explicit environment overrides after parameter substitution.
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub environment: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl BenchmarkResult {
|
||||
|
||||
@@ -215,6 +215,7 @@ impl Executor for ShellExecutor<'_> {
|
||||
) -> Result<Measurement> {
|
||||
let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
|
||||
let mut command_builder = self.shell.command();
|
||||
command_builder.envs(command.get_environment());
|
||||
command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
|
||||
|
||||
// Windows needs special treatment for its behavior on parsing cmd arguments
|
||||
|
||||
@@ -61,8 +61,9 @@ impl<'a> Benchmark<'a> {
|
||||
output_policy: &CommandOutputPolicy,
|
||||
iteration: executor::BenchmarkIteration,
|
||||
) -> Result<Measurement> {
|
||||
let command = command.clone().with_environment_from(self.command);
|
||||
self.executor
|
||||
.run_intermediate_command_and_measure(command, iteration, output_policy)
|
||||
.run_intermediate_command_and_measure(&command, iteration, output_policy)
|
||||
.map_err(|error| {
|
||||
let hint = if cfg!(windows) {
|
||||
"Append ' || exit /b 0' to the command if this failure can be ignored."
|
||||
@@ -443,6 +444,11 @@ impl<'a> Benchmark<'a> {
|
||||
Ok(BenchmarkResult {
|
||||
command,
|
||||
name,
|
||||
environment: self
|
||||
.command
|
||||
.get_environment()
|
||||
.map(|(name, value)| (name.to_owned(), value))
|
||||
.collect(),
|
||||
display_name: self.command.get_name_with_unused_parameters(),
|
||||
measurements,
|
||||
parameters: self
|
||||
|
||||
@@ -69,7 +69,7 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
|
||||
assert_eq!(options.executor_kind, ExecutorKind::Mock(None));
|
||||
|
||||
let commands = Commands::from_cli_arguments(&cli_arguments)?;
|
||||
let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
|
||||
let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?;
|
||||
|
||||
options.validate_against_command_list(&commands)?;
|
||||
|
||||
|
||||
+39
-3
@@ -50,6 +50,19 @@ fn build_command() -> Command {
|
||||
.value_hint(ValueHint::CommandString)
|
||||
.value_parser(NonEmptyStringValueParser::new()),
|
||||
)
|
||||
// Retain the removed option to provide migration guidance for old invocations.
|
||||
.arg(
|
||||
Arg::new("reference")
|
||||
.long("reference")
|
||||
.hide(true)
|
||||
.value_name("CMD")
|
||||
.num_args(0..=1)
|
||||
.default_missing_value("")
|
||||
.value_parser(|_: &str| -> Result<String, String> {
|
||||
Err("The --reference option has been removed. Put your reference command first instead.".into())
|
||||
})
|
||||
.help("Removed: put your reference command first instead."),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("metrics")
|
||||
.long("metrics")
|
||||
@@ -58,7 +71,7 @@ fn build_command() -> Command {
|
||||
.hide_default_value(true)
|
||||
.help("Performance metrics to measure, in order.\n\n\
|
||||
This is either a preset name, or a comma-separated list of metrics with optional units. \
|
||||
The first metric is used for non-JSON exports, \
|
||||
CSV exports all selected metrics; markup exports use the first metric. \
|
||||
JSON always includes all collected metrics.\n\n\
|
||||
Presets (use one on its own):\n \
|
||||
default Wall-clock time and peak RSS\n \
|
||||
@@ -119,6 +132,28 @@ fn build_command() -> Command {
|
||||
.help("Perform exactly NUM runs for each command. If this option is not specified, \
|
||||
hyperfine automatically determines the number of runs."),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("env")
|
||||
.long("env")
|
||||
.action(ArgAction::Append)
|
||||
.num_args(1)
|
||||
.value_name("ASSIGNMENTS")
|
||||
.help("Set environment variables for benchmark commands and their respective setup, \
|
||||
prepare, conclude, and cleanup commands. Assignments are space-separated and \
|
||||
support shell-style quoting, but no shell expansion:\n\n \
|
||||
hyperfine \\\n \
|
||||
--env 'OMP_NUM_THREADS=8 CFLAGS=\"-O3 -march=native\"' \\\n \
|
||||
'my_command'\n\n\
|
||||
Specify once for all commands or once per benchmark command:\n\n \
|
||||
hyperfine \\\n \
|
||||
--env 'OMP_NUM_THREADS=8' './benchmark-cpp' \\\n \
|
||||
--env 'RAYON_NUM_THREADS=8' './benchmark-rust'\n\n\
|
||||
Use an empty --env='' for no overrides. Values can contain parameter placeholders. \
|
||||
For example, the following benchmarks 'my_command' with OMP_NUM_THREADS ranging \
|
||||
from 1 to 8:\n\n \
|
||||
hyperfine -P threads 1 8 \\\n \
|
||||
--env 'OMP_NUM_THREADS={threads}' 'my_command'"),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("setup")
|
||||
.long("setup")
|
||||
@@ -303,9 +338,10 @@ fn build_command() -> Command {
|
||||
.action(ArgAction::Set)
|
||||
.value_name("FILE")
|
||||
.value_hint(ValueHint::FilePath)
|
||||
.help("Export the primary metric summary statistics as CSV to the given FILE. If you need \
|
||||
.help("Export summary statistics for all selected metrics as CSV to the given FILE. If you need \
|
||||
all metrics for each individual run, use the JSON export format. \
|
||||
Use the explicitly selected unit, or the base unit when unspecified."),
|
||||
Column names are prefixed with the metric name. \
|
||||
Use the explicitly selected units, or base units when unspecified."),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("export-json")
|
||||
|
||||
+91
-6
@@ -14,9 +14,11 @@ use crate::{
|
||||
|
||||
use clap::{parser::ValuesRef, ArgMatches};
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use anyhow::{bail, ensure, Context, Result};
|
||||
use rust_decimal::Decimal;
|
||||
|
||||
type Environment = Vec<(String, String)>;
|
||||
|
||||
/// A command that should be benchmarked.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Command<'a> {
|
||||
@@ -28,6 +30,9 @@ pub struct Command<'a> {
|
||||
|
||||
/// Zero or more parameter values.
|
||||
parameters: Vec<ParameterNameAndValue<'a>>,
|
||||
|
||||
/// Environment overrides, before parameter substitution.
|
||||
environment: Environment,
|
||||
}
|
||||
|
||||
impl<'a> Command<'a> {
|
||||
@@ -36,6 +41,7 @@ impl<'a> Command<'a> {
|
||||
name,
|
||||
expression,
|
||||
parameters: Vec::new(),
|
||||
environment: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,14 +54,26 @@ impl<'a> Command<'a> {
|
||||
name,
|
||||
expression,
|
||||
parameters: parameters.into_iter().collect(),
|
||||
environment: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_name(&self) -> String {
|
||||
self.name.map_or_else(
|
||||
|| self.get_command_line(),
|
||||
|name| self.replace_parameters_in(name),
|
||||
)
|
||||
match self.name {
|
||||
Some(name) => self.replace_parameters_in(name),
|
||||
None => {
|
||||
let environment = self
|
||||
.get_environment()
|
||||
.map(|(name, value)| format!("{name}={}", shell_words::quote(&value)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
if environment.is_empty() {
|
||||
self.get_command_line()
|
||||
} else {
|
||||
format!("{environment} {}", self.get_command_line())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_name_with_unused_parameters(&self) -> String {
|
||||
@@ -87,6 +105,7 @@ impl<'a> Command<'a> {
|
||||
if let Some(program_name) = tokens.next() {
|
||||
let mut command_builder = std::process::Command::new(program_name);
|
||||
command_builder.args(tokens);
|
||||
command_builder.envs(self.get_environment());
|
||||
Ok(command_builder)
|
||||
} else {
|
||||
bail!("Can not execute empty command")
|
||||
@@ -97,8 +116,24 @@ impl<'a> Command<'a> {
|
||||
&self.parameters
|
||||
}
|
||||
|
||||
pub fn get_environment(&self) -> impl Iterator<Item = (&str, String)> {
|
||||
self.environment
|
||||
.iter()
|
||||
.map(move |(name, value)| (name.as_str(), self.replace_parameters_in(value)))
|
||||
}
|
||||
|
||||
pub fn with_environment_from(mut self, command: &Command<'_>) -> Self {
|
||||
self.environment = command.environment.clone();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn is_parameter_unused(&self, parameter: &str) -> bool {
|
||||
!self.expression.contains(&format!("{{{parameter}}}"))
|
||||
let placeholder = format!("{{{parameter}}}");
|
||||
!self.expression.contains(&placeholder)
|
||||
&& !self
|
||||
.environment
|
||||
.iter()
|
||||
.any(|(_, value)| value.contains(&placeholder))
|
||||
}
|
||||
|
||||
pub fn get_unused_parameters(&self) -> impl Iterator<Item = &(&'a str, ParameterValue)> {
|
||||
@@ -137,8 +172,58 @@ impl<'a> Command<'a> {
|
||||
/// A collection of commands that should be benchmarked
|
||||
pub struct Commands<'a>(Vec<Command<'a>>);
|
||||
|
||||
fn parse_environment(value: &str) -> Result<Environment> {
|
||||
let mut environment = Environment::new();
|
||||
for assignment in shell_words::split(value).context("Failed to parse '--env' assignments")? {
|
||||
let (name, value) = assignment.split_once('=').with_context(|| {
|
||||
format!(
|
||||
"Invalid value '{assignment}' for '--env': expected an assignment like NAME=value or NAME=\"quoted value\"."
|
||||
)
|
||||
})?;
|
||||
ensure!(
|
||||
!name.is_empty()
|
||||
&& name.bytes().enumerate().all(|(index, byte)| {
|
||||
byte == b'_'
|
||||
|| byte.is_ascii_alphabetic()
|
||||
|| (index > 0 && byte.is_ascii_digit())
|
||||
}),
|
||||
"Invalid environment variable name '{name}' in '--env'"
|
||||
);
|
||||
// Keep only the last assignment, including case variants on Windows.
|
||||
environment.retain(|(key, _)| {
|
||||
if cfg!(windows) {
|
||||
!key.eq_ignore_ascii_case(name)
|
||||
} else {
|
||||
key != name
|
||||
}
|
||||
});
|
||||
environment.push((name.to_owned(), value.to_owned()));
|
||||
}
|
||||
Ok(environment)
|
||||
}
|
||||
|
||||
impl<'a> Commands<'a> {
|
||||
pub fn from_cli_arguments(matches: &'a ArgMatches) -> Result<Commands<'a>> {
|
||||
let mut commands = Self::build_from_cli_arguments(matches)?;
|
||||
if let Some(values) = matches.get_many::<String>("env") {
|
||||
let environments = values
|
||||
.map(|value| parse_environment(value))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let num_commands = commands.num_commands();
|
||||
ensure!(
|
||||
environments.len() == 1 || environments.len() == num_commands,
|
||||
"The '--env' option has to be provided just once or N times, where N={num_commands} is the \
|
||||
number of benchmark commands."
|
||||
);
|
||||
for (index, command) in commands.0.iter_mut().enumerate() {
|
||||
command.environment =
|
||||
environments[if environments.len() == 1 { 0 } else { index }].clone();
|
||||
}
|
||||
}
|
||||
Ok(commands)
|
||||
}
|
||||
|
||||
fn build_from_cli_arguments(matches: &'a ArgMatches) -> Result<Commands<'a>> {
|
||||
let command_names = matches.get_many::<String>("command-name");
|
||||
let command_strings = matches
|
||||
.get_many::<String>("command")
|
||||
|
||||
+56
-49
@@ -4,23 +4,34 @@ use super::Exporter;
|
||||
use crate::benchmark::benchmark_result::BenchmarkResult;
|
||||
use crate::metric::MetricSelection;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use anyhow::Result;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CsvExporter {}
|
||||
|
||||
impl Exporter for CsvExporter {
|
||||
fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
|
||||
// CSV never chooses units automatically: an omitted suffix means base units.
|
||||
let unit = primary.csv_unit();
|
||||
fn serialize(
|
||||
&self,
|
||||
results: &[BenchmarkResult],
|
||||
metrics: &[MetricSelection],
|
||||
) -> Result<Vec<u8>> {
|
||||
// Omit metrics unavailable for all commands (possible with `--metrics all`).
|
||||
let metrics: Vec<_> = metrics
|
||||
.iter()
|
||||
.filter(|selection| {
|
||||
results
|
||||
.iter()
|
||||
.any(|result| selection.metric.summarize(&result.measurements).is_some())
|
||||
})
|
||||
.collect();
|
||||
let mut writer = WriterBuilder::new().from_writer(vec![]);
|
||||
let mut headers: Vec<String> = [
|
||||
"command", "metric", "unit", "mean", "stddev", "median", "min", "max",
|
||||
]
|
||||
.iter()
|
||||
.copied()
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
let mut headers = vec!["command".to_owned()];
|
||||
for selection in &metrics {
|
||||
headers.extend(
|
||||
["unit", "mean", "stddev", "median", "min", "max"]
|
||||
.map(|field| format!("{}_{field}", selection.metric.name())),
|
||||
);
|
||||
}
|
||||
if let Some(result) = results.first() {
|
||||
headers.extend(
|
||||
result
|
||||
@@ -32,36 +43,28 @@ impl Exporter for CsvExporter {
|
||||
writer.write_record(headers)?;
|
||||
|
||||
for result in results {
|
||||
let stats = primary
|
||||
.metric
|
||||
.summarize(&result.measurements)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Metric '{}' is unavailable for '{}'",
|
||||
primary.metric.name(),
|
||||
result.get_name()
|
||||
)
|
||||
})?;
|
||||
let mut fields = vec![
|
||||
result.get_name().to_owned(),
|
||||
primary.metric.name().to_owned(),
|
||||
unit.symbol.to_owned(),
|
||||
];
|
||||
// Preserve numeric precision rather than using rounded display values.
|
||||
fields.extend(
|
||||
[
|
||||
Some(stats.mean),
|
||||
stats.stddev,
|
||||
Some(stats.median),
|
||||
Some(stats.min),
|
||||
Some(stats.max),
|
||||
]
|
||||
.map(|value| {
|
||||
value
|
||||
.map(|value| (value / unit.scale).to_string())
|
||||
.unwrap_or_default()
|
||||
}),
|
||||
);
|
||||
let mut fields = vec![result.get_name().to_owned()];
|
||||
for selection in &metrics {
|
||||
// CSV never chooses units automatically: an omitted suffix means base units.
|
||||
let unit = selection.csv_unit();
|
||||
let stats = selection.metric.summarize(&result.measurements);
|
||||
fields.push(unit.symbol.to_owned());
|
||||
// Preserve numeric precision rather than using rounded display values.
|
||||
fields.extend(
|
||||
[
|
||||
stats.map(|stats| stats.mean),
|
||||
stats.and_then(|stats| stats.stddev),
|
||||
stats.map(|stats| stats.median),
|
||||
stats.map(|stats| stats.min),
|
||||
stats.map(|stats| stats.max),
|
||||
]
|
||||
.map(|value| {
|
||||
value
|
||||
.map(|value| (value / unit.scale).to_string())
|
||||
.unwrap_or_default()
|
||||
}),
|
||||
);
|
||||
}
|
||||
fields.extend(
|
||||
result
|
||||
.parameters
|
||||
@@ -88,6 +91,7 @@ fn test_csv() {
|
||||
|
||||
let results = vec![
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("echo command_a"),
|
||||
name: Some(String::from("command_a")),
|
||||
display_name: String::from("command_a"),
|
||||
@@ -138,6 +142,7 @@ fn test_csv() {
|
||||
},
|
||||
},
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("command_b"),
|
||||
name: None,
|
||||
display_name: String::from("command_b"),
|
||||
@@ -191,15 +196,18 @@ fn test_csv() {
|
||||
|
||||
let actual = String::from_utf8(
|
||||
exporter
|
||||
.serialize(&results, MetricSelection::default())
|
||||
.serialize(
|
||||
&results,
|
||||
&MetricSelection::parse_list("time_wall_clock,memory_peak_resident").unwrap(),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
insta::assert_snapshot!(actual, @r#"
|
||||
command,metric,unit,mean,stddev,median,min,max,parameter_bar,parameter_foo
|
||||
command_a,time_wall_clock,s,9,2.6457513110645907,8,7,12,two,one
|
||||
command_b,time_wall_clock,s,18,1,18,17,19,seven,one
|
||||
command,time_wall_clock_unit,time_wall_clock_mean,time_wall_clock_stddev,time_wall_clock_median,time_wall_clock_min,time_wall_clock_max,memory_peak_resident_unit,memory_peak_resident_mean,memory_peak_resident_stddev,memory_peak_resident_median,memory_peak_resident_min,memory_peak_resident_max,parameter_bar,parameter_foo
|
||||
command_a,s,9,2.6457513110645907,8,7,12,B,1024,0,1024,1024,1024,two,one
|
||||
command_b,s,18,1,18,17,19,B,1024,0,1024,1024,1024,seven,one
|
||||
"#);
|
||||
|
||||
for (selection, unit, mean) in [
|
||||
@@ -207,12 +215,11 @@ fn test_csv() {
|
||||
("memory_peak_resident:KiB", "KiB", "1"),
|
||||
] {
|
||||
let output = exporter
|
||||
.serialize(&results, MetricSelection::parse_list(selection).unwrap()[0])
|
||||
.serialize(&results, &MetricSelection::parse_list(selection).unwrap())
|
||||
.unwrap();
|
||||
let mut reader = csv::Reader::from_reader(output.as_slice());
|
||||
let row = reader.records().next().unwrap().unwrap();
|
||||
assert_eq!(&row[1], "memory_peak_resident");
|
||||
assert_eq!(&row[2], unit);
|
||||
assert_eq!(&row[3], mean);
|
||||
assert_eq!(&row[1], unit);
|
||||
assert_eq!(&row[2], mean);
|
||||
}
|
||||
}
|
||||
+6
-1
@@ -65,7 +65,12 @@ pub struct JsonExporter {
|
||||
}
|
||||
|
||||
impl Exporter for JsonExporter {
|
||||
fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
|
||||
fn serialize(
|
||||
&self,
|
||||
results: &[BenchmarkResult],
|
||||
metrics: &[MetricSelection],
|
||||
) -> Result<Vec<u8>> {
|
||||
let primary = metrics[0];
|
||||
let mut output = to_vec_pretty(&HyperfineSummary {
|
||||
schema_version: JSON_SCHEMA_VERSION,
|
||||
metadata: &self.metadata,
|
||||
|
||||
@@ -77,7 +77,12 @@ pub trait MarkupExporter {
|
||||
}
|
||||
|
||||
impl<T: MarkupExporter> Exporter for T {
|
||||
fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
|
||||
fn serialize(
|
||||
&self,
|
||||
results: &[BenchmarkResult],
|
||||
metrics: &[MetricSelection],
|
||||
) -> Result<Vec<u8>> {
|
||||
let primary = metrics[0];
|
||||
let stats: Vec<_> = results
|
||||
.iter()
|
||||
.map(|result| {
|
||||
|
||||
+10
-6
@@ -44,8 +44,12 @@ pub enum ExportType {
|
||||
|
||||
/// Interface for different exporters.
|
||||
trait Exporter {
|
||||
/// Export the given entries in the serialized form.
|
||||
fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>>;
|
||||
/// Export the given entries using a nonempty list of selected metrics, primary first.
|
||||
fn serialize(
|
||||
&self,
|
||||
results: &[BenchmarkResult],
|
||||
metrics: &[MetricSelection],
|
||||
) -> Result<Vec<u8>>;
|
||||
}
|
||||
|
||||
pub enum ExportTarget {
|
||||
@@ -61,16 +65,16 @@ struct ExporterWithTarget {
|
||||
/// Handles the management of multiple file exporters.
|
||||
pub struct ExportManager {
|
||||
exporters: Vec<ExporterWithTarget>,
|
||||
primary: MetricSelection,
|
||||
metrics: Vec<MetricSelection>,
|
||||
}
|
||||
|
||||
impl ExportManager {
|
||||
/// Build the ExportManager that will export the results specified
|
||||
/// in the given ArgMatches
|
||||
pub fn from_cli_arguments(matches: &ArgMatches, primary: MetricSelection) -> Result<Self> {
|
||||
pub fn from_cli_arguments(matches: &ArgMatches, metrics: &[MetricSelection]) -> Result<Self> {
|
||||
let mut export_manager = Self {
|
||||
exporters: vec![],
|
||||
primary,
|
||||
metrics: metrics.to_vec(),
|
||||
};
|
||||
{
|
||||
let mut add_exporter = |flag, exporttype| -> Result<()> {
|
||||
@@ -120,7 +124,7 @@ impl ExportManager {
|
||||
/// order not to clutter the output of hyperfine with intermediate results).
|
||||
pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
|
||||
for e in &self.exporters {
|
||||
let content = || e.exporter.serialize(results, self.primary);
|
||||
let content = || e.exporter.serialize(results, &self.metrics);
|
||||
|
||||
match e.target {
|
||||
ExportTarget::File(ref filename) => {
|
||||
|
||||
+10
-2
@@ -15,10 +15,10 @@ fn get_output<E: Exporter + Default>(results: &[BenchmarkResult], unit: Option<U
|
||||
exporter
|
||||
.serialize(
|
||||
results,
|
||||
MetricSelection {
|
||||
&[MetricSelection {
|
||||
unit,
|
||||
..MetricSelection::default()
|
||||
},
|
||||
}],
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -35,6 +35,7 @@ fn get_output<E: Exporter + Default>(results: &[BenchmarkResult], unit: Option<U
|
||||
fn test_markup_export_auto_ms() {
|
||||
let results = [
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 0.1"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 0.1"),
|
||||
@@ -70,6 +71,7 @@ fn test_markup_export_auto_ms() {
|
||||
parameters: BTreeMap::new(),
|
||||
},
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 2"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 2"),
|
||||
@@ -147,6 +149,7 @@ fn test_markup_export_auto_ms() {
|
||||
fn test_markup_export_auto_s() {
|
||||
let results = [
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 2"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 2"),
|
||||
@@ -182,6 +185,7 @@ fn test_markup_export_auto_s() {
|
||||
parameters: BTreeMap::new(),
|
||||
},
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 0.1"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 0.1"),
|
||||
@@ -259,6 +263,7 @@ fn test_markup_export_auto_s() {
|
||||
fn test_markup_export_manual_ms() {
|
||||
let timing_results = [
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 2"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 2"),
|
||||
@@ -294,6 +299,7 @@ fn test_markup_export_manual_ms() {
|
||||
parameters: BTreeMap::new(),
|
||||
},
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 0.1"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 0.1"),
|
||||
@@ -370,6 +376,7 @@ fn test_markup_export_manual_ms() {
|
||||
fn test_markup_export_manual_s() {
|
||||
let results = [
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 2"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 2"),
|
||||
@@ -405,6 +412,7 @@ fn test_markup_export_manual_s() {
|
||||
parameters: BTreeMap::new(),
|
||||
},
|
||||
BenchmarkResult {
|
||||
environment: BTreeMap::new(),
|
||||
command: String::from("sleep 0.1"),
|
||||
name: None,
|
||||
display_name: String::from("sleep 0.1"),
|
||||
|
||||
+9
-1
@@ -47,6 +47,14 @@ fn run() -> Result<()> {
|
||||
{
|
||||
for command in commands.iter() {
|
||||
let command_line = command.get_command_line();
|
||||
if shell_syntax::starts_with_assignment(&command_line) {
|
||||
bail!(
|
||||
"Cannot run '{command_line}' directly because it starts with a variable assignment.\n\
|
||||
Use the --env option to set environment variables, for example:\n\n \
|
||||
hyperfine --env 'OMP_NUM_THREADS=8 CFLAGS=\"-O3 -march=native\"' 'my_command'\n\n\
|
||||
Alternatively, use -S / --shell=default to run the command in a shell."
|
||||
);
|
||||
}
|
||||
if let Some(operator) = shell_syntax::first_unquoted_operator(&command_line) {
|
||||
bail!(
|
||||
concat!(
|
||||
@@ -62,7 +70,7 @@ fn run() -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
|
||||
let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?;
|
||||
|
||||
let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
|
||||
scheduler.run_benchmarks()?;
|
||||
|
||||
+49
-2
@@ -1,8 +1,32 @@
|
||||
//! Conservative detection of shell operators in commands run without a shell.
|
||||
//! Conservative detection of shell syntax in commands run without a shell.
|
||||
|
||||
use std::iter::Peekable;
|
||||
use std::str::CharIndices;
|
||||
|
||||
/// Detects the variable assignment at the start of a command like
|
||||
/// `OMP_NUM_THREADS=8 my_command`.
|
||||
pub fn starts_with_assignment(command: &str) -> bool {
|
||||
let Some(token) = tokenize(command).next() else {
|
||||
return false;
|
||||
};
|
||||
let mut bytes = token.bytes();
|
||||
if !bytes
|
||||
.next()
|
||||
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
for byte in bytes {
|
||||
if byte == b'=' {
|
||||
return true;
|
||||
}
|
||||
if !byte.is_ascii_alphanumeric() && byte != b'_' {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Find the first standalone operator that is neither quoted nor escaped.
|
||||
///
|
||||
/// Operators attached to another word (such as `hello>out`), expansions, and
|
||||
@@ -90,7 +114,30 @@ impl<'a> Iterator for Tokens<'a> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{first_unquoted_operator, tokenize};
|
||||
use super::{first_unquoted_operator, starts_with_assignment, tokenize};
|
||||
|
||||
#[test]
|
||||
fn detects_leading_assignments() {
|
||||
assert!(starts_with_assignment("A=1 command"));
|
||||
assert!(starts_with_assignment("A=$HOME command"));
|
||||
assert!(starts_with_assignment("A= command"));
|
||||
assert!(starts_with_assignment("_A2='hello world' B=2 command"));
|
||||
assert!(starts_with_assignment("A=1"));
|
||||
assert!(starts_with_assignment(" # comment\n A=1 command"));
|
||||
|
||||
assert!(!starts_with_assignment(""));
|
||||
assert!(!starts_with_assignment("# A=1 command"));
|
||||
assert!(!starts_with_assignment("command A=1"));
|
||||
assert!(!starts_with_assignment("env A=1 command"));
|
||||
assert!(!starts_with_assignment("'A=1' command"));
|
||||
assert!(!starts_with_assignment("\"A\"=1 command"));
|
||||
assert!(!starts_with_assignment(r"A\=1 command"));
|
||||
assert!(!starts_with_assignment("./A=1 command"));
|
||||
assert!(!starts_with_assignment("1A=1 command"));
|
||||
assert!(!starts_with_assignment("A-B=1 command"));
|
||||
assert!(!starts_with_assignment("=1 command"));
|
||||
assert!(!starts_with_assignment("A"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenizer_preserves_words_quotes_and_escapes() {
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::process::Command;
|
||||
pub fn hyperfine_raw_command() -> Command {
|
||||
let mut cmd = Command::new(assert_cmd::cargo::cargo_bin!("hyperfine"));
|
||||
cmd.current_dir("tests/");
|
||||
cmd.env_remove("CLICOLOR_FORCE");
|
||||
cmd
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,25 @@ fn runs_successfully() {
|
||||
.success();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_removed_reference_option_with_migration_guidance() {
|
||||
for args in [
|
||||
vec!["--reference", "echo reference", "echo comparison"],
|
||||
vec!["echo comparison", "--reference=echo reference"],
|
||||
vec!["--reference", "echo reference"],
|
||||
vec!["--reference"],
|
||||
] {
|
||||
hyperfine()
|
||||
.args(args)
|
||||
.assert()
|
||||
.code(2)
|
||||
.stdout("")
|
||||
.stderr(predicate::str::contains(
|
||||
"The --reference option has been removed. Put your reference command first instead.",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_run_is_supported() {
|
||||
hyperfine()
|
||||
@@ -53,6 +72,66 @@ fn one_run_is_supported() {
|
||||
.success();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn environment_basic_usage() {
|
||||
let command = if cfg!(windows) {
|
||||
"echo %HYPERFINE_TEST%"
|
||||
} else {
|
||||
"echo \"$HYPERFINE_TEST\""
|
||||
};
|
||||
hyperfine()
|
||||
.args(["--shell=default", "--runs=1", "--show-output"])
|
||||
.arg("--command-name=env-test")
|
||||
.args(["--env", r#"HYPERFINE_TEST="hello, world""#])
|
||||
.arg(command)
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("hello, world"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn environment_parameter_expansion() {
|
||||
let command = if cfg!(windows) {
|
||||
"echo %HYPERFINE_TEST%"
|
||||
} else {
|
||||
"echo \"$HYPERFINE_TEST\""
|
||||
};
|
||||
hyperfine()
|
||||
.args(["--shell=default", "--runs=1", "--show-output"])
|
||||
.arg("--command-name=env-test")
|
||||
.args(["-P", "value", "1", "2"])
|
||||
.args(["--env", "HYPERFINE_TEST=hello-{value}"])
|
||||
.arg(command)
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("hello-1"))
|
||||
.stdout(predicate::str::contains("hello-2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_environment() {
|
||||
hyperfine()
|
||||
.args(["--env", "OMP_NUM_THREADS"])
|
||||
.arg("my_command")
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"Invalid value 'OMP_NUM_THREADS' for '--env': expected an assignment like NAME=value or NAME=\"quoted value\".",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requires_explicit_shell_for_environment_assignments() {
|
||||
// Without an explicit shell choice, report the leading variable assignment.
|
||||
hyperfine()
|
||||
.args(["--runs=1", "SOME_ENV_VAR=$HOME my_command"])
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"starts with a variable assignment",
|
||||
));
|
||||
}
|
||||
|
||||
/// Regression test: hyperfine must not panic when writing to a closed
|
||||
/// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process
|
||||
/// that terminates early). A `BrokenPipe` error should result in a quiet
|
||||
|
||||
Reference in new issue
Block a user