Compare commits

...
16 Commits
Author SHA1 Message Date
sharkdp b089eda3a3 Bump version 2018-12-12 23:13:35 +01:00
sharkdp 5637af48ba Move snapcraft file 2018-12-12 20:39:03 +01:00
David Peter d83d9ffd0d Add link to "scripts" folder 2018-12-12 20:37:51 +01:00
David Peter 6267a26ccb Create README.md 2018-12-12 20:30:19 +01:00
sharkdp 2f51d22e4d Add a simple script to show advanced statistics 2018-12-12 20:27:14 +01:00
sharkdp 0ba8a20bb8 Add script to plot benchmark results 2018-12-12 19:54:57 +01:00
sharkdp 51fe31e7d2 Improve progress bar 'experience' 2018-12-12 19:45:15 +01:00
Alan Pope 266105b52c Add support for building a snap via snapcraft 2018-12-12 11:05:46 +01:00
dependabot[bot] 4298c83028 Bump serde from 1.0.81 to 1.0.82
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.81 to 1.0.82.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.81...v1.0.82)

Signed-off-by: dependabot[bot] <support@dependabot.com>
2018-12-12 11:05:17 +01:00
dependabot[bot] e16b7305d7 Bump serde_derive from 1.0.80 to 1.0.82
Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.80 to 1.0.82.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.80...v1.0.82)

Signed-off-by: dependabot[bot] <support@dependabot.com>
2018-12-11 21:16:54 +01:00
dependabot[bot] fb914c41db Bump serde from 1.0.80 to 1.0.81
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.80 to 1.0.81.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.80...v1.0.81)

Signed-off-by: dependabot[bot] <support@dependabot.com>
2018-12-11 09:08:16 +01:00
tcmal 048506bed4 Show (dimmed) # of runs along with range.
Fixes #78
2018-12-05 14:43:29 +01:00
Peter Steinbach 1dcefd9082 refined help text of export flags to separate summary results from summary+raw timings 2018-12-05 14:37:16 +01:00
sharkdp c540aab8e3 Revert "Bump indicatif from 0.9.0 to 0.10.1"
This reverts commit 8f0c0209e8.
2018-12-03 18:18:36 +01:00
dependabot[bot] 8f0c0209e8 Bump indicatif from 0.9.0 to 0.10.1
Bumps [indicatif](https://github.com/mitsuhiko/indicatif) from 0.9.0 to 0.10.1.
- [Release notes](https://github.com/mitsuhiko/indicatif/releases)
- [Commits](https://github.com/mitsuhiko/indicatif/commits)

Signed-off-by: dependabot[bot] <support@dependabot.com>
2018-11-29 23:06:29 +01:00
David Peter ebc2ea6d6d Update README.md 2018-11-21 20:29:00 +01:00
9 changed files with 119 additions and 23 deletions

No files matched your search

+27
View File
@@ -0,0 +1,27 @@
name: hyperfine
version: git
summary: "A command-line benchmarking tool"
description: |
A command-line benchmarking tool (inspired by bench).
Features
* Statistical analysis across multiple runs.
* Support for arbitrary shell commands.
* Constant feedback about the benchmark progress and current estimates.
* Warmup runs can be executed before the actual benchmark.
* Cache-clearing commands can be set up before each timing run.
* Statistical outlier detection.
* Export results to various formats: CSV, JSON, Markdown.
* Parameterized benchmarks.
* Cross-platform
grade: stable
confinement: classic
parts:
hyperfine:
plugin: rust
source: ./
apps:
hyperfine:
command: bin/hyperfine
Generated
+12 -12
View File
@@ -105,7 +105,7 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -132,7 +132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "hyperfine"
version = "1.4.0"
version = "1.5.0"
dependencies = [
"approx 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -142,8 +142,8 @@ dependencies = [
"csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"indicatif 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
"statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -455,17 +455,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde"
version = "1.0.80"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde_derive"
version = "1.0.80"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.21 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.22 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -475,7 +475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -507,7 +507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "syn"
version = "0.15.21"
version = "0.15.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -680,14 +680,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
"checksum serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "15c141fc7027dd265a47c090bf864cf62b42c4d228bbcf4e51a0c9e2b0d3f7ef"
"checksum serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "225de307c6302bec3898c51ca302fc94a7a1697ef0845fcee6448f33c032249c"
"checksum serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)" = "6fa52f19aee12441d5ad11c9a00459122bd8f98707cadf9778c540674f1935b6"
"checksum serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)" = "96a7f9496ac65a2db5929afa087b54f8fc5008dcfbe48a8874ed20049b0d6154"
"checksum serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)" = "c37ccd6be3ed1fdf419ee848f7c758eb31b054d7cd3ae3600e3bae0adf569811"
"checksum smallvec 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "622df2d454c29a4d89b30dc3b27b42d7d90d6b9e587dbf8f67652eb7514da484"
"checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8"
"checksum statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c139942f46d96c53b28420a2cdfb374629f122656bd9daef7fc221ed4d8ec228"
"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
"checksum syn 0.15.21 (registry+https://github.com/rust-lang/crates.io-index)" = "816b7af21405b011a23554ea2dc3f6576dc86ca557047c34098c1d741f10f823"
"checksum syn 0.15.22 (registry+https://github.com/rust-lang/crates.io-index)" = "ae8b29eb5210bc5cf63ed6149cbf9adfc82ac0be023d8735c176ee74a2db4da7"
"checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625"
+3 -3
View File
@@ -7,7 +7,7 @@ license = "MIT/Apache-2.0"
name = "hyperfine"
readme = "README.md"
repository = "https://github.com/sharkdp/hyperfine"
version = "1.4.0"
version = "1.5.0"
[dependencies]
colored = "1.6"
@@ -16,8 +16,8 @@ statistical = "0.1"
atty = "0.2.2"
cfg-if = "0.1.6"
csv = "1.0.0"
serde = "1.0.80"
serde_derive = "1.0.80"
serde = "1.0.82"
serde_derive = "1.0.82"
serde_json = "1.0.33"
[target.'cfg(not(windows))'.dependencies]
+5 -2
View File
@@ -83,6 +83,9 @@ option that will create tables like this:
| `find . -iname '*[0-9].jpg'` | 304.9 ± 3.1 | 299.8…309.3 |
| `fd -HI '.*[0-9]\.jpg$'` | 66.2 ± 5.8 | 62.5…86.3 |
The JSON output is useful if you want to analyze the benchmark results in more detail. See the
[`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder for some examples.
## Installation
### On macOS
@@ -97,8 +100,8 @@ brew install hyperfine
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.3.0/hyperfine_1.3.0_amd64.deb
sudo dpkg -i hyperfine_1.3.0_amd64.deb
wget https://github.com/sharkdp/hyperfine/releases/download/v1.4.0/hyperfine_1.4.0_amd64.deb
sudo dpkg -i hyperfine_1.4.0_amd64.deb
```
### On Arch Linux
+8
View File
@@ -0,0 +1,8 @@
This folder contains scripts that can be used in combination with hyperfines `--export-json` option.
### Example:
``` bash
> hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
> python plot_benchmark_results.py sleep.json
```
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/python
import argparse
import json
import numpy as np
parser = argparse.ArgumentParser()
parser.add_argument("file", help="JSON file with benchmark results")
args = parser.parse_args()
with open(args.file) as f:
results = json.load(f)["results"]
commands = [b["command"] for b in results]
times = [b["times"] for b in results]
for command, ts in zip(commands, times):
p05 = np.percentile(ts, 5)
p25 = np.percentile(ts, 25)
p75 = np.percentile(ts, 75)
p95 = np.percentile(ts, 95)
iqr = p75 - p25
print("Command '{}'".format(command))
print(" mean: {:8.3f} s".format(np.mean(ts)))
print(" stddev: {:8.3f} s".format(np.std(ts, ddof=1)))
print(" median: {:8.3f} s".format(np.median(ts)))
print()
print(" percentiles:")
print(" P_05 .. P_95: {:.3f} s .. {:.3f} s".format(p05, p95))
print(" P_25 .. P_75: {:.3f} s .. {:.3f} s "
"(IQR = {:.3f} s)".format(p25, p75, iqr))
print()
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/python
import argparse
import json
import matplotlib.pyplot as plt
parser = argparse.ArgumentParser()
parser.add_argument("file", help="JSON file with benchmark results")
args = parser.parse_args()
with open(args.file) as f:
results = json.load(f)["results"]
commands = [b["command"] for b in results]
times = [b["times"] for b in results]
plt.boxplot(times, labels=commands)
plt.ylabel("Time [s]")
plt.ylim(0, None)
plt.show()
+3 -3
View File
@@ -138,21 +138,21 @@ fn build_app() -> App<'static, 'static> {
.long("export-csv")
.takes_value(true)
.value_name("FILE")
.help("Export the timing results as CSV to the given FILE."),
.help("Export the timing summary statistics as CSV to the given FILE."),
)
.arg(
Arg::with_name("export-json")
.long("export-json")
.takes_value(true)
.value_name("FILE")
.help("Export the timing results as JSON to the given FILE."),
.help("Export the timing summary statistics and timings of individual runs as JSON to the given FILE."),
)
.arg(
Arg::with_name("export-markdown")
.long("export-markdown")
.takes_value(true)
.value_name("FILE")
.help("Export the timing results as a Markdown table to the given FILE."),
.help("Export the timing summary statistics as a Markdown table to the given FILE."),
)
.arg(
Arg::with_name("show-output")
+7 -3
View File
@@ -258,7 +258,8 @@ pub fn run_benchmark(
all_succeeded = all_succeeded && success;
// Re-configure the progress bar
progress_bar.set_length(count_remaining);
progress_bar.set_length(count);
progress_bar.inc(1);
// Gather statistics
for _ in 0..count_remaining {
@@ -289,6 +290,7 @@ pub fn run_benchmark(
progress_bar.finish_and_clear();
// Compute statistical quantities
let t_num = times_real.len();
let t_mean = mean(&times_real);
let t_stddev = standard_deviation(&times_real, Some(t_mean));
let t_min = min(&times_real);
@@ -302,6 +304,7 @@ pub fn run_benchmark(
let stddev_str = format_duration(t_stddev, Some(unit_mean));
let min_str = format_duration(t_min, Some(unit_mean));
let max_str = format_duration(t_max, Some(unit_mean));
let num_str = format!("{} runs", t_num);
let (user_str, user_unit) = format_duration_unit(user_mean, None);
let system_str = format_duration(system_mean, Some(user_unit));
@@ -317,11 +320,12 @@ pub fn run_benchmark(
);
println!(
" Range ({} … {}): {:>8} … {:>8}",
" Range ({} … {}): {:>8} … {:>8} {}",
"min".cyan(),
"max".purple(),
min_str.cyan(),
max_str.purple()
max_str.purple(),
num_str.dimmed()
);
// Warnings