Compare commits

..
Author SHA1 Message Date
David Peter 49ebab2eb4 Correct the interpretation of Welch's t-test results 2026-10-02 11:29:44 +02:00
3 changed files with 9 additions and 6 deletions

No files matched your search

+1 -1
View File
@@ -141,7 +141,7 @@ string '{\fIVAR\fP}' in each command by the current parameter value.
.RS
Example:
.RS
\fBhyperfine\fR \fB\-\-prepare\fR 'make clean' \fB\-P\fR threads 1 8 'make \-j {threads}'
\fBhyperfine\fR \fB\-P\fR threads 1 8 'make \-j {threads}'
.RE
.RE
.IP
+7 -4
View File
@@ -7,8 +7,7 @@
# ///
"""This script performs Welch's t-test on a JSON export file with two
benchmark results to test whether or not the two distributions are
the same."""
benchmark results to test for a difference in their population mean runtimes."""
import argparse
import json
@@ -40,6 +39,10 @@ print(f"t = {t:.3}, p = {p:.3}")
print()
if dispose:
print(f"There is a difference between the two benchmarks (p < {th}).")
print(
f"A statistically significant difference in mean runtime was detected (p < {th})."
)
else:
print(f"The two benchmarks are almost the same (p >= {th}).")
print(
f"No statistically significant difference in mean runtime was detected (p >= {th})."
)
+1 -1
View File
@@ -161,7 +161,7 @@ fn build_command() -> Command {
.help(
"Perform benchmark runs for each value in the range MIN..MAX. Replaces the \
string '{VAR}' in each command by the current parameter value.\n\n \
Example: hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\
Example: hyperfine -P threads 1 8 'make -j {threads}'\n\n\
This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\
To have the value increase following different patterns, use shell arithmetics.\n\n \
Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\