Correct the interpretation of Welch's t-test results

CHANGELOG.md

@@ -247,7 +247,7 @@ Package authors: note that Hyperfine now comes with a set of shell completion fi
   necessarily required numeric arguments.
 
   ``` bash
-  hyperfine --parameter-list compiler "g++,clang++" \
+  hyperfine --parameter-list compiler "gcc,clang" \
       "{compiler} -O2 main.cpp"
   ```
 

README.md

@@ -84,7 +84,7 @@ This runs `sleep 0.3`, `sleep 0.5` and `sleep 0.7`.
 For non-numeric parameters, you can also supply a list of values with the `-L`/`--parameter-list`
 option:
 ```
-hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'
+hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'
 ```
 
 ### Intermediate shell

doc/hyperfine.1

@@ -184,11 +184,11 @@ Replaces the string '{\fIVAR\fP}' in each command by the current parameter value
 .RS
 Example:
 .RS
-\fBhyperfine\fR \fB\-L\fR compiler g++,clang++ '{compiler} \-O2 main.cpp'
+\fBhyperfine\fR \fB\-L\fR compiler gcc,clang '{compiler} \-O2 main.cpp'
 .RE
 .RE
 .IP
-This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'.
+This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'.
 .IP
 The option can be specified multiple times to run benchmarks for all
 possible parameter combinations.

scripts/welch_ttest.py

@@ -7,8 +7,7 @@
 # ///
 
 """This script performs Welch's t-test on a JSON export file with two
-benchmark results to test whether or not the two distributions are
-the same."""
+benchmark results to test for a difference in their population mean runtimes."""
 
 import argparse
 import json
@@ -40,6 +39,10 @@ print(f"t = {t:.3}, p = {p:.3}")
 print()
 
 if dispose:
-    print(f"There is a difference between the two benchmarks (p < {th}).")
+    print(
+        f"A statistically significant difference in mean runtime was detected (p < {th})."
+    )
 else:
-    print(f"The two benchmarks are almost the same (p >= {th}).")
+    print(
+        f"No statistically significant difference in mean runtime was detected (p >= {th})."
+    )

src/cli.rs

@@ -194,8 +194,8 @@ fn build_command() -> Command {
                 .help(
                     "Perform benchmark runs for each value in the comma-separated list VALUES. \
                      Replaces the string '{VAR}' in each command by the current parameter value\
-                     .\n\nExample:  hyperfine -L compiler g++,clang++ '{compiler} -O2 main.cpp'\n\n\
-                     This performs benchmarks for 'g++ -O2 main.cpp' and 'clang++ -O2 main.cpp'.\n\n\
+                     .\n\nExample:  hyperfine -L compiler gcc,clang '{compiler} -O2 main.cpp'\n\n\
+                     This performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp'.\n\n\
                      The option can be specified multiple times to run benchmarks for all \
                      possible parameter combinations.\n"
                 ),