Compare commits

...
Author SHA1 Message Date
David Peter e8ca561a24 Initial work on tracking memory usage 2024-12-01 21:49:28 +01:00
David Peter f78f482d8f Add --time-unit option to 'advanced_statistics.py' 2024-12-01 21:13:31 +01:00
dependabot[bot] 4e1e9f2e48 Bump serde_json from 1.0.132 to 1.0.133
Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.132 to 1.0.133.
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.132...v1.0.133)

---
updated-dependencies:
- dependency-name: serde_json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-01 12:38:15 +01:00
dependabot[bot] e8ebfc4913 Bump serde from 1.0.214 to 1.0.215
Bumps [serde](https://github.com/serde-rs/serde) from 1.0.214 to 1.0.215.
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.214...v1.0.215)

---
updated-dependencies:
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-12-01 12:37:50 +01:00
David Peter 87d77c861f Fix version in README 2024-11-13 15:00:12 +01:00
16 changed files with 119 additions and 22 deletions

No files matched your search

Generated
+6 -6
View File
@@ -964,18 +964,18 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "serde"
version = "1.0.214"
version = "1.0.215"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.214"
version = "1.0.215"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
dependencies = [
"proc-macro2",
"quote",
@@ -984,9 +984,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.132"
version = "1.0.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
dependencies = [
"itoa",
"memchr",
+2 -2
View File
@@ -170,8 +170,8 @@ like `--warmup`, `--prepare <cmd>`, `--setup <cmd>` or `--cleanup <cmd>`:
Download the appropriate `.deb` package from the [Release page](https://github.com/sharkdp/hyperfine/releases)
and install it via `dpkg`:
```
wget https://github.com/sharkdp/hyperfine/releases/download/v1.16.1/hyperfine_1.16.1_amd64.deb
sudo dpkg -i hyperfine_1.16.1_amd64.deb
wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb
sudo dpkg -i hyperfine_1.19.0_amd64.deb
```
### On Fedora
+43 -7
View File
@@ -8,13 +8,45 @@
import argparse
import json
from enum import Enum
import numpy as np
class Unit(Enum):
SECOND = 1
MILLISECOND = 2
def factor(self):
match self:
case Unit.SECOND:
return 1
case Unit.MILLISECOND:
return 1e3
def __str__(self):
match self:
case Unit.SECOND:
return "s"
case Unit.MILLISECOND:
return "ms"
parser = argparse.ArgumentParser()
parser.add_argument("file", help="JSON file with benchmark results")
parser.add_argument(
"--time-unit",
help="The unit of time.",
default="second",
action="store",
choices=["second", "millisecond"],
dest="unit",
)
args = parser.parse_args()
unit = Unit.MILLISECOND if args.unit == "millisecond" else Unit.SECOND
unit_str = str(unit)
with open(args.file) as f:
results = json.load(f)["results"]
@@ -22,6 +54,8 @@ commands = [b["command"] for b in results]
times = [b["times"] for b in results]
for command, ts in zip(commands, times):
ts = [t * unit.factor() for t in ts]
p05 = np.percentile(ts, 5)
p25 = np.percentile(ts, 25)
p75 = np.percentile(ts, 75)
@@ -31,13 +65,15 @@ for command, ts in zip(commands, times):
print(f"Command '{command}'")
print(f" runs: {len(ts):8d}")
print(f" mean: {np.mean(ts):8.3f} s")
print(f" stddev: {np.std(ts, ddof=1):8.3f} s")
print(f" median: {np.median(ts):8.3f} s")
print(f" min: {np.min(ts):8.3f} s")
print(f" max: {np.max(ts):8.3f} s")
print(f" mean: {np.mean(ts):8.3f} {unit_str}")
print(f" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}")
print(f" median: {np.median(ts):8.3f} {unit_str}")
print(f" min: {np.min(ts):8.3f} {unit_str}")
print(f" max: {np.max(ts):8.3f} {unit_str}")
print()
print(" percentiles:")
print(f" P_05 .. P_95: {p05:.3f} s .. {p95:.3f} s")
print(f" P_25 .. P_75: {p25:.3f} s .. {p75:.3f} s (IQR = {iqr:.3f} s)")
print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}")
print(
f" P_25 .. P_75: {p25:.3f} {unit_str} .. {p75:.3f} {unit_str} (IQR = {iqr:.3f} {unit_str})"
)
print()
+4
View File
@@ -42,6 +42,10 @@ pub struct BenchmarkResult {
#[serde(skip_serializing_if = "Option::is_none")]
pub times: Option<Vec<Second>>,
/// Maximum memory usage of the process, in bytes
#[serde(skip_serializing_if = "Option::is_none")]
pub memory_usage_byte: Option<Vec<u64>>,
/// Exit codes of all command invocations
pub exit_codes: Vec<Option<i32>>,
+4
View File
@@ -133,6 +133,7 @@ impl Executor for RawExecutor<'_> {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
@@ -204,6 +205,7 @@ impl Executor for ShellExecutor<'_> {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
@@ -268,6 +270,7 @@ impl Executor for ShellExecutor<'_> {
time_real: mean(&times_real),
time_user: mean(&times_user),
time_system: mean(&times_system),
memory_usage_byte: 0,
});
Ok(())
@@ -323,6 +326,7 @@ impl Executor for MockExecutor {
time_real: Self::extract_time(command.get_command_line()),
time_user: 0.0,
time_system: 0.0,
memory_usage_byte: 0,
},
status,
))
+4
View File
@@ -151,6 +151,7 @@ impl<'a> Benchmark<'a> {
let mut times_real: Vec<Second> = vec![];
let mut times_user: Vec<Second> = vec![];
let mut times_system: Vec<Second> = vec![];
let mut memory_usage_byte: Vec<u64> = vec![];
let mut exit_codes: Vec<Option<i32>> = vec![];
let mut all_succeeded = true;
@@ -279,6 +280,7 @@ impl<'a> Benchmark<'a> {
times_real.push(res.time_real);
times_user.push(res.time_user);
times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(status));
all_succeeded = all_succeeded && success;
@@ -315,6 +317,7 @@ impl<'a> Benchmark<'a> {
times_real.push(res.time_real);
times_user.push(res.time_user);
times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(status));
all_succeeded = all_succeeded && success;
@@ -451,6 +454,7 @@ impl<'a> Benchmark<'a> {
min: t_min,
max: t_max,
times: Some(times_real),
memory_usage_byte: Some(memory_usage_byte),
exit_codes,
parameters: self
.command
+1
View File
@@ -133,6 +133,7 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult {
min: mean,
max: mean,
times: None,
memory_usage_byte: None,
exit_codes: Vec::new(),
parameters: BTreeMap::new(),
}
+3
View File
@@ -11,4 +11,7 @@ pub struct TimingResult {
/// Time spent in kernel mode
pub time_system: Second,
/// Maximum amount of memory used, in bytes
pub memory_usage_byte: u64,
}
+4
View File
@@ -105,6 +105,7 @@ fn test_asciidoc_format_s() {
min: 5.0,
max: 6.0,
times: Some(vec![7.0, 8.0, 9.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: {
let mut params = BTreeMap::new();
@@ -124,6 +125,7 @@ fn test_asciidoc_format_s() {
min: 15.0,
max: 16.0,
times: Some(vec![17.0, 18.0, 19.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: {
let mut params = BTreeMap::new();
@@ -180,6 +182,7 @@ fn test_asciidoc_format_ms() {
min: 0.015,
max: 0.016,
times: Some(vec![0.017, 0.018, 0.019]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: {
let mut params = BTreeMap::new();
@@ -199,6 +202,7 @@ fn test_asciidoc_format_ms() {
min: 5.0,
max: 6.0,
times: Some(vec![7.0, 8.0, 9.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: {
let mut params = BTreeMap::new();
+2
View File
@@ -77,6 +77,7 @@ fn test_csv() {
min: 5.0,
max: 6.0,
times: Some(vec![7.0, 8.0, 9.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: {
let mut params = BTreeMap::new();
@@ -96,6 +97,7 @@ fn test_csv() {
min: 15.0,
max: 16.5,
times: Some(vec![17.0, 18.0, 19.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: {
let mut params = BTreeMap::new();
+8
View File
@@ -82,6 +82,7 @@ fn test_markdown_format_ms() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -96,6 +97,7 @@ fn test_markdown_format_ms() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -139,6 +141,7 @@ fn test_markdown_format_s() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -153,6 +156,7 @@ fn test_markdown_format_s() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -196,6 +200,7 @@ fn test_markdown_format_time_unit_s() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -210,6 +215,7 @@ fn test_markdown_format_time_unit_s() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -274,6 +280,7 @@ fn test_markdown_format_time_unit_ms() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -288,6 +295,7 @@ fn test_markdown_format_time_unit_ms() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
+8
View File
@@ -136,6 +136,7 @@ fn test_determine_unit_from_results_unit_given_s() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -150,6 +151,7 @@ fn test_determine_unit_from_results_unit_given_s() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -178,6 +180,7 @@ fn test_determine_unit_from_results_unit_given_ms() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -192,6 +195,7 @@ fn test_determine_unit_from_results_unit_given_ms() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -220,6 +224,7 @@ fn test_determine_unit_from_results_unit_first_s() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -234,6 +239,7 @@ fn test_determine_unit_from_results_unit_first_s() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -262,6 +268,7 @@ fn test_determine_unit_from_results_unit_first_ms() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -276,6 +283,7 @@ fn test_determine_unit_from_results_unit_first_ms() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
+4
View File
@@ -88,6 +88,7 @@ fn test_orgmode_format_ms() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -102,6 +103,7 @@ fn test_orgmode_format_ms() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -147,6 +149,7 @@ fn test_orgmode_format_s() {
min: 2.0020,
max: 2.0080,
times: Some(vec![2.0, 2.0, 2.0]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
@@ -161,6 +164,7 @@ fn test_orgmode_format_s() {
min: 0.1023,
max: 0.1080,
times: Some(vec![0.1, 0.1, 0.1]),
memory_usage_byte: None,
exit_codes: vec![Some(0), Some(0), Some(0)],
parameters: BTreeMap::new(),
},
+6 -2
View File
@@ -32,6 +32,9 @@ struct CPUTimes {
/// Total amount of time spent executing in kernel mode
pub system_usec: i64,
/// Maximum amount of memory used by the process, in bytes
pub memory_usage_byte: u64,
}
/// Used to indicate the result of running a command
@@ -40,7 +43,7 @@ pub struct TimerResult {
pub time_real: Second,
pub time_user: Second,
pub time_system: Second,
pub memory_usage_byte: u64,
/// The exit status of the process
pub status: ExitStatus,
}
@@ -106,12 +109,13 @@ pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
let status = child.wait()?;
let time_real = wallclock_timer.stop();
let (time_user, time_system) = cpu_timer.stop();
let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
Ok(TimerResult {
time_real,
time_user,
time_system,
memory_usage_byte,
status,
})
}
+17 -2
View File
@@ -1,5 +1,6 @@
#![cfg(not(windows))]
use std::convert::TryFrom;
use std::mem;
use crate::timer::CPUTimes;
@@ -25,10 +26,14 @@ impl CPUTimer {
}
}
pub fn stop(&self) -> (Second, Second) {
pub fn stop(&self) -> (Second, Second, u64) {
let end_cpu = get_cpu_times();
let cpu_interval = cpu_time_interval(&self.start_cpu, &end_cpu);
(cpu_interval.user, cpu_interval.system)
(
cpu_interval.user,
cpu_interval.system,
end_cpu.memory_usage_byte,
)
}
}
@@ -45,12 +50,20 @@ fn get_cpu_times() -> CPUTimes {
const MICROSEC_PER_SEC: i64 = 1000 * 1000;
// Linux and *BSD return the value in KibiBytes, Darwin flavors in bytes
let max_rss_byte = if cfg!(target_os = "macos") || cfg!(target_os = "ios") {
result.ru_maxrss
} else {
result.ru_maxrss * 1024
};
#[allow(clippy::useless_conversion)]
CPUTimes {
user_usec: i64::from(result.ru_utime.tv_sec) * MICROSEC_PER_SEC
+ i64::from(result.ru_utime.tv_usec),
system_usec: i64::from(result.ru_stime.tv_sec) * MICROSEC_PER_SEC
+ i64::from(result.ru_stime.tv_usec),
memory_usage_byte: u64::try_from(max_rss_byte).unwrap_or(0),
}
}
@@ -70,11 +83,13 @@ fn test_cpu_time_interval() {
let t_a = CPUTimes {
user_usec: 12345,
system_usec: 54321,
memory_usage_byte: 0,
};
let t_b = CPUTimes {
user_usec: 20000,
system_usec: 70000,
memory_usage_byte: 0,
};
let t_zero = cpu_time_interval(&t_a, &t_a);
+3 -3
View File
@@ -86,7 +86,7 @@ impl CPUTimer {
Self { job_object }
}
pub fn stop(&self) -> (Second, Second) {
pub fn stop(&self) -> (Second, Second, u64) {
let mut job_object_info =
mem::MaybeUninit::<JOBOBJECT_BASIC_ACCOUNTING_INFORMATION>::uninit();
@@ -114,9 +114,9 @@ impl CPUTimer {
// for all active processes associated with the job, as well as all terminated
// processes no longer associated with the job, in 100-nanosecond ticks."
let kernel: i64 = job_object_info.TotalKernelTime / HUNDRED_NS_PER_MS;
(user as f64 * 1e-6, kernel as f64 * 1e-6)
(user as f64 * 1e-6, kernel as f64 * 1e-6, 0)
} else {
(0.0, 0.0)
(0.0, 0.0, 0)
}
}
}