more black_box + choices for release profile

README.md

@@ -62,3 +62,10 @@ More functions in API:
 - `ge, gt, le, lt, eq, ne` ?
 - `min, max, min_by, max_by, min_by_key, max_by_key` ?
 - `product, sum, reduce` ?
+
+Micro benchmark measurement weirdness:
+
+- Preemptive2 seems stable around 16% overhead in release and release-with-debug
+- Preemptive is unstable 3% in release and 45% in release-with-debug
+- TODO: keep only the Preemptive1 and try with/without debug, look and compare
+  asm maybe insert a call to a function with inline never to compare more easily

examples/overhead.rs

@@ -8,9 +8,9 @@ use std::ops::Range;
 macro_rules! timeof {
     ( $function:expr, $( $arg:expr ),*) => {
         {
-        let start = std::time::Instant::now();
+        let start = std::hint::black_box(std::time::Instant::now());
         std::hint::black_box($function($(std::hint::black_box($arg),)*));
-        start.elapsed()
+        std::hint::black_box(start.elapsed())
         }
     };
 }
@@ -19,9 +19,9 @@ macro_rules! timeof {
 macro_rules! timeofasync {
     ( $function:expr, $( $arg:expr ),*) => {
         embassy_futures::block_on(async {
-            let start = std::time::Instant::now();
+            let start = std::hint::black_box(std::time::Instant::now());
             std::hint::black_box($function($(std::hint::black_box($arg),)*)).await;
-            start.elapsed()
+            std::hint::black_box(start.elapsed())
         })
     };
 }
@@ -55,15 +55,15 @@ struct Args {
 fn main() {
     let args = Args::parse();
     for _ in 0..args.repeat {
-        let time_ref = timeof!(Range::for_each, 0..args.n_iter, |_| work(
-            args.duration_task
-        ));
         let time_auto = timeofasync!(
             Range::preemptive_for_each,
             0..args.n_iter,
             |_| work(args.duration_task),
             Duration::from_micros(args.reactivity_us)
         );
+        let time_ref = timeof!(Range::for_each, 0..args.n_iter, |_| work(
+            args.duration_task
+        ));
         let time_manual = timeofasync!(
             Range::preemptive_for_each_fixed_block,
             0..args.n_iter,

scripts/heatmap.py

@@ -3,89 +3,98 @@
 import argparse
 import subprocess
 import itertools
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from bisect import bisect_left
 import os
 import re
+from dataclasses import dataclass
 import pandas
 
 import matplotlib.pyplot as plt
 import numpy as np
 
-PARAMETERS = {
+DEFAULT_PARAMETERS = {
     "Reactivity": [10],
-    "DurationTask": [1, 10, 100, 1_000],
+    "DurationTask": [1, 10, 100],
     "N_Iter": [100, 1_000, 10_000],
 }
-CSV_COLS = [
-    "Reactivity",
-    "DurationTask",
-    "N_Iter",
-    "Baseline",
-    "Preemptive",
-    "PreemptiveFixed",
-    "Preemptive2",
-]
-N_REPEAT = 1000
-
-EXEC = "target/release/examples/overhead"
-REGEX_BASELINE = re.compile(r"Baseline: (\d+)")
-REGEX_PREEMPT = re.compile(r"Preemptive: (\d+)")
-REGEX_PREEMPT2 = re.compile(r"PreemptiveFixed: (\d+)")
-REGEX_PREEMPT3 = re.compile(r"Preemptive2: (\d+)")
-
-
-def collect_heatmap(out_csv: str, repeat: int):
-    if os.path.exists(out_csv):
-        user_input = input("Remove old csv (y/n) ?")
-        if user_input.startswith("y"):
-            os.remove(out_csv)
-        else:
-            print("Do nothing")
-            return
-    with open(out_csv, "w") as f:
-        f.write(",".join(CSV_COLS) + "\n")
-        # this should be in the same order than declared since dict garanties this
-        for reac, dur, niter in itertools.product(*PARAMETERS.values()):
-            out = subprocess.run(
-                [
-                    EXEC,
-                    "--duration-task",
-                    str(dur),
-                    "--reactivity-us",
-                    str(reac),
-                    "--n-iter",
-                    str(niter),
-                    "--repeat",
-                    str(repeat),
-                    "--output-ns",
-                ],
-                check=True,
-                text=True,
-                stdout=subprocess.PIPE,
-            )
-            assert out.stdout
-            baselines = re.finditer(REGEX_BASELINE, out.stdout)
-            premptives = re.finditer(REGEX_PREEMPT, out.stdout)
-            premptives2 = re.finditer(REGEX_PREEMPT2, out.stdout)
-            premptives3 = re.finditer(REGEX_PREEMPT3, out.stdout)
-            for cb, cp, cp2, cp3 in zip(
-                baselines, premptives, premptives2, premptives3
-            ):
-                f.write(
-                    f"{reac},{dur},{niter},{cb.group(1)},{cp.group(1)},{cp2.group(1)},{cp3.group(1)}\n"
+DEFAULT_MEASUREMENTS = ["Baseline", "Preemptive", "PreemptiveFixed", "Preemptive2"]
+DEFAULT_REPEAT = 1000
+
+
+@dataclass
+class CollectorCSV:
+    out_csv: str
+    repeat: int
+    parameters: dict[str, list[int]]
+    measurements: list[str]
+    profile: str
+
+    def params_product(self) -> Iterable[dict[str, int]]:
+        tmp: list[list[tuple[str, int]]] = [
+            [(name, v) for v in values] for name, values in self.parameters.items()
+        ]
+        products = itertools.product(*tmp)
+        return ({k: v for k, v in prod} for prod in products)
+
+    def collect(self) -> None:
+        if os.path.exists(self.out_csv):
+            user_input = input("Remove old csv (y/n) ?")
+            if user_input.startswith("y"):
+                os.remove(self.out_csv)
+            else:
+                print("Do nothing")
+                return
+        subprocess.run(
+            ["cargo", "build", "--profile", self.profile, "--example", "overhead"],
+            check=True,
+        )
+        assert self.measurements
+        csv_cols = list(self.parameters) + self.measurements
+        with open(self.out_csv, "w") as f:
+            f.write(",".join(csv_cols) + "\n")
+            # this should be in the same order than declared since dict garanties this
+            for param in self.params_product():
+                out = subprocess.run(
+                    [
+                        f"target/{self.profile}/examples/overhead",
+                        "--duration-task",
+                        str(param["DurationTask"]),
+                        "--reactivity-us",
+                        str(param["Reactivity"]),
+                        "--n-iter",
+                        str(param["N_Iter"]),
+                        "--repeat",
+                        str(self.repeat),
+                        "--output-ns",
+                    ],
+                    check=True,
+                    text=True,
+                    stdout=subprocess.PIPE,
                 )
-            print(f"{reac=} {dur=} {niter=} done")
+                assert out.stdout
+                iterators = [
+                    re.finditer(f"{k}: (\\d+)", out.stdout) for k in self.measurements
+                ]
+                params = ",".join(map(str, param.values()))
+                for _ in range(self.repeat):
+                    meas = ",".join(next(it).group(1) for it in iterators)
+                    f.write(f"{params},{meas}\n")
+                print(f"{param=} done")
 
 
 def plot_heatmap(path: str, rate: float):
     df = pandas.read_csv(path)
     print(df)
     fig, ax = plt.subplots()
+    niters = df["N_Iter"].unique()
+    durs = df["DurationTask"].unique()
+    reacs = df["Reactivity"].unique()
+    assert len(reacs) == 1  # for now
 
-    data = np.zeros((len(PARAMETERS["N_Iter"]), len(PARAMETERS["DurationTask"])))
-    for i, niter in enumerate(PARAMETERS["N_Iter"]):
-        for j, dur in enumerate(PARAMETERS["DurationTask"]):
+    data = np.zeros((len(niters), len(durs)))
+    for i, niter in enumerate(niters):
+        for j, dur in enumerate(durs):
             df_filtered = df[(df["N_Iter"] == niter) & (df["DurationTask"] == dur)]
             baselines = df_filtered["Baseline"].values
             preemptives = df_filtered["Preemptive"].values
@@ -105,62 +114,62 @@ def plot_heatmap(path: str, rate: float):
     ax.imshow(data)
     fig.tight_layout()
 
-    def settick(set_func: Callable, key: str):
-        set_func(list(range(len(PARAMETERS[key]))), list(map(str, PARAMETERS[key])))
+    def settick(set_func: Callable, arr: np.ndarray):
+        set_func(list(range(len(arr))), list(map(str, arr)))
 
-    settick(ax.set_xticks, "DurationTask")
-    settick(ax.set_yticks, "N_Iter")
+    settick(ax.set_xticks, durs)
+    settick(ax.set_yticks, niters)
     ax.set_xlabel("Duration of each iteration in approx e-7 secs.")
     ax.set_ylabel("Number of iterations.")
     ax.set_title(
-        f"Overhead in % of the iteration using preemptive iter vs native iter. (reactivity = {PARAMETERS['Reactivity'][0]} μs, confidence = {rate*100:.2f}%)"
+        f"Overhead in % of the iteration using preemptive iter vs native iter. (reactivity = {reacs[0]} μs, confidence = {rate*100:.2f}%)"
     )
     plt.show()
 
 
-def get_overhead(baselines: np.ndarray, preemptives: np.ndarray) -> np.ndarray:
-    overhead2 = np.zeros(len(baselines) * len(preemptives))
-    i = 0
-    for base in baselines:
-        for t in preemptives:
-            overhead2[i] = (t - base) / base * 100.0
-            i += 1
-    overhead2.sort()
-    return overhead2
+def get_overhead(base: np.ndarray, other: np.ndarray) -> np.ndarray:
+    overhead = np.fromiter(
+        ((o - b) / b * 100.0 for o, b in itertools.product(other, base)), np.float64
+    )
+    # remove outliers
+    lower_bound, upper_bound = -20.0, 1000.0
+    filter_arr = (overhead > lower_bound) & (overhead < upper_bound)
+    overhead = overhead[filter_arr]
+    # sort for hist and cdf purposes
+    overhead.sort()
+    print(len(overhead))
+    return overhead
 
 
-def plot_likelyhood(path: str, n_iter: int, task_dur: int, reac: int):
+def plot_likelyhood(
+    path: str, n_iter: int, task_dur: int, reac: int, measurements: list[str]
+):
     print(f"{task_dur=} {n_iter=} {reac=}")
     df = pandas.read_csv(path)
-    df_filtered = df[
+    df = df[
         (df["N_Iter"] == n_iter)
         & (df["DurationTask"] == task_dur)
         & (df["Reactivity"] == reac)
     ]
-    assert len(df_filtered) > 0
-    print(len(df_filtered))
-    baselines = df_filtered["Baseline"].values
-    preemptives = df_filtered["Preemptive"].values
-    overhead2 = get_overhead(baselines, preemptives)
-    preemptives_fixed = df_filtered["PreemptiveFixed"].values
-    overhead2fixed = get_overhead(baselines, preemptives_fixed)
-    preemptives2 = df_filtered["Preemptive2"].values
-    overhead3 = get_overhead(baselines, preemptives2)
-    ecdf = np.linspace(0, 1, len(overhead2))
-    plt.plot(overhead2, ecdf, label="ecdf-preemptive-auto")
-    plt.plot(overhead2fixed, ecdf, label="ecdf-preemptive-fixed")
-    plt.plot(overhead3, ecdf, label="ecdf-preemptive-auto2")
+    assert len(df) > 0
+    print("repeat =", len(df))
+    assert "Baseline" in df.columns
+    base = df["Baseline"].values
+    overheads = {
+        m: get_overhead(base, df[m].values) for m in measurements if m != "Baseline"
+    }
+    for label, overhead in overheads.items():
+        plt.plot(overhead, np.linspace(0, 1, len(overhead)), label=label)
     plt.xlabel("x in %")
-    plt.xlim([-5, 50])
     plt.ylabel("P(X < x)")
     plt.legend()
     plt.title("Probability than the real overhead is below x")
     for rate in (0.85, 0.90, 0.95, 0.99):
-        x_rate = overhead2[bisect_left(ecdf, rate)]
-        x_rate_fix = overhead2fixed[bisect_left(ecdf, rate)]
-        print(
-            f"P(Overhead < x) < {100*rate:.0f}% for x={x_rate:.2f}% (auto) x={x_rate_fix:.2f}% (fixed)"
-        )
+        print(f"Probability than overhead is below x with {100*rate:.0f}% confidence :")
+        for label, sorted_data in overheads.items():
+            idx = int(rate * len(sorted_data))
+            x_rate = sorted_data[idx]
+            print(f"\t{label} x={x_rate:.2f}%")
     plt.show()
 
 
@@ -168,19 +177,55 @@ def main():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
+    parser.add_argument(
+        "--measurements",
+        nargs="+",
+        help="List of Measurements to consider.",
+        type=str,
+        default=DEFAULT_MEASUREMENTS,
+    )
     subparsers = parser.add_subparsers(dest="subparser_name")
     parser_collect = subparsers.add_parser(
         "collect", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
     parser_collect.add_argument("output_csv", help="File to generate.")
+    parser_collect.add_argument(
+        "--profile",
+        default="release",
+        help="Profile to use with executable.",
+        choices=("release", "release-with-debug"),
+    )
     parser_collect.add_argument(
         "--repeat",
         type=int,
         help="Number of times to repeat each run.",
-        default=N_REPEAT,
+        default=DEFAULT_REPEAT,
+    )
+    parser_collect.add_argument(
+        "--n-iters",
+        nargs="+",
+        help="List of n-iter to benchmarks.",
+        type=int,
+        default=DEFAULT_PARAMETERS["N_Iter"],
+    )
+    parser_collect.add_argument(
+        "--task-durs",
+        nargs="+",
+        help="List of DurationTask to benchmarks.",
+        type=int,
+        default=DEFAULT_PARAMETERS["DurationTask"],
+    )
+    parser_collect.add_argument(
+        "--reacs",
+        nargs="+",
+        help="List of Reactivity to benchmarks.",
+        type=int,
+        default=DEFAULT_PARAMETERS["Reactivity"],
     )
 
-    parser_plot = subparsers.add_parser("plot")
+    parser_plot = subparsers.add_parser(
+        "plot", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
     parser_plot.add_argument("input_csv", help="File to use for plot.")
     parser_plot.add_argument(
         "--confidence-rate",
@@ -189,35 +234,46 @@ def main():
         help="Confidence rate for the value showed on the heatmap. It displays the value x such that P(Overhead < x) = rate.",
     )
 
-    parser_likelyhood = subparsers.add_parser("likelyhood")
+    parser_likelyhood = subparsers.add_parser(
+        "likelyhood", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
     parser_likelyhood.add_argument("input_csv", help="File to use for plot.")
     parser_likelyhood.add_argument(
         "--n-iter",
         help="Value of N_Iter to fix.",
         type=int,
-        default=PARAMETERS["N_Iter"][0],
+        default=DEFAULT_PARAMETERS["N_Iter"][0],
     )
     parser_likelyhood.add_argument(
         "--task-dur",
         help="Value of DurationTask to fix.",
         type=int,
-        default=PARAMETERS["DurationTask"][0],
+        default=DEFAULT_PARAMETERS["DurationTask"][0],
     )
     parser_likelyhood.add_argument(
         "--reac",
         help="Value of Reactivity to fix.",
         type=int,
-        default=PARAMETERS["Reactivity"][0],
+        default=DEFAULT_PARAMETERS["Reactivity"][0],
     )
     args = parser.parse_args()
 
     match args.subparser_name:
         case "collect":
-            collect_heatmap(args.output_csv, args.repeat)
+            params = {
+                "Reactivity": args.reacs,
+                "DurationTask": args.task_durs,
+                "N_Iter": args.n_iters,
+            }
+            CollectorCSV(
+                args.output_csv, args.repeat, params, args.measurements, args.profile
+            ).collect()
         case "plot":
             plot_heatmap(args.input_csv, args.confidence_rate)
         case "likelyhood":
-            plot_likelyhood(args.input_csv, args.n_iter, args.task_dur, args.reac)
+            plot_likelyhood(
+                args.input_csv, args.n_iter, args.task_dur, args.reac, args.measurements
+            )
         case other:
             print(f"Unknown command: {other}")
 

src/adapt.rs

@@ -17,6 +17,7 @@ impl Divisible for std::ops::Range<u32> {
     }
 }
 
+#[allow(async_fn_in_trait)]
 pub trait AsyncIterator: Iterator + Divisible {
     async fn async_for_each<F: FnMut(Self::Item)>(self, mut f: F, reactivity: Duration) {
         self.async_fold((), move |(), item| f(item), reactivity)