defk-rt2.9: Defk.Examples.Result, the example report read as one record

dev/defk/examples.ex

@@ -19,7 +19,7 @@ defmodule Defk.Examples do
           name: String.t(),
           status: :ok | :incorrect | :unexpected_verdict | :crashed | :timeout,
           secs: float(),
-          results: [map()],
+          results: [Defk.Examples.Result.t()],
           out: String.t(),
           err: String.t(),
           note: String.t()
@@ -277,13 +277,13 @@ defmodule Defk.Examples do
 
   defp verdict_of(mode, _status, [first | _] = results) do
     cond do
-      Enum.any?(results, &(&1["correct"] not in [true, "skipped"])) ->
+      not Enum.all?(results, &Defk.Examples.Result.correct?/1) ->
         %{status: :incorrect, note: "an agreement check returned #{inspect(wrong(results))}"}
 
-      mode == :bench and first["verdict"] != first["expected"] ->
+      mode == :bench and first.verdict != first.expected ->
         %{
           status: :unexpected_verdict,
-          note: "measured #{first["verdict"]}, expected #{first["expected"]}"
+          note: "measured #{first.verdict}, expected #{first.expected}"
         }
 
       true ->
@@ -297,7 +297,7 @@ defmodule Defk.Examples do
   defp noun(:bench, _), do: "reports"
 
   defp wrong(results),
-    do: Enum.find(results, &(&1["correct"] not in [true, "skipped"]))["correct"]
+    do: Enum.find(results, &(not Defk.Examples.Result.correct?(&1))).correct
 
   defp judge_app(name, ran, secs) do
     base = %{name: name, secs: secs, err: "", results: []}
@@ -322,11 +322,11 @@ defmodule Defk.Examples do
     end
   end
 
-  @doc "Every `DEFK_RESULT` line of an example's stdout, decoded, in order."
+  @doc "Every `DEFK_RESULT` line of an example's stdout as a `Defk.Examples.Result`, in order."
   def parse_results(out) do
     for [_, json] <- Regex.scan(~r/^DEFK_RESULT (.*)$/m, out),
         {:ok, r} <- [JSON.decode(json)],
-        do: r
+        do: Defk.Examples.Result.decode(r)
   end
 
   # ---- running a process ----------------------------------------------

dev/defk/examples/result.ex

@@ -0,0 +1,117 @@
+defmodule Defk.Examples.Result do
+  @moduledoc false
+  # One report of an example: what one `DEFK_RESULT` line of
+  # `Defk.Bench.report/1` (or its check-only form) says, decoded once. The
+  # runner, the scoreboard, the stored scoreboard and the README tables read
+  # a report through this record and its questions, never by key. The
+  # line's JSON map is kept in `raw`: the stored scoreboard holds the lines
+  # as the harness wrote them (Defk.Examples.Snapshot).
+
+  defstruct raw: %{},
+            example: nil,
+            expected: nil,
+            verdict: nil,
+            baseline: nil,
+            correct: nil,
+            reason: nil,
+            check_only: false,
+            speedup: nil,
+            elixir_ms: nil,
+            defk_ms: nil,
+            native_ms: nil,
+            native_label: nil,
+            native_speedup: nil,
+            realistic_size: nil,
+            crossover: :unmeasured,
+            gaps: [],
+            cache: nil,
+            measured_at: nil,
+            stamp: nil
+
+  @type t :: %__MODULE__{}
+
+  @fields ~w(example expected verdict baseline correct reason speedup elixir_ms defk_ms
+             native_ms native_label native_speedup realistic_size cache measured_at stamp)
+
+  @doc "The record of one decoded `DEFK_RESULT` line (a map with string keys)."
+  def decode(%__MODULE__{} = r), do: r
+
+  def decode(map) when is_map(map) do
+    fields = for k <- @fields, Map.has_key?(map, k), do: {String.to_existing_atom(k), map[k]}
+
+    struct(
+      __MODULE__,
+      fields ++
+        [
+          raw: map,
+          check_only: map["check_only"] == true,
+          crossover: if(Map.has_key?(map, "crossover"), do: map["crossover"], else: :unmeasured),
+          gaps: gaps(map)
+        ]
+    )
+  end
+
+  @doc "The stand-in for an example that crashed before its first line."
+  def crashed(name),
+    do: %__MODULE__{example: name, expected: "?", verdict: "crashed", correct: false}
+
+  # the tick gaps a report measured, `{side, max, p99}` for each side it has
+  defp gaps(map) do
+    for {k, side} <- [
+          {"elixir_gap_ms", "elixir"},
+          {"defk_gap_ms", "defk"},
+          {"native_gap_ms", "native"}
+        ],
+        %{"max" => m, "p99" => p} <- [map[k]],
+        do: {side, m, p}
+  end
+
+  @doc "The agreement check passed, or the example skipped it with a reason."
+  def correct?(%__MODULE__{correct: c}), do: c in [true, "skipped"]
+
+  def skipped?(%__MODULE__{correct: c}), do: c == "skipped"
+
+  @doc "The report's result came from the cache, not from a measurement."
+  def reused?(%__MODULE__{cache: c}), do: c == "reused"
+
+  @doc "The report was measured but not stored: the machine was not quiet."
+  def unstored?(%__MODULE__{cache: c}), do: c == "not_stored"
+
+  @doc "The report's stamp says the machine was quiet before and after it."
+  def quiet?(%__MODULE__{stamp: stamp}) when is_map(stamp),
+    do: get_in(stamp, ["quiet", "quiet"]) == true
+
+  def quiet?(%__MODULE__{}), do: false
+
+  @doc "A `good` verdict over a baseline not yet optimised: the claim is provisional."
+  def provisional?(%__MODULE__{verdict: "good", baseline: b}), do: b != "optimized"
+  def provisional?(%__MODULE__{}), do: false
+
+  @doc "The report ran a size sweep (its crossover may still be none, `nil`)."
+  def crossover?(%__MODULE__{crossover: c}), do: c != :unmeasured
+
+  @doc "The day the report was measured, or `?`."
+  def measured_on(%__MODULE__{measured_at: at}) when is_binary(at), do: String.slice(at, 0, 10)
+  def measured_on(%__MODULE__{}), do: "?"
+
+  @doc "Verdicts in the order the tables list them: `good` first, an unknown one last."
+  def rank(%__MODULE__{verdict: v}), do: rank(v)
+  def rank("good"), do: 0
+  def rank("marginal"), do: 1
+  def rank("poor"), do: 2
+  def rank("out_of_scope"), do: 3
+  def rank(_other), do: 4
+
+  @doc "The verdict's mark in a table."
+  def mark(%__MODULE__{verdict: v}), do: mark(v)
+  def mark("good"), do: "✅"
+  def mark("marginal"), do: "🟡"
+  def mark("poor"), do: "❌"
+  def mark("out_of_scope"), do: "⛔"
+  def mark(other), do: "#{other || "?"}"
+
+  @doc "The verdict in words."
+  def label(%__MODULE__{verdict: v}), do: label(v)
+  def label("out_of_scope"), do: "out of scope"
+  def label(other), do: "#{other || "?"}"
+end

dev/defk/examples/scoreboard.ex

@@ -1,9 +1,11 @@
 defmodule Defk.Examples.Scoreboard do
   @moduledoc false
   # The `--bench` scoreboard: what `examples/run_all.exs` printed, over the
-  # first `DEFK_RESULT` line of each example. Also the check-only
+  # first report (Defk.Examples.Result) of each example. Also the check-only
   # summary.
 
+  alias Defk.Examples.Result
+
   @doc "The check-only summary: one line per outcome."
   def print_check(outcomes) do
     IO.puts("\n" <> pad("example", 34) <> pad("result", 20) <> pad("secs", 9) <> "note")
@@ -33,13 +35,13 @@ defmodule Defk.Examples.Scoreboard do
 
     for {o, r} <- Enum.zip(outcomes, results) do
       IO.puts(
-        pad(r["example"], 26) <>
-          pad(r["expected"], 14) <>
-          pad(r["verdict"], 14) <>
+        pad(r.example, 26) <>
+          pad(r.expected, 14) <>
+          pad(r.verdict, 14) <>
           pad(speedup(r), 10) <>
-          pad(num(r["elixir_ms"]), 11) <>
-          pad(num(r["defk_ms"]), 10) <>
-          pad(r["baseline"] || "?", 12) <> if(o.status == :ok, do: "yes", else: "NO")
+          pad(num(r.elixir_ms), 11) <>
+          pad(num(r.defk_ms), 10) <>
+          pad(r.baseline || "?", 12) <> if(o.status == :ok, do: "yes", else: "NO")
       )
 
       case extra_line(r) do
@@ -60,85 +62,70 @@ defmodule Defk.Examples.Scoreboard do
   # the first report's
   defp print_cache(outcomes) do
     reports = for o <- outcomes, r <- o.results, do: r
-    reused = Enum.filter(reports, &(&1["cache"] == "reused"))
-    unstored = Enum.filter(reports, &(&1["cache"] == "not_stored"))
+    reused = Enum.filter(reports, &Result.reused?/1)
+    unstored = Enum.filter(reports, &Result.unstored?/1)
 
     if reused != [] do
       IO.puts(
         "#{length(reused)} report(s) reused a cached result (inputs unchanged; --fresh measures them): " <>
-          Enum.map_join(reused, ", ", &"#{&1["example"]} (#{measured_on(&1)})")
+          Enum.map_join(reused, ", ", &"#{&1.example} (#{Result.measured_on(&1)})")
       )
     end
 
     if unstored != [] do
       IO.puts(
         "#{length(unstored)} report(s) not cached, the machine was not quiet: " <>
-          Enum.map_join(unstored, ", ", & &1["example"])
+          Enum.map_join(unstored, ", ", & &1.example)
       )
     end
   end
 
-  defp measured_on(r), do: String.slice(r["measured_at"] || "?", 0, 10)
-
-  # the first result line, or a stand-in for an example that crashed
+  # the first report, or a stand-in for an example that crashed
   defp row(%{results: [first | _], status: status}) when status != :crashed, do: first
-
-  defp row(%{name: name}),
-    do: %{"example" => name, "expected" => "?", "verdict" => "crashed", "correct" => false}
+  defp row(%{name: name}), do: Result.crashed(name)
 
   defp print_provisional(results) do
-    provisional =
-      Enum.filter(results, &(&1["verdict"] == "good" and &1["baseline"] != "optimized"))
+    provisional = Enum.filter(results, &Result.provisional?/1)
 
     if provisional != [] do
       IO.puts(
         "#{length(provisional)} 'good' claim(s) are provisional: the Elixir baseline is not optimised yet " <>
-          "(#{Enum.map_join(provisional, ", ", & &1["example"])})"
+          "(#{Enum.map_join(provisional, ", ", & &1.example)})"
       )
     end
   end
 
   defp print_skipped(outcomes) do
-    skipped =
-      for o <- outcomes, r <- o.results, r["correct"] == "skipped", do: r
+    skipped = for o <- outcomes, r <- o.results, Result.skipped?(r), do: r
 
     if skipped != [] do
       IO.puts(
         "#{length(skipped)} check(s) skipped by their examples:\n" <>
-          Enum.map_join(skipped, "\n", &"  #{&1["example"]}: #{&1["reason"]}")
+          Enum.map_join(skipped, "\n", &"  #{&1.example}: #{&1.reason}")
       )
     end
   end
 
-  # second line per example, only for the keys a script reports
+  # second line per example, only for what a script reports
   defp extra_line(r) do
     parts =
       Enum.filter(
         [
-          r["realistic_size"] && "realistic size #{r["realistic_size"]}",
-          r["native_speedup"] &&
-            "vs #{r["native_label"]}: defk #{r["native_speedup"]}x (native #{r["native_ms"]} ms)",
-          r["native_ms"] && is_nil(r["native_speedup"]) &&
-            "native #{r["native_ms"]} ms [#{r["native_label"]}]",
+          r.realistic_size && "realistic size #{r.realistic_size}",
+          r.native_speedup &&
+            "vs #{r.native_label}: defk #{r.native_speedup}x (native #{r.native_ms} ms)",
+          r.native_ms && is_nil(r.native_speedup) &&
+            "native #{r.native_ms} ms [#{r.native_label}]",
           gaps(r) != [] && "tick gap ms: " <> Enum.join(gaps(r), "; "),
-          Map.has_key?(r, "crossover") && "crossover #{r["crossover"] || "none"}"
+          Result.crossover?(r) && "crossover #{r.crossover || "none"}"
         ],
         & &1
       )
 
     if parts == [], do: "", else: "    " <> Enum.join(parts, " | ")
   end
 
-  defp gaps(r) do
-    for {k, label} <- [
-          {"elixir_gap_ms", "elixir"},
-          {"defk_gap_ms", "defk"},
-          {"native_gap_ms", "native"}
-        ],
-        %{"max" => m, "p99" => p} <- [r[k]] do
-      "#{label} max #{m} / p99 #{p}"
-    end
-  end
+  defp gaps(r), do: for({side, m, p} <- r.gaps, do: "#{side} max #{m} / p99 #{p}")
 
   # The markdown table for the scoreboard pages (examples/README.md,
   # docs/verdict.md): the verdict mark first, rows sorted by verdict and
@@ -150,41 +137,32 @@ defmodule Defk.Examples.Scoreboard do
         "Tick gap ms, max / p99 | Crossover |\n|---|---|---|---|---|---|---|---|---|"
     )
 
-    for r <- Enum.sort_by(results, &{verdict_rank(&1["verdict"]), -(&1["speedup"] || 0)}) do
+    for r <- Enum.sort_by(results, &{Result.rank(&1), -(&1.speedup || 0)}) do
       cells = [
-        verdict_mark(r["verdict"]),
-        if(r["cache"] == "reused",
-          do: "#{r["example"]} (reused, measured #{measured_on(r)})",
-          else: r["example"]
+        verdict_mark(r),
+        if(Result.reused?(r),
+          do: "#{r.example} (reused, measured #{Result.measured_on(r)})",
+          else: r.example
         ),
-        cell(r["realistic_size"]),
+        cell(r.realistic_size),
         speedup(r),
-        cell(r["elixir_ms"]),
-        cell(r["defk_ms"]),
+        cell(r.elixir_ms),
+        cell(r.defk_ms),
         native_cell(r),
         gap_cell(r),
-        if(Map.has_key?(r, "crossover"), do: r["crossover"] || "none", else: "-")
+        if(Result.crossover?(r), do: r.crossover || "none", else: "-")
       ]
 
       IO.puts("| " <> Enum.join(cells, " | ") <> " |")
     end
   end
 
-  defp verdict_rank("good"), do: 0
-  defp verdict_rank("marginal"), do: 1
-  defp verdict_rank("poor"), do: 2
-  defp verdict_rank("out_of_scope"), do: 3
-  defp verdict_rank(_), do: 4
-
-  defp verdict_mark("good"), do: "✅ good"
-  defp verdict_mark("marginal"), do: "🟡 marginal"
-  defp verdict_mark("poor"), do: "❌ poor"
-  defp verdict_mark("out_of_scope"), do: "⛔ out of scope"
-  defp verdict_mark(other), do: "#{other || "?"}"
+  # a known verdict with its mark; another word as it is
+  defp verdict_mark(r),
+    do: if(Result.rank(r) < 4, do: "#{Result.mark(r)} #{Result.label(r)}", else: Result.label(r))
 
-  defp native_cell(%{"native_speedup" => nil}), do: "-"
-  defp native_cell(%{"native_speedup" => x, "native_label" => l}), do: "#{x}x (#{l})"
-  defp native_cell(_), do: "-"
+  defp native_cell(%Result{native_speedup: nil}), do: "-"
+  defp native_cell(%Result{native_speedup: x, native_label: l}), do: "#{x}x (#{l})"
 
   defp gap_cell(r) do
     case gaps(r) do
@@ -193,9 +171,8 @@ defmodule Defk.Examples.Scoreboard do
     end
   end
 
-  defp speedup(%{"speedup" => nil}), do: "-"
-  defp speedup(%{"speedup" => s}), do: "#{s}x"
-  defp speedup(_), do: "-"
+  defp speedup(%Result{speedup: nil}), do: "-"
+  defp speedup(%Result{speedup: s}), do: "#{s}x"
 
   defp cell(nil), do: "-"
   defp cell(x), do: to_string(x)