diff --git a/dev/defk/examples.ex b/dev/defk/examples.ex index 9585197f..6a003ba7 100644 --- a/dev/defk/examples.ex +++ b/dev/defk/examples.ex @@ -19,7 +19,7 @@ defmodule Defk.Examples do name: String.t(), status: :ok | :incorrect | :unexpected_verdict | :crashed | :timeout, secs: float(), - results: [map()], + results: [Defk.Examples.Result.t()], out: String.t(), err: String.t(), note: String.t() @@ -277,13 +277,13 @@ defmodule Defk.Examples do defp verdict_of(mode, _status, [first | _] = results) do cond do - Enum.any?(results, &(&1["correct"] not in [true, "skipped"])) -> + not Enum.all?(results, &Defk.Examples.Result.correct?/1) -> %{status: :incorrect, note: "an agreement check returned #{inspect(wrong(results))}"} - mode == :bench and first["verdict"] != first["expected"] -> + mode == :bench and first.verdict != first.expected -> %{ status: :unexpected_verdict, - note: "measured #{first["verdict"]}, expected #{first["expected"]}" + note: "measured #{first.verdict}, expected #{first.expected}" } true -> @@ -297,7 +297,7 @@ defmodule Defk.Examples do defp noun(:bench, _), do: "reports" defp wrong(results), - do: Enum.find(results, &(&1["correct"] not in [true, "skipped"]))["correct"] + do: Enum.find(results, &(not Defk.Examples.Result.correct?(&1))).correct defp judge_app(name, ran, secs) do base = %{name: name, secs: secs, err: "", results: []} @@ -322,11 +322,11 @@ defmodule Defk.Examples do end end - @doc "Every `DEFK_RESULT` line of an example's stdout, decoded, in order." + @doc "Every `DEFK_RESULT` line of an example's stdout as a `Defk.Examples.Result`, in order." def parse_results(out) do for [_, json] <- Regex.scan(~r/^DEFK_RESULT (.*)$/m, out), {:ok, r} <- [JSON.decode(json)], - do: r + do: Defk.Examples.Result.decode(r) end # ---- running a process ---------------------------------------------- diff --git a/dev/defk/examples/result.ex b/dev/defk/examples/result.ex new file mode 100644 index 00000000..751617f7 --- /dev/null +++ b/dev/defk/examples/result.ex @@ -0,0 +1,117 @@ +defmodule Defk.Examples.Result do + @moduledoc false + # One report of an example: what one `DEFK_RESULT` line of + # `Defk.Bench.report/1` (or its check-only form) says, decoded once. The + # runner, the scoreboard, the stored scoreboard and the README tables read + # a report through this record and its questions, never by key. The + # line's JSON map is kept in `raw`: the stored scoreboard holds the lines + # as the harness wrote them (Defk.Examples.Snapshot). + + defstruct raw: %{}, + example: nil, + expected: nil, + verdict: nil, + baseline: nil, + correct: nil, + reason: nil, + check_only: false, + speedup: nil, + elixir_ms: nil, + defk_ms: nil, + native_ms: nil, + native_label: nil, + native_speedup: nil, + realistic_size: nil, + crossover: :unmeasured, + gaps: [], + cache: nil, + measured_at: nil, + stamp: nil + + @type t :: %__MODULE__{} + + @fields ~w(example expected verdict baseline correct reason speedup elixir_ms defk_ms + native_ms native_label native_speedup realistic_size cache measured_at stamp) + + @doc "The record of one decoded `DEFK_RESULT` line (a map with string keys)." + def decode(%__MODULE__{} = r), do: r + + def decode(map) when is_map(map) do + fields = for k <- @fields, Map.has_key?(map, k), do: {String.to_existing_atom(k), map[k]} + + struct( + __MODULE__, + fields ++ + [ + raw: map, + check_only: map["check_only"] == true, + crossover: if(Map.has_key?(map, "crossover"), do: map["crossover"], else: :unmeasured), + gaps: gaps(map) + ] + ) + end + + @doc "The stand-in for an example that crashed before its first line." + def crashed(name), + do: %__MODULE__{example: name, expected: "?", verdict: "crashed", correct: false} + + # the tick gaps a report measured, `{side, max, p99}` for each side it has + defp gaps(map) do + for {k, side} <- [ + {"elixir_gap_ms", "elixir"}, + {"defk_gap_ms", "defk"}, + {"native_gap_ms", "native"} + ], + %{"max" => m, "p99" => p} <- [map[k]], + do: {side, m, p} + end + + @doc "The agreement check passed, or the example skipped it with a reason." + def correct?(%__MODULE__{correct: c}), do: c in [true, "skipped"] + + def skipped?(%__MODULE__{correct: c}), do: c == "skipped" + + @doc "The report's result came from the cache, not from a measurement." + def reused?(%__MODULE__{cache: c}), do: c == "reused" + + @doc "The report was measured but not stored: the machine was not quiet." + def unstored?(%__MODULE__{cache: c}), do: c == "not_stored" + + @doc "The report's stamp says the machine was quiet before and after it." + def quiet?(%__MODULE__{stamp: stamp}) when is_map(stamp), + do: get_in(stamp, ["quiet", "quiet"]) == true + + def quiet?(%__MODULE__{}), do: false + + @doc "A `good` verdict over a baseline not yet optimised: the claim is provisional." + def provisional?(%__MODULE__{verdict: "good", baseline: b}), do: b != "optimized" + def provisional?(%__MODULE__{}), do: false + + @doc "The report ran a size sweep (its crossover may still be none, `nil`)." + def crossover?(%__MODULE__{crossover: c}), do: c != :unmeasured + + @doc "The day the report was measured, or `?`." + def measured_on(%__MODULE__{measured_at: at}) when is_binary(at), do: String.slice(at, 0, 10) + def measured_on(%__MODULE__{}), do: "?" + + @doc "Verdicts in the order the tables list them: `good` first, an unknown one last." + def rank(%__MODULE__{verdict: v}), do: rank(v) + def rank("good"), do: 0 + def rank("marginal"), do: 1 + def rank("poor"), do: 2 + def rank("out_of_scope"), do: 3 + def rank(_other), do: 4 + + @doc "The verdict's mark in a table." + def mark(%__MODULE__{verdict: v}), do: mark(v) + def mark("good"), do: "✅" + def mark("marginal"), do: "🟡" + def mark("poor"), do: "❌" + def mark("out_of_scope"), do: "⛔" + def mark(other), do: "#{other || "?"}" + + @doc "The verdict in words." + def label(%__MODULE__{verdict: v}), do: label(v) + def label("out_of_scope"), do: "out of scope" + def label(other), do: "#{other || "?"}" +end diff --git a/dev/defk/examples/scoreboard.ex b/dev/defk/examples/scoreboard.ex index de6cdaa8..eb3625df 100644 --- a/dev/defk/examples/scoreboard.ex +++ b/dev/defk/examples/scoreboard.ex @@ -1,9 +1,11 @@ defmodule Defk.Examples.Scoreboard do @moduledoc false # The `--bench` scoreboard: what `examples/run_all.exs` printed, over the - # first `DEFK_RESULT` line of each example. Also the check-only + # first report (Defk.Examples.Result) of each example. Also the check-only # summary. + alias Defk.Examples.Result + @doc "The check-only summary: one line per outcome." def print_check(outcomes) do IO.puts("\n" <> pad("example", 34) <> pad("result", 20) <> pad("secs", 9) <> "note") @@ -33,13 +35,13 @@ defmodule Defk.Examples.Scoreboard do for {o, r} <- Enum.zip(outcomes, results) do IO.puts( - pad(r["example"], 26) <> - pad(r["expected"], 14) <> - pad(r["verdict"], 14) <> + pad(r.example, 26) <> + pad(r.expected, 14) <> + pad(r.verdict, 14) <> pad(speedup(r), 10) <> - pad(num(r["elixir_ms"]), 11) <> - pad(num(r["defk_ms"]), 10) <> - pad(r["baseline"] || "?", 12) <> if(o.status == :ok, do: "yes", else: "NO") + pad(num(r.elixir_ms), 11) <> + pad(num(r.defk_ms), 10) <> + pad(r.baseline || "?", 12) <> if(o.status == :ok, do: "yes", else: "NO") ) case extra_line(r) do @@ -60,68 +62,62 @@ defmodule Defk.Examples.Scoreboard do # the first report's defp print_cache(outcomes) do reports = for o <- outcomes, r <- o.results, do: r - reused = Enum.filter(reports, &(&1["cache"] == "reused")) - unstored = Enum.filter(reports, &(&1["cache"] == "not_stored")) + reused = Enum.filter(reports, &Result.reused?/1) + unstored = Enum.filter(reports, &Result.unstored?/1) if reused != [] do IO.puts( "#{length(reused)} report(s) reused a cached result (inputs unchanged; --fresh measures them): " <> - Enum.map_join(reused, ", ", &"#{&1["example"]} (#{measured_on(&1)})") + Enum.map_join(reused, ", ", &"#{&1.example} (#{Result.measured_on(&1)})") ) end if unstored != [] do IO.puts( "#{length(unstored)} report(s) not cached, the machine was not quiet: " <> - Enum.map_join(unstored, ", ", & &1["example"]) + Enum.map_join(unstored, ", ", & &1.example) ) end end - defp measured_on(r), do: String.slice(r["measured_at"] || "?", 0, 10) - - # the first result line, or a stand-in for an example that crashed + # the first report, or a stand-in for an example that crashed defp row(%{results: [first | _], status: status}) when status != :crashed, do: first - - defp row(%{name: name}), - do: %{"example" => name, "expected" => "?", "verdict" => "crashed", "correct" => false} + defp row(%{name: name}), do: Result.crashed(name) defp print_provisional(results) do - provisional = - Enum.filter(results, &(&1["verdict"] == "good" and &1["baseline"] != "optimized")) + provisional = Enum.filter(results, &Result.provisional?/1) if provisional != [] do IO.puts( "#{length(provisional)} 'good' claim(s) are provisional: the Elixir baseline is not optimised yet " <> - "(#{Enum.map_join(provisional, ", ", & &1["example"])})" + "(#{Enum.map_join(provisional, ", ", & &1.example)})" ) end end defp print_skipped(outcomes) do - skipped = - for o <- outcomes, r <- o.results, r["correct"] == "skipped", do: r + skipped = for o <- outcomes, r <- o.results, Result.skipped?(r), do: r if skipped != [] do IO.puts( "#{length(skipped)} check(s) skipped by their examples:\n" <> - Enum.map_join(skipped, "\n", &" #{&1["example"]}: #{&1["reason"]}") + Enum.map_join(skipped, "\n", &" #{&1.example}: #{&1.reason}") ) end end - # second line per example, only for the keys a script reports + # second line per example, only for what a script reports defp extra_line(r) do parts = Enum.filter( [ - r["realistic_size"] && "realistic size #{r["realistic_size"]}", - r["native_speedup"] && - "vs #{r["native_label"]}: defk #{r["native_speedup"]}x (native #{r["native_ms"]} ms)", - r["native_ms"] && is_nil(r["native_speedup"]) && - "native #{r["native_ms"]} ms [#{r["native_label"]}]", + r.realistic_size && "realistic size #{r.realistic_size}", + r.native_speedup && + "vs #{r.native_label}: defk #{r.native_speedup}x (native #{r.native_ms} ms)", + r.native_ms && is_nil(r.native_speedup) && + "native #{r.native_ms} ms [#{r.native_label}]", gaps(r) != [] && "tick gap ms: " <> Enum.join(gaps(r), "; "), - Map.has_key?(r, "crossover") && "crossover #{r["crossover"] || "none"}" + Result.crossover?(r) && "crossover #{r.crossover || "none"}" ], & &1 ) @@ -129,16 +125,7 @@ defmodule Defk.Examples.Scoreboard do if parts == [], do: "", else: " " <> Enum.join(parts, " | ") end - defp gaps(r) do - for {k, label} <- [ - {"elixir_gap_ms", "elixir"}, - {"defk_gap_ms", "defk"}, - {"native_gap_ms", "native"} - ], - %{"max" => m, "p99" => p} <- [r[k]] do - "#{label} max #{m} / p99 #{p}" - end - end + defp gaps(r), do: for({side, m, p} <- r.gaps, do: "#{side} max #{m} / p99 #{p}") # The markdown table for the scoreboard pages (examples/README.md, # docs/verdict.md): the verdict mark first, rows sorted by verdict and @@ -150,41 +137,32 @@ defmodule Defk.Examples.Scoreboard do "Tick gap ms, max / p99 | Crossover |\n|---|---|---|---|---|---|---|---|---|" ) - for r <- Enum.sort_by(results, &{verdict_rank(&1["verdict"]), -(&1["speedup"] || 0)}) do + for r <- Enum.sort_by(results, &{Result.rank(&1), -(&1.speedup || 0)}) do cells = [ - verdict_mark(r["verdict"]), - if(r["cache"] == "reused", - do: "#{r["example"]} (reused, measured #{measured_on(r)})", - else: r["example"] + verdict_mark(r), + if(Result.reused?(r), + do: "#{r.example} (reused, measured #{Result.measured_on(r)})", + else: r.example ), - cell(r["realistic_size"]), + cell(r.realistic_size), speedup(r), - cell(r["elixir_ms"]), - cell(r["defk_ms"]), + cell(r.elixir_ms), + cell(r.defk_ms), native_cell(r), gap_cell(r), - if(Map.has_key?(r, "crossover"), do: r["crossover"] || "none", else: "-") + if(Result.crossover?(r), do: r.crossover || "none", else: "-") ] IO.puts("| " <> Enum.join(cells, " | ") <> " |") end end - defp verdict_rank("good"), do: 0 - defp verdict_rank("marginal"), do: 1 - defp verdict_rank("poor"), do: 2 - defp verdict_rank("out_of_scope"), do: 3 - defp verdict_rank(_), do: 4 - - defp verdict_mark("good"), do: "✅ good" - defp verdict_mark("marginal"), do: "🟡 marginal" - defp verdict_mark("poor"), do: "❌ poor" - defp verdict_mark("out_of_scope"), do: "⛔ out of scope" - defp verdict_mark(other), do: "#{other || "?"}" + # a known verdict with its mark; another word as it is + defp verdict_mark(r), + do: if(Result.rank(r) < 4, do: "#{Result.mark(r)} #{Result.label(r)}", else: Result.label(r)) - defp native_cell(%{"native_speedup" => nil}), do: "-" - defp native_cell(%{"native_speedup" => x, "native_label" => l}), do: "#{x}x (#{l})" - defp native_cell(_), do: "-" + defp native_cell(%Result{native_speedup: nil}), do: "-" + defp native_cell(%Result{native_speedup: x, native_label: l}), do: "#{x}x (#{l})" defp gap_cell(r) do case gaps(r) do @@ -193,9 +171,8 @@ defmodule Defk.Examples.Scoreboard do end end - defp speedup(%{"speedup" => nil}), do: "-" - defp speedup(%{"speedup" => s}), do: "#{s}x" - defp speedup(_), do: "-" + defp speedup(%Result{speedup: nil}), do: "-" + defp speedup(%Result{speedup: s}), do: "#{s}x" defp cell(nil), do: "-" defp cell(x), do: to_string(x) diff --git a/dev/defk/examples/snapshot.ex b/dev/defk/examples/snapshot.ex index 84781ad1..39b3185e 100644 --- a/dev/defk/examples/snapshot.ex +++ b/dev/defk/examples/snapshot.ex @@ -11,6 +11,7 @@ defmodule Defk.Examples.Snapshot do # generated: never edit it by hand, measure again. alias Defk.Bench.Cache + alias Defk.Examples.Result # 3: quiet is per row, from the reports' stamps; the run keeps the limit, # not the load average @@ -43,7 +44,7 @@ defmodule Defk.Examples.Snapshot do do: %{ "name" => o.name, "status" => to_string(o.status), - "results" => o.results, + "results" => Enum.map(o.results, &Result.decode(&1).raw), "quiet" => quiet?(o.results), "run" => run } @@ -52,7 +53,7 @@ defmodule Defk.Examples.Snapshot do end @doc "Whether every report of a row passed the quiet test." - def quiet?(results), do: Enum.all?(results, &(get_in(&1, ["stamp", "quiet", "quiet"]) == true)) + def quiet?(results), do: Enum.all?(results, &Result.quiet?(Result.decode(&1))) @doc """ `{:ok, snapshot}`: `old` with the rows of `new` in place of its rows of @@ -91,7 +92,7 @@ defmodule Defk.Examples.Snapshot do def stale(snapshot, digests) do for %{"name" => name, "results" => results} <- snapshot["examples"], Map.has_key?(digests, name) do - stamps = Enum.map(results, & &1["stamp"]) + stamps = Enum.map(results, &Result.decode(&1).stamp) if stamps == [] or nil in stamps do %{name: name, status: :unstamped} diff --git a/dev/defk/examples/tables.ex b/dev/defk/examples/tables.ex index 3988accf..44cae26c 100644 --- a/dev/defk/examples/tables.ex +++ b/dev/defk/examples/tables.ex @@ -11,6 +11,8 @@ defmodule Defk.Examples.Tables do # The numbers come only from the snapshot, so a number a report measures is # never typed into a table. + alias Defk.Examples.Result + @text "examples/_tables.exs" @targets [{"examples/README.md", "scoreboard"}, {"README.md", "summary"}] @@ -47,12 +49,16 @@ defmodule Defk.Examples.Tables do defp rows(snapshot) do for %{"name" => name, "results" => [first | _]} <- snapshot["examples"], into: %{}, - do: {name, first} + do: {name, Result.decode(first)} end # report name => report; an example name no report has => its first report defp index(snapshot) do - reports = for e <- snapshot["examples"], r <- e["results"], into: %{}, do: {r["example"], r} + reports = + for e <- snapshot["examples"], r <- Enum.map(e["results"], &Result.decode/1), into: %{} do + {r.example, r} + end + Map.merge(rows(snapshot), reports) end @@ -84,7 +90,7 @@ defmodule Defk.Examples.Tables do {line, p2} = fill(t.line, name, index) cells = [ - mark(r["verdict"]), + Result.mark(r), "[`#{name}`](#{name}.exs)" <> note(r, runs[name], newest, loaded == :some and quiet[name] != true), t.job, @@ -93,15 +99,15 @@ defmodule Defk.Examples.Tables do line ] - {{{rank(r["verdict"]), 0, -(r["speedup"] || 0)}, cells}, acc ++ p1 ++ p2} + {{{Result.rank(r), 0, -(r.speedup || 0)}, cells}, acc ++ p1 ++ p2} end) # an app's row comes after the measured rows of its verdict apps = for a <- text.apps do - {{rank(a.verdict), 1, 0}, + {{Result.rank(a.verdict), 1, 0}, [ - mark(a.verdict), + Result.mark(a.verdict), "[`#{a.name}/`](#{a.name})", a.job, a.value, @@ -179,7 +185,7 @@ defmodule Defk.Examples.Tables do # its report's stamp; a row not measured quietly among quiet ones says # provisional. defp note(r, run, newest, provisional?) do - earlier? = run != newest or r["cache"] == "reused" + earlier? = run != newest or Result.reused?(r) parts = if(earlier?, do: ["measured " <> measured(r, run)], else: []) ++ @@ -188,19 +194,19 @@ defmodule Defk.Examples.Tables do if parts == [], do: "", else: " (" <> Enum.join(parts, ", ") <> ")" end - defp measured(%{"stamp" => %{"measured_at" => at} = stamp}, _run), + defp measured(%Result{stamp: %{"measured_at" => at} = stamp}, _run), do: String.slice(at, 0, 10) <> at_commit(stamp["commit"]) - defp measured(%{"cache" => "reused", "measured_at" => at}, _run), do: String.slice(at, 0, 10) + defp measured(%Result{cache: "reused", measured_at: at}, _run), do: String.slice(at, 0, 10) defp measured(_r, run), do: day(run) <> at_commit(run["commit"]) defp at_commit(nil), do: "" defp at_commit(commit), do: " at `#{commit}`" - defp speedup_cell(%{"speedup" => s, "verdict" => "good"}) when is_number(s), + defp speedup_cell(%Result{speedup: s, verdict: "good"}) when is_number(s), do: "**#{times(s)}**" - defp speedup_cell(%{"speedup" => s}) when is_number(s), do: times(s) + defp speedup_cell(%Result{speedup: s}) when is_number(s), do: times(s) defp speedup_cell(_), do: "–" # ---- README.md ------------------------------------------------------ @@ -210,7 +216,7 @@ defmodule Defk.Examples.Tables do Enum.map_reduce(text.summary, [], fn g, acc -> {verdict, value, p1} = group(g, rows, text) {job, p2} = fill(g.text, nil, index) - {"| #{mark(verdict)} | #{job} | #{value} |", acc ++ p1 ++ p2} + {"| #{Result.mark(verdict)} | #{job} | #{value} |", acc ++ p1 ++ p2} end) block = @@ -234,7 +240,7 @@ defmodule Defk.Examples.Tables do [] -> found = Enum.map(names, &rows[&1]) - case Enum.uniq(Enum.map(found, & &1["verdict"])) do + case Enum.uniq(Enum.map(found, & &1.verdict)) do [verdict] -> {verdict, Map.get_lazy(g, :value, fn -> range(found) end), []} @@ -251,7 +257,7 @@ defmodule Defk.Examples.Tables do end defp range(reports) do - case reports |> Enum.map(& &1["speedup"]) |> Enum.filter(&is_number/1) |> Enum.sort() do + case reports |> Enum.map(& &1.speedup) |> Enum.filter(&is_number/1) |> Enum.sort() do [] -> "–" @@ -289,19 +295,19 @@ defmodule Defk.Examples.Tables do defp value(_field, nil), do: {:error, "names no report"} defp value("crossover", r) do - case Map.fetch(r, "crossover") do - {:ok, nil} -> {:ok, "none"} - {:ok, c} -> {:ok, to_string(c)} - :error -> {:error, "but report #{r["example"]} has no sweep"} + cond do + not Result.crossover?(r) -> {:error, "but report #{r.example} has no sweep"} + is_nil(r.crossover) -> {:ok, "none"} + true -> {:ok, to_string(r.crossover)} end end defp value(field, r) do - key = if field == "native", do: "native_speedup", else: "speedup" + key = if field == "native", do: :native_speedup, else: :speedup - case r[key] do + case Map.fetch!(r, key) do x when is_number(x) -> {:ok, times(x)} - _ -> {:error, "but report #{r["example"]} recorded no #{key}"} + _ -> {:error, "but report #{r.example} recorded no #{key}"} end end @@ -312,16 +318,4 @@ defmodule Defk.Examples.Tables do defp number(x) when x >= 10, do: Integer.to_string(round(x)) defp number(x) when x >= 1, do: :erlang.float_to_binary(x * 1.0, decimals: 1) defp number(x), do: :erlang.float_to_binary(x * 1.0, decimals: 2) - - defp rank("good"), do: 0 - defp rank("marginal"), do: 1 - defp rank("poor"), do: 2 - defp rank("out_of_scope"), do: 3 - defp rank(_), do: 4 - - defp mark("good"), do: "✅" - defp mark("marginal"), do: "🟡" - defp mark("poor"), do: "❌" - defp mark("out_of_scope"), do: "⛔" - defp mark(other), do: "#{other || "?"}" end diff --git a/dev/mix/tasks/defk.examples.ex b/dev/mix/tasks/defk.examples.ex index ac7b0aa7..0a249de9 100644 --- a/dev/mix/tasks/defk.examples.ex +++ b/dev/mix/tasks/defk.examples.ex @@ -259,7 +259,7 @@ defmodule Mix.Tasks.Defk.Examples do # a reused result must be visible where the example's line is read defp reused_note(o) do - case Enum.count(o.results, &(&1["cache"] == "reused")) do + case Enum.count(o.results, &Defk.Examples.Result.reused?/1) do 0 -> "" n -> " CACHED: #{n} of #{length(o.results)} report(s) reused, not measured" end diff --git a/docs/agents/compiler-map.md b/docs/agents/compiler-map.md index 334e9921..0792bd18 100644 --- a/docs/agents/compiler-map.md +++ b/docs/agents/compiler-map.md @@ -126,7 +126,11 @@ of rows the lowering declares as it emits; defk-ch4), not from the C text. test (`wait_quiet/0`, `other_cpu/0`, `quiet?/1`); rules in `examples/README.md` and AGENTS.md. The result cache of `mix defk.examples --bench` is the runner's adapter, `dev/defk/bench/cache.ex`, - loaded into each example's process with `elixir -r` (ADR-0011). + loaded into each example's process with `elixir -r` (ADR-0011). The + runner, the scoreboard, the stored scoreboard and the README tables read + each report through `Defk.Examples.Result` (`dev/defk/examples/result.ex`): + one `DEFK_RESULT` line decoded once, then asked (`reused?`, `quiet?`, + `provisional?`, `rank`, `mark`), never indexed by key. - `lib/defk/report.ex`, `lib/defk/bits.ex`, `lib/defk/test.ex` (`assert_agrees`): user-facing helpers. - `lib/defk/deprecations.ex`: the deprecation table and the renames it diff --git a/lib/defk/bench.ex b/lib/defk/bench.ex index 52942efc..722eadca 100644 --- a/lib/defk/bench.ex +++ b/lib/defk/bench.ex @@ -314,7 +314,7 @@ defmodule Defk.Bench do c -> IO.puts(" correct #{c} [check only, not timed]") end - IO.puts("\nDEFK_RESULT " <> JSON.encode!(result)) + put_result(result) # the returned map claims the expected verdict: scripts assert their own # verdicts after `report/1`, and nothing was measured to contradict them %{result | verdict: expected} @@ -709,9 +709,13 @@ defmodule Defk.Bench do print_gaps(r) print_sweep(r) if cache_line, do: IO.puts(cache_line) - IO.puts("\nDEFK_RESULT " <> JSON.encode!(r)) + put_result(r) end + # the one line the catalog's runner reads, after a report's table + # (Defk.Examples.Result decodes it); a check-only report prints it too + defp put_result(r), do: IO.puts("\nDEFK_RESULT " <> JSON.encode!(r)) + defp native_line(r) do vs = if r.native_speedup, do: " (defk #{r.native_speedup}x vs native)", else: "" " native #{r.native_ms} ms [#{r.native_label}]#{vs}" diff --git a/test/examples_result_test.exs b/test/examples_result_test.exs new file mode 100644 index 00000000..7ed2380d --- /dev/null +++ b/test/examples_result_test.exs @@ -0,0 +1,64 @@ +defmodule Defk.ExamplesResultTest do + # One report of an example as the runner reads it (Defk.Examples.Result): + # decoded once from its DEFK_RESULT line, then asked, never indexed by key. + use ExUnit.Case, async: true + + alias Defk.Examples.Result + + @line %{ + "example" => "crc16", + "expected" => "good", + "verdict" => "good", + "baseline" => "first_pass", + "correct" => true, + "speedup" => 12.5, + "elixir_ms" => 10.0, + "defk_ms" => 0.8, + "cache" => "reused", + "measured_at" => "2026-10-19T20:00:00Z", + "stamp" => %{"quiet" => %{"quiet" => true}}, + "defk_gap_ms" => %{"max" => 1.5, "p99" => 0.2} + } + + test "decode keeps the line and answers its questions" do + r = Result.decode(@line) + assert r.raw == @line + assert r.example == "crc16" and r.speedup == 12.5 + assert Result.correct?(r) + refute Result.skipped?(r) + assert Result.reused?(r) + refute Result.unstored?(r) + assert Result.quiet?(r) + assert Result.provisional?(r) + assert Result.measured_on(r) == "2026-10-19" + assert r.gaps == [{"defk", 1.5, 0.2}] + assert Result.decode(r) == r + end + + test "a crossover is unmeasured, none, or a size" do + refute Result.crossover?(Result.decode(@line)) + assert %{crossover: nil} = r = Result.decode(Map.put(@line, "crossover", nil)) + assert Result.crossover?(r) + assert Result.decode(Map.put(@line, "crossover", 4096)).crossover == 4096 + end + + test "a skipped check is correct; a failed one is not; an unquiet or unstamped report is not quiet" do + assert Result.skipped?(Result.decode(%{"correct" => "skipped"})) + assert Result.correct?(Result.decode(%{"correct" => "skipped"})) + refute Result.correct?(Result.decode(%{"correct" => false})) + refute Result.quiet?(Result.decode(%{"stamp" => %{"quiet" => %{"quiet" => false}}})) + refute Result.quiet?(Result.decode(%{})) + refute Result.provisional?(Result.decode(%{"verdict" => "good", "baseline" => "optimized"})) + refute Result.provisional?(Result.decode(%{"verdict" => "poor"})) + end + + test "the verdicts rank good first and have a mark and a label" do + verdicts = ~w(good marginal poor out_of_scope crashed) + assert Enum.map(verdicts, &Result.rank/1) == [0, 1, 2, 3, 4] + assert Enum.map(verdicts, &Result.mark/1) == ["✅", "🟡", "❌", "⛔", "crashed"] + assert Result.label("out_of_scope") == "out of scope" + assert Result.label(nil) == "?" + assert Result.rank(Result.crashed("x")) == 4 + refute Result.correct?(Result.crashed("x")) + end +end diff --git a/test/examples_task_test.exs b/test/examples_task_test.exs index 964bdab1..4b85718b 100644 --- a/test/examples_task_test.exs +++ b/test/examples_task_test.exs @@ -110,7 +110,11 @@ defmodule Defk.ExamplesTaskTest do out = ~s(noise\nDEFK_RESULT {"example":"a","correct":true}\nDEFK_RESULT not json\nDEFK_RESULT {"example":"b"}\n) - assert [%{"example" => "a"}, %{"example" => "b"}] = Examples.parse_results(out) + assert [ + %Defk.Examples.Result{example: "a", correct: true}, + %Defk.Examples.Result{example: "b"} + ] = + Examples.parse_results(out) end describe "the C of an example's kernels (--stale)" do