diff --git a/bench/keccak_yield.exs b/bench/keccak_yield.exs new file mode 100644 index 00000000..fef493e8 --- /dev/null +++ b/bench/keccak_yield.exs @@ -0,0 +1,114 @@ +# Yield bookkeeping in a hot short loop: the Keccak round loop (24 trips, no +# safepoint inside) in the yielding kernel against the same kernel with +# yield: false. Two signals: the stack spills in the hottest loop of the +# compiled C (cc -S with the build's flags; the basic block with the most +# `ror` instructions; loads and stores against [sp]), which do not depend +# on load, and the time for one 16 MB message, interleaved. The kernels +# are examples/keccak.exs' own, read from the file and compiled twice. +# docs/performance.md 6.8 and 6.46. +# +# mix run bench/keccak_yield.exs [rounds] +rounds = String.to_integer(List.first(System.argv()) || "7") + +src = File.read!("examples/keccak.exs") +{c0, _} = :binary.match(src, "defmodule Keccak.Constants") +{k0, _} = :binary.match(src, "kernels =\n quote do") +{k1, _} = :binary.match(src, "\nModule.create(Keccak.Kernels") +constants = binary_part(src, c0, k0 - c0) +kernels = binary_part(src, k0, k1 - k0) +# every kernel head gets yield: false in the blocking twin +blocking = Regex.replace(~r/(defk \w+\(.*\) :: \S+) do$/m, kernels, "\\1, yield: false do") + +{yielding_ast, _} = Code.eval_string(constants <> "\n" <> kernels) +{blocking_ast, _} = Code.eval_string(blocking) + +Module.create(KY.Yield, yielding_ast, Macro.Env.location(__ENV__)) +Module.create(KY.Blocking, blocking_ast, Macro.Env.location(__ENV__)) + +defmodule KY.Spills do + # The stack loads and stores in the hottest loop of a module's C: the + # basic block of the assembly with the most `ror` instructions. + def count(mod, name) do + dir = Path.join("tmp", "keccak_yield") + File.mkdir_p!(dir) + c = Path.join(dir, name <> ".c") + s = Path.join(dir, name <> ".s") + File.write!(c, mod.__defk_c_source__()) + + flags = + ~w(-O3 -ffp-contract=off -std=gnu11 -fPIC -Wall -Wno-unused-label -Wno-unused-variable -fstack-protector-strong -D_FORTIFY_SOURCE=2 -fno-strict-overflow -fno-builtin) + + {_, 0} = + System.cmd( + "cc", + ["-S"] ++ flags ++ ["-I", Defk.Build.erts_include(), "-I", "priv", "-o", s, c], + stderr_to_stdout: true + ) + + blocks = + s + |> File.read!() + |> String.split("\n") + |> Enum.chunk_while( + [], + fn line, acc -> + if Regex.match?(~r/^\S+:/, line), + do: {:cont, Enum.reverse(acc), [line]}, + else: {:cont, [line | acc]} + end, + fn acc -> {:cont, Enum.reverse(acc), []} end + ) + + {block, rors} = + blocks + |> Enum.map(&{&1, Enum.count(&1, fn l -> l =~ ~r/^\s+ror/ end)}) + |> Enum.max_by(&elem(&1, 1)) + + # the loop body ends at its back-edge; what follows in the same block + # (no label) is the exit path, run once per enclosing iteration + block = Enum.take_while(block, &(not (&1 =~ ~r/^\s+b\.\w+/))) ++ ["b."] + + stores = Enum.count(block, &(&1 =~ ~r/^\s+st[pr]\b.*\[sp/)) + loads = Enum.count(block, &(&1 =~ ~r/^\s+ld[pr]\b.*\[sp/)) + + %{ + rors: rors, + stores: stores, + loads: loads, + instructions: Enum.count(block, &(&1 =~ ~r/^\s+[a-z]/)) + } + end +end + +big = :crypto.strong_rand_bytes(16 * 1024 * 1024) +true = KY.Yield.keccak256(big) == KY.Blocking.keccak256(big) + +for {name, mod} <- [{"yielding", KY.Yield}, {"blocking", KY.Blocking}] do + %{rors: r, stores: st, loads: ld, instructions: n} = KY.Spills.count(mod, name) + + IO.puts( + "#{String.pad_trailing(name, 10)} round loop: #{st} stack stores, #{ld} stack loads, #{r} ror, #{n} instructions" + ) +end + +t = fn f -> elem(:timer.tc(f), 0) / 1000 end + +IO.puts( + "\n16 MB keccak256, best and median of #{rounds} interleaved (other CPU: #{Float.round(Defk.Bench.other_cpu() * 1.0, 2)} cores)" +) + +pairs = + for _ <- 1..rounds do + {t.(fn -> KY.Yield.keccak256(big) end), t.(fn -> KY.Blocking.keccak256(big) end)} + end + +med = fn l -> Enum.at(Enum.sort(l), div(length(l), 2)) end +ys = Enum.map(pairs, &elem(&1, 0)) +bs = Enum.map(pairs, &elem(&1, 1)) + +IO.puts(" yielding #{Float.round(Enum.min(ys), 2)} / #{Float.round(med.(ys), 2)} ms") +IO.puts(" blocking #{Float.round(Enum.min(bs), 2)} / #{Float.round(med.(bs), 2)} ms") + +IO.puts( + " ratio #{Float.round(Enum.min(ys) / Enum.min(bs), 3)}x (best), #{Float.round(med.(ys) / med.(bs), 3)}x (median)" +) diff --git a/docs/performance.md b/docs/performance.md index aa81d869..5dabdc24 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1727,7 +1727,8 @@ for LLVM. Resuming through each loop's header fixed it. "a safepoint every iteration" on the word loop. Each word is now a few ns of work, and yielding costs +19% (4.91-4.99 vs 4.11-4.20 ms, ~17 yields per query); `DEFK_SLICE_US=10000000` kept the gap. Under load defk gives - ~0.79x the blocking build's req/s. Open as `defk-w8r.1`. + ~0.79x the blocking build's req/s. Quiet it was +30%; one countdown + instead of two counters took it to +16% (6.46). - **Conditions:** `defk-nua`/`defk-j3h` load not recorded; `defk-klw` quiet (load average 3-6, no builds); `defk-l2g` load average 4-7; Linux not measured. - **Source:** `defk-nua` comment 2026-10-05, `defk-j3h` comment 2026-10-05, `defk-klw`, `defk-l2g`, [`fuzzy_search/README.md`](https://github.com/DanielLuning/defk/blob/main/examples/fuzzy_search/README.md), [results snapshot](https://github.com/DanielLuning/defk/blob/main/docs/project.md). - **Status:** measured; the Myers figures provisional. @@ -1839,7 +1840,9 @@ changed speed by 1.5x. (9-11 stores per round); `yield: false` 18.2-18.6 ms with 0 spills. Side effects: `xxh64` 4 MB 0.36-0.40 -> 0.175-0.182 ms (about 2x); siphash 1k keys 0.054-0.057 -> 0.043-0.044 ms. The remaining ~10% against `yield: - false` is yield bookkeeping held in registers (`defk-w8l`, open). + false` was read as yield bookkeeping held in registers; 6.46 measured it: + the counters are none of it, the safepoint's saves and the resume + dispatch are, and quiet the gap is 1.3x. - **Conditions:** under load (spill counts do not depend on load). - **Source:** `defk-zep` comment 2026-10-10 (f81c2cd, merge 522e7f2), [`keccak.exs`](https://github.com/DanielLuning/defk/blob/main/examples/keccak.exs). - **Status:** provisional (timings); the older "28 ms" and "0.55x of ex_keccak" figures are superseded. @@ -3033,6 +3036,94 @@ stays is about array contents. dropped check, and the agreement properties). - **Status:** measured. +### 6.46 Yield bookkeeping: one countdown instead of two counters halves fuzzy_search's per-word cost; Keccak's spills come from the safepoint's saves and the resume dispatch, not from the counters + +Two hot short loops paid for yielding where the yields themselves are +free (6.4): Keccak's round loop (24 trips, no safepoint inside; the +block loop around it has one per iteration) spilled per round in the +yielding kernel and not under `yield: false`, and fuzzy_search's word loop +(a safepoint per word, as each word holds a short inner loop) ran +19-31% +over its `yield: false` twin. `bench/keccak_yield.exs` compiles the +example's kernels twice and counts the stack loads and stores in the +hottest round block of the assembly (`cc -S` with the build's flags, the +block with the most `ror`, cut at its back-edge: the stores after it are +the loop's exit path, once per block) beside the 16 MB time; +`examples/fuzzy_search/bench/yield_cost.exs` gives the per-word gap. + +- **Before** (quiet): Keccak 12 stack stores and 12 loads per round, 164 + instructions, against 0 / 0 and 109 under `yield: false`; 16 MB 23.1 / + 23.3 ms (best / median of 7) against 17.2 / 17.6, 1.32-1.34x. 6.8's + 9-11 spills were counted on the compiler of 2026-10-10 and its 20.7 vs + 18.3 ms under load; today, with the exit path left out, the count is 12 + / 12, and quiet the gap is a third. fuzzy_search: + yielding 5.1-5.4 ms a query against 3.9-4.1 blocking, +30-31% (three + runs; `defk-l2g`'s +19% was under load), 18-19 yields a query: about + 5 ns per word on 17. +- **The counters** (`defk-w8l`'s experiment (a); each variant changes + every yielding kernel's C): + - *V1, one countdown.* `defk_ticks` counts down from the budget and a + safepoint tests `<= 0`; the budget lives only in the clock + (`defk_clk.budget`), where `defk_tick` keeps it anyway. Keccak: 12 / 12 + spills, 23.3 / 23.5 ms, unchanged. fuzzy_search: 4.6-4.8 ms, +16-17% + (three runs: +17.4, +16.2, +15.7), 16.5 yields a query: about 3 ns + per word. **Adopted**: it removes one live value and one compare per + safepoint, and the semantics are the same (ticks >= budget is + budget - ticks <= 0; the resume restores the saved budget as before). + - *V2, charge a plain short loop on entry* (as a loop a halt can leave + already is): Keccak 12 / 12, 23.0 / 23.2 ms; fuzzy_search +21%. Alone + or with V1 (+17.5%) nothing beyond V1; not adopted. + - *V3, no bookkeeping register at all*: the countdown in the frame + (`F->ticks`), the dispatch test on `F->resume`, `env` in the clock so + the tick call needs no register. Keccak 12 / 12 spills, 22.6 / 23.1 + ms: unchanged. fuzzy_search +19.6% (two runs): the memory + read-modify-write per word costs more than the register it frees. + Not adopted. +- **Where Keccak's spills come from** (the yielding kernel's C with one + piece of the safepoint removed, assembled, the same count; the C of V3, + so no counter is in a register): as emitted 12 stores / 12 loads; the + tick call removed 15 / 16; the saves (`DEFK_SAVE`, the stores of the + enclosing loops' live locals and the 25 lanes into the frame at a yield) + removed 4 / 4; call and saves removed 4 / 4; no safepoint at all but the + resume dispatch kept 4 / 4; no dispatch either 1 / 1 and 118 + instructions, the `yield: false` shape. So 8 of the 12 spills are the + saves and 3 the dispatch (the `if (resuming) switch` at each loop body's + top and the arrival labels in front of the loops), and the counters are + none of them. The 25 lanes are the block loop's accumulators, live in + both kernels; what the saves change is clang's allocation around a cold + block that reads 35 values. Neither can go without a redesign (the saves + are the resume; the dispatch is how a resume re-enters the loops through + their headers, 6.4), so Keccak keeps its 1.3x and 6.8's "bookkeeping + held in registers" is superseded by this entry. +- **fuzzy_search's per-word cost, what remains after V1** (timing only, + the variants break resuming; two runs each): the resume + dispatch compiled out (`DEFK_RESUMING 0`: a resumed call re-enters only + through the kernel's top dispatch) +25-26%, worse, with more yields a + query (17.6): not a measurement of the test's cost, so it stays at "a + load and a branch per word"; the inner loop's strip form compiled out + (`DEFK_SHORT(R) 1`, so each word's loop is the plain loop and its trips + are charged on entry) +15-16% against V1's +17.6-18.4% in the same + session: about 2 points, inside the noise. What remains is the shape of + the word loop's body in a kernel that can yield, as in Keccak. +- **A cheaper shape for the outer loop:** one safepoint per N words would + let N x DEFK_STRIP inner trips run unchecked, past the tick-gap + contract; charging the outer loop in strips by the inner loop's trip + count is what the per-iteration safepoint already does (the inner loop + charges its trips on entry, the outer safepoint tests the countdown), + and after V1 that test is one subtract and one branch per word. The + per-word cost that remains is code shape, as in Keccak, not the check. +- **Checks:** `bench/tick_gap.exs` (3 runs a side): the yielding kernel's p99 + gap 2.27 ms on the branch against 2.25 on `main`, worst 2.3-2.5 against + 2.3; the early-exit kernel 2.27 / 2.27; unchanged. `mix test` green, the + forced-yield tests included (2309 passed). +- **Conditions:** M-series Mac, on battery, other processes at 0.2-0.8 + cores before each Keccak block (one block at 2.0: V1+V2, whose numbers + are not used); fuzzy_search runs on 12 schedulers, 20 queries x 5 + rounds, medians. +- **Source:** `defk-w8l` (and `defk-w8r.1`, merged into it), [`bench/keccak_yield.exs`](https://github.com/DanielLuning/defk/blob/main/bench/keccak_yield.exs), + [`examples/fuzzy_search/bench/yield_cost.exs`](https://github.com/DanielLuning/defk/blob/main/examples/fuzzy_search/bench/yield_cost.exs); `DEFK_SP` and + `DEFK_TICKS` in `lib/defk/compiler.ex` (`sp_macro/3`, `run_preamble/1`). +- **Status:** measured. + --- ## 7. When a hand-written or wrapper NIF is still the answer @@ -3378,4 +3469,5 @@ example or close a workload ticket, add its entries and its line here. `defk-o08.2`, `defk-o08.3`, `defk-o08.4`, `defk-o08.5`, `defk-o08.6`, `defk-w8r.2`, `defk-25s` ([`bench/call_cost.exs`](https://github.com/DanielLuning/defk/blob/main/bench/call_cost.exs)), `defk-vcx.2` ([`bench/elixir_mode.exs`](https://github.com/DanielLuning/defk/blob/main/bench/elixir_mode.exs)), - `defk-w8r.3` ([`bench/gather_layer.exs`](https://github.com/DanielLuning/defk/blob/main/bench/gather_layer.exs)). + `defk-w8r.3` ([`bench/gather_layer.exs`](https://github.com/DanielLuning/defk/blob/main/bench/gather_layer.exs)), + `defk-w8l` ([`bench/keccak_yield.exs`](https://github.com/DanielLuning/defk/blob/main/bench/keccak_yield.exs)). diff --git a/lib/defk/compiler.ex b/lib/defk/compiler.ex index 138dee5e..5d2fd080 100644 --- a/lib/defk/compiler.ex +++ b/lib/defk/compiler.ex @@ -5852,7 +5852,7 @@ defmodule Defk.Compiler do defp run_preamble(yields?) do if yields?, do: - "int defk_rs = F->resume;\nint64_t defk_ticks = 0;\nint64_t defk_budget = defk_rs ? F->budget : DEFK_BUDGET0;\ndefk_clock defk_clk;\ndefk_clock_init(&defk_clk, ctx);\n", + "int defk_rs = F->resume;\ndefk_clock defk_clk;\ndefk_clock_init(&defk_clk, ctx);\ndefk_clk.budget = defk_rs ? F->budget : DEFK_BUDGET0;\nint64_t defk_ticks = defk_clk.budget;\n", else: "" end @@ -5935,8 +5935,8 @@ defmodule Defk.Compiler do inputs = k |> input_aliases(saved) |> Enum.flat_map(&elem(&1, 1)) |> Enum.uniq() "#define DEFK_SAVE#{n} #{Enum.map(saved ++ inputs, &"F->#{&1} = #{&1}; ")}\n" end) <> - "#define DEFK_SP(N, W) if (DEFK_UNLIKELY((defk_ticks += (W)) >= defk_budget)) { defk_ticks = 0; if (!(defk_budget = defk_tick(env, &defk_clk, defk_budget))) { if (DEFK_UNLIKELY(F->resume)) { defk_lost_resume(ctx, F->resume, N); return DEFK_ERR; } DEFK_SAVE##N F->resume = N; F->budget = defk_clk.budget; return DEFK_YIELD; } }\n" <> - "#define DEFK_SHORT(R) DEFK_LIKELY((R) < DEFK_STRIP)\n#define DEFK_TICKS(W) defk_ticks += (W);\n#define DEFK_RESUMING DEFK_UNLIKELY(defk_rs)\n" + "#define DEFK_SP(N, W) if (DEFK_UNLIKELY((defk_ticks -= (W)) <= 0)) { if (!(defk_ticks = defk_tick(env, &defk_clk, defk_clk.budget))) { if (DEFK_UNLIKELY(F->resume)) { defk_lost_resume(ctx, F->resume, N); return DEFK_ERR; } DEFK_SAVE##N F->resume = N; F->budget = defk_clk.budget; return DEFK_YIELD; } }\n" <> + "#define DEFK_SHORT(R) DEFK_LIKELY((R) < DEFK_STRIP)\n#define DEFK_TICKS(W) defk_ticks -= (W);\n#define DEFK_RESUMING DEFK_UNLIKELY(defk_rs)\n" else @no_yield_macros end diff --git a/priv/defk_runtime.h b/priv/defk_runtime.h index 857d4734..10fc5aa8 100644 --- a/priv/defk_runtime.h +++ b/priv/defk_runtime.h @@ -62,7 +62,7 @@ * * kernel:start * DEFK_SP DEFK_SAVE DEFK_SAVE DEFK_SHORT DEFK_TICKS DEFK_RESUMING defk_rs - * defk_ticks defk_budget defk_clk defk_funcs defk_slot defk_z0 defk_a + * defk_ticks defk_clk defk_funcs defk_slot defk_z0 defk_a * defk_l defk_halt defk_r defk_rp defk_s_ defk_lit_ * kernel:end */