From 0d672bf32a7f044365ff8daa3795b830a0fc7dec Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 16:41:26 +0000 Subject: [PATCH] Run NNS.seas's lag scan as R's own C++ instead of pure Python nns_seas scored every lag up to n / 5 with a per-element Python loop, so the O(n^2) scan that R runs in compiled code took the interpreter about 50 minutes on the 222,040-point tides series in gists/timeseries/tides (R 14: 25 seconds). #193's exact-sum pass made it roughly twice as slow again. The scan is now R's seas_score from src/NNS_seas.cpp, copied verbatim into src/nns/_r14_seas.cpp and scored across lags with nns::parallel_for, as R splits it with RcppParallel. It builds into the NNS-core library target: nanobind compiles its module with -Os, which made the same loop three times slower than R's. It now matches R's speed (4.7 s single-threaded, 1.7 s on four threads for 60,000 points; R: 4.7 s and 1.7 s). The overlay is registered with its hashes in sync/nns_source.json. Without the native extension, a vectorised NumPy port of the same walk runs instead: one stable sort, then per lag every component's terms in ascending order, summed row by row from 0.0 as R's running sums are. Native and fallback scores match the previous implementation bit for bit (16,260 lag scores, at 1 and 3 threads), and the live R sweeps stay exact (72 NNS.seas and 32 seeded p-value cases, both modes). A new invariant test pins both paths to a line-by-line transcription of R's loop. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Wws6NpuQ72C5sfEYVoVmGt --- CMakeLists.txt | 6 +- docs/conventions.md | 6 +- src/nns/_nnscore_bindings.cpp | 15 ++++ src/nns/_r14_seas.cpp | 78 ++++++++++++++++ src/nns/_r14_seas.hpp | 15 ++++ src/nns/_seas_r14.py | 105 ++++++++++++++-------- sync/nns_source.json | 8 ++ tests/invariants/test_r14_shared_paths.py | 2 +- tests/invariants/test_seasonality.py | 61 +++++++++++++ 9 files changed, 257 insertions(+), 39 deletions(-) create mode 100644 src/nns/_r14_seas.cpp create mode 100644 src/nns/_r14_seas.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d8b009d..e67d424e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,8 +14,9 @@ set(NNSCORE_BUILD_PYTHON OFF CACHE BOOL "Build NNS-core Python bindings" FORCE) add_subdirectory(extern/NNS-core) # Preserve the accepted NNS-core vendor tree; explicitly overlay the R14 -# central-tendency kernels that its R13.3 snapshot does not yet supply, and the -# multivariate prediction kernel with R's own nmath densities. +# central-tendency kernels that its R13.3 snapshot does not yet supply, the +# multivariate prediction kernel with R's own nmath densities, and R's NNS.seas +# lag scan (kept out of the nanobind module, which is compiled for size). get_target_property(NNS_CORE_SOURCES nnscore SOURCES) set(NNS_R14_CORE_SOURCES "") foreach(source IN LISTS NNS_CORE_SOURCES) @@ -25,6 +26,7 @@ foreach(source IN LISTS NNS_CORE_SOURCES) endforeach() list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_central_tendencies.cpp") list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_mreg_predict.cpp") +list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_seas.cpp") set_property(TARGET nnscore PROPERTY SOURCES "${NNS_R14_CORE_SOURCES}") # The Python extension is a shared object. The vendored NNS-core target builds diff --git a/docs/conventions.md b/docs/conventions.md index df0c88bb..9672f779 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -520,7 +520,11 @@ covers periods up to `min(n / 2, n / 5)` (or `max_period`), `modulo` adds multiples as in R, and the reported period is the fundamental one: the smallest plausible divisor of the strongest period within two paired standard errors of it. Inputs shorter than five observations return R's sentinel period `0`. The -scores follow R's arithmetic and agree with R to the last bit. `p_value=True` +scores follow R's arithmetic and agree with R to the last bit. The scan is +R's own C++ loop (one sort, then one pass per lag, split across threads), so +like R it costs O(n^2): about 25 seconds for 220,000 observations on four +threads, as in R. Without the native extension a vectorised NumPy port of the +same loop runs instead, more slowly. `p_value=True` adds R's permutation p-value of each period (`NaN` for period 1): its dispersion ratio against the best ratio of each of `nperm` shuffles of the detrended series; `periods` then lists the periods with p-value at most 0.05 (else 1). With diff --git a/src/nns/_nnscore_bindings.cpp b/src/nns/_nnscore_bindings.cpp index 4d7619c2..5c605650 100644 --- a/src/nns/_nnscore_bindings.cpp +++ b/src/nns/_nnscore_bindings.cpp @@ -18,6 +18,7 @@ #include "nns/nns.hpp" #include "_r14_dependence.hpp" +#include "_r14_seas.hpp" namespace nb = nanobind; @@ -709,6 +710,20 @@ NB_MODULE(_nnscore, m) { nb::arg("p"), nb::arg("yhat"), nb::arg("xtest"), nb::arg("m"), nb::arg("kmax"), nb::arg("dist_code"), nb::arg("mins"), nb::arg("maxs"), nb::arg("is_class")); m.def("nns_dep_pair", &nns_dep_pair, nb::arg("x"), nb::arg("y"), nb::arg("asym")); + m.def( + "seas_scores_r14", + [](const Vector& x, const IntVector& lags, bool use_cv) { + const std::size_t n = x.shape(0); + if (n == 0) throw std::invalid_argument("x must be non-empty."); + if (n > static_cast(INT_MAX)) throw std::invalid_argument("x is too long."); + for (std::size_t i = 0; i < lags.shape(0); ++i) { + if (lags.data()[i] < 1) throw std::invalid_argument("lags must be >= 1."); + } + nb::gil_scoped_release release; + return nns_r14::seas_scores(x.data(), static_cast(n), lags.data(), lags.shape(0), + use_cv, call_threads()); + }, + nb::arg("x"), nb::arg("lags"), nb::arg("use_cv")); // Versioned aliases prevent new Python code from loading the old R13 // statistic when an installation accidentally retains a stale extension. m.attr("gravity_r14") = m.attr("gravity"); diff --git a/src/nns/_r14_seas.cpp b/src/nns/_r14_seas.cpp new file mode 100644 index 00000000..ce41e39d --- /dev/null +++ b/src/nns/_r14_seas.cpp @@ -0,0 +1,78 @@ +// R14 NNS.seas lag scan. +// Source: OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae +// src/NNS_seas.cpp: SeasComponent, seas_score and the SeasScanWorker split, +// copied verbatim. RcppParallel::parallelFor becomes nns::parallel_for and +// R vectors become plain arrays; each lag's arithmetic is R's, in R's order, +// and every lag writes its own output slot, so results never depend on the +// thread count. +// SPDX-License-Identifier: GPL-3.0-only +#include +#include +#include +#include +#include + +#include "nns/parallel.hpp" +#include "_r14_seas.hpp" + +namespace nns_r14 { + +// Running state of one component series while the sorted series is walked. +struct SeasComponent { int rank; int len; double first; double last; double sum; double acc; }; + +// Score of lag m: the length-weighted mean over its m component series of the +// Gini mean difference, divided by the component mean when use_cv. xs holds the +// series in increasing order and pos the original index of each value, so each +// value lands in its component (pos mod m) at that component's next rank. +static double seas_score(int m, int n, const double* xs, const int* pos, bool use_cv, + std::vector& comp){ + if ((int)comp.size() < m) comp.resize(m); + const double inv_m = 1.0 / (double)m; + for(int j=0;j= m) j -= m; else if (j < 0) j += m; + const double v = xs[t]; + SeasComponent& c = comp[j]; + if (c.rank == 0) c.first = v; + c.last = v; + // The weights sum to 0, so measuring from the component minimum avoids cancelling a large level. + c.acc += (2.0 * c.rank + 1.0 - c.len) * (v - c.first); + c.sum += v; + ++c.rank; + } + double wsum = 0.0; + for(int j=0;j seas_scores(const double* x, int n, const int* lags, + std::size_t n_lags, bool use_cv, int n_threads){ + std::vector ord(n); + std::iota(ord.begin(), ord.end(), 0); + std::stable_sort(ord.begin(), ord.end(), [&](int a, int b){ return x[a] < x[b]; }); + std::vector xs(n); + for(int t=0;t out(n_lags); + nns::parallel_for( + 0, n_lags, + [&](std::size_t begin, std::size_t end){ + std::vector comp; + for(std::size_t idx = begin; idx < end; ++idx) + out[idx] = seas_score((int)lags[idx], n, xs.data(), ord.data(), use_cv, comp); + }, + n_threads, static_cast(n)); + return out; +} + +} // namespace nns_r14 diff --git a/src/nns/_r14_seas.hpp b/src/nns/_r14_seas.hpp new file mode 100644 index 00000000..1ccca3ce --- /dev/null +++ b/src/nns/_r14_seas.hpp @@ -0,0 +1,15 @@ +// R14 NNS.seas lag scan: declaration. The definition lives in _r14_seas.cpp, +// which builds into the NNS-core library target with its Release optimisation +// (the nanobind module itself is compiled for size). +// SPDX-License-Identifier: GPL-3.0-only +#pragma once +#include +#include + +namespace nns_r14 { + +// R's seas_score (src/NNS_seas.cpp) for each requested lag of the series x. +std::vector seas_scores(const double* x, int n, const int* lags, std::size_t n_lags, + bool use_cv, int n_threads); + +} // namespace nns_r14 diff --git a/src/nns/_seas_r14.py b/src/nns/_seas_r14.py index 8ad300b2..99a13551 100644 --- a/src/nns/_seas_r14.py +++ b/src/nns/_seas_r14.py @@ -14,6 +14,7 @@ from numpy.typing import NDArray from nns._helpers import _fast_lm +from nns._native import native_fn from nns._rmean import r_cumsum, r_mean, r_sum from nns._rrng import RRNG @@ -45,37 +46,69 @@ def _as_float_series(values: F, dispersion: str, detrend: bool) -> F: return x -def _gini_component(values: F) -> tuple[float, float]: - """Unbiased Gini difference of observations, including tied observations, - and the component's sum, both accumulated over the values in ascending order - as R's C++ ``seas_score`` visits them.""" - n = int(values.size) - sorted_values = np.sort(values, kind="stable") - total = 0.0 - for value in sorted_values: - total += float(value) - if n < 2 or sorted_values[0] == sorted_values[-1]: - return 0.0, total - first = float(sorted_values[0]) - acc = 0.0 - for i, value in enumerate(sorted_values): - acc += (2.0 * i + 1.0 - n) * (float(value) - first) - return max(0.0, 2.0 * acc / (n * (n - 1.0))), total +def _lag_scores(x: F, lags: Sequence[int] | IntArray, use_cv: bool) -> F: + """R's ``seas_score`` for each lag (``NNS_seas_cpp``): the length-weighted + mean over the lag's component series of the unbiased Gini mean difference, + divided by the component mean when ``use_cv``. + The native kernel is R's C++ loop itself, scored across threads. The NumPy + fallback performs the same walk: one stable sort of the series, then per lag + every component's terms in ascending order, summed sequentially from 0.0 as + R's running ``acc`` and ``sum`` are. + """ + lag_array = np.asarray(lags, dtype=np.int64) + if lag_array.size == 0: + return np.empty(0, dtype=np.float64) + native = native_fn("seas_scores_r14") + if native is not None: + scores = native( + np.ascontiguousarray(x, dtype=np.float64), + np.ascontiguousarray(lag_array, dtype=np.int32), + bool(use_cv), + ) + return np.asarray(scores, dtype=np.float64) + positions = np.argsort(x, kind="stable") + ordered = np.asarray(x, dtype=np.float64)[positions] + return np.array( + [_lag_score_sorted(ordered, positions, int(lag), use_cv) for lag in lag_array], + dtype=np.float64, + ) -def _lag_score(x: F, lag: int, use_cv: bool) -> float: - """Length-weighted component Gini dispersion / mean (when one-sided).""" - n = x.size - acc = 0.0 - for j in range(lag): - values = x[j::lag] - g, total = _gini_component(values) - if use_cv and g != 0.0: - score = g / abs(total / float(values.size)) + +def _lag_score_sorted(ordered: F, positions: IntArray, lag: int, use_cv: bool) -> float: + """One lag of R's ``seas_score`` from the sorted series and its positions.""" + n = int(ordered.size) + component = positions % lag + lengths = (n - 1 - np.arange(lag)) // lag + 1 + # Group by component, keeping the ascending (stable) order within each one. + key = component.astype(np.uint16) if lag <= np.iinfo(np.uint16).max else component + grouped = np.argsort(key, kind="stable") + owner = component[grouped] + values = ordered[grouped] + offsets = np.concatenate(([0], np.cumsum(lengths)[:-1])) + rank = np.arange(n) - offsets[owner] + first = values[offsets] + last = values[offsets + lengths - 1] + terms = (2.0 * rank + 1.0 - lengths[owner].astype(np.float64)) * (values - first[owner]) + # Column 0 is R's 0.0 starting value; shorter rows end in zeros, which + # leave a running sum that started from +0.0 unchanged. + width = int(lengths[0]) + 1 + grid = np.zeros((lag, width), dtype=np.float64) + grid[owner, rank + 1] = terms + acc = np.cumsum(grid, axis=1)[:, -1] + grid[:] = 0.0 + grid[owner, rank + 1] = values + total = np.cumsum(grid, axis=1)[:, -1] + k = lengths.astype(np.float64) + varies = (lengths >= 2) & (first != last) + with np.errstate(divide="ignore", invalid="ignore"): + g = np.where(varies, np.maximum(0.0, 2.0 * acc / (k * (k - 1.0))), 0.0) + if use_cv: + score = np.where(g == 0.0, 0.0, g / np.abs(total / k)) else: score = g - acc += values.size * score - return acc / n + weighted = np.cumsum(np.concatenate(([0.0], k * score)))[-1] + return float(weighted / n) def _delete_one_scores(x: F, lag: int, use_cv: bool) -> F | None: @@ -250,8 +283,9 @@ def _seas_scan( max_lag = min(n // 2, n // 5) if max_period is not None: max_lag = max(1, min(max_lag, int(max_period))) - base = _lag_score(x, 1, use_cv) - scored = {lag: _lag_score(x, lag, use_cv) for lag in range(1, max_lag + 1)} + scan = _lag_scores(x, np.arange(1, max_lag + 1), use_cv) + base = float(scan[0]) + scored = {lag: float(score) for lag, score in zip(range(1, max_lag + 1), scan, strict=True)} current = {lag: score for lag, score in scored.items() if score < base} if not current: current = {1: base} @@ -276,12 +310,13 @@ def _seas_scan( added.add(1) # R std::set iterates in ascending integer order. In a stable sort # over equal-VarCV injected rows, insertion order is user-visible. - for lag in sorted(added): - if lag not in current and lag <= n // 2: - # The scan already rejected in-range lags not in current; - # R's added_cv gives them the baseline, not their worse score. - score = min(_lag_score(x, lag, use_cv), base) if lag > max_lag else base - current[lag] = score + pending = [lag for lag in sorted(added) if lag not in current and lag <= n // 2] + beyond = [lag for lag in pending if lag > max_lag] + beyond_scores = dict(zip(beyond, _lag_scores(x, beyond, use_cv), strict=True)) + for lag in pending: + # The scan already rejected in-range lags not in current; + # R's added_cv gives them the baseline, not their worse score. + current[lag] = min(float(beyond_scores[lag]), base) if lag > max_lag else base if not current: current = {1: base} current = {lag: score for lag, score in current.items() if lag < n // 2} diff --git a/sync/nns_source.json b/sync/nns_source.json index fa81f5bf..1583f1ca 100644 --- a/sync/nns_source.json +++ b/sync/nns_source.json @@ -41,6 +41,14 @@ "r_source_sha256": "e0fac52e99f33cd29fbb07b51711ded56c016d9e1a4b4a8eac8a9b4b9ed98571", "r_source_commit": "7b07a837c130ba7220dbe16c584221508757d5ae", "adaptation": "The vendored NNS-core multivariate prediction kernel (src/mreg_predict.cpp, sha256 c98d3fbb93c3165109870cb3d2cc1a55c8cb0fa35770b3ff0204441fd20e1e04) with its Student-t and normal ensemble weights evaluated by R's own nmath dt and dnorm4 (src/nns/_rnmath.hpp) instead of the vendor's lgamma-based closed form, so the weights and predictions match R's NNS_mreg_predict_v2_cpp bit for bit. CMake replaces only the vendor translation unit." + }, + { + "path": "src/nns/_r14_seas.cpp", + "sha256": "2b956af7f064a575d1acd1283ec5a6f4ced8e91b227e294634ce3ab7fd83420e", + "r_source_path": "src/NNS_seas.cpp", + "r_source_sha256": "cf950b8e1b92bc942c132135dfb6a4909ca9f77c60da186e23eac0d30b2ed23e", + "r_source_commit": "7b07a837c130ba7220dbe16c584221508757d5ae", + "adaptation": "R14 NNS.seas lag scan: SeasComponent and seas_score copied verbatim, scored across lags with nns::parallel_for in place of RcppParallel (each lag writes its own slot, so results never depend on the thread count). Built into the NNS-core library target for its Release optimisation; bound as seas_scores_r14." } ], "r_cache_summary": { diff --git a/tests/invariants/test_r14_shared_paths.py b/tests/invariants/test_r14_shared_paths.py index a82474f0..10a59ee1 100644 --- a/tests/invariants/test_r14_shared_paths.py +++ b/tests/invariants/test_r14_shared_paths.py @@ -90,7 +90,7 @@ def test_native_r14_overlays_have_separate_source_provenance() -> None: root = Path(__file__).resolve().parents[2] manifest = json.loads((root / "sync/nns_source.json").read_text()) assert manifest["r_version"] == "14.0" - assert len(manifest["r14_native_overlays"]) == 3 + assert len(manifest["r14_native_overlays"]) == 4 for overlay in manifest["r14_native_overlays"]: assert overlay["r_source_commit"] == manifest["r_commit"] assert not overlay["path"].startswith("extern/") diff --git a/tests/invariants/test_seasonality.py b/tests/invariants/test_seasonality.py index 3fc5a170..bab61255 100644 --- a/tests/invariants/test_seasonality.py +++ b/tests/invariants/test_seasonality.py @@ -4,6 +4,7 @@ from typing import Any, cast import numpy as np +import pytest from nns import nns_seas from nns.seasonality import _SD_TOL, _component_score @@ -93,3 +94,63 @@ def test_constant_series_and_components_compare_raw_values() -> None: # 0.1 repeated does not centre to exactly 0, so the sd was rounding noise. assert nns_seas(np.full(20, 0.1))["best.period"] == 1 assert nns_seas(np.tile([0.1, 1.0], 10))["best.period"] == 2 + + +def _r_seas_score(lag: int, ordered: np.ndarray, positions: np.ndarray, use_cv: bool) -> float: + """R's seas_score (src/NNS_seas.cpp) transcribed line by line.""" + n = ordered.size + rank = [0] * lag + length = [(n - 1 - j) // lag + 1 for j in range(lag)] + first = [0.0] * lag + last = [0.0] * lag + total = [0.0] * lag + acc = [0.0] * lag + for t in range(n): + j = int(positions[t]) % lag + v = float(ordered[t]) + if rank[j] == 0: + first[j] = v + last[j] = v + acc[j] += (2.0 * rank[j] + 1.0 - length[j]) * (v - first[j]) + total[j] += v + rank[j] += 1 + wsum = 0.0 + for j in range(lag): + k = float(length[j]) + varies = length[j] >= 2 and first[j] != last[j] + g = max(0.0, 2.0 * acc[j] / (k * (k - 1.0))) if varies else 0.0 + wsum += k * ((0.0 if g == 0.0 else g / abs(total[j] / k)) if use_cv else g) + return wsum / n + + +@pytest.mark.parametrize("kind", ["continuous", "ties", "positive", "mixed"]) +@pytest.mark.parametrize("n", [5, 37, 400]) +def test_seasonality_lag_scores_match_r_loop_bitwise(kind: str, n: int) -> None: + # The native scan (R's C++ itself) and the vectorised NumPy fallback both + # reproduce R's per-lag walk to the last bit, at any thread count. + import nns + from nns._native import native_fn + from nns._seas_r14 import _lag_score_sorted, _lag_scores + + rng = np.random.default_rng(n) + x = rng.normal(size=n) + if kind == "ties": + x = np.round(x * 2.0) + elif kind == "positive": + x = np.abs(x) + 0.5 + elif kind == "mixed": + x = np.round(rng.uniform(-3.0, 3.0, n), 1) + use_cv = bool(x.min() > 0.0 or x.max() < 0.0) + lags = np.arange(1, max(1, n // 5) + 1) + positions = np.argsort(x, kind="stable") + ordered = x[positions] + expected = np.array([_r_seas_score(int(m), ordered, positions, use_cv) for m in lags]) + fallback = np.array([_lag_score_sorted(ordered, positions, int(m), use_cv) for m in lags]) + np.testing.assert_array_equal(fallback, expected) + if native_fn("seas_scores_r14") is not None: + for threads in (1, 3): + nns.set_num_threads(threads) + try: + np.testing.assert_array_equal(_lag_scores(x, lags, use_cv), expected) + finally: + nns.set_num_threads(None)