diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d8b009d..e67d424e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,8 +14,9 @@ set(NNSCORE_BUILD_PYTHON OFF CACHE BOOL "Build NNS-core Python bindings" FORCE) add_subdirectory(extern/NNS-core) # Preserve the accepted NNS-core vendor tree; explicitly overlay the R14 -# central-tendency kernels that its R13.3 snapshot does not yet supply, and the -# multivariate prediction kernel with R's own nmath densities. +# central-tendency kernels that its R13.3 snapshot does not yet supply, the +# multivariate prediction kernel with R's own nmath densities, and R's NNS.seas +# lag scan (kept out of the nanobind module, which is compiled for size). get_target_property(NNS_CORE_SOURCES nnscore SOURCES) set(NNS_R14_CORE_SOURCES "") foreach(source IN LISTS NNS_CORE_SOURCES) @@ -25,6 +26,7 @@ foreach(source IN LISTS NNS_CORE_SOURCES) endforeach() list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_central_tendencies.cpp") list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_mreg_predict.cpp") +list(APPEND NNS_R14_CORE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/nns/_r14_seas.cpp") set_property(TARGET nnscore PROPERTY SOURCES "${NNS_R14_CORE_SOURCES}") # The Python extension is a shared object. The vendored NNS-core target builds diff --git a/docs/conventions.md b/docs/conventions.md index df0c88bb..9672f779 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -520,7 +520,11 @@ covers periods up to `min(n / 2, n / 5)` (or `max_period`), `modulo` adds multiples as in R, and the reported period is the fundamental one: the smallest plausible divisor of the strongest period within two paired standard errors of it. Inputs shorter than five observations return R's sentinel period `0`. The -scores follow R's arithmetic and agree with R to the last bit. `p_value=True` +scores follow R's arithmetic and agree with R to the last bit. The scan is +R's own C++ loop (one sort, then one pass per lag, split across threads), so +like R it costs O(n^2): about 25 seconds for 220,000 observations on four +threads, as in R. Without the native extension a vectorised NumPy port of the +same loop runs instead, more slowly. `p_value=True` adds R's permutation p-value of each period (`NaN` for period 1): its dispersion ratio against the best ratio of each of `nperm` shuffles of the detrended series; `periods` then lists the periods with p-value at most 0.05 (else 1). With diff --git a/src/nns/_nnscore_bindings.cpp b/src/nns/_nnscore_bindings.cpp index 4d7619c2..5c605650 100644 --- a/src/nns/_nnscore_bindings.cpp +++ b/src/nns/_nnscore_bindings.cpp @@ -18,6 +18,7 @@ #include "nns/nns.hpp" #include "_r14_dependence.hpp" +#include "_r14_seas.hpp" namespace nb = nanobind; @@ -709,6 +710,20 @@ NB_MODULE(_nnscore, m) { nb::arg("p"), nb::arg("yhat"), nb::arg("xtest"), nb::arg("m"), nb::arg("kmax"), nb::arg("dist_code"), nb::arg("mins"), nb::arg("maxs"), nb::arg("is_class")); m.def("nns_dep_pair", &nns_dep_pair, nb::arg("x"), nb::arg("y"), nb::arg("asym")); + m.def( + "seas_scores_r14", + [](const Vector& x, const IntVector& lags, bool use_cv) { + const std::size_t n = x.shape(0); + if (n == 0) throw std::invalid_argument("x must be non-empty."); + if (n > static_cast(INT_MAX)) throw std::invalid_argument("x is too long."); + for (std::size_t i = 0; i < lags.shape(0); ++i) { + if (lags.data()[i] < 1) throw std::invalid_argument("lags must be >= 1."); + } + nb::gil_scoped_release release; + return nns_r14::seas_scores(x.data(), static_cast(n), lags.data(), lags.shape(0), + use_cv, call_threads()); + }, + nb::arg("x"), nb::arg("lags"), nb::arg("use_cv")); // Versioned aliases prevent new Python code from loading the old R13 // statistic when an installation accidentally retains a stale extension. m.attr("gravity_r14") = m.attr("gravity"); diff --git a/src/nns/_r14_seas.cpp b/src/nns/_r14_seas.cpp new file mode 100644 index 00000000..ce41e39d --- /dev/null +++ b/src/nns/_r14_seas.cpp @@ -0,0 +1,78 @@ +// R14 NNS.seas lag scan. +// Source: OVVO-Financial/NNS@7b07a837c130ba7220dbe16c584221508757d5ae +// src/NNS_seas.cpp: SeasComponent, seas_score and the SeasScanWorker split, +// copied verbatim. RcppParallel::parallelFor becomes nns::parallel_for and +// R vectors become plain arrays; each lag's arithmetic is R's, in R's order, +// and every lag writes its own output slot, so results never depend on the +// thread count. +// SPDX-License-Identifier: GPL-3.0-only +#include +#include +#include +#include +#include + +#include "nns/parallel.hpp" +#include "_r14_seas.hpp" + +namespace nns_r14 { + +// Running state of one component series while the sorted series is walked. +struct SeasComponent { int rank; int len; double first; double last; double sum; double acc; }; + +// Score of lag m: the length-weighted mean over its m component series of the +// Gini mean difference, divided by the component mean when use_cv. xs holds the +// series in increasing order and pos the original index of each value, so each +// value lands in its component (pos mod m) at that component's next rank. +static double seas_score(int m, int n, const double* xs, const int* pos, bool use_cv, + std::vector& comp){ + if ((int)comp.size() < m) comp.resize(m); + const double inv_m = 1.0 / (double)m; + for(int j=0;j= m) j -= m; else if (j < 0) j += m; + const double v = xs[t]; + SeasComponent& c = comp[j]; + if (c.rank == 0) c.first = v; + c.last = v; + // The weights sum to 0, so measuring from the component minimum avoids cancelling a large level. + c.acc += (2.0 * c.rank + 1.0 - c.len) * (v - c.first); + c.sum += v; + ++c.rank; + } + double wsum = 0.0; + for(int j=0;j seas_scores(const double* x, int n, const int* lags, + std::size_t n_lags, bool use_cv, int n_threads){ + std::vector ord(n); + std::iota(ord.begin(), ord.end(), 0); + std::stable_sort(ord.begin(), ord.end(), [&](int a, int b){ return x[a] < x[b]; }); + std::vector xs(n); + for(int t=0;t out(n_lags); + nns::parallel_for( + 0, n_lags, + [&](std::size_t begin, std::size_t end){ + std::vector comp; + for(std::size_t idx = begin; idx < end; ++idx) + out[idx] = seas_score((int)lags[idx], n, xs.data(), ord.data(), use_cv, comp); + }, + n_threads, static_cast(n)); + return out; +} + +} // namespace nns_r14 diff --git a/src/nns/_r14_seas.hpp b/src/nns/_r14_seas.hpp new file mode 100644 index 00000000..1ccca3ce --- /dev/null +++ b/src/nns/_r14_seas.hpp @@ -0,0 +1,15 @@ +// R14 NNS.seas lag scan: declaration. The definition lives in _r14_seas.cpp, +// which builds into the NNS-core library target with its Release optimisation +// (the nanobind module itself is compiled for size). +// SPDX-License-Identifier: GPL-3.0-only +#pragma once +#include +#include + +namespace nns_r14 { + +// R's seas_score (src/NNS_seas.cpp) for each requested lag of the series x. +std::vector seas_scores(const double* x, int n, const int* lags, std::size_t n_lags, + bool use_cv, int n_threads); + +} // namespace nns_r14 diff --git a/src/nns/_seas_r14.py b/src/nns/_seas_r14.py index 8ad300b2..99a13551 100644 --- a/src/nns/_seas_r14.py +++ b/src/nns/_seas_r14.py @@ -14,6 +14,7 @@ from numpy.typing import NDArray from nns._helpers import _fast_lm +from nns._native import native_fn from nns._rmean import r_cumsum, r_mean, r_sum from nns._rrng import RRNG @@ -45,37 +46,69 @@ def _as_float_series(values: F, dispersion: str, detrend: bool) -> F: return x -def _gini_component(values: F) -> tuple[float, float]: - """Unbiased Gini difference of observations, including tied observations, - and the component's sum, both accumulated over the values in ascending order - as R's C++ ``seas_score`` visits them.""" - n = int(values.size) - sorted_values = np.sort(values, kind="stable") - total = 0.0 - for value in sorted_values: - total += float(value) - if n < 2 or sorted_values[0] == sorted_values[-1]: - return 0.0, total - first = float(sorted_values[0]) - acc = 0.0 - for i, value in enumerate(sorted_values): - acc += (2.0 * i + 1.0 - n) * (float(value) - first) - return max(0.0, 2.0 * acc / (n * (n - 1.0))), total +def _lag_scores(x: F, lags: Sequence[int] | IntArray, use_cv: bool) -> F: + """R's ``seas_score`` for each lag (``NNS_seas_cpp``): the length-weighted + mean over the lag's component series of the unbiased Gini mean difference, + divided by the component mean when ``use_cv``. + The native kernel is R's C++ loop itself, scored across threads. The NumPy + fallback performs the same walk: one stable sort of the series, then per lag + every component's terms in ascending order, summed sequentially from 0.0 as + R's running ``acc`` and ``sum`` are. + """ + lag_array = np.asarray(lags, dtype=np.int64) + if lag_array.size == 0: + return np.empty(0, dtype=np.float64) + native = native_fn("seas_scores_r14") + if native is not None: + scores = native( + np.ascontiguousarray(x, dtype=np.float64), + np.ascontiguousarray(lag_array, dtype=np.int32), + bool(use_cv), + ) + return np.asarray(scores, dtype=np.float64) + positions = np.argsort(x, kind="stable") + ordered = np.asarray(x, dtype=np.float64)[positions] + return np.array( + [_lag_score_sorted(ordered, positions, int(lag), use_cv) for lag in lag_array], + dtype=np.float64, + ) -def _lag_score(x: F, lag: int, use_cv: bool) -> float: - """Length-weighted component Gini dispersion / mean (when one-sided).""" - n = x.size - acc = 0.0 - for j in range(lag): - values = x[j::lag] - g, total = _gini_component(values) - if use_cv and g != 0.0: - score = g / abs(total / float(values.size)) + +def _lag_score_sorted(ordered: F, positions: IntArray, lag: int, use_cv: bool) -> float: + """One lag of R's ``seas_score`` from the sorted series and its positions.""" + n = int(ordered.size) + component = positions % lag + lengths = (n - 1 - np.arange(lag)) // lag + 1 + # Group by component, keeping the ascending (stable) order within each one. + key = component.astype(np.uint16) if lag <= np.iinfo(np.uint16).max else component + grouped = np.argsort(key, kind="stable") + owner = component[grouped] + values = ordered[grouped] + offsets = np.concatenate(([0], np.cumsum(lengths)[:-1])) + rank = np.arange(n) - offsets[owner] + first = values[offsets] + last = values[offsets + lengths - 1] + terms = (2.0 * rank + 1.0 - lengths[owner].astype(np.float64)) * (values - first[owner]) + # Column 0 is R's 0.0 starting value; shorter rows end in zeros, which + # leave a running sum that started from +0.0 unchanged. + width = int(lengths[0]) + 1 + grid = np.zeros((lag, width), dtype=np.float64) + grid[owner, rank + 1] = terms + acc = np.cumsum(grid, axis=1)[:, -1] + grid[:] = 0.0 + grid[owner, rank + 1] = values + total = np.cumsum(grid, axis=1)[:, -1] + k = lengths.astype(np.float64) + varies = (lengths >= 2) & (first != last) + with np.errstate(divide="ignore", invalid="ignore"): + g = np.where(varies, np.maximum(0.0, 2.0 * acc / (k * (k - 1.0))), 0.0) + if use_cv: + score = np.where(g == 0.0, 0.0, g / np.abs(total / k)) else: score = g - acc += values.size * score - return acc / n + weighted = np.cumsum(np.concatenate(([0.0], k * score)))[-1] + return float(weighted / n) def _delete_one_scores(x: F, lag: int, use_cv: bool) -> F | None: @@ -250,8 +283,9 @@ def _seas_scan( max_lag = min(n // 2, n // 5) if max_period is not None: max_lag = max(1, min(max_lag, int(max_period))) - base = _lag_score(x, 1, use_cv) - scored = {lag: _lag_score(x, lag, use_cv) for lag in range(1, max_lag + 1)} + scan = _lag_scores(x, np.arange(1, max_lag + 1), use_cv) + base = float(scan[0]) + scored = {lag: float(score) for lag, score in zip(range(1, max_lag + 1), scan, strict=True)} current = {lag: score for lag, score in scored.items() if score < base} if not current: current = {1: base} @@ -276,12 +310,13 @@ def _seas_scan( added.add(1) # R std::set iterates in ascending integer order. In a stable sort # over equal-VarCV injected rows, insertion order is user-visible. - for lag in sorted(added): - if lag not in current and lag <= n // 2: - # The scan already rejected in-range lags not in current; - # R's added_cv gives them the baseline, not their worse score. - score = min(_lag_score(x, lag, use_cv), base) if lag > max_lag else base - current[lag] = score + pending = [lag for lag in sorted(added) if lag not in current and lag <= n // 2] + beyond = [lag for lag in pending if lag > max_lag] + beyond_scores = dict(zip(beyond, _lag_scores(x, beyond, use_cv), strict=True)) + for lag in pending: + # The scan already rejected in-range lags not in current; + # R's added_cv gives them the baseline, not their worse score. + current[lag] = min(float(beyond_scores[lag]), base) if lag > max_lag else base if not current: current = {1: base} current = {lag: score for lag, score in current.items() if lag < n // 2} diff --git a/sync/nns_source.json b/sync/nns_source.json index fa81f5bf..1583f1ca 100644 --- a/sync/nns_source.json +++ b/sync/nns_source.json @@ -41,6 +41,14 @@ "r_source_sha256": "e0fac52e99f33cd29fbb07b51711ded56c016d9e1a4b4a8eac8a9b4b9ed98571", "r_source_commit": "7b07a837c130ba7220dbe16c584221508757d5ae", "adaptation": "The vendored NNS-core multivariate prediction kernel (src/mreg_predict.cpp, sha256 c98d3fbb93c3165109870cb3d2cc1a55c8cb0fa35770b3ff0204441fd20e1e04) with its Student-t and normal ensemble weights evaluated by R's own nmath dt and dnorm4 (src/nns/_rnmath.hpp) instead of the vendor's lgamma-based closed form, so the weights and predictions match R's NNS_mreg_predict_v2_cpp bit for bit. CMake replaces only the vendor translation unit." + }, + { + "path": "src/nns/_r14_seas.cpp", + "sha256": "2b956af7f064a575d1acd1283ec5a6f4ced8e91b227e294634ce3ab7fd83420e", + "r_source_path": "src/NNS_seas.cpp", + "r_source_sha256": "cf950b8e1b92bc942c132135dfb6a4909ca9f77c60da186e23eac0d30b2ed23e", + "r_source_commit": "7b07a837c130ba7220dbe16c584221508757d5ae", + "adaptation": "R14 NNS.seas lag scan: SeasComponent and seas_score copied verbatim, scored across lags with nns::parallel_for in place of RcppParallel (each lag writes its own slot, so results never depend on the thread count). Built into the NNS-core library target for its Release optimisation; bound as seas_scores_r14." } ], "r_cache_summary": { diff --git a/tests/invariants/test_r14_shared_paths.py b/tests/invariants/test_r14_shared_paths.py index a82474f0..10a59ee1 100644 --- a/tests/invariants/test_r14_shared_paths.py +++ b/tests/invariants/test_r14_shared_paths.py @@ -90,7 +90,7 @@ def test_native_r14_overlays_have_separate_source_provenance() -> None: root = Path(__file__).resolve().parents[2] manifest = json.loads((root / "sync/nns_source.json").read_text()) assert manifest["r_version"] == "14.0" - assert len(manifest["r14_native_overlays"]) == 3 + assert len(manifest["r14_native_overlays"]) == 4 for overlay in manifest["r14_native_overlays"]: assert overlay["r_source_commit"] == manifest["r_commit"] assert not overlay["path"].startswith("extern/") diff --git a/tests/invariants/test_seasonality.py b/tests/invariants/test_seasonality.py index 3fc5a170..bab61255 100644 --- a/tests/invariants/test_seasonality.py +++ b/tests/invariants/test_seasonality.py @@ -4,6 +4,7 @@ from typing import Any, cast import numpy as np +import pytest from nns import nns_seas from nns.seasonality import _SD_TOL, _component_score @@ -93,3 +94,63 @@ def test_constant_series_and_components_compare_raw_values() -> None: # 0.1 repeated does not centre to exactly 0, so the sd was rounding noise. assert nns_seas(np.full(20, 0.1))["best.period"] == 1 assert nns_seas(np.tile([0.1, 1.0], 10))["best.period"] == 2 + + +def _r_seas_score(lag: int, ordered: np.ndarray, positions: np.ndarray, use_cv: bool) -> float: + """R's seas_score (src/NNS_seas.cpp) transcribed line by line.""" + n = ordered.size + rank = [0] * lag + length = [(n - 1 - j) // lag + 1 for j in range(lag)] + first = [0.0] * lag + last = [0.0] * lag + total = [0.0] * lag + acc = [0.0] * lag + for t in range(n): + j = int(positions[t]) % lag + v = float(ordered[t]) + if rank[j] == 0: + first[j] = v + last[j] = v + acc[j] += (2.0 * rank[j] + 1.0 - length[j]) * (v - first[j]) + total[j] += v + rank[j] += 1 + wsum = 0.0 + for j in range(lag): + k = float(length[j]) + varies = length[j] >= 2 and first[j] != last[j] + g = max(0.0, 2.0 * acc[j] / (k * (k - 1.0))) if varies else 0.0 + wsum += k * ((0.0 if g == 0.0 else g / abs(total[j] / k)) if use_cv else g) + return wsum / n + + +@pytest.mark.parametrize("kind", ["continuous", "ties", "positive", "mixed"]) +@pytest.mark.parametrize("n", [5, 37, 400]) +def test_seasonality_lag_scores_match_r_loop_bitwise(kind: str, n: int) -> None: + # The native scan (R's C++ itself) and the vectorised NumPy fallback both + # reproduce R's per-lag walk to the last bit, at any thread count. + import nns + from nns._native import native_fn + from nns._seas_r14 import _lag_score_sorted, _lag_scores + + rng = np.random.default_rng(n) + x = rng.normal(size=n) + if kind == "ties": + x = np.round(x * 2.0) + elif kind == "positive": + x = np.abs(x) + 0.5 + elif kind == "mixed": + x = np.round(rng.uniform(-3.0, 3.0, n), 1) + use_cv = bool(x.min() > 0.0 or x.max() < 0.0) + lags = np.arange(1, max(1, n // 5) + 1) + positions = np.argsort(x, kind="stable") + ordered = x[positions] + expected = np.array([_r_seas_score(int(m), ordered, positions, use_cv) for m in lags]) + fallback = np.array([_lag_score_sorted(ordered, positions, int(m), use_cv) for m in lags]) + np.testing.assert_array_equal(fallback, expected) + if native_fn("seas_scores_r14") is not None: + for threads in (1, 3): + nns.set_num_threads(threads) + try: + np.testing.assert_array_equal(_lag_scores(x, lags, use_cv), expected) + finally: + nns.set_num_threads(None)