diff --git a/ccflow/utils/tokenize.py b/ccflow/utils/tokenize.py index 91444488..26e8f700 100644 --- a/ccflow/utils/tokenize.py +++ b/ccflow/utils/tokenize.py @@ -159,6 +159,13 @@ def _normalize_date(obj): @normalize_token.register(datetime) def _normalize_datetime(obj): + cls = type(obj) + # pandas.Timestamp subclasses datetime and must not share a token with an equal plain datetime. + # Matching by name keeps `import ccflow` from paying for pandas just to register one handler. + # DataFrame/Series/Index have no structural handler and fall through to cloudpickle; adding them + # (as dask does) would require real lazy registration rather than this name check. + if cls is not datetime and any(c.__name__ == "Timestamp" and _module_in(c.__module__, ("pandas",)) for c in cls.__mro__): + return ("pd_timestamp", obj.isoformat()) return ("datetime", obj.isoformat()) @@ -371,22 +378,7 @@ def _normalize_np_scalar(obj): return ("np_scalar", str(type(obj).__name__), obj.item()) -def _register_pandas() -> None: - try: - import pandas as pd - except ImportError: # pragma: no cover - return - - # Only Timestamp has a structural handler; DataFrame/Series/Index fall through to the cloudpickle - # fallback. That works today but is fragile across pandas version upgrades — a follow-up could add - # structural handlers (matching dask) for stability and a perf win on large frames. - @normalize_token.register(pd.Timestamp) - def _normalize_pd_timestamp(obj): - return ("pd_timestamp", obj.isoformat()) - - _register_numpy() -_register_pandas() def tokenize(*args: Any, **kwargs: Any) -> str: