From 7fc6bc9ddca2c9fdbcaad2bbfed1b272a3500715 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Fri, 6 Nov 2026 13:03:08 +0000 Subject: [PATCH] TST: Test missing categorical values in fitted models Follow-up to #858, which makes a missing value of a categorical column a missing value of its dummy variables, so that the observation is dropped instead of being kept as a row of zeros. - IV2SLS and PooledOLS fit with a categorical column that has missing values drop those observations with a MissingValueWarning, and their estimates equal those of a least-squares fit of the same model without the observations, with dummies from pandas.get_dummies. - A categorical column whose values are all missing keeps its dummy columns, and every observation is missing. This is the branch that codecov reported as uncovered in IVData and PanelData. - The missing values of several categorical columns of IVData are combined. The five new tests fail before #858. The convert_dummies parameter of IVData and PanelData now says how missing categorical values are converted, with a versionchanged note for 7.2. --- linearmodels/iv/data.py | 9 ++++- linearmodels/panel/data.py | 9 ++++- linearmodels/tests/iv/test_data.py | 53 ++++++++++++++++++++++++++- linearmodels/tests/panel/test_data.py | 45 ++++++++++++++++++++++- 4 files changed, 111 insertions(+), 5 deletions(-) diff --git a/linearmodels/iv/data.py b/linearmodels/iv/data.py index 86d0fb4bd8..8be86272a6 100644 --- a/linearmodels/iv/data.py +++ b/linearmodels/iv/data.py @@ -90,7 +90,14 @@ class IVData: then nobs is used to check the number of observations in `x`. convert_dummies : bool, optional Flag indicating whether pandas categoricals or string input data - should be converted to dummy variables. The default is True. + should be converted to dummy variables. The default is True. A + missing categorical value is a missing value of each of its dummy + variables, so that the observation is dropped as missing. + + .. versionchanged:: 7.2 + + A missing categorical value was a row of zeros, which kept the + observation (in the first category when ``drop_first`` is True). drop_first : bool, optional Flag indicating to drop the first dummy category. The default is True. diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py index d7587fb45a..770674f9d5 100644 --- a/linearmodels/panel/data.py +++ b/linearmodels/panel/data.py @@ -214,7 +214,14 @@ class PanelData: xarray DataArrays. The default is ``"x"``. convert_dummies : bool, optional Flag indicating whether pandas categoricals or string input data - should be converted to dummy variables. The default is True. + should be converted to dummy variables. The default is True. A + missing categorical value is a missing value of each of its dummy + variables, so that the observation is dropped as missing. + + .. versionchanged:: 7.2 + + A missing categorical value was a row of zeros, which kept the + observation (in the first category when ``drop_first`` is True). drop_first : bool, optional Flag indicating to drop first dummy category when converting. The default is True. diff --git a/linearmodels/tests/iv/test_data.py b/linearmodels/tests/iv/test_data.py index 6b0c671fd8..98ebd4f0ca 100644 --- a/linearmodels/tests/iv/test_data.py +++ b/linearmodels/tests/iv/test_data.py @@ -1,12 +1,14 @@ from typing import cast import numpy as np -from numpy.testing import assert_equal +from numpy.testing import assert_allclose, assert_equal import pandas as pd from pandas.testing import assert_frame_equal, assert_series_equal import pytest +from linearmodels.iv import IV2SLS from linearmodels.iv.data import IVData +from linearmodels.shared.exceptions import MissingValueWarning from linearmodels.typing.data import BoolArray try: @@ -273,3 +275,52 @@ def test_categorical_missing_values_with_unused_category(drop_first: bool) -> No data = IVData(values, drop_first=drop_first) assert "group.unused" not in data.cols assert_equal(data.isnull.to_numpy(), [False, True, False, False]) + + +def test_categorical_all_missing_values() -> None: + # No dummy column is used, so the columns are kept and every observation is + # missing + values = pd.Series( + pd.Categorical([None, None, None], categories=["a", "b"]), name="group" + ) + data = IVData(values, drop_first=False) + assert data.cols == ["group.a", "group.b"] + assert data.isnull.all() + assert np.isnan(data.ndarray).all() + + +def test_categorical_missing_values_in_several_columns() -> None: + frame = pd.DataFrame( + { + "g": pd.Categorical(["a", None, "b", "a", "b"]), + "h": pd.Categorical(["x", "y", None, "y", "x"]), + "v": [1.0, 2.0, 3.0, 4.0, 5.0], + } + ) + data = IVData(frame) + assert_equal(data.isnull.to_numpy(), [False, True, True, False, False]) + + +def test_categorical_missing_values_in_a_model() -> None: + # The observations with a missing category are dropped with a warning, and the + # estimates are those of the model without them + rng = np.random.default_rng(858) + nobs = 200 + labels = rng.choice(["a", "b", "c"], nobs).astype(object) + labels[rng.choice(nobs, 20, replace=False)] = None + x = rng.standard_normal(nobs) + y = 1 + x + 0.5 * (labels == "b") - 0.7 * (labels == "c") + y = y + rng.standard_normal(nobs) + exog = pd.DataFrame({"const": 1.0, "x": x, "g": pd.Categorical(labels)}) + with pytest.warns(MissingValueWarning, match="missing values"): + res = IV2SLS(y, exog, None, None).fit() + + keep = exog["g"].notna().to_numpy() + dummies = pd.get_dummies( + exog["g"][keep], prefix="g", prefix_sep=".", drop_first=True, dtype=float + ) + regressors = np.column_stack([np.ones(keep.sum()), x[keep], dummies]) + expected = np.linalg.lstsq(regressors, y[keep], rcond=None)[0] + assert res.nobs == keep.sum() + assert list(res.params.index) == ["const", "x", "g.b", "g.c"] + assert_allclose(res.params, expected, rtol=1e-10) diff --git a/linearmodels/tests/panel/test_data.py b/linearmodels/tests/panel/test_data.py index 7fb3430ee9..80decf91dc 100644 --- a/linearmodels/tests/panel/test_data.py +++ b/linearmodels/tests/panel/test_data.py @@ -10,8 +10,8 @@ import pytest from linearmodels.panel.data import PanelData, _Panel -from linearmodels.panel.model import PanelOLS, panel_structure_stats -from linearmodels.shared.exceptions import UnknownOptionError +from linearmodels.panel.model import PanelOLS, PooledOLS, panel_structure_stats +from linearmodels.shared.exceptions import MissingValueWarning, UnknownOptionError from linearmodels.shared.utility import panel_to_frame from linearmodels.tests._utility import warns_if_missing from linearmodels.tests.panel._utility import MISSING_XARRAY, datatypes, generate_data @@ -990,3 +990,44 @@ def test_categorical_missing_values_with_unused_category(drop_first): data = PanelData(values, drop_first=drop_first) assert "group.unused" not in data.dataframe assert_equal(data.isnull.to_numpy(), [False, True, False, False]) + + +def test_categorical_all_missing_values(): + # No dummy column is used, so the columns are kept and every observation is + # missing + index = MultiIndex.from_product([["a", "b"], [0, 1]]) + values = DataFrame( + {"group": Categorical([None] * 4, categories=["first", "second"])}, + index=index, + ) + data = PanelData(values, drop_first=False) + assert list(data.dataframe.columns) == ["group.first", "group.second"] + assert data.isnull.all() + + +def test_categorical_missing_values_in_a_model(): + # The observations with a missing category are dropped with a warning, and the + # estimates are those of the model without them + rng = np.random.default_rng(858) + index = MultiIndex.from_product([range(20), range(10)]) + nobs = len(index) + labels = rng.choice(["a", "b", "c"], nobs).astype(object) + labels[rng.choice(nobs, 20, replace=False)] = None + x = rng.standard_normal(nobs) + y = 1 + x + 0.5 * (labels == "b") - 0.7 * (labels == "c") + y = y + rng.standard_normal(nobs) + frame = DataFrame( + {"y": y, "const": 1.0, "x": x, "g": Categorical(labels)}, index=index + ) + with pytest.warns(MissingValueWarning, match="missing values"): + res = PooledOLS(frame.y, frame[["const", "x", "g"]]).fit() + + keep = frame["g"].notna().to_numpy() + dummies = get_dummies( + frame["g"][keep], prefix="g", prefix_sep=".", drop_first=True, dtype=float + ) + regressors = np.column_stack([np.ones(keep.sum()), x[keep], dummies]) + expected = lstsq(regressors, y[keep], rcond=None)[0] + assert res.nobs == keep.sum() + assert list(res.params.index) == ["const", "x", "g.b", "g.c"] + assert_allclose(res.params, expected, rtol=1e-10)