diff --git a/linearmodels/iv/data.py b/linearmodels/iv/data.py index 86d0fb4bd8..8be86272a6 100644 --- a/linearmodels/iv/data.py +++ b/linearmodels/iv/data.py @@ -90,7 +90,14 @@ class IVData: then nobs is used to check the number of observations in `x`. convert_dummies : bool, optional Flag indicating whether pandas categoricals or string input data - should be converted to dummy variables. The default is True. + should be converted to dummy variables. The default is True. A + missing categorical value is a missing value of each of its dummy + variables, so that the observation is dropped as missing. + + .. versionchanged:: 7.2 + + A missing categorical value was a row of zeros, which kept the + observation (in the first category when ``drop_first`` is True). drop_first : bool, optional Flag indicating to drop the first dummy category. The default is True. diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py index d7587fb45a..770674f9d5 100644 --- a/linearmodels/panel/data.py +++ b/linearmodels/panel/data.py @@ -214,7 +214,14 @@ class PanelData: xarray DataArrays. The default is ``"x"``. convert_dummies : bool, optional Flag indicating whether pandas categoricals or string input data - should be converted to dummy variables. The default is True. + should be converted to dummy variables. The default is True. A + missing categorical value is a missing value of each of its dummy + variables, so that the observation is dropped as missing. + + .. versionchanged:: 7.2 + + A missing categorical value was a row of zeros, which kept the + observation (in the first category when ``drop_first`` is True). drop_first : bool, optional Flag indicating to drop first dummy category when converting. The default is True. diff --git a/linearmodels/tests/iv/test_data.py b/linearmodels/tests/iv/test_data.py index 6b0c671fd8..98ebd4f0ca 100644 --- a/linearmodels/tests/iv/test_data.py +++ b/linearmodels/tests/iv/test_data.py @@ -1,12 +1,14 @@ from typing import cast import numpy as np -from numpy.testing import assert_equal +from numpy.testing import assert_allclose, assert_equal import pandas as pd from pandas.testing import assert_frame_equal, assert_series_equal import pytest +from linearmodels.iv import IV2SLS from linearmodels.iv.data import IVData +from linearmodels.shared.exceptions import MissingValueWarning from linearmodels.typing.data import BoolArray try: @@ -273,3 +275,52 @@ def test_categorical_missing_values_with_unused_category(drop_first: bool) -> No data = IVData(values, drop_first=drop_first) assert "group.unused" not in data.cols assert_equal(data.isnull.to_numpy(), [False, True, False, False]) + + +def test_categorical_all_missing_values() -> None: + # No dummy column is used, so the columns are kept and every observation is + # missing + values = pd.Series( + pd.Categorical([None, None, None], categories=["a", "b"]), name="group" + ) + data = IVData(values, drop_first=False) + assert data.cols == ["group.a", "group.b"] + assert data.isnull.all() + assert np.isnan(data.ndarray).all() + + +def test_categorical_missing_values_in_several_columns() -> None: + frame = pd.DataFrame( + { + "g": pd.Categorical(["a", None, "b", "a", "b"]), + "h": pd.Categorical(["x", "y", None, "y", "x"]), + "v": [1.0, 2.0, 3.0, 4.0, 5.0], + } + ) + data = IVData(frame) + assert_equal(data.isnull.to_numpy(), [False, True, True, False, False]) + + +def test_categorical_missing_values_in_a_model() -> None: + # The observations with a missing category are dropped with a warning, and the + # estimates are those of the model without them + rng = np.random.default_rng(858) + nobs = 200 + labels = rng.choice(["a", "b", "c"], nobs).astype(object) + labels[rng.choice(nobs, 20, replace=False)] = None + x = rng.standard_normal(nobs) + y = 1 + x + 0.5 * (labels == "b") - 0.7 * (labels == "c") + y = y + rng.standard_normal(nobs) + exog = pd.DataFrame({"const": 1.0, "x": x, "g": pd.Categorical(labels)}) + with pytest.warns(MissingValueWarning, match="missing values"): + res = IV2SLS(y, exog, None, None).fit() + + keep = exog["g"].notna().to_numpy() + dummies = pd.get_dummies( + exog["g"][keep], prefix="g", prefix_sep=".", drop_first=True, dtype=float + ) + regressors = np.column_stack([np.ones(keep.sum()), x[keep], dummies]) + expected = np.linalg.lstsq(regressors, y[keep], rcond=None)[0] + assert res.nobs == keep.sum() + assert list(res.params.index) == ["const", "x", "g.b", "g.c"] + assert_allclose(res.params, expected, rtol=1e-10) diff --git a/linearmodels/tests/panel/test_data.py b/linearmodels/tests/panel/test_data.py index 7fb3430ee9..80decf91dc 100644 --- a/linearmodels/tests/panel/test_data.py +++ b/linearmodels/tests/panel/test_data.py @@ -10,8 +10,8 @@ import pytest from linearmodels.panel.data import PanelData, _Panel -from linearmodels.panel.model import PanelOLS, panel_structure_stats -from linearmodels.shared.exceptions import UnknownOptionError +from linearmodels.panel.model import PanelOLS, PooledOLS, panel_structure_stats +from linearmodels.shared.exceptions import MissingValueWarning, UnknownOptionError from linearmodels.shared.utility import panel_to_frame from linearmodels.tests._utility import warns_if_missing from linearmodels.tests.panel._utility import MISSING_XARRAY, datatypes, generate_data @@ -990,3 +990,44 @@ def test_categorical_missing_values_with_unused_category(drop_first): data = PanelData(values, drop_first=drop_first) assert "group.unused" not in data.dataframe assert_equal(data.isnull.to_numpy(), [False, True, False, False]) + + +def test_categorical_all_missing_values(): + # No dummy column is used, so the columns are kept and every observation is + # missing + index = MultiIndex.from_product([["a", "b"], [0, 1]]) + values = DataFrame( + {"group": Categorical([None] * 4, categories=["first", "second"])}, + index=index, + ) + data = PanelData(values, drop_first=False) + assert list(data.dataframe.columns) == ["group.first", "group.second"] + assert data.isnull.all() + + +def test_categorical_missing_values_in_a_model(): + # The observations with a missing category are dropped with a warning, and the + # estimates are those of the model without them + rng = np.random.default_rng(858) + index = MultiIndex.from_product([range(20), range(10)]) + nobs = len(index) + labels = rng.choice(["a", "b", "c"], nobs).astype(object) + labels[rng.choice(nobs, 20, replace=False)] = None + x = rng.standard_normal(nobs) + y = 1 + x + 0.5 * (labels == "b") - 0.7 * (labels == "c") + y = y + rng.standard_normal(nobs) + frame = DataFrame( + {"y": y, "const": 1.0, "x": x, "g": Categorical(labels)}, index=index + ) + with pytest.warns(MissingValueWarning, match="missing values"): + res = PooledOLS(frame.y, frame[["const", "x", "g"]]).fit() + + keep = frame["g"].notna().to_numpy() + dummies = get_dummies( + frame["g"][keep], prefix="g", prefix_sep=".", drop_first=True, dtype=float + ) + regressors = np.column_stack([np.ones(keep.sum()), x[keep], dummies]) + expected = lstsq(regressors, y[keep], rcond=None)[0] + assert res.nobs == keep.sum() + assert list(res.params.index) == ["const", "x", "g.b", "g.c"] + assert_allclose(res.params, expected, rtol=1e-10)