TST: Test missing categorical values in fitted models

linearmodels/iv/data.py

@@ -90,7 +90,14 @@ class IVData:
         then nobs is used to check the number of observations in `x`.
     convert_dummies : bool, optional
         Flag indicating whether pandas categoricals or string input data
-        should be converted to dummy variables. The default is True.
+        should be converted to dummy variables. The default is True. A
+        missing categorical value is a missing value of each of its dummy
+        variables, so that the observation is dropped as missing.
+
+        .. versionchanged:: 7.2
+
+           A missing categorical value was a row of zeros, which kept the
+           observation (in the first category when ``drop_first`` is True).
     drop_first : bool, optional
         Flag indicating to drop the first dummy category. The default is True.
 

linearmodels/panel/data.py

@@ -214,7 +214,14 @@ class PanelData:
         xarray DataArrays. The default is ``"x"``.
     convert_dummies : bool, optional
         Flag indicating whether pandas categoricals or string input data
-        should be converted to dummy variables. The default is True.
+        should be converted to dummy variables. The default is True. A
+        missing categorical value is a missing value of each of its dummy
+        variables, so that the observation is dropped as missing.
+
+        .. versionchanged:: 7.2
+
+           A missing categorical value was a row of zeros, which kept the
+           observation (in the first category when ``drop_first`` is True).
     drop_first : bool, optional
         Flag indicating to drop first dummy category when converting. The
         default is True.

linearmodels/tests/iv/test_data.py

@@ -1,12 +1,14 @@
 from typing import cast
 
 import numpy as np
-from numpy.testing import assert_equal
+from numpy.testing import assert_allclose, assert_equal
 import pandas as pd
 from pandas.testing import assert_frame_equal, assert_series_equal
 import pytest
 
+from linearmodels.iv import IV2SLS
 from linearmodels.iv.data import IVData
+from linearmodels.shared.exceptions import MissingValueWarning
 from linearmodels.typing.data import BoolArray
 
 try:
@@ -273,3 +275,52 @@ def test_categorical_missing_values_with_unused_category(drop_first: bool) -> No
     data = IVData(values, drop_first=drop_first)
     assert "group.unused" not in data.cols
     assert_equal(data.isnull.to_numpy(), [False, True, False, False])
+
+
+def test_categorical_all_missing_values() -> None:
+    # No dummy column is used, so the columns are kept and every observation is
+    # missing
+    values = pd.Series(
+        pd.Categorical([None, None, None], categories=["a", "b"]), name="group"
+    )
+    data = IVData(values, drop_first=False)
+    assert data.cols == ["group.a", "group.b"]
+    assert data.isnull.all()
+    assert np.isnan(data.ndarray).all()
+
+
+def test_categorical_missing_values_in_several_columns() -> None:
+    frame = pd.DataFrame(
+        {
+            "g": pd.Categorical(["a", None, "b", "a", "b"]),
+            "h": pd.Categorical(["x", "y", None, "y", "x"]),
+            "v": [1.0, 2.0, 3.0, 4.0, 5.0],
+        }
+    )
+    data = IVData(frame)
+    assert_equal(data.isnull.to_numpy(), [False, True, True, False, False])
+
+
+def test_categorical_missing_values_in_a_model() -> None:
+    # The observations with a missing category are dropped with a warning, and the
+    # estimates are those of the model without them
+    rng = np.random.default_rng(858)
+    nobs = 200
+    labels = rng.choice(["a", "b", "c"], nobs).astype(object)
+    labels[rng.choice(nobs, 20, replace=False)] = None
+    x = rng.standard_normal(nobs)
+    y = 1 + x + 0.5 * (labels == "b") - 0.7 * (labels == "c")
+    y = y + rng.standard_normal(nobs)
+    exog = pd.DataFrame({"const": 1.0, "x": x, "g": pd.Categorical(labels)})
+    with pytest.warns(MissingValueWarning, match="missing values"):
+        res = IV2SLS(y, exog, None, None).fit()
+
+    keep = exog["g"].notna().to_numpy()
+    dummies = pd.get_dummies(
+        exog["g"][keep], prefix="g", prefix_sep=".", drop_first=True, dtype=float
+    )
+    regressors = np.column_stack([np.ones(keep.sum()), x[keep], dummies])
+    expected = np.linalg.lstsq(regressors, y[keep], rcond=None)[0]
+    assert res.nobs == keep.sum()
+    assert list(res.params.index) == ["const", "x", "g.b", "g.c"]
+    assert_allclose(res.params, expected, rtol=1e-10)

linearmodels/tests/panel/test_data.py

@@ -10,8 +10,8 @@
 import pytest
 
 from linearmodels.panel.data import PanelData, _Panel
-from linearmodels.panel.model import PanelOLS, panel_structure_stats
-from linearmodels.shared.exceptions import UnknownOptionError
+from linearmodels.panel.model import PanelOLS, PooledOLS, panel_structure_stats
+from linearmodels.shared.exceptions import MissingValueWarning, UnknownOptionError
 from linearmodels.shared.utility import panel_to_frame
 from linearmodels.tests._utility import warns_if_missing
 from linearmodels.tests.panel._utility import MISSING_XARRAY, datatypes, generate_data
@@ -990,3 +990,44 @@ def test_categorical_missing_values_with_unused_category(drop_first):
     data = PanelData(values, drop_first=drop_first)
     assert "group.unused" not in data.dataframe
     assert_equal(data.isnull.to_numpy(), [False, True, False, False])
+
+
+def test_categorical_all_missing_values():
+    # No dummy column is used, so the columns are kept and every observation is
+    # missing
+    index = MultiIndex.from_product([["a", "b"], [0, 1]])
+    values = DataFrame(
+        {"group": Categorical([None] * 4, categories=["first", "second"])},
+        index=index,
+    )
+    data = PanelData(values, drop_first=False)
+    assert list(data.dataframe.columns) == ["group.first", "group.second"]
+    assert data.isnull.all()
+
+
+def test_categorical_missing_values_in_a_model():
+    # The observations with a missing category are dropped with a warning, and the
+    # estimates are those of the model without them
+    rng = np.random.default_rng(858)
+    index = MultiIndex.from_product([range(20), range(10)])
+    nobs = len(index)
+    labels = rng.choice(["a", "b", "c"], nobs).astype(object)
+    labels[rng.choice(nobs, 20, replace=False)] = None
+    x = rng.standard_normal(nobs)
+    y = 1 + x + 0.5 * (labels == "b") - 0.7 * (labels == "c")
+    y = y + rng.standard_normal(nobs)
+    frame = DataFrame(
+        {"y": y, "const": 1.0, "x": x, "g": Categorical(labels)}, index=index
+    )
+    with pytest.warns(MissingValueWarning, match="missing values"):
+        res = PooledOLS(frame.y, frame[["const", "x", "g"]]).fit()
+
+    keep = frame["g"].notna().to_numpy()
+    dummies = get_dummies(
+        frame["g"][keep], prefix="g", prefix_sep=".", drop_first=True, dtype=float
+    )
+    regressors = np.column_stack([np.ones(keep.sum()), x[keep], dummies])
+    expected = lstsq(regressors, y[keep], rcond=None)[0]
+    assert res.nobs == keep.sum()
+    assert list(res.params.index) == ["const", "x", "g.b", "g.c"]
+    assert_allclose(res.params, expected, rtol=1e-10)