From 3616bbea8f0b4a2a3af4094b743de58d06339eaf Mon Sep 17 00:00:00 2001 From: "Tianlang (Jackie) Chen" <305351710+jackiectl2@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:59:39 -0400 Subject: [PATCH 1/2] BUG: Preserve missing categorical observations in dummy conversion Signed-off-by: Tianlang (Jackie) Chen <305351710+jackiectl2@users.noreply.github.com> --- linearmodels/iv/data.py | 2 ++ linearmodels/panel/data.py | 2 ++ linearmodels/tests/iv/test_data.py | 12 ++++++++++++ linearmodels/tests/panel/test_data.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/linearmodels/iv/data.py b/linearmodels/iv/data.py index 14c1ad2ed5..794993b3d0 100644 --- a/linearmodels/iv/data.py +++ b/linearmodels/iv/data.py @@ -36,6 +36,8 @@ def convert_columns(s: pd.Series, drop_first: bool) -> linearmodels.typing.AnyPa """ if isinstance(s.dtype, pd.CategoricalDtype): out = pd.get_dummies(s, drop_first=drop_first) + if s.hasnans: + out = out.astype(float).where(s.notna(), axis=0) # TODO: Remove once pandas typing fixed out.columns = pd.Index([str(s.name) + "." + str(c) for c in out]) return out diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py index 456f43ed01..518a38b21f 100644 --- a/linearmodels/panel/data.py +++ b/linearmodels/panel/data.py @@ -162,6 +162,8 @@ def convert_columns(s: Series, drop_first: bool) -> linearmodels.typing.AnyPanda if isinstance(s.dtype, pd.CategoricalDtype): out = get_dummies(s, drop_first=drop_first) + if s.hasnans: + out = out.astype(float).where(s.notna(), axis=0) # TODO: Remove once pandas typing fixed out.columns = Index([str(s.name) + "." + str(c) for c in out]) return out diff --git a/linearmodels/tests/iv/test_data.py b/linearmodels/tests/iv/test_data.py index 73da5beec1..2369fc119f 100644 --- a/linearmodels/tests/iv/test_data.py +++ b/linearmodels/tests/iv/test_data.py @@ -250,3 +250,15 @@ def test_duplicate_column_names(): x = pd.DataFrame(np.ones((3, 2)), columns=["x", "x"]) with pytest.raises(ValueError, match=r"DataFrame contains duplicate column names"): IVData(x) + + +@pytest.mark.parametrize("drop_first", [True, False]) +def test_categorical_missing_values(drop_first: bool) -> None: + values = pd.Series(pd.Categorical(["a", None, "b", "a"]), name="group") + data = IVData(values, drop_first=drop_first) + assert_equal(data.isnull.to_numpy(), [False, True, False, False]) + assert np.isnan(data.ndarray[1]).all() + assert_equal(data.pandas["group.b"].iloc[[0, 2, 3]].to_numpy(), [0, 1, 0]) + data.drop(data.isnull) + assert data.rows == [0, 2, 3] + assert not data.isnull.any() diff --git a/linearmodels/tests/panel/test_data.py b/linearmodels/tests/panel/test_data.py index 1e20a4f9d3..cd1709dedf 100644 --- a/linearmodels/tests/panel/test_data.py +++ b/linearmodels/tests/panel/test_data.py @@ -958,3 +958,18 @@ def test_fake_panel_properties(mi_df): assert_index_equal(panel.minor_axis, mi_df.index.levels[0]) df = panel.to_frame() assert_frame_equal(df, mi_df) + + +@pytest.mark.parametrize("drop_first", [True, False]) +def test_categorical_missing_values(drop_first): + index = MultiIndex.from_product([["a", "b"], [0, 1]]) + values = DataFrame( + {"group": Categorical(["first", None, "second", "first"])}, index=index + ) + data = PanelData(values, drop_first=drop_first) + assert_equal(data.isnull.to_numpy(), [False, True, False, False]) + assert np.isnan(data.values2d[1]).all() + assert_equal(data.dataframe["group.second"].iloc[[0, 2, 3]].to_numpy(), [0, 1, 0]) + data.drop(data.isnull) + assert data.dataframe.index.tolist() == [("a", 0), ("b", 0), ("b", 1)] + assert not data.isnull.any() From 46645aa55f985bdf713bb51d8061f49d4a48e844 Mon Sep 17 00:00:00 2001 From: "Tianlang (Jackie) Chen" <305351710+jackiectl2@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:43:44 +0000 Subject: [PATCH 2/2] Drop unused dummy columns after naming them Signed-off-by: Tianlang (Jackie) Chen <305351710+jackiectl2@users.noreply.github.com> --- linearmodels/iv/data.py | 7 +++++-- linearmodels/panel/data.py | 7 +++++-- linearmodels/tests/iv/test_data.py | 11 +++++++++++ linearmodels/tests/panel/test_data.py | 17 +++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/linearmodels/iv/data.py b/linearmodels/iv/data.py index 794993b3d0..86d0fb4bd8 100644 --- a/linearmodels/iv/data.py +++ b/linearmodels/iv/data.py @@ -36,10 +36,13 @@ def convert_columns(s: pd.Series, drop_first: bool) -> linearmodels.typing.AnyPa """ if isinstance(s.dtype, pd.CategoricalDtype): out = pd.get_dummies(s, drop_first=drop_first) - if s.hasnans: - out = out.astype(float).where(s.notna(), axis=0) # TODO: Remove once pandas typing fixed out.columns = pd.Index([str(s.name) + "." + str(c) for c in out]) + if s.isna().any(): + used = out.ne(0).any(axis=0) + if used.any(): + out = out.loc[:, used] + out = out.astype(float).where(s.notna(), axis=0) return out return s diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py index 518a38b21f..d7587fb45a 100644 --- a/linearmodels/panel/data.py +++ b/linearmodels/panel/data.py @@ -162,10 +162,13 @@ def convert_columns(s: Series, drop_first: bool) -> linearmodels.typing.AnyPanda if isinstance(s.dtype, pd.CategoricalDtype): out = get_dummies(s, drop_first=drop_first) - if s.hasnans: - out = out.astype(float).where(s.notna(), axis=0) # TODO: Remove once pandas typing fixed out.columns = Index([str(s.name) + "." + str(c) for c in out]) + if s.isna().any(): + used = out.ne(0).any(axis=0) + if used.any(): + out = out.loc[:, used] + out = out.astype(float).where(s.notna(), axis=0) return out return s diff --git a/linearmodels/tests/iv/test_data.py b/linearmodels/tests/iv/test_data.py index 2369fc119f..6b0c671fd8 100644 --- a/linearmodels/tests/iv/test_data.py +++ b/linearmodels/tests/iv/test_data.py @@ -262,3 +262,14 @@ def test_categorical_missing_values(drop_first: bool) -> None: data.drop(data.isnull) assert data.rows == [0, 2, 3] assert not data.isnull.any() + + +@pytest.mark.parametrize("drop_first", [True, False]) +def test_categorical_missing_values_with_unused_category(drop_first: bool) -> None: + values = pd.Series( + pd.Categorical(["a", None, "b", "a"], categories=["a", "b", "unused"]), + name="group", + ) + data = IVData(values, drop_first=drop_first) + assert "group.unused" not in data.cols + assert_equal(data.isnull.to_numpy(), [False, True, False, False]) diff --git a/linearmodels/tests/panel/test_data.py b/linearmodels/tests/panel/test_data.py index cd1709dedf..7fb3430ee9 100644 --- a/linearmodels/tests/panel/test_data.py +++ b/linearmodels/tests/panel/test_data.py @@ -973,3 +973,20 @@ def test_categorical_missing_values(drop_first): data.drop(data.isnull) assert data.dataframe.index.tolist() == [("a", 0), ("b", 0), ("b", 1)] assert not data.isnull.any() + + +@pytest.mark.parametrize("drop_first", [True, False]) +def test_categorical_missing_values_with_unused_category(drop_first): + index = MultiIndex.from_product([["a", "b"], [0, 1]]) + values = DataFrame( + { + "group": Categorical( + ["first", None, "second", "first"], + categories=["first", "second", "unused"], + ) + }, + index=index, + ) + data = PanelData(values, drop_first=drop_first) + assert "group.unused" not in data.dataframe + assert_equal(data.isnull.to_numpy(), [False, True, False, False])