From 0cb45b07079cf0b04519a1ac9ce60dd6c8164380 Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:11:57 +0200 Subject: [PATCH] fix(onehot): encode categories before onehot_from_ordinal ``categorical_method="onehot_from_ordinal"`` appended only ``OneHotFromOrdinalTransformer``, which requires input that is already ordinal encoded. The Preprocessor hands it raw column values, so any string column died inside ``np.max(X, axis=0).astype(int)`` with a bare ValueError: invalid literal for int() with base 10: 'c' that says nothing about the chosen method. Since ``_detect_column_types`` routes every non-numeric column to the categorical side, this was the common case. Insert ``ContinuousOrdinalTransformer`` ahead of it so the method does what the Preprocessor docstring already claimed -- integer codes, then one-hot. Because that encoder numbers categories from 1 and reserves 0 for unseen values, a column with k categories expands to k + 1 columns and categories unseen at fit time land in the reserved column instead of producing an all-zero row. Docs reconciled: the Preprocessor docstring, the user-guide table and the class itself previously gave three different accounts of this method. Closes #17 Co-Authored-By: Claude Opus 5 (1M context) --- docs/user_guide/preprocessing.md | 2 +- pretab/pipeline/categorical.py | 5 ++++ pretab/preprocessor.py | 8 ++++-- tests/test_categorical_pipeline.py | 44 ++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/user_guide/preprocessing.md b/docs/user_guide/preprocessing.md index 5f3d52c..272924c 100644 --- a/docs/user_guide/preprocessing.md +++ b/docs/user_guide/preprocessing.md @@ -77,7 +77,7 @@ preprocessor = Preprocessor( | --------------------- | ------------------------------ | ----- | | `int` | `ContinuousOrdinalTransformer` | Integer/ordinal encoding (default) | | `one-hot` | `OneHotEncoder` | One-hot encoding | -| `onehot_from_ordinal` | `OneHotFromOrdinalTransformer` | One-hot from pre-encoded ordinals | +| `onehot_from_ordinal` | `ContinuousOrdinalTransformer` -> `OneHotFromOrdinalTransformer` | Integer codes, then one-hot (reserves column `0` for unseen categories) | | `pretrained` | `LanguageEmbeddingTransformer` | Pretrained language embeddings | | `custombin` | `CustomBinTransformer` | Binning of categorical codes | | `none` | `NoTransformer` | Pass-through | diff --git a/pretab/pipeline/categorical.py b/pretab/pipeline/categorical.py index a860be2..2f63020 100644 --- a/pretab/pipeline/categorical.py +++ b/pretab/pipeline/categorical.py @@ -49,6 +49,11 @@ def get_categorical_transformer_steps( bin_kwargs.setdefault("output_dim", output_dim) steps.append(("custombin", CustomBinTransformer(**bin_kwargs))) elif method == "onehot_from_ordinal": + # ``OneHotFromOrdinalTransformer`` expects integer codes; the Preprocessor + # hands it raw column values, which are usually strings. Encode first so + # the method matches its documented "integer codes then one-hot" + # behaviour instead of dying in ``np.max(...).astype(int)``. + steps.append(("continuous_ordinal", ContinuousOrdinalTransformer())) steps.append(("onehot_from_ordinal", OneHotFromOrdinalTransformer())) else: raise invalid_param_error( diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index fcf228a..22c9b36 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -52,9 +52,11 @@ class Preprocessor(TransformerMixin, BaseEstimator): categorical_method : str, default="int" Preprocessing strategy applied to every categorical column unless overridden per feature. Choices: ``"int"`` (contiguous integer codes), ``"one-hot"`` (dummy columns), - ``"onehot_from_ordinal"`` (integer codes then one-hot), ``"pretrained"`` (sentence-transformer - language embeddings), and ``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to - ``"none"``) to leave categorical columns unchanged. + ``"onehot_from_ordinal"`` (contiguous integer codes, then one-hot; reserves the first + column for categories unseen at fit time, so a column with ``k`` categories expands to + ``k + 1`` columns), ``"pretrained"`` (sentence-transformer language embeddings), and + ``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to ``"none"``) to leave + categorical columns unchanged. feature_preprocessing : dict, optional Mapping of individual column names to a method, overriding the global ``numerical_method`` / ``categorical_method`` for those columns only, e.g. diff --git a/tests/test_categorical_pipeline.py b/tests/test_categorical_pipeline.py index 1f8ad2e..66b2c00 100644 --- a/tests/test_categorical_pipeline.py +++ b/tests/test_categorical_pipeline.py @@ -1,7 +1,11 @@ +from typing import cast + import numpy as np +import pandas as pd import pytest from sklearn.pipeline import Pipeline +from pretab import Preprocessor from pretab.pipeline import get_categorical_transformer_steps @@ -28,3 +32,43 @@ def test_one_hot_handle_unknown_override(): with pytest.raises(ValueError): pipe.transform(np.array([["C"]])) + + +# --------------------------------------------------------------------------- # +# ``onehot_from_ordinal`` must accept raw (string) categoricals. +# +# The pipeline appended only ``OneHotFromOrdinalTransformer``, which requires +# already-ordinal input, so ``np.max(X, axis=0).astype(int)`` died with a bare +# ``ValueError: invalid literal for int() with base 10: 'c'``. +# --------------------------------------------------------------------------- # +def test_onehot_from_ordinal_encodes_string_categories(): + frame = pd.DataFrame({"c": ["a", "b", "c"] * 30}) + pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none") + + out = cast("np.ndarray", pre.fit_transform(frame, return_array=True)) + + # 3 categories plus the reserved column 0 for unseen values. + assert out.shape == (90, 4) + np.testing.assert_array_equal(out[:3], np.eye(4)[[1, 2, 3]]) + + +def test_onehot_from_ordinal_pipeline_encodes_before_one_hot(): + steps = [name for name, _ in get_categorical_transformer_steps("onehot_from_ordinal")] + assert steps.index("continuous_ordinal") < steps.index("onehot_from_ordinal") + + +def test_onehot_from_ordinal_sends_unseen_categories_to_the_reserved_column(): + frame = pd.DataFrame({"c": ["a", "b", "c"] * 30}) + pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none").fit(frame) + + out = cast("np.ndarray", pre.transform(pd.DataFrame({"c": ["a", "ZZZ", "c"]}), return_array=True)) + + np.testing.assert_array_equal(out[1], [1.0, 0.0, 0.0, 0.0]) + + +def test_onehot_from_ordinal_feature_names_match_width(): + frame = pd.DataFrame({"c": ["a", "b", "c"] * 30}) + pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none").fit(frame) + + transformed = cast("np.ndarray", pre.transform(frame, return_array=True)) + assert len(pre.get_feature_names_out()) == transformed.shape[1]