diff --git a/docs/user_guide/preprocessing.md b/docs/user_guide/preprocessing.md index 5f3d52c..272924c 100644 --- a/docs/user_guide/preprocessing.md +++ b/docs/user_guide/preprocessing.md @@ -77,7 +77,7 @@ preprocessor = Preprocessor( | --------------------- | ------------------------------ | ----- | | `int` | `ContinuousOrdinalTransformer` | Integer/ordinal encoding (default) | | `one-hot` | `OneHotEncoder` | One-hot encoding | -| `onehot_from_ordinal` | `OneHotFromOrdinalTransformer` | One-hot from pre-encoded ordinals | +| `onehot_from_ordinal` | `ContinuousOrdinalTransformer` -> `OneHotFromOrdinalTransformer` | Integer codes, then one-hot (reserves column `0` for unseen categories) | | `pretrained` | `LanguageEmbeddingTransformer` | Pretrained language embeddings | | `custombin` | `CustomBinTransformer` | Binning of categorical codes | | `none` | `NoTransformer` | Pass-through | diff --git a/pretab/pipeline/categorical.py b/pretab/pipeline/categorical.py index a860be2..2f63020 100644 --- a/pretab/pipeline/categorical.py +++ b/pretab/pipeline/categorical.py @@ -49,6 +49,11 @@ def get_categorical_transformer_steps( bin_kwargs.setdefault("output_dim", output_dim) steps.append(("custombin", CustomBinTransformer(**bin_kwargs))) elif method == "onehot_from_ordinal": + # ``OneHotFromOrdinalTransformer`` expects integer codes; the Preprocessor + # hands it raw column values, which are usually strings. Encode first so + # the method matches its documented "integer codes then one-hot" + # behaviour instead of dying in ``np.max(...).astype(int)``. + steps.append(("continuous_ordinal", ContinuousOrdinalTransformer())) steps.append(("onehot_from_ordinal", OneHotFromOrdinalTransformer())) else: raise invalid_param_error( diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index fcf228a..22c9b36 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -52,9 +52,11 @@ class Preprocessor(TransformerMixin, BaseEstimator): categorical_method : str, default="int" Preprocessing strategy applied to every categorical column unless overridden per feature. Choices: ``"int"`` (contiguous integer codes), ``"one-hot"`` (dummy columns), - ``"onehot_from_ordinal"`` (integer codes then one-hot), ``"pretrained"`` (sentence-transformer - language embeddings), and ``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to - ``"none"``) to leave categorical columns unchanged. + ``"onehot_from_ordinal"`` (contiguous integer codes, then one-hot; reserves the first + column for categories unseen at fit time, so a column with ``k`` categories expands to + ``k + 1`` columns), ``"pretrained"`` (sentence-transformer language embeddings), and + ``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to ``"none"``) to leave + categorical columns unchanged. feature_preprocessing : dict, optional Mapping of individual column names to a method, overriding the global ``numerical_method`` / ``categorical_method`` for those columns only, e.g. diff --git a/tests/test_categorical_pipeline.py b/tests/test_categorical_pipeline.py index 1f8ad2e..66b2c00 100644 --- a/tests/test_categorical_pipeline.py +++ b/tests/test_categorical_pipeline.py @@ -1,7 +1,11 @@ +from typing import cast + import numpy as np +import pandas as pd import pytest from sklearn.pipeline import Pipeline +from pretab import Preprocessor from pretab.pipeline import get_categorical_transformer_steps @@ -28,3 +32,43 @@ def test_one_hot_handle_unknown_override(): with pytest.raises(ValueError): pipe.transform(np.array([["C"]])) + + +# --------------------------------------------------------------------------- # +# ``onehot_from_ordinal`` must accept raw (string) categoricals. +# +# The pipeline appended only ``OneHotFromOrdinalTransformer``, which requires +# already-ordinal input, so ``np.max(X, axis=0).astype(int)`` died with a bare +# ``ValueError: invalid literal for int() with base 10: 'c'``. +# --------------------------------------------------------------------------- # +def test_onehot_from_ordinal_encodes_string_categories(): + frame = pd.DataFrame({"c": ["a", "b", "c"] * 30}) + pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none") + + out = cast("np.ndarray", pre.fit_transform(frame, return_array=True)) + + # 3 categories plus the reserved column 0 for unseen values. + assert out.shape == (90, 4) + np.testing.assert_array_equal(out[:3], np.eye(4)[[1, 2, 3]]) + + +def test_onehot_from_ordinal_pipeline_encodes_before_one_hot(): + steps = [name for name, _ in get_categorical_transformer_steps("onehot_from_ordinal")] + assert steps.index("continuous_ordinal") < steps.index("onehot_from_ordinal") + + +def test_onehot_from_ordinal_sends_unseen_categories_to_the_reserved_column(): + frame = pd.DataFrame({"c": ["a", "b", "c"] * 30}) + pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none").fit(frame) + + out = cast("np.ndarray", pre.transform(pd.DataFrame({"c": ["a", "ZZZ", "c"]}), return_array=True)) + + np.testing.assert_array_equal(out[1], [1.0, 0.0, 0.0, 0.0]) + + +def test_onehot_from_ordinal_feature_names_match_width(): + frame = pd.DataFrame({"c": ["a", "b", "c"] * 30}) + pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none").fit(frame) + + transformed = cast("np.ndarray", pre.transform(frame, return_array=True)) + assert len(pre.get_feature_names_out()) == transformed.shape[1]