Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/user_guide/preprocessing.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ preprocessor = Preprocessor(
| --------------------- | ------------------------------ | ----- |
| `int` | `ContinuousOrdinalTransformer` | Integer/ordinal encoding (default) |
| `one-hot` | `OneHotEncoder` | One-hot encoding |
| `onehot_from_ordinal` | `OneHotFromOrdinalTransformer` | One-hot from pre-encoded ordinals |
| `onehot_from_ordinal` | `ContinuousOrdinalTransformer` -> `OneHotFromOrdinalTransformer` | Integer codes, then one-hot (reserves column `0` for unseen categories) |
| `pretrained` | `LanguageEmbeddingTransformer` | Pretrained language embeddings |
| `custombin` | `CustomBinTransformer` | Binning of categorical codes |
| `none` | `NoTransformer` | Pass-through |
Expand Down
5 changes: 5 additions & 0 deletions pretab/pipeline/categorical.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ def get_categorical_transformer_steps(
bin_kwargs.setdefault("output_dim", output_dim)
steps.append(("custombin", CustomBinTransformer(**bin_kwargs)))
elif method == "onehot_from_ordinal":
# ``OneHotFromOrdinalTransformer`` expects integer codes; the Preprocessor
# hands it raw column values, which are usually strings. Encode first so
# the method matches its documented "integer codes then one-hot"
# behaviour instead of dying in ``np.max(...).astype(int)``.
steps.append(("continuous_ordinal", ContinuousOrdinalTransformer()))
steps.append(("onehot_from_ordinal", OneHotFromOrdinalTransformer()))
else:
raise invalid_param_error(
Expand Down
8 changes: 5 additions & 3 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,11 @@ class Preprocessor(TransformerMixin, BaseEstimator):
categorical_method : str, default="int"
Preprocessing strategy applied to every categorical column unless overridden per feature.
Choices: ``"int"`` (contiguous integer codes), ``"one-hot"`` (dummy columns),
``"onehot_from_ordinal"`` (integer codes then one-hot), ``"pretrained"`` (sentence-transformer
language embeddings), and ``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to
``"none"``) to leave categorical columns unchanged.
``"onehot_from_ordinal"`` (contiguous integer codes, then one-hot; reserves the first
column for categories unseen at fit time, so a column with ``k`` categories expands to
``k + 1`` columns), ``"pretrained"`` (sentence-transformer language embeddings), and
``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to ``"none"``) to leave
categorical columns unchanged.
feature_preprocessing : dict, optional
Mapping of individual column names to a method, overriding the global ``numerical_method`` /
``categorical_method`` for those columns only, e.g.
Expand Down
44 changes: 44 additions & 0 deletions tests/test_categorical_pipeline.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
from typing import cast

import numpy as np
import pandas as pd
import pytest
from sklearn.pipeline import Pipeline

from pretab import Preprocessor
from pretab.pipeline import get_categorical_transformer_steps


Expand All @@ -28,3 +32,43 @@ def test_one_hot_handle_unknown_override():
with pytest.raises(ValueError):
pipe.transform(np.array([["C"]]))



# --------------------------------------------------------------------------- #
# ``onehot_from_ordinal`` must accept raw (string) categoricals.
#
# The pipeline appended only ``OneHotFromOrdinalTransformer``, which requires
# already-ordinal input, so ``np.max(X, axis=0).astype(int)`` died with a bare
# ``ValueError: invalid literal for int() with base 10: 'c'``.
# --------------------------------------------------------------------------- #
def test_onehot_from_ordinal_encodes_string_categories():
frame = pd.DataFrame({"c": ["a", "b", "c"] * 30})
pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none")

out = cast("np.ndarray", pre.fit_transform(frame, return_array=True))

# 3 categories plus the reserved column 0 for unseen values.
assert out.shape == (90, 4)
np.testing.assert_array_equal(out[:3], np.eye(4)[[1, 2, 3]])


def test_onehot_from_ordinal_pipeline_encodes_before_one_hot():
steps = [name for name, _ in get_categorical_transformer_steps("onehot_from_ordinal")]
assert steps.index("continuous_ordinal") < steps.index("onehot_from_ordinal")


def test_onehot_from_ordinal_sends_unseen_categories_to_the_reserved_column():
frame = pd.DataFrame({"c": ["a", "b", "c"] * 30})
pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none").fit(frame)

out = cast("np.ndarray", pre.transform(pd.DataFrame({"c": ["a", "ZZZ", "c"]}), return_array=True))

np.testing.assert_array_equal(out[1], [1.0, 0.0, 0.0, 0.0])


def test_onehot_from_ordinal_feature_names_match_width():
frame = pd.DataFrame({"c": ["a", "b", "c"] * 30})
pre = Preprocessor(categorical_method="onehot_from_ordinal", numerical_method="none").fit(frame)

transformed = cast("np.ndarray", pre.transform(frame, return_array=True))
assert len(pre.get_feature_names_out()) == transformed.shape[1]
Loading