diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..9512d4011 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,59 @@ +name: CodSpeed + +on: + push: + branches: + - main + # Only fire on the events the job's `if:` below actually needs: `labeled` + # (adding the `runcodespeed` label) and `synchronize` (further pushes, + # which re-benchmark for as long as the label stays attached). Doc-only + # changes are skipped entirely, since they can't affect performance. + pull_request: + types: [labeled, synchronize] + paths-ignore: + - '**.md' + - '**.rst' + - 'docs/**' + - 'paper/**' + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Cancel a stale in-progress run when a PR gets pushed to again, but let + # every push to main finish, since each one records a CodSpeed baseline. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + id-token: write # for OpenID Connect authentication with CodSpeed + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + # Always run for push (main) and workflow_dispatch. For pull_request + # events, only run while the PR carries the `runcodespeed` label. + if: > + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'runcodespeed') + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install pytest pytest-codspeed + + - name: Run benchmarks + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: pytest benchmarks/ --codspeed diff --git a/README.md b/README.md index 85ddcd6b0..8f04b87e0 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ | **Meta** | [![GitHub contributors](https://img.shields.io/github/contributors/feature-engine/feature_engine?logo=GitHub)](https://github.com/feature-engine/feature_engine/graphs/contributors) [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat)](https://www.firsttimersonly.com/) | | **Documentation** | [![Read the Docs](https://img.shields.io/readthedocs/feature_engine?logo=readthedocs)](https://feature-engine.readthedocs.io/en/latest/index.html) | | **Citation** | [![DOI](https://zenodo.org/badge/163630824.svg)](https://zenodo.org/badge/latestdoi/163630824) [![JOSS](https://joss.theoj.org/papers/10.21105/joss.03642/status.svg)](https://doi.org/10.21105/joss.03642) | -| **Testing** | [![CircleCI](https://img.shields.io/circleci/build/github/feature-engine/feature_engine/main?logo=CircleCI)](https://app.circleci.com/pipelines/github/feature-engine/feature_engine) [![Codecov](https://img.shields.io/codecov/c/github/feature-engine/feature_engine?logo=CodeCov&token=ZBKKSN6ERL)](https://codecov.io/github/feature-engine/feature_engine) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) | +| **Testing** | [![CircleCI](https://img.shields.io/circleci/build/github/feature-engine/feature_engine/main?logo=CircleCI)](https://app.circleci.com/pipelines/github/feature-engine/feature_engine) [![Codecov](https://img.shields.io/codecov/c/github/feature-engine/feature_engine?logo=CodeCov&token=ZBKKSN6ERL)](https://codecov.io/github/feature-engine/feature_engine) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/feature-engine/feature_engine?utm_source=badge) |
diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..afa8c63f5 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,60 @@ +# Benchmarks + +This folder contains the performance benchmarks of Feature-engine. They are +written with [pytest-codspeed](https://github.com/CodSpeedHQ/pytest-codspeed) +and run on every push and pull request by the `CodSpeed` GitHub Actions +workflow, which reports the results to +[CodSpeed](https://app.codspeed.io/feature-engine/feature_engine). + +## What is covered + +One module per transformer family, benchmarking `fit` and `transform` +separately, since they have very different performance profiles: + +| File | Covers | +| -------------------------- | ------------------------------------------------------------- | +| `test_imputation.py` | Missing data imputers | +| `test_encoding.py` | Categorical encoders | +| `test_discretisation.py` | Discretisers | +| `test_outliers.py` | Outlier cappers and trimmers | +| `test_transformation.py` | Mathematical transformers and scalers | +| `test_creation.py` | Feature creation transformers | +| `test_datetime.py` | Datetime feature extraction | +| `test_timeseries.py` | Lag, window and expanding window features | +| `test_selection.py` | Feature selectors | +| `test_variable_handling.py`| Variable handling helpers, called by every transformer's `fit` | +| `test_pipeline.py` | End to end pipelines and the preprocessing transformers | + +The data is synthetic and built in `conftest.py` fixtures, so data generation is +never part of what is measured. Dataframes are session scoped and shared by all +benchmarks. + +## Running them locally + +```bash +pip install -e . +pip install pytest pytest-codspeed + +# quick check that the benchmarks run, with walltime measurements +pytest benchmarks/ --codspeed + +# same measurements as CI, requires the CodSpeed CLI +codspeed run --mode simulation -- pytest benchmarks/ --codspeed +``` + +Running a single file or benchmark works as with any other pytest test: + +```bash +pytest benchmarks/test_encoding.py --codspeed +pytest benchmarks/test_encoding.py::test_woe_encoder_fit --codspeed +``` + +## Adding a benchmark + +- Reuse the dataframe fixtures from `conftest.py`. Use `df_big` for the + vectorised transformers, `df_small` for the ones that train models + (decision trees, cross-validation) and `df_tiny` for the row-wise ones. +- Do the `fit` outside of the measured section when benchmarking `transform`. +- Keep a single benchmark in the millisecond range: the whole suite runs under + CPU simulation in CI, which is roughly two orders of magnitude slower than a + plain run. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 000000000..4332b3de7 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,133 @@ +"""Shared data fixtures for the benchmark suite. + +The dataframes built here are synthetic but representative of the kind of data +Feature-engine transformers are used on: a mix of numerical, categorical and +datetime variables, with missing values. + +Data generation happens in fixtures so that it is never included in the +measured section of a benchmark. +""" + +import numpy as np +import pandas as pd +import pytest + +# Number of rows used for the transformers whose fit/transform is cheap. +BIG_N = 10_000 + +# Number of rows used for the transformers that train models under the hood +# (decision trees, cross-validation, ...) so benchmarks stay in the millisecond +# to low second range. +SMALL_N = 1_000 + +# Number of rows used for the row-wise transformers, which are an order of +# magnitude slower per row than the vectorised ones. +TINY_N = 500 + +N_NUMERICAL = 8 +N_CATEGORICAL = 4 + + +def _make_dataframe(n_rows: int, seed: int = 0, with_na: bool = False): + rng = np.random.default_rng(seed) + + data = { + f"num_{i}": rng.normal(loc=i, scale=i + 1, size=n_rows) + for i in range(N_NUMERICAL) + } + + # A couple of strictly positive variables, needed by log/box-cox style + # transformers. + data["pos_0"] = rng.gamma(shape=2.0, scale=3.0, size=n_rows) + 0.1 + data["pos_1"] = rng.gamma(shape=5.0, scale=1.0, size=n_rows) + 0.1 + + # A variable bounded between 0 and 1, needed by the arcsin transformer. + data["frac_0"] = rng.uniform(0.0, 1.0, size=n_rows) + + # Categorical variables with a decreasing cardinality, including rare + # categories to exercise the rare label encoder. + for i in range(N_CATEGORICAL): + n_categories = 5 * (i + 1) + weights = np.linspace(1.0, 0.02, num=n_categories) + weights = weights / weights.sum() + data[f"cat_{i}"] = rng.choice( + [f"cat_{i}_value_{j}" for j in range(n_categories)], + size=n_rows, + p=weights, + ) + + data["date_0"] = pd.date_range("2015-01-01", periods=n_rows, freq="h") + data["date_1"] = pd.date_range("2018-06-15", periods=n_rows, freq="7min") + + df = pd.DataFrame(data) + + if with_na: + for column in ["num_0", "num_1", "pos_0", "cat_0", "cat_1"]: + mask = rng.random(n_rows) < 0.15 + df.loc[mask, column] = np.nan + + return df + + +def numerical_vars(): + return [f"num_{i}" for i in range(N_NUMERICAL)] + + +def categorical_vars(): + return [f"cat_{i}" for i in range(N_CATEGORICAL)] + + +@pytest.fixture(scope="session") +def df_big(): + """Complete dataframe, no missing data.""" + return _make_dataframe(BIG_N, seed=0) + + +@pytest.fixture(scope="session") +def df_big_na(): + """Complete dataframe with missing data in numerical and categorical vars.""" + return _make_dataframe(BIG_N, seed=1, with_na=True) + + +@pytest.fixture(scope="session") +def df_small(): + """Smaller dataframe, for the estimator based transformers.""" + return _make_dataframe(SMALL_N, seed=2) + + +@pytest.fixture(scope="session") +def df_tiny(): + """Smallest dataframe, for the row-wise transformers.""" + return _make_dataframe(TINY_N, seed=7) + + +@pytest.fixture(scope="session") +def y_binary(): + """Binary target aligned with ``df_small``.""" + rng = np.random.default_rng(3) + return pd.Series(rng.integers(0, 2, size=SMALL_N), name="target") + + +@pytest.fixture(scope="session") +def y_binary_big(): + """Binary target aligned with ``df_big``.""" + rng = np.random.default_rng(4) + return pd.Series(rng.integers(0, 2, size=BIG_N), name="target") + + +@pytest.fixture(scope="session") +def y_continuous(): + """Continuous target aligned with ``df_small``.""" + rng = np.random.default_rng(5) + return pd.Series(rng.normal(size=SMALL_N), name="target") + + +@pytest.fixture(scope="session") +def df_timeseries(): + """Time indexed dataframe with numerical variables only.""" + rng = np.random.default_rng(6) + index = pd.date_range("2020-01-01", periods=BIG_N, freq="15min") + return pd.DataFrame( + {f"num_{i}": rng.normal(size=BIG_N).cumsum() for i in range(4)}, + index=index, + ) diff --git a/benchmarks/test_creation.py b/benchmarks/test_creation.py new file mode 100644 index 000000000..2992e8e3e --- /dev/null +++ b/benchmarks/test_creation.py @@ -0,0 +1,64 @@ +"""Benchmarks for the feature creation transformers.""" + +import pytest + +from feature_engine.creation import ( + CyclicalFeatures, + DecisionTreeFeatures, + MathFeatures, + RelativeFeatures, +) + +from .conftest import numerical_vars + +NUM_VARS = numerical_vars() + + +@pytest.mark.parametrize( + "func", [["sum", "mean"], ["sum", "mean", "std", "min", "max"]] +) +def test_math_features_transform(benchmark, df_tiny, func): + # MathFeatures aggregates row-wise, which is orders of magnitude slower per + # row than the vectorised transformers, hence the smallest dataframe. + creator = MathFeatures(variables=NUM_VARS, func=func) + creator.fit(df_tiny) + benchmark(creator.transform, df_tiny) + + +def test_relative_features_transform(benchmark, df_big): + creator = RelativeFeatures( + variables=NUM_VARS[:4], + reference=["num_4"], + func=["sub", "div"], + ) + creator.fit(df_big) + benchmark(creator.transform, df_big) + + +def test_cyclical_features_transform(benchmark, df_big): + creator = CyclicalFeatures(variables=NUM_VARS) + creator.fit(df_big) + benchmark(creator.transform, df_big) + + +def test_decision_tree_features_fit(benchmark, df_small, y_continuous): + creator = DecisionTreeFeatures( + variables=NUM_VARS[:3], + features_to_combine=2, + regression=True, + cv=2, + random_state=0, + ) + benchmark(creator.fit, df_small, y_continuous) + + +def test_decision_tree_features_transform(benchmark, df_small, y_continuous): + creator = DecisionTreeFeatures( + variables=NUM_VARS[:3], + features_to_combine=2, + regression=True, + cv=2, + random_state=0, + ) + creator.fit(df_small, y_continuous) + benchmark(creator.transform, df_small) diff --git a/benchmarks/test_datetime.py b/benchmarks/test_datetime.py new file mode 100644 index 000000000..32c5f0c58 --- /dev/null +++ b/benchmarks/test_datetime.py @@ -0,0 +1,51 @@ +"""Benchmarks for the datetime feature extraction transformers.""" + +import pytest + +from feature_engine.datetime import ( + DatetimeFeatures, + DatetimeOrdinal, + DatetimeSubtraction, +) + +DATE_VARS = ["date_0", "date_1"] + + +@pytest.mark.parametrize( + "features_to_extract", + [ + ["year", "month", "day_of_month"], + None, + "all", + ], + ids=["basic", "default", "all"], +) +def test_datetime_features_transform(benchmark, df_big, features_to_extract): + transformer = DatetimeFeatures( + variables=DATE_VARS, features_to_extract=features_to_extract + ) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_datetime_features_from_string_transform(benchmark, df_big): + # Dates stored as strings: parsing dominates the runtime. + df = df_big.copy() + df["date_0"] = df["date_0"].astype(str) + transformer = DatetimeFeatures( + variables=["date_0"], features_to_extract=["year", "month", "day_of_month"] + ) + transformer.fit(df) + benchmark(transformer.transform, df) + + +def test_datetime_subtraction_transform(benchmark, df_big): + transformer = DatetimeSubtraction(variables=["date_0"], reference=["date_1"]) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_datetime_ordinal_transform(benchmark, df_big): + transformer = DatetimeOrdinal(variables=DATE_VARS) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) diff --git a/benchmarks/test_discretisation.py b/benchmarks/test_discretisation.py new file mode 100644 index 000000000..199bb1d40 --- /dev/null +++ b/benchmarks/test_discretisation.py @@ -0,0 +1,69 @@ +"""Benchmarks for the discretisation transformers.""" + +import pytest + +from feature_engine.discretisation import ( + ArbitraryDiscretiser, + DecisionTreeDiscretiser, + EqualFrequencyDiscretiser, + EqualWidthDiscretiser, + GeometricWidthDiscretiser, +) + +from .conftest import numerical_vars + +NUM_VARS = numerical_vars() +POS_VARS = ["pos_0", "pos_1"] + + +def test_equal_frequency_discretiser_fit(benchmark, df_big): + disc = EqualFrequencyDiscretiser(q=10, variables=NUM_VARS) + benchmark(disc.fit, df_big) + + +@pytest.mark.parametrize("return_boundaries", [False, True]) +def test_equal_frequency_discretiser_transform(benchmark, df_big, return_boundaries): + disc = EqualFrequencyDiscretiser( + q=10, variables=NUM_VARS, return_boundaries=return_boundaries + ) + disc.fit(df_big) + benchmark(disc.transform, df_big) + + +def test_equal_width_discretiser_fit(benchmark, df_big): + disc = EqualWidthDiscretiser(bins=10, variables=NUM_VARS) + benchmark(disc.fit, df_big) + + +def test_equal_width_discretiser_transform(benchmark, df_big): + disc = EqualWidthDiscretiser(bins=10, variables=NUM_VARS) + disc.fit(df_big) + benchmark(disc.transform, df_big) + + +def test_geometric_width_discretiser_transform(benchmark, df_big): + disc = GeometricWidthDiscretiser(bins=10, variables=POS_VARS) + disc.fit(df_big) + benchmark(disc.transform, df_big) + + +def test_arbitrary_discretiser_transform(benchmark, df_big): + limits = {var: [-1000, -1, 0, 1, 1000] for var in NUM_VARS} + disc = ArbitraryDiscretiser(binning_dict=limits) + disc.fit(df_big) + benchmark(disc.transform, df_big) + + +def test_decision_tree_discretiser_fit(benchmark, df_small, y_continuous): + disc = DecisionTreeDiscretiser( + variables=NUM_VARS, regression=True, cv=2, random_state=0 + ) + benchmark(disc.fit, df_small, y_continuous) + + +def test_decision_tree_discretiser_transform(benchmark, df_small, y_continuous): + disc = DecisionTreeDiscretiser( + variables=NUM_VARS, regression=True, cv=2, random_state=0 + ) + disc.fit(df_small, y_continuous) + benchmark(disc.transform, df_small) diff --git a/benchmarks/test_encoding.py b/benchmarks/test_encoding.py new file mode 100644 index 000000000..42b773627 --- /dev/null +++ b/benchmarks/test_encoding.py @@ -0,0 +1,120 @@ +"""Benchmarks for the categorical encoding transformers.""" + +import pytest + +from feature_engine.encoding import ( + CountEncoder, + DecisionTreeEncoder, + MeanEncoder, + OneHotEncoder, + OrdinalEncoder, + RareLabelEncoder, + StringSimilarityEncoder, + WoEEncoder, +) + +from .conftest import categorical_vars + +CAT_VARS = categorical_vars() + + +@pytest.mark.parametrize("encoding_method", ["count", "frequency"]) +def test_count_encoder_fit(benchmark, df_big, encoding_method): + encoder = CountEncoder(encoding_method=encoding_method, variables=CAT_VARS) + benchmark(encoder.fit, df_big) + + +@pytest.mark.parametrize("encoding_method", ["count", "frequency"]) +def test_count_encoder_transform(benchmark, df_big, encoding_method): + encoder = CountEncoder(encoding_method=encoding_method, variables=CAT_VARS) + encoder.fit(df_big) + benchmark(encoder.transform, df_big) + + +@pytest.mark.parametrize("encoding_method", ["ordered", "arbitrary"]) +def test_ordinal_encoder_fit(benchmark, df_big, y_binary_big, encoding_method): + encoder = OrdinalEncoder(encoding_method=encoding_method, variables=CAT_VARS) + benchmark(encoder.fit, df_big, y_binary_big) + + +def test_ordinal_encoder_transform(benchmark, df_big, y_binary_big): + encoder = OrdinalEncoder(encoding_method="ordered", variables=CAT_VARS) + encoder.fit(df_big, y_binary_big) + benchmark(encoder.transform, df_big) + + +def test_mean_encoder_fit(benchmark, df_big, y_binary_big): + encoder = MeanEncoder(variables=CAT_VARS) + benchmark(encoder.fit, df_big, y_binary_big) + + +def test_mean_encoder_transform(benchmark, df_big, y_binary_big): + encoder = MeanEncoder(variables=CAT_VARS) + encoder.fit(df_big, y_binary_big) + benchmark(encoder.transform, df_big) + + +def test_mean_encoder_smoothing_fit(benchmark, df_big, y_binary_big): + encoder = MeanEncoder(variables=CAT_VARS, smoothing="auto") + benchmark(encoder.fit, df_big, y_binary_big) + + +def test_woe_encoder_fit(benchmark, df_big, y_binary_big): + encoder = WoEEncoder(variables=CAT_VARS) + benchmark(encoder.fit, df_big, y_binary_big) + + +def test_woe_encoder_transform(benchmark, df_big, y_binary_big): + encoder = WoEEncoder(variables=CAT_VARS) + encoder.fit(df_big, y_binary_big) + benchmark(encoder.transform, df_big) + + +@pytest.mark.parametrize("drop_last", [False, True]) +def test_one_hot_encoder_transform(benchmark, df_big, drop_last): + encoder = OneHotEncoder(variables=CAT_VARS, drop_last=drop_last) + encoder.fit(df_big) + benchmark(encoder.transform, df_big) + + +def test_one_hot_encoder_top_categories_transform(benchmark, df_big): + encoder = OneHotEncoder(variables=CAT_VARS, top_categories=5) + encoder.fit(df_big) + benchmark(encoder.transform, df_big) + + +def test_rare_label_encoder_fit(benchmark, df_big): + encoder = RareLabelEncoder(tol=0.05, n_categories=2, variables=CAT_VARS) + benchmark(encoder.fit, df_big) + + +def test_rare_label_encoder_transform(benchmark, df_big): + encoder = RareLabelEncoder(tol=0.05, n_categories=2, variables=CAT_VARS) + encoder.fit(df_big) + benchmark(encoder.transform, df_big) + + +def test_decision_tree_encoder_fit(benchmark, df_small, y_binary): + encoder = DecisionTreeEncoder( + variables=CAT_VARS, regression=False, cv=2, random_state=0 + ) + benchmark(encoder.fit, df_small, y_binary) + + +def test_decision_tree_encoder_transform(benchmark, df_small, y_binary): + encoder = DecisionTreeEncoder( + variables=CAT_VARS, regression=False, cv=2, random_state=0 + ) + encoder.fit(df_small, y_binary) + benchmark(encoder.transform, df_small) + + +def test_string_similarity_encoder_fit(benchmark, df_small): + encoder = StringSimilarityEncoder(variables=CAT_VARS) + benchmark(encoder.fit, df_small) + + +def test_string_similarity_encoder_transform(benchmark, df_small): + encoder = StringSimilarityEncoder(variables=CAT_VARS) + encoder.fit(df_small) + benchmark(encoder.transform, df_small) diff --git a/benchmarks/test_imputation.py b/benchmarks/test_imputation.py new file mode 100644 index 000000000..a439e957e --- /dev/null +++ b/benchmarks/test_imputation.py @@ -0,0 +1,80 @@ +"""Benchmarks for the missing data imputation transformers.""" + +import pytest + +from feature_engine.imputation import ( + ArbitraryImputer, + CategoricalImputer, + DropMissingData, + EndTailImputer, + MeanImputer, + MissingIndicator, + RandomSampleImputer, +) + +from .conftest import categorical_vars, numerical_vars + +NUM_VARS = numerical_vars() +CAT_VARS = categorical_vars() + + +@pytest.mark.parametrize("method", ["mean", "median"]) +def test_mean_imputer_fit(benchmark, df_big_na, method): + imputer = MeanImputer(imputation_method=method, variables=NUM_VARS) + benchmark(imputer.fit, df_big_na) + + +@pytest.mark.parametrize("method", ["mean", "median"]) +def test_mean_imputer_transform(benchmark, df_big_na, method): + imputer = MeanImputer(imputation_method=method, variables=NUM_VARS) + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) + + +def test_arbitrary_imputer_transform(benchmark, df_big_na): + imputer = ArbitraryImputer(arbitrary_number=-999, variables=NUM_VARS) + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) + + +@pytest.mark.parametrize("method", ["gaussian", "iqr"]) +def test_end_tail_imputer_fit(benchmark, df_big_na, method): + imputer = EndTailImputer(imputation_method=method, variables=NUM_VARS) + benchmark(imputer.fit, df_big_na) + + +def test_end_tail_imputer_transform(benchmark, df_big_na): + imputer = EndTailImputer(imputation_method="gaussian", variables=NUM_VARS) + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) + + +@pytest.mark.parametrize("method", ["frequent", "missing"]) +def test_categorical_imputer_fit(benchmark, df_big_na, method): + imputer = CategoricalImputer(imputation_method=method, variables=CAT_VARS) + benchmark(imputer.fit, df_big_na) + + +@pytest.mark.parametrize("method", ["frequent", "missing"]) +def test_categorical_imputer_transform(benchmark, df_big_na, method): + imputer = CategoricalImputer(imputation_method=method, variables=CAT_VARS) + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) + + +def test_random_sample_imputer_transform(benchmark, df_big_na): + imputer = RandomSampleImputer(variables=NUM_VARS + CAT_VARS, random_state=0) + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) + + +def test_missing_indicator_transform(benchmark, df_big_na): + imputer = MissingIndicator(missing_only=True) + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) + + +def test_drop_missing_data_transform(benchmark, df_big_na): + imputer = DropMissingData() + imputer.fit(df_big_na) + benchmark(imputer.transform, df_big_na) diff --git a/benchmarks/test_outliers.py b/benchmarks/test_outliers.py new file mode 100644 index 000000000..ad5a91318 --- /dev/null +++ b/benchmarks/test_outliers.py @@ -0,0 +1,51 @@ +"""Benchmarks for the outlier capping and trimming transformers.""" + +import pytest + +from feature_engine.outliers import ( + ArbitraryOutlierCapper, + OutlierTrimmer, + Winsoriser, +) + +from .conftest import numerical_vars + +NUM_VARS = numerical_vars() + + +@pytest.mark.parametrize("capping_method", ["gaussian", "iqr", "quantiles", "mad"]) +def test_winsoriser_fit(benchmark, df_big, capping_method): + capper = Winsoriser(capping_method=capping_method, tail="both", variables=NUM_VARS) + benchmark(capper.fit, df_big) + + +@pytest.mark.parametrize("add_indicators", [False, True]) +def test_winsoriser_transform(benchmark, df_big, add_indicators): + capper = Winsoriser( + capping_method="iqr", + tail="both", + variables=NUM_VARS, + add_indicators=add_indicators, + ) + capper.fit(df_big) + benchmark(capper.transform, df_big) + + +def test_arbitrary_outlier_capper_transform(benchmark, df_big): + capper = ArbitraryOutlierCapper( + max_capping_dict={var: 10 for var in NUM_VARS}, + min_capping_dict={var: -10 for var in NUM_VARS}, + ) + capper.fit(df_big) + benchmark(capper.transform, df_big) + + +def test_outlier_trimmer_fit(benchmark, df_big): + trimmer = OutlierTrimmer(capping_method="iqr", tail="both", variables=NUM_VARS) + benchmark(trimmer.fit, df_big) + + +def test_outlier_trimmer_transform(benchmark, df_big): + trimmer = OutlierTrimmer(capping_method="iqr", tail="both", variables=NUM_VARS) + trimmer.fit(df_big) + benchmark(trimmer.transform, df_big) diff --git a/benchmarks/test_pipeline.py b/benchmarks/test_pipeline.py new file mode 100644 index 000000000..9c0e4681d --- /dev/null +++ b/benchmarks/test_pipeline.py @@ -0,0 +1,77 @@ +"""End to end benchmarks: several transformers chained in a Pipeline. + +These are the closest thing to a real user workflow and catch regressions that +only show up when transformers are combined. +""" + +import pytest + +from feature_engine.discretisation import EqualFrequencyDiscretiser +from feature_engine.encoding import OneHotEncoder, RareLabelEncoder, WoEEncoder +from feature_engine.imputation import CategoricalImputer, MeanImputer +from feature_engine.outliers import Winsoriser +from feature_engine.pipeline import Pipeline +from feature_engine.preprocessing import MatchCategories, MatchVariables +from feature_engine.selection import DropConstantFeatures, DropCorrelatedFeatures +from feature_engine.transformation import YeoJohnsonTransformer + +from .conftest import categorical_vars, numerical_vars + +NUM_VARS = numerical_vars() +CAT_VARS = categorical_vars() + + +def _build_pipeline(): + return Pipeline( + [ + ("cat_imputer", CategoricalImputer(variables=CAT_VARS)), + ("num_imputer", MeanImputer()), + ("rare_label", RareLabelEncoder(tol=0.05, n_categories=2)), + ( + "winsorizer", + Winsoriser(capping_method="iqr", tail="both", variables=NUM_VARS), + ), + ("yeo_johnson", YeoJohnsonTransformer(variables=NUM_VARS)), + ("one_hot", OneHotEncoder(variables=CAT_VARS, drop_last=True)), + ("drop_constant", DropConstantFeatures(tol=0.998)), + ] + ) + + +def test_pipeline_fit(benchmark, df_big_na, y_binary_big): + pipe = _build_pipeline() + benchmark(pipe.fit, df_big_na, y_binary_big) + + +def test_pipeline_transform(benchmark, df_big_na, y_binary_big): + pipe = _build_pipeline() + pipe.fit(df_big_na, y_binary_big) + benchmark(pipe.transform, df_big_na) + + +def test_credit_scoring_pipeline_fit(benchmark, df_big, y_binary_big): + pipe = Pipeline( + [ + ( + "discretiser", + EqualFrequencyDiscretiser(q=10, variables=NUM_VARS, return_object=True), + ), + ("rare_label", RareLabelEncoder(tol=0.02, n_categories=2)), + ("woe", WoEEncoder(variables=NUM_VARS + CAT_VARS)), + ("drop_correlated", DropCorrelatedFeatures(threshold=0.9)), + ] + ) + benchmark(pipe.fit, df_big, y_binary_big) + + +@pytest.mark.parametrize("match_dtypes", [False, True]) +def test_match_variables_transform(benchmark, df_big, match_dtypes): + matcher = MatchVariables(match_dtypes=match_dtypes, verbose=False) + matcher.fit(df_big) + benchmark(matcher.transform, df_big) + + +def test_match_categories_transform(benchmark, df_big): + matcher = MatchCategories(variables=CAT_VARS) + matcher.fit(df_big) + benchmark(matcher.transform, df_big) diff --git a/benchmarks/test_selection.py b/benchmarks/test_selection.py new file mode 100644 index 000000000..6a3da451f --- /dev/null +++ b/benchmarks/test_selection.py @@ -0,0 +1,93 @@ +"""Benchmarks for the feature selection transformers. + +Selectors do most of their work in ``fit``, so these benchmarks focus on it. +The estimator based selectors run on the smaller dataframe and with a light +estimator and 2 folds to keep the runtime reasonable. +""" + +import pytest +from sklearn.tree import DecisionTreeClassifier + +from feature_engine.selection import ( + DropConstantFeatures, + DropCorrelatedFeatures, + DropDuplicateFeatures, + DropFeatures, + DropHighPSIFeatures, + SelectByInformationValue, + SelectBySingleFeaturePerformance, + SelectByTargetEncoding, + SmartCorrelatedSelection, +) + +from .conftest import categorical_vars, numerical_vars + +NUM_VARS = numerical_vars() +CAT_VARS = categorical_vars() + + +def _estimator(): + return DecisionTreeClassifier(max_depth=3, random_state=0) + + +def test_drop_features_transform(benchmark, df_big): + selector = DropFeatures(features_to_drop=NUM_VARS[:3]) + selector.fit(df_big) + benchmark(selector.transform, df_big) + + +def test_drop_constant_features_fit(benchmark, df_big): + selector = DropConstantFeatures(tol=0.998) + benchmark(selector.fit, df_big) + + +def test_drop_duplicate_features_fit(benchmark, df_small): + # Compares every pair of columns, so it runs on the smaller dataframe. + selector = DropDuplicateFeatures() + benchmark(selector.fit, df_small) + + +@pytest.mark.parametrize("method", ["pearson", "spearman"]) +def test_drop_correlated_features_fit(benchmark, df_big, method): + selector = DropCorrelatedFeatures(variables=NUM_VARS, method=method, threshold=0.8) + benchmark(selector.fit, df_big) + + +def test_smart_correlated_selection_fit(benchmark, df_big, y_binary_big): + selector = SmartCorrelatedSelection( + variables=NUM_VARS, + selection_method="variance", + threshold=0.8, + ) + benchmark(selector.fit, df_big, y_binary_big) + + +def test_drop_high_psi_features_fit(benchmark, df_big): + selector = DropHighPSIFeatures(variables=NUM_VARS, bins=10, split_frac=0.5) + benchmark(selector.fit, df_big) + + +def test_select_by_information_value_fit(benchmark, df_big, y_binary_big): + selector = SelectByInformationValue(variables=CAT_VARS, threshold=0.2) + benchmark(selector.fit, df_big, y_binary_big) + + +def test_select_by_target_encoding_fit(benchmark, df_small, y_binary): + selector = SelectByTargetEncoding( + variables=NUM_VARS[:4] + CAT_VARS[:2], + bins=5, + cv=2, + scoring="roc_auc", + regression=False, + ) + benchmark(selector.fit, df_small, y_binary) + + +def test_select_by_single_feature_performance_fit(benchmark, df_small, y_binary): + selector = SelectBySingleFeaturePerformance( + estimator=_estimator(), + variables=NUM_VARS, + scoring="roc_auc", + cv=2, + ) + benchmark(selector.fit, df_small, y_binary) diff --git a/benchmarks/test_timeseries.py b/benchmarks/test_timeseries.py new file mode 100644 index 000000000..351d4f0f1 --- /dev/null +++ b/benchmarks/test_timeseries.py @@ -0,0 +1,39 @@ +"""Benchmarks for the time series forecasting feature transformers.""" + +import pytest + +from feature_engine.timeseries.forecasting import ( + ExpandingWindowFeatures, + LagFeatures, + WindowFeatures, +) + +TS_VARS = [f"num_{i}" for i in range(4)] + + +@pytest.mark.parametrize("periods", [1, [1, 3, 6, 12]], ids=["single", "multiple"]) +def test_lag_features_transform(benchmark, df_timeseries, periods): + transformer = LagFeatures(variables=TS_VARS, periods=periods) + transformer.fit(df_timeseries) + benchmark(transformer.transform, df_timeseries) + + +def test_lag_features_freq_transform(benchmark, df_timeseries): + transformer = LagFeatures(variables=TS_VARS, freq=["1h", "1D"]) + transformer.fit(df_timeseries) + benchmark(transformer.transform, df_timeseries) + + +@pytest.mark.parametrize("window", [3, [3, 12]], ids=["single", "multiple"]) +def test_window_features_transform(benchmark, df_timeseries, window): + transformer = WindowFeatures( + variables=TS_VARS, window=window, functions=["mean", "std"] + ) + transformer.fit(df_timeseries) + benchmark(transformer.transform, df_timeseries) + + +def test_expanding_window_features_transform(benchmark, df_timeseries): + transformer = ExpandingWindowFeatures(variables=TS_VARS, functions=["mean", "max"]) + transformer.fit(df_timeseries) + benchmark(transformer.transform, df_timeseries) diff --git a/benchmarks/test_transformation.py b/benchmarks/test_transformation.py new file mode 100644 index 000000000..79d0ccb01 --- /dev/null +++ b/benchmarks/test_transformation.py @@ -0,0 +1,92 @@ +"""Benchmarks for the mathematical variable transformers and scalers.""" + +from feature_engine.scaling import MeanNormalisationScaler +from feature_engine.transformation import ( + ArcSinhTransformer, + ArcsinTransformer, + BoxCoxTransformer, + LogTransformer, + PowerTransformer, + ReciprocalTransformer, + YeoJohnsonTransformer, +) + +from .conftest import numerical_vars + +NUM_VARS = numerical_vars() +POS_VARS = ["pos_0", "pos_1"] + + +def test_log_transformer_transform(benchmark, df_big): + transformer = LogTransformer(variables=POS_VARS) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_log_transformer_auto_c_fit(benchmark, df_big): + # C="auto" makes fit learn the shift needed to make the variables positive. + transformer = LogTransformer(variables=NUM_VARS, C="auto") + benchmark(transformer.fit, df_big) + + +def test_log_transformer_auto_c_transform(benchmark, df_big): + transformer = LogTransformer(variables=NUM_VARS, C="auto") + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_power_transformer_transform(benchmark, df_big): + transformer = PowerTransformer(variables=POS_VARS, exp=0.5) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_reciprocal_transformer_transform(benchmark, df_big): + transformer = ReciprocalTransformer(variables=POS_VARS) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_box_cox_transformer_fit(benchmark, df_big): + transformer = BoxCoxTransformer(variables=POS_VARS) + benchmark(transformer.fit, df_big) + + +def test_box_cox_transformer_transform(benchmark, df_big): + transformer = BoxCoxTransformer(variables=POS_VARS) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_yeo_johnson_transformer_fit(benchmark, df_big): + transformer = YeoJohnsonTransformer(variables=NUM_VARS) + benchmark(transformer.fit, df_big) + + +def test_yeo_johnson_transformer_transform(benchmark, df_big): + transformer = YeoJohnsonTransformer(variables=NUM_VARS) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_arcsin_transformer_transform(benchmark, df_big): + transformer = ArcsinTransformer(variables=["frac_0"]) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_arcsinh_transformer_transform(benchmark, df_big): + transformer = ArcSinhTransformer(variables=NUM_VARS) + transformer.fit(df_big) + benchmark(transformer.transform, df_big) + + +def test_mean_normalisation_scaler_fit(benchmark, df_big): + scaler = MeanNormalisationScaler(variables=NUM_VARS) + benchmark(scaler.fit, df_big) + + +def test_mean_normalisation_scaler_transform(benchmark, df_big): + scaler = MeanNormalisationScaler(variables=NUM_VARS) + scaler.fit(df_big) + benchmark(scaler.transform, df_big) diff --git a/benchmarks/test_variable_handling.py b/benchmarks/test_variable_handling.py new file mode 100644 index 000000000..7799d75b1 --- /dev/null +++ b/benchmarks/test_variable_handling.py @@ -0,0 +1,55 @@ +"""Benchmarks for the variable handling helpers. + +These functions are called by every transformer during fit, so they are on the +hot path of the whole library. +""" + +from feature_engine.variable_handling import ( + check_numerical_variables, + find_all_variables, + find_categorical_and_numerical_variables, + find_categorical_variables, + find_datetime_variables, + find_numerical_variables, + retain_variables_if_in_df, +) + +from .conftest import numerical_vars + +NUM_VARS = numerical_vars() + + +def test_find_numerical_variables(benchmark, df_big): + benchmark(find_numerical_variables, df_big) + + +def test_find_categorical_variables(benchmark, df_big): + benchmark(find_categorical_variables, df_big) + + +def test_find_datetime_variables(benchmark, df_big): + benchmark(find_datetime_variables, df_big) + + +def test_find_all_variables(benchmark, df_big): + benchmark(find_all_variables, df_big) + + +def test_find_categorical_and_numerical_variables(benchmark, df_big): + benchmark(find_categorical_and_numerical_variables, df_big) + + +def test_check_numerical_variables(benchmark, df_big): + benchmark(check_numerical_variables, df_big, NUM_VARS) + + +def test_retain_variables_if_in_df(benchmark, df_big): + benchmark(retain_variables_if_in_df, df_big, NUM_VARS + ["not_in_df"]) + + +def test_find_datetime_variables_object_dtype(benchmark, df_big): + # Datetimes cast as strings: the check needs to try parsing the columns. + df = df_big.copy() + df["date_0"] = df["date_0"].astype(str) + df["date_1"] = df["date_1"].astype(str) + benchmark(find_datetime_variables, df) diff --git a/pyproject.toml b/pyproject.toml index 2a385d74d..9e5fb0199 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,9 @@ exclude = '''(?x)( )''' [tool.pytest.ini_options] +# The benchmarks folder is excluded from the default test paths: it requires +# pytest-codspeed and is run by the CodSpeed workflow, not by the test suite. +testpaths = ["tests"] filterwarnings = [ "ignore::sklearn.exceptions.SkipTestWarning", "ignore::UserWarning",