diff --git a/.circleci/config.yml b/.circleci/config.yml index 127b33bef..0d31c7f44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ orbs: defaults: &defaults docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project prepare_tox: &prepare_tox @@ -42,58 +42,6 @@ jobs: # Test matrix # ------------------------ - test_feature_engine_py39: - docker: - - image: cimg/python:3.9.0 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.9) - command: | - tox -e py39 - - test_feature_engine_py310: - docker: - - image: cimg/python:3.10.0 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.10) - command: | - tox -e py310 - - test_feature_engine_py311_sklearn150: - docker: - - image: cimg/python:3.11.7 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.11, scikit-learn 1.5) - command: | - tox -e py311-sklearn150 - - test_feature_engine_py311_sklearn160: - docker: - - image: cimg/python:3.11.7 - working_directory: ~/project - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Run tests (Python 3.11, scikit-learn 1.6) - command: | - tox -e py311-sklearn160 - test_feature_engine_py311_sklearn170: docker: - image: cimg/python:3.11.7 @@ -166,7 +114,7 @@ jobs: test_style: docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project steps: - checkout: @@ -179,7 +127,7 @@ jobs: test_docs: docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project steps: - checkout: @@ -192,7 +140,7 @@ jobs: test_type: docker: - - image: cimg/python:3.10.0 + - image: cimg/python:3.12.1 working_directory: ~/project steps: - checkout: @@ -277,10 +225,6 @@ workflows: test-all: jobs: - - test_feature_engine_py39 - - test_feature_engine_py310 - - test_feature_engine_py311_sklearn150 - - test_feature_engine_py311_sklearn160 - test_feature_engine_py311_sklearn170 - test_feature_engine_py312_pandas230 - test_feature_engine_py312_pandas300 @@ -298,10 +242,6 @@ workflows: - package_and_upload_to_pypi: requires: - - test_feature_engine_py39 - - test_feature_engine_py310 - - test_feature_engine_py311_sklearn150 - - test_feature_engine_py311_sklearn160 - test_feature_engine_py311_sklearn170 - test_feature_engine_py312_pandas230 - test_feature_engine_py312_pandas300 diff --git a/README.md b/README.md index 76a9056bc..7542bfba7 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ Feature-engine documentation is built using [Sphinx](https://www.sphinx-doc.org) To build the documentation make sure you have the dependencies installed: from the root directory: ``` -pip install -r docs/requirements.txt +pip install -e ".[docs]" ``` Now you can build the docs using: diff --git a/docs/contribute/contribute_code.rst b/docs/contribute/contribute_code.rst index 4e85515ec..d5413e051 100644 --- a/docs/contribute/contribute_code.rst +++ b/docs/contribute/contribute_code.rst @@ -395,7 +395,7 @@ To do this, first make sure you have all the documentation dependencies installe set up the environment as we described previously, they should be installed. Alternatively, from the windows cmd or mac terminal, run:: - $ pip install -r docs/requirements.txt + $ pip install -e ".[docs]" Make sure you are within the feature_engine module when you run the previous command. diff --git a/docs/contribute/contribute_docs.rst b/docs/contribute/contribute_docs.rst index 856e22d85..ee16ed9f4 100644 --- a/docs/contribute/contribute_docs.rst +++ b/docs/contribute/contribute_docs.rst @@ -79,7 +79,7 @@ dependencies. If you set up the development environment as we described in the Alternatively, first activate your environment. Then navigate to the root folder of feature-engine. And now install the requirements for the documentation:: - $ pip install -r docs/requirements.txt + $ pip install -e ".[docs]" To build the documentation (and test if it is working properly) run:: diff --git a/docs/index.rst b/docs/index.rst index 6a05f7e9e..9e5a18dd0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -127,7 +127,7 @@ The following characteristics make feature-engine unique: Installation ------------ -Feature-engine is a Python 3 package and works well with 3.9 or later. +Feature-engine is a Python 3 package and works well with 3.11 or later. The simplest way to install feature-engine is from PyPI with pip: diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index fe313a627..ae3be3c7b 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,120 +2,79 @@ transform(). """ -from typing import List, Tuple, Union +from typing import List, Union +import narwhals as nw +import narwhals.dependencies as nwd import numpy as np -import pandas as pd -from scipy.sparse import issparse +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.utils.validation import _check_y, check_consistent_length, column_or_1d -from feature_engine.variable_handling._variable_type_checks import is_object - -def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: +def check_X(X: IntoDataFrame): """ - Checks if the input is a DataFrame and then creates a copy. This is an important - step not to accidentally transform the original dataset entered by the user. - - If the input is a numpy array, it converts it to a pandas Dataframe. The column - names are strings representing the column index starting at 0. - - Feature-engine was originally designed to work with pandas dataframes. However, - allowing numpy arrays as input allows 2 things: - - We can use the Scikit-learn tests for transformers provided by the - `check_estimator` function to test the compatibility of our transformers with - sklearn functionality. - - Feature-engine transformers can be used within a Scikit-learn Pipeline together - with Scikit-learn transformers like the `SimpleImputer`, which return by default - Numpy arrays. + Checks that X is a dataframe from any library supported by narwhals (for example + pandas, polars, modin, cuDF, or PyArrow). Parameters ---------- - X : pandas Dataframe or numpy array. - The input to check and copy or transform. + X : dataframe (pandas, polars, or any other library supported by narwhals). + The input to check and transform. Raises ------ TypeError - If the input is not a Pandas DataFrame or a numpy array. + If the input is not a recognised dataframe. ValueError - If the input is an empty dataframe. + If the input has duplicated column names, or 0 columns or rows. Returns ------- - X : pandas Dataframe. - A copy of original DataFrame or a converted Numpy array. + X : dataframe. + The validated dataframe in its native format. """ - if isinstance(X, pd.DataFrame): - if not X.columns.is_unique: - raise ValueError("Input data contains duplicated variable names.") - X = X.copy() - - elif isinstance(X, (np.generic, np.ndarray)): - # If input is scalar raise error - if X.ndim == 0: - raise ValueError( - "Expected 2D array, got scalar array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) - ) - # If input is 1D raise error - if X.ndim == 1: + if nwd.is_into_dataframe(X): + # from_native() raises narwhals.exceptions.DuplicateError, a ValueError + # subclass, when the dataframe has duplicated column names. + nw_X = nw.from_native(X, eager_only=True) + if nw_X.is_empty() or nw_X.shape[1] == 0: raise ValueError( - "Expected 2D array, got 1D array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) + f"Found array with 0 feature(s) (shape={nw_X.shape}) while a " + "minimum of 1 is required." ) - if np.any(np.iscomplex(X)): - raise TypeError("Complex data not supported by this transformer.") - - X = pd.DataFrame(X) - X.columns = [f"x{i}" for i in range(X.shape[1])] - - elif issparse(X): - raise TypeError("This transformer does not support sparse matrices.") - else: raise TypeError( - f"X must be a numpy array or pandas dataframe. Got {type(X)} instead." - ) - - if X.empty: - raise ValueError( - "0 feature(s) (shape=%s) while a minimum of %d is required." % (X.shape, 1) + "X must be a dataframe from a library supported by narwhals " + f"(e.g. pandas, polars, PyArrow). Got {type(X)} instead." ) - return X + return nw_X.to_native() def check_y( - y: Union[np.generic, np.ndarray, pd.Series, pd.DataFrame, List], + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, -) -> pd.Series: +): """ - Checks that y is a series or a dataframe, or alternatively, if it can be converted - to a series or dataframe. + Checks that y is a Series or DataFrame from a library supported by narwhals (for + example pandas or polars), or alternatively, if it can be converted to a numpy + array. Parameters ---------- - y : pd.Series, pd.DataFrame, np.array, list - The input to check and copy or transform. + y : Series or DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list + The input to check. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Returns ------- - y: pd.Series or pd.DataFrame + y: Series, DataFrame, or numpy array """ - if y is None: raise ValueError( "requires y to be passed, but the target y is None", @@ -123,110 +82,89 @@ def check_y( "y should be a 1d array", ) - elif isinstance(y, pd.Series): - if y.isnull().any(): + if nwd.is_into_series(y): + nw_y = nw.from_native(y, series_only=True) + if nw_y.is_null().any(): raise ValueError("y contains NaN values.") - if not is_object(y) and not np.isfinite(y).all(): - raise ValueError("y contains infinity values.") - if y_numeric and is_object(y): - y = y.astype("float64") - y = y.copy() - - elif isinstance(y, pd.DataFrame): - if y.isnull().any().any(): + if nw_y.dtype.is_numeric(): + if not np.isfinite(nw_y.to_numpy()).all(): + raise ValueError("y contains infinity values.") + elif y_numeric: + nw_y = nw_y.cast(nw.Float64()) + return nw_y.to_native() + + if nwd.is_into_dataframe(y): + nw_y = nw.from_native(y, eager_only=True) + if nw_y.select(nw.all().is_null().any()).to_numpy().any(): raise ValueError("y contains NaN values.") - if not np.isfinite(y).all().all(): + if not np.isfinite(nw_y.to_numpy()).all(): raise ValueError("y contains infinity values.") - y = y.copy() + return nw_y.to_native() - else: - try: - y = column_or_1d(y) - y = _check_y(y, multi_output=False, y_numeric=y_numeric) - y = pd.Series(y).copy() - except ValueError: - y = _check_y(y, multi_output=True, y_numeric=y_numeric) - y = pd.DataFrame(y).copy() - return y + try: + y = column_or_1d(y) + return _check_y(y, multi_output=False, y_numeric=y_numeric) + except ValueError: + return _check_y(y, multi_output=True, y_numeric=y_numeric) def check_X_y( - X: Union[np.generic, np.ndarray, pd.DataFrame], - y: Union[np.generic, np.ndarray, pd.Series, List], + X: IntoDataFrame, + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, -) -> Tuple[pd.DataFrame, pd.Series]: +): """ - Ensures X and y are compatible pandas DataFrame and Series. If both are pandas - objects, checks that their indexes match. If any is a numpy array, converts to - pandas object with compatible index. - - This transformer ensures that we can concatenate X and y using `pandas.concat`, - functionality needed in the encoders. + Ensures X and y are compatible dataframe/array-like objects with a consistent + number of rows. If both are pandas objects, checks that their indexes match. Parameters ---------- - X: Pandas DataFrame or numpy ndarray - The input to check and copy or transform. + X: dataframe (pandas, polars, or any other library supported by narwhals) + The input to check. - y: pd.Series, np.array, list - The input to check and copy or transform. + y: Series, DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list + The input to check. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Raises ------ - ValueError: if X and y are pandas objects with inconsistent indexes. - TypeError: if X is sparse matrix, empty dataframe or not a dataframe. - TypeError: if y can't be parsed as pandas Series. + TypeError + If X is not a recognised dataframe. + ValueError + If X has duplicated column names, 0 columns, or 0 rows; if y is None, or + contains NaN or infinity values; if X and y have a different number of + rows; or if X and y are pandas objects with mismatched indexes. Returns ------- - X: Pandas DataFrame - y: Pandas Series + X: dataframe + y: Series, DataFrame, or numpy array """ + X = check_X(X) + y = check_y(y, y_numeric=y_numeric) + check_consistent_length(X, y) - def _check_X_y(X, y): - X = check_X(X) - y = check_y(y, y_numeric=y_numeric) - check_consistent_length(X, y) - return X, y - - # case 1: both are pandas objects - if isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - # Check that their indexes match. - if X.index.equals(y.index) is False: - raise ValueError("The indexes of X and y do not match.") - - # case 2: X is dataframe and y is something else - if isinstance(X, pd.DataFrame) and not isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - y.index = X.index - - # case 3: X is not a dataframe and y is a series - elif not isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - X.index = y.index - - # all other cases - else: - X, y = _check_X_y(X, y) + if nwd.is_pandas_dataframe(X): + if nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y): + if not X.index.equals(y.index): + raise ValueError("The indexes of X and y do not match.") return X, y -def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: +def _check_X_matches_training_df(X: IntoDataFrame, reference: int) -> None: """ - Checks that DataFrame to transform has the same number of columns that the - DataFrame used with the fit() method. + Checks that the dataframe to transform has the same number of columns as the + dataframe used with the fit() method. Parameters ---------- - X : Pandas DataFrame - The df to be checked + X : dataframe (pandas, polars, or any other library supported by narwhals) + The df to be checked. reference : int The number of columns in the dataframe that was used with the fit() method. @@ -234,92 +172,71 @@ def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: ------ ValueError If the number of columns does not match. - - Returns - ------- - None """ - if X.shape[1] != reference: raise ValueError( "The number of columns in this dataset is different from the one used to " "fit this transformer (when using the fit() method)." ) - return None - def _check_contains_na( - X: pd.DataFrame, + X: IntoDataFrame, variables: List[Union[str, int]], + error_msg: str = "simple", ) -> None: """ - Checks if DataFrame contains null values in the selected columns. + Checks if the dataframe contains null values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : dataframe variables : List The selected group of variables in which null values will be examined. - Raises - ------ - ValueError - If the variable(s) contain null values. - """ - - if X[variables].isnull().any().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer." - ) - - -def _check_optional_contains_na( - X: pd.DataFrame, variables: List[Union[str, int]] -) -> None: - """ - Checks if DataFrame contains null values in the selected columns. - - Parameters - ---------- - X : Pandas DataFrame - - variables : List - The selected group of variables in which null values will be examined. + error_msg : str, default="simple" + The message in the error. Some transformers can ignore null values. Raises ------ ValueError If the variable(s) contain null values. """ - - if X[variables].isnull().any().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) - - -def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> None: + error_msg_simple = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." + ) + error_msg_ignore = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." + ) + nw_X = nw.from_native(X, eager_only=True) + if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any(): + if error_msg == "simple": + raise ValueError(error_msg_simple) + else: + raise ValueError(error_msg_ignore) + + +def _check_contains_inf(X: IntoDataFrame, variables: List[Union[str, int]]) -> None: """ - Checks if DataFrame contains inf values in the selected columns. + Checks if the dataframe contains inf values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : dataframe variables : List - The selected group of variables in which null values will be examined. + The selected group of variables in which infinite values will be examined. Raises ------ ValueError If the variable(s) contain np.inf values """ - - if np.isinf(X[variables]).any().any(): + values = nw.from_native(X, eager_only=True).select(nw.col(variables)).to_numpy() + if np.isinf(values.astype(float)).any(): raise ValueError( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 35427f260..53eca3095 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -20,7 +20,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -123,7 +123,7 @@ class CategoricalMethodsMixin(TransformerMixin, BaseEstimator, GetFeatureNamesOu def _check_na(self, X: pd.DataFrame, variables): if self.missing_values == "raise": - _check_optional_contains_na(X, variables) + _check_contains_na(X, variables, error_msg="optional") def _check_or_select_variables(self, X: pd.DataFrame): """ @@ -225,7 +225,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") X = self._encode(X) diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 84c1f6910..2bbd2bf73 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -23,7 +23,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -255,7 +255,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") with_nan = [] else: with_nan = [np.nan] diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index c438487d6..f15f87003 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -17,7 +17,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixin, CategoricalMethodsMixin, @@ -247,7 +247,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # if data contains nan, fail before running any logic if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.encoder_dict_ = {} @@ -317,7 +317,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: check_is_fitted(self) X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") if len(self.variables_) == 0: return X diff --git a/feature_engine/preprocessing/match_categories.py b/feature_engine/preprocessing/match_categories.py index e0e863c1f..9d66c3d6c 100644 --- a/feature_engine/preprocessing/match_categories.py +++ b/feature_engine/preprocessing/match_categories.py @@ -19,7 +19,7 @@ ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -148,7 +148,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): variables_ = self._check_or_select_variables(X) if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.category_dict_ = dict() for var in variables_: @@ -175,7 +175,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") for feature, levels in self.category_dict_.items(): X[feature] = pd.Categorical( diff --git a/feature_engine/scaling/mean_normalization.py b/feature_engine/scaling/mean_normalization.py index 320358838..3ecd1b3ea 100644 --- a/feature_engine/scaling/mean_normalization.py +++ b/feature_engine/scaling/mean_normalization.py @@ -1,9 +1,11 @@ # Authors: Vasco Schiavo # License: BSD 3 clause +import warnings from typing import List, Optional, Union -import pandas as pd +import narwhals as nw +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._check_init_parameters.check_init_input_params import ( @@ -37,9 +39,9 @@ fit_transform=_fit_transform_docstring, inverse_transform=_inverse_transform_docstring, ) -class MeanNormalizationScaler(BaseNumericalTransformer): +class MeanNormalisationScaler(BaseNumericalTransformer): """ - MeanNormalizationScaler() applies mean normalisation, which consists of subtracting + MeanNormalisationScaler() applies mean normalisation, which consists of subtracting the mean of each feature and then dividing the result by the value range, that is, the difference between its maximum and minimum value. The method aims to center the variables at 0, and rescale the distribution between -1 and 1. @@ -51,7 +53,6 @@ class MeanNormalizationScaler(BaseNumericalTransformer): More details in the :ref:`User Guide `. - Parameters ---------- {variables} @@ -89,10 +90,10 @@ class MeanNormalizationScaler(BaseNumericalTransformer): >>> import numpy as np >>> import pandas as pd - >>> from feature_engine.scaling import MeanNormalizationScaler + >>> from feature_engine.scaling import MeanNormalisationScaler >>> np.random.seed(42) >>> X = pd.DataFrame(dict(x = np.random.lognormal(size = 100))) - >>> mns = MeanNormalizationScaler() + >>> mns = MeanNormalisationScaler() >>> mns.fit(X) >>> X = mns.transform(X) >>> X.head() @@ -115,24 +116,32 @@ def __init__( self.variables = _check_variables_input_value(variables) self.return_empty = return_empty - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Finds the mean and value range of each variable. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features]. + X: dataframe of shape = [n_samples, n_features]. The training input samples. Can be the entire dataframe, not just the - variables to transform. + variables to transform. Accepts dataframes from libraries supported by + narwhals (e.g. pandas, polars, PyArrow). - y: pandas Series, default=None + y: series, default=None It is not needed in this transformer. You can pass y or None. """ - # check input dataframe + # check input dataframe / select numerical variables (shared base) X = super().fit(X) - self.mean_ = X[self.variables_].mean().to_dict() - self.range_ = (X[self.variables_].max() - X[self.variables_].min()).to_dict() + + nw_X = nw.from_native(X, eager_only=True) + selected = nw_X.select(self.variables_) + + self.mean_ = {col: float(selected[col].mean()) for col in self.variables_} + self.range_ = { + col: float(selected[col].max() - selected[col].min()) + for col in self.variables_ + } # check for constant columns constant_columns = [col for col, value in self.range_.items() if value == 0] @@ -144,48 +153,78 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Transform the variables using mean normalisation. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] - The data to be transformed. + X: dataframe of shape = [n_samples, n_features] + The data to be transformed. Accepts dataframes from libraries supported by + narwhals (e.g. pandas, polars, PyArrow). Returns ------- - X_new: pandas dataframe - The dataframe with the transformed variables. + X_new: dataframe + The dataframe with the transformed variables, in the same library as the + input. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) - # transformation - X[self.variables_] = (X[self.variables_] - self.mean_) / self.range_ + nw_X = nw.from_native(X, eager_only=True) + nw_X = nw_X.with_columns( + [ + ((nw.col(col) - self.mean_[col]) / self.range_[col]).alias(col) + for col in self.variables_ + ] + ) + return nw_X.to_native() - return X - - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] - The data to be transformed. + X: dataframe of shape = [n_samples, n_features] + The data to be transformed. Accepts dataframes from libraries supported by + narwhals (e.g. pandas, polars, PyArrow). Returns ------- - X_tr: pandas dataframe - The dataframe with the transformed variables. + X_tr: dataframe + The dataframe with the transformed variables, in the same library as the + input. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) - # inverse transform - X[self.variables_] = X[self.variables_] * self.range_ + self.mean_ + nw_X = nw.from_native(X, eager_only=True) + nw_X = nw_X.with_columns( + [ + (nw.col(col) * self.range_[col] + self.mean_[col]).alias(col) + for col in self.variables_ + ] + ) + return nw_X.to_native() + - return X +# TODO: remove in version 2.1.0 +class MeanNormalizationScaler(MeanNormalisationScaler): + def __init__( + self, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + ) -> None: + warnings.warn( + "MeanNormalizationScaler was deprecated in favour of " + "MeanNormalisationScaler in version 2.0.0 and will be removed in " + "version 2.1.0. To silence this warning, use MeanNormalisationScaler " + "instead.", + FutureWarning, + stacklevel=2, + ) + super().__init__(variables=variables, return_empty=return_empty) diff --git a/feature_engine/text/text_features.py b/feature_engine/text/text_features.py index c6f2692d4..659938419 100644 --- a/feature_engine/text/text_features.py +++ b/feature_engine/text/text_features.py @@ -12,7 +12,7 @@ _check_param_missing_values, ) from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -236,7 +236,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) # Set features to extract if self.features is None: @@ -278,7 +280,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) else: X[self.variables_] = X[self.variables_].fillna("") diff --git a/pyproject.toml b/pyproject.toml index 2a385d74d..b862c76b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,18 +10,16 @@ license = {text = "BSD 3 clause"} authors = [ { name = "Soledad Galli", email = "solegalli@protonmail.com" } ] -requires-python = ">=3.9.0" +requires-python = ">=3.11.0" dependencies = [ "numpy>=1.18.2", - "pandas>=2.2.0", - "scikit-learn>=1.4.0", + "scikit-learn>=1.7.0", "scipy>=1.4.1", + "narwhals>=2.0.0", ] classifiers = [ "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -39,10 +37,13 @@ docs = [ "pydata_sphinx_theme>=0.7.2", "sphinx_autodoc_typehints>=1.11.1,<=1.21.3", "numpydoc>=0.9.2", + "pandas>=2.2.0", ] tests = [ "pytest>=5.4.1", + "pandas>=2.2.0", + "polars>=1.0.0", # repo maintenance tooling "black>=21.5b1", diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6df436a29..000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -numpy>=1.18.2 -pandas>=2.2.0 -scikit-learn>=1.4.0 -scipy>=1.4.1 diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 711b1aea7..085a82fad 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,138 +1,286 @@ import numpy as np import pandas as pd +import polars as pl import pytest from pandas.testing import assert_frame_equal, assert_series_equal +from polars.testing import assert_frame_equal as pl_assert_frame_equal +from polars.testing import assert_series_equal as pl_assert_series_equal from scipy.sparse import csr_matrix from feature_engine.dataframe_checks import ( _check_contains_inf, _check_contains_na, - _check_optional_contains_na, _check_X_matches_training_df, check_X, check_X_y, check_y, ) +# ------------------------ +# test check_X +# ------------------------ -def test_check_X_returns_df(df_vartypes): - assert_frame_equal(check_X(df_vartypes), df_vartypes) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_unchanged(make_df, assert_equal_fn): + df = make_df({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) + X = check_X(df) + assert isinstance(X, type(df)) + assert_equal_fn(X, df) -def test_check_X_converts_numpy_to_pandas(): - a1D = np.array([1, 2, 3, 4]) - a2D = np.array([[1, 2], [3, 4]]) - a3D = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) - - df_2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - assert_frame_equal(df_2D, check_X(a2D)) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_with_mixed_dtypes(make_df, assert_equal_fn): + data = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="min"), + } + df = make_df(data) + assert_equal_fn(check_X(df), df) + + +@pytest.mark.parametrize( + "df", + [ + pd.DataFrame([]), + pd.DataFrame({"a": []}), + pl.DataFrame({"a": []}), + ], +) +def test_raises_error_if_empty_df(df): with pytest.raises(ValueError): - check_X(a3D) + check_X(df) + + +def test_check_X_raises_error_if_0_columns(): + # A dataframe with rows but no columns is not caught by `is_empty()`, which + # only looks at the row count, so it needs its own explicit check. Polars has + # no representation for "rows with 0 columns", so this case is pandas-only. + df = pd.DataFrame(index=range(3)) + assert df.shape == (3, 0) with pytest.raises(ValueError): - check_X(a1D) + check_X(df) -def test_check_X_raises_error_sparse_matrix(): - sparse_mx = csr_matrix([[5]]) - with pytest.raises(TypeError): - assert check_X(sparse_mx) +def test_check_X_raises_error_on_duplicated_column_names(): + # only relevant for pandas + df = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + } + ) + df.columns = ["var_A", "var_A", "var_B", "var_C"] + msg = "Expected unique column names" + with pytest.raises(ValueError, match=msg): + check_X(df) -def test_check_X_raises_error_with_complex_data(): - msg = "Complex data not supported" - rng = np.random.RandomState(0) - X = rng.uniform(size=10) + 1j * rng.uniform(size=10) - X = X.reshape(-1, 1) - with pytest.raises(TypeError, match=msg): - assert check_X(X) +@pytest.mark.parametrize( + "X", + [ + np.array([[1, 2], [3, 4]]), + np.array([1, 2, 3]), + np.array(1), + [1, 2, 3], + {"a": [1, 2, 3]}, + "not a dataframe", + None, + csr_matrix([[1, 2], [3, 4]]), + ], +) +def test_check_X_raises_error_on_non_dataframe_input(X): + with pytest.raises(TypeError) as record: + check_X(X) + assert record.match("X must be a dataframe from a library supported by narwhals") -def test_raises_error_if_empty_df(): - df = pd.DataFrame([]) - with pytest.raises(ValueError): - check_X(df) +# ------------------------ +# test check_y +# ------------------------ + +# --- series input --- + + +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_returns_values_unchanged(make_series, assert_equal_fn): + s = make_series([0, 1, 2, 3, 4]) + assert_equal_fn(check_y(s), s) -def test_check_y_returns_series(): - s = pd.Series([0, 1, 2, 3, 4]) - assert_series_equal(check_y(s), s) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_nan_error(make_series): + s = make_series([0.0, None, 2.0]) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(s) -def test_check_y_returns_dataframe(): - d = pd.DataFrame({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) - assert_frame_equal(check_y(d), d) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_inf_error(make_series): + s = make_series([0.0, float("inf"), 2.0]) + with pytest.raises(ValueError, match="y contains infinity values."): + check_y(s) -def test_check_y_converts_np_array(): - a1D = np.array([1, 2, 3, 4]) - s = pd.Series(a1D) - assert_series_equal(check_y(a1D), s) +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_converts_string_to_number_when_y_numeric( + make_series, assert_equal_fn +): + s = make_series(["0", "1", "2"]) + y = check_y(s, y_numeric=True) + expected = make_series([0.0, 1.0, 2.0]) + assert_equal_fn(y, expected) + + +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_leaves_non_numeric_unchanged_by_default( + make_series, assert_equal_fn +): + # y_numeric defaults to False: a non-numeric series (e.g. classification + # labels) should be returned as-is, without being cast to float. + s = make_series(["a", "b", "c"]) + assert_equal_fn(check_y(s), s) -def test_check_y_converts_np_array_2D(): - a2D = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4) - d = pd.DataFrame(a2D) - assert_frame_equal(check_y(a2D), d) +# --- dataframe (multioutput) input --- -def test_check_y_raises_none_error(): - with pytest.raises(ValueError): - check_y(None) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_y_dataframe_returns_values_unchanged(make_df, assert_equal_fn): + d = make_df({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) + assert_equal_fn(check_y(d), d) -def test_check_y_raises_nan_error(): - msg = "y contains NaN values." +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_nan_error(make_df): + d = make_df({"t1": [0.0, None, 2.0], "t2": [5.0, 6.0, 7.0]}) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(d) - # y is series - s = pd.Series([0, np.nan, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg - # y is multioutput - d = pd.DataFrame(np.array([1, np.nan, 3, 4, 5, 6, np.nan, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_inf_error(make_df): + d = make_df({"t1": [0.0, 0.4, 2.0], "t2": [5.0, float("inf"), 7.0]}) + with pytest.raises(ValueError, match="y contains infinity values."): check_y(d) - assert str(record.value) == msg -def test_check_y_raises_inf_error(): - msg = "y contains infinity values." +# --- array-like input --- - # y is series - s = pd.Series([0, np.inf, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg - # y is multioutput - d = pd.DataFrame(np.array([1, np.inf, 3, 4, 5, 6, np.inf, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: - check_y(d) - assert str(record.value) == msg +@pytest.mark.parametrize( + "a", + [ + np.array([1, 2, 3, 4]), + np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4), + [1, 2, 3, 4], + ], +) +def test_check_y_array_returns_unchanged(a): + y = check_y(a) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(a, y) -def test_check_y_converts_string_to_number(): - s = pd.Series(["0", "1", "2", "3", "4"]) - assert_series_equal(check_y(s, y_numeric=True), s.astype("float")) +def test_check_y_raises_none_error(): + msg = "requires y to be passed, but the target y" + with pytest.raises(ValueError, match=msg): + check_y(None) -def test_check_x_y_returns_pandas_from_pandas(df_vartypes): - # when s is series - s = pd.Series([0, 1, 2, 3]) - x, y = check_X_y(df_vartypes, s) - assert_frame_equal(df_vartypes, x) - assert_series_equal(s, y) +# ------------------------ +# test check_X_y +# ------------------------ + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn, make_series, assert_series_fn", + [ + (pd.DataFrame, assert_frame_equal, pd.Series, assert_series_equal), + (pl.DataFrame, pl_assert_frame_equal, pl.Series, pl_assert_series_equal), + ], +) +def test_check_X_y_returns_df_and_series_unchanged( + make_df, assert_frame_fn, make_series, assert_series_fn +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + s = make_series([0, 1, 2]) + X, y = check_X_y(df, s) + assert isinstance(X, type(df)) and isinstance(y, type(s)) + assert_frame_fn(X, df) + assert_series_fn(y, s) + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_y_returns_df_and_multioutput_y_unchanged(make_df, assert_frame_fn): + df = make_df({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + d = make_df({"t1": [1, 2, 3, 4], "t2": [5, 6, 7, 8]}) + X, y = check_X_y(df, d) + assert_frame_fn(X, df) + assert_frame_fn(y, d) - # when y is multioutput - d = pd.DataFrame(np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2)) - x, y = check_X_y(df_vartypes, d) - assert_frame_equal(df_vartypes, x) - assert_frame_equal(d, y) +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +@pytest.mark.parametrize( + "y", + [ + np.array([0, 1, 2]), + [0, 1, 2], + np.array([[0, 1], [2, 3], [4, 5]]), + ], +) +def test_check_X_y_with_array_like_y_returns_check_y_output( + make_df, assert_frame_fn, y +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + X, y_out = check_X_y(df, y) + assert_frame_fn(X, df) + np.testing.assert_array_equal(y_out, check_y(y)) -def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): + +def test_check_X_y_returns_pandas_with_non_typical_index(): + # only relevant for pandas: polars has no index to reconcile df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) x, y = check_X_y(df, s) @@ -141,164 +289,112 @@ def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): def test_check_X_y_raises_error_when_pandas_index_dont_match(): + # only relevant for pandas: polars has no index to reconcile msg = "The indexes of X and y do not match." df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, s) - assert str(record.value) == msg # when y is multioutput d = pd.DataFrame( np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2), index=[22, 99, 101, 999] ) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, d) - assert str(record.value) == msg - - -def test_check_x_y_reassings_index_when_only_one_input_is_pandas(): - # X is dataframe, y is 1D array - df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) - s = np.array([1, 2, 3, 4]) - s_exp = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert_series_equal(s_exp.astype(int), y.astype(int)) - # X is dataframe, y is 2d array - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s_exp = pd.DataFrame(s, index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert_frame_equal(s_exp.astype(int), y.astype(int)) - # X is not a df, y is a series - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index - x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) - assert_series_equal(s, y) - - # X is not a df, y is a dataframe - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s = pd.DataFrame(s, index=[22, 99, 101, 212]) - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index - x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) - assert_frame_equal(s, y) - - -def test_check_x_y_converts_numpy_to_pandas(): - a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) - df2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - - a1D = np.array([1, 2, 3, 4]) - s1D = pd.Series(a1D) +@pytest.mark.parametrize( + "make_df, make_series", + [(pd.DataFrame, pd.Series), (pl.DataFrame, pl.Series)], +) +def test_check_x_y_raises_error_when_inconsistent_length(make_df, make_series): + df = make_df({"a": [1, 2, 3]}) + s = make_series([0, 1]) + with pytest.raises(ValueError): + check_X_y(df, s) - # X is df and y is array - x, y = check_X_y(df2D, a1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) - # X is array and y is series - x, y = check_X_y(a2D, s1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) +# ----------------------------------- +# test _check_X_matches_training_df +# ----------------------------------- - # X is df and y is 2d array - y2D = pd.DataFrame(a2D, columns=[0, 1]) - x, y = check_X_y(df2D, a2D) - assert_frame_equal(df2D, x) - assert_frame_equal(y2D, y) - # X is array and y multioutput df - x, y = check_X_y(a2D, df2D) - assert_frame_equal(df2D, x) - assert_frame_equal(df2D, y) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_passes_when_columns_match(make_df): + df = make_df({"a": [1, 2], "b": [3, 4]}) + assert _check_X_matches_training_df(df, 2) is None -def test_check_x_y_raises_error_when_inconsistent_length(df_vartypes): - s = pd.Series([0, 1, 2, 3, 5]) - with pytest.raises(ValueError): - check_X_y(df_vartypes, s) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_raises_error_when_columns_dont_match(make_df): + msg = "The number of columns in this dataset is different from" + df = make_df({"a": [1, 2], "b": [3, 4]}) + with pytest.raises(ValueError, match=msg): + _check_X_matches_training_df(df, 3) -def test_check_X_matches_training_df(df_vartypes): - with pytest.raises(ValueError): - assert _check_X_matches_training_df(df_vartypes, 4) +# ------------------------- +# test _check_contains_na +# ------------------------- -def test_contains_na(df_na): - msg = ( +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_raises_when_nan(make_df): + msg1 = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer." ) - - with pytest.raises(ValueError) as record: - assert _check_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg - - -def test_optional_contains_na(df_na): - msg = ( + msg2 = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer or set the parameter " "`missing_values='ignore'` when initialising this transformer." ) - with pytest.raises(ValueError) as record: - assert _check_optional_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + with pytest.raises(ValueError, match=msg1): + _check_contains_na(df, ["Name", "City"]) + + with pytest.raises(ValueError, match=msg2): + _check_contains_na(df, ["Name", "City"], error_msg="other") + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_passes_when_no_nan(make_df): + df = make_df({"Name": ["tom", "nick"], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["Name", "City"]) is None -def test_contains_inf_raises_on_inf(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_ignores_columns_not_in_variables(make_df): + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["City"]) is None + + +# -------------------------- +# test _check_contains_inf +# -------------------------- + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_raises_on_inf(make_df): msg = ( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." ) - df = pd.DataFrame({"A": [1.1, np.inf, 3.3]}) + df = make_df({"A": [1.1, np.inf, 3.3]}) with pytest.raises(ValueError, match=msg): _check_contains_inf(df, ["A"]) -def test_contains_inf_passes_without_inf(): - df = pd.DataFrame({"A": [1.1, 2.2, 3.3]}) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_passes_without_inf(make_df): + df = make_df({"A": [1.1, 2.2, 3.3]}) assert _check_contains_inf(df, ["A"]) is None -def test_check_X_raises_error_on_duplicated_column_names(): - df = pd.DataFrame( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - } - ) - df.columns = ["var_A", "var_A", "var_B", "var_C"] - with pytest.raises(ValueError) as err_txt: - check_X(df) - assert err_txt.match("Input data contains duplicated variable names.") - - -def test_check_X_errors(): - # Test scalar array error (line 58) - with pytest.raises(ValueError) as record: - check_X(np.array(1)) - assert record.match("Expected 2D array, got scalar array instead") - - # Test 1D array error (line 65) - with pytest.raises(ValueError) as record: - check_X(np.array([1, 2, 3])) - assert record.match("Expected 2D array, got 1D array instead") - - # Test incorrect type error (line 80) - with pytest.raises(TypeError) as record: - check_X("not a dataframe") - assert record.match("X must be a numpy array or pandas dataframe") +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_ignores_columns_not_in_variables(make_df): + df = make_df({"A": [1.1, float("inf"), 3.3], "B": [1.0, 2.0, 3.0]}) + assert _check_contains_inf(df, ["B"]) is None diff --git a/tox.ini b/tox.ini index c31b3edd1..4a21bd8ee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,5 @@ [tox] envlist = - py39 - py310 - py311-sklearn150 - py311-sklearn160 py311-sklearn170 py312-pandas230 py312-pandas300 @@ -32,14 +28,6 @@ commands = # Python versions # ------------------------- -[testenv:py39] -deps = - .[tests] - -[testenv:py310] -deps = - .[tests] - [testenv:py313] deps = .[tests] @@ -53,16 +41,6 @@ deps = # scikit-learn matrix # ------------------------- -[testenv:py311-sklearn150] -deps = - .[tests] - scikit-learn==1.5.1 - -[testenv:py311-sklearn160] -deps = - .[tests] - scikit-learn==1.6.1 - [testenv:py311-sklearn170] deps = .[tests]