0.27.12 importance plots

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotpython"
-version = "0.27.11"
+version = "0.27.12"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotpython/plot/importance.py

@@ -0,0 +1,188 @@
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.inspection import permutation_importance
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def generate_mdi(X, y, feature_names=None, random_state=42) -> pd.DataFrame:
+    """
+    Generates a DataFrame with Gini importances from a RandomForestRegressor.
+
+    Notes:
+     There are two limitations of impurity-based feature importances:
+        - impurity-based importances are biased towards high cardinality features;
+        - impurity-based importances are computed on training set statistics
+        and therefore do not reflect the ability of feature to be useful to
+        make predictions that generalize to the test set. Permutation
+        importances can mitigate the last limitation, because ti can be computed on the
+        test set.
+
+    Args:
+        X (pd.DataFrame or np.ndarray): The feature set.
+        y (pd.Series or np.ndarray): The target variable.
+        feature_names (list, optional): List of feature names for labeling. Defaults to None.
+        random_state (int, optional): Random state for the RandomForestRegressor. Defaults to 42.
+
+    Returns:
+        pd.DataFrame: DataFrame with 'Feature' and 'Importance' columns.
+
+    Examples:
+        >>> from spotpython.plot.importance import generate_mdi
+        >>> import pandas as pd
+        >>> from sklearn.datasets import make_regression
+        >>> X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+        >>> X_df = pd.DataFrame(X)
+        >>> y_series = pd.Series(y)
+        >>> result = generate_mdi(X_df, y_series)
+        >>> print(result)
+
+    """
+    # Convert X and y to pandas DataFrames if they are not already
+    if not isinstance(X, pd.DataFrame):
+        X = pd.DataFrame(X)
+    if not isinstance(y, pd.Series):
+        y = pd.Series(np.ravel(y))  # Use np.ravel instead of flatten
+
+    # Train a Random Forest Regressor
+    rf = RandomForestRegressor(random_state=random_state)
+    rf.fit(X, y)
+
+    # Get feature importances
+    importances = rf.feature_importances_
+
+    # Create a DataFrame
+    if feature_names is None:
+        df_mdi = pd.DataFrame({"Feature": X.columns, "Importance": importances})
+    else:
+        df_mdi = pd.DataFrame({"Feature": feature_names, "Importance": importances})
+    df_mdi = df_mdi.sort_values("Importance", ascending=False).reset_index(drop=True)
+
+    return df_mdi
+
+
+def generate_imp(X_train, X_test, y_train, y_test, random_state=42, n_repeats=10, use_test=True) -> permutation_importance:
+    """
+    Generates permutation importances from a RandomForestRegressor.
+
+    Args:
+        X_train (pd.DataFrame or np.ndarray): The training feature set.
+        X_test (pd.DataFrame or np.ndarray): The test feature set.
+        y_train (pd.Series or np.ndarray): The training target variable.
+        y_test (pd.Series or np.ndarray): The test target variable.
+        random_state (int, optional): Random state for the RandomForestRegressor. Defaults to 42.
+        n_repeats (int, optional): Number of repeats for permutation importance. Defaults to 10.
+        use_test (bool, optional): If True, computes permutation importance on the test set. If False, uses the training set. Defaults to True.
+
+    Returns:
+        permutation_importance: Permutation importances object.
+
+    Examples:
+        >>> from spotpython.plot.importance import generate_imp
+        >>> import pandas as pd
+        >>> from sklearn.datasets import make_regression
+        >>> X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+        >>> X_train, X_test = X[:80], X[80:]
+        >>> y_train, y_test = y[:80], y[80:]
+        >>> X_train_df = pd.DataFrame(X_train)
+        >>> X_test_df = pd.DataFrame(X_test)
+        >>> y_train_series = pd.Series(y_train)
+        >>> y_test_series = pd.Series(y_test)
+        >>> perm_imp = generate_imp(X_train_df, X_test_df, y_train_series, y_test_series)
+        >>> print(perm_imp)
+    """
+    # Convert inputs to pandas DataFrames/Series if they are not already
+    if not isinstance(X_train, pd.DataFrame):
+        X_train = pd.DataFrame(X_train)
+    if not isinstance(X_test, pd.DataFrame):
+        X_test = pd.DataFrame(X_test)
+    if not isinstance(y_train, pd.Series):
+        y_train = pd.Series(np.ravel(y_train))  # Use np.ravel instead of flatten
+    if not isinstance(y_test, pd.Series):
+        y_test = pd.Series(np.ravel(y_test))  # Use np.ravel instead of flatten
+
+    # Train a Random Forest Regressor
+    rf = RandomForestRegressor(random_state=random_state)
+    rf.fit(X_train, y_train)
+
+    # Select the dataset for permutation importance
+    X_eval = X_test if use_test else X_train
+    y_eval = y_test if use_test else y_train
+
+    # Calculate permutation importances
+    perm_imp = permutation_importance(rf, X_eval, y_eval, n_repeats=n_repeats, random_state=random_state)
+
+    return perm_imp
+
+
+def plot_importances(df_mdi, perm_imp, X_test, target_name=None, feature_names=None, k=10, show=True) -> None:
+    """
+    Plots the impurity-based and permutation-based feature importances for a given classifier.
+
+    Args:
+        df_mdi (pd.DataFrame):
+            DataFrame with Gini importances.
+        perm_imp (object):
+            Permutation importances object.
+        X_test (pd.DataFrame):
+            The test feature set for permutation importance.
+        target_name (str, optional):
+            Name of the target variable for labeling. Defaults to None.
+        feature_names (list, optional):
+            List of feature names for labeling. Defaults to None.
+        k (int, optional):
+            Number of top features to display based on importance. Default is 10.
+        show (bool, optional):
+            If True, displays the plot immediately. Default is True.
+
+    Returns:
+        None
+
+    Examples:
+        >>> from spotpython.plot.importance import generate_mdi, generate_imp, plot_importances
+        >>> import pandas as pd
+        >>> from sklearn.datasets import make_regression
+        >>> X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+        >>> X_train, X_test = X[:80], X[80:]
+        >>> y_train, y_test = y[:80], y[80:]
+        >>> X_train_df = pd.DataFrame(X_train)
+        >>> X_test_df = pd.DataFrame(X_test)
+        >>> y_train_series = pd.Series(y_train)
+        >>> y_test_series = pd.Series(y_test)
+        >>> df_mdi = generate_mdi(X_train_df, y_train_series)
+        >>> perm_imp = generate_imp(X_train_df, X_test_df, y_train_series, y_test_series)
+        >>> plot_importances(df_mdi, perm_imp, X_test_df)
+
+    """
+
+    # Plot impurity-based importances for top-k features
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8))
+
+    sorted_mdi_importances = df_mdi.set_index("Feature")["Importance"]
+    sorted_mdi_importances[:k].sort_values().plot.barh(ax=ax1)
+    ax1.set_xlabel("Gini importance")
+    if target_name:
+        ax1.set_title(f"Impurity-based feature importances for target: {target_name}")
+    else:
+        ax1.set_title("Impurity-based feature importances")
+
+    # Ensure X_test is a DataFrame
+    if not isinstance(X_test, pd.DataFrame):
+        X_test = pd.DataFrame(X_test)
+
+    perm_sorted_idx = perm_imp.importances_mean.argsort()[-k:]
+    if feature_names is not None:
+        ax2.boxplot(perm_imp.importances[perm_sorted_idx].T, vert=False, labels=np.array(feature_names)[perm_sorted_idx])
+    else:
+        ax2.boxplot(perm_imp.importances[perm_sorted_idx].T, vert=False, labels=X_test.columns[perm_sorted_idx])
+    ax2.axvline(x=0, color="k", linestyle="--")
+    if target_name:
+        ax2.set_xlabel(f"Decrease in mse for target: {target_name}")
+    else:
+        ax2.set_xlabel("Decrease in mse")
+    ax2.set_title("Permutation-based feature importances")
+
+    # fig.suptitle("Impurity-based vs. permutation importances")
+    fig.tight_layout()
+    if show:
+        plt.show()

test/test_importance_imp.py

@@ -0,0 +1,31 @@
+import pytest
+import pandas as pd
+import numpy as np
+from sklearn.datasets import make_regression
+from sklearn.utils import Bunch
+from spotpython.plot.importance import generate_imp
+
+def test_generate_imp():
+    # Generate synthetic regression data
+    X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+    X_train, X_test = X[:80], X[80:]
+    y_train, y_test = y[:80], y[80:]
+
+    # Convert to DataFrame/Series for testing
+    feature_names = [f"Feature_{i}" for i in range(X.shape[1])]
+    X_train_df = pd.DataFrame(X_train, columns=feature_names)
+    X_test_df = pd.DataFrame(X_test, columns=feature_names)
+    y_train_series = pd.Series(y_train)
+    y_test_series = pd.Series(y_test)
+
+    # Test permutation importance on the test set (default behavior)
+    perm_imp_test = generate_imp(X_train_df, X_test_df, y_train_series, y_test_series, use_test=True)
+    assert isinstance(perm_imp_test, Bunch), "Output should be a Bunch object"
+    assert perm_imp_test.importances_mean.shape[0] == X.shape[1], "Number of importances should match the number of features"
+    assert np.all(perm_imp_test.importances_mean >= 0), "All importances should be non-negative"
+
+    # Test permutation importance on the training set
+    perm_imp_train = generate_imp(X_train_df, X_test_df, y_train_series, y_test_series, use_test=False)
+    assert isinstance(perm_imp_train, Bunch), "Output should be a Bunch object"
+    assert perm_imp_train.importances_mean.shape[0] == X.shape[1], "Number of importances should match the number of features"
+    assert np.all(perm_imp_train.importances_mean >= 0), "All importances should be non-negative"

test/test_importance_mdi.py

@@ -0,0 +1,50 @@
+import pytest
+import pandas as pd
+import numpy as np
+from sklearn.datasets import make_regression
+from spotpython.plot.importance import generate_mdi
+
+def test_generate_mdi_with_dataframe():
+    # Generate synthetic data
+    X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+    feature_names = [f"Feature_{i}" for i in range(X.shape[1])]
+    X_df = pd.DataFrame(X, columns=feature_names)
+    y_series = pd.Series(y)
+
+    # Call the function
+    result = generate_mdi(X_df, y_series)
+
+    # Assertions
+    assert isinstance(result, pd.DataFrame), "Result should be a DataFrame"
+    assert list(result.columns) == ["Feature", "Importance"], "DataFrame should have 'Feature' and 'Importance' columns"
+    assert len(result) == X_df.shape[1], "Number of rows should match the number of features"
+    assert result["Importance"].sum() > 0, "Feature importances should be greater than zero"
+
+def test_generate_mdi_with_ndarray():
+    # Generate synthetic data
+    X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+
+    # Call the function
+    result = generate_mdi(X, y)
+
+    # Assertions
+    assert isinstance(result, pd.DataFrame), "Result should be a DataFrame"
+    assert list(result.columns) == ["Feature", "Importance"], "DataFrame should have 'Feature' and 'Importance' columns"
+    assert len(result) == X.shape[1], "Number of rows should match the number of features"
+    assert result["Importance"].sum() > 0, "Feature importances should be greater than zero"
+
+def test_generate_mdi_with_custom_feature_names():
+    # Generate synthetic data
+    X, y = make_regression(n_samples=100, n_features=5, noise=0.1, random_state=42)
+    feature_names = [f"Custom_Feature_{i}" for i in range(X.shape[1])]
+    X_df = pd.DataFrame(X)
+
+    # Call the function
+    result = generate_mdi(X_df, y, feature_names=feature_names)
+
+    # Assertions
+    assert isinstance(result, pd.DataFrame), "Result should be a DataFrame"
+    assert list(result.columns) == ["Feature", "Importance"], "DataFrame should have 'Feature' and 'Importance' columns"
+    assert len(result) == len(feature_names), "Number of rows should match the number of custom feature names"
+    assert set(result["Feature"].values) == set(feature_names), "Feature names should match the custom feature names"
+    assert result["Importance"].sum() > 0, "Feature importances should be greater than zero"

test/test_importance_plot.py

@@ -0,0 +1,46 @@
+import pytest
+import pandas as pd
+import numpy as np
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.inspection import permutation_importance
+from spotpython.plot.importance import plot_importances
+
+@pytest.fixture
+def sample_data():
+    # Generate sample data
+    np.random.seed(42)
+    X_train = pd.DataFrame(np.random.rand(100, 5), columns=[f"Feature_{i}" for i in range(5)])
+    X_test = pd.DataFrame(np.random.rand(20, 5), columns=[f"Feature_{i}" for i in range(5)])
+    y_train = pd.Series(np.random.rand(100))
+    y_test = pd.Series(np.random.rand(20))
+    return X_train, X_test, y_train, y_test
+
+@pytest.fixture
+def mdi_importances(sample_data):
+    # Generate MDI importances
+    X_train, _, y_train, _ = sample_data
+    rf = RandomForestRegressor(random_state=42)
+    rf.fit(X_train, y_train)
+    importances = rf.feature_importances_
+    df_mdi = pd.DataFrame({"Feature": X_train.columns, "Importance": importances}).sort_values("Importance", ascending=False)
+    return df_mdi
+
+@pytest.fixture
+def perm_importances(sample_data):
+    # Generate permutation importances
+    X_train, X_test, y_train, y_test = sample_data
+    rf = RandomForestRegressor(random_state=42)
+    rf.fit(X_train, y_train)
+    perm_imp = permutation_importance(rf, X_test, y_test, n_repeats=10, random_state=42)
+    return perm_imp
+
+def test_plot_importances(sample_data, mdi_importances, perm_importances):
+    X_train, X_test, y_train, y_test = sample_data
+    df_mdi = mdi_importances
+    perm_imp = perm_importances
+
+    # Test if the function runs without errors
+    try:
+        plot_importances(df_mdi, perm_imp, X_test, target_name="Test Target", feature_names=X_train.columns, k=3, show=False)
+    except Exception as e:
+        pytest.fail(f"plot_importances raised an exception: {e}")