implementations of new xai consistency metrics

Author: ahinterl94-th

src/spotpython/light/trainmodel.py

@@ -1,7 +1,7 @@
 import lightning as L
 from spotpython.data.lightdatamodule import LightDataModule, PadSequenceManyToMany
 from spotpython.utils.eda import generate_config_id
-from spotpython.utils.metrics import calculate_xai_consistency
+from spotpython.utils.metrics import calculate_xai_consistency_corr, calculate_xai_consistency_cosine, calculate_xai_consistency_euclidean
 from pytorch_lightning.loggers import TensorBoardLogger
 from lightning.pytorch.callbacks.early_stopping import EarlyStopping
 from lightning.pytorch.callbacks import ModelCheckpoint
@@ -694,38 +694,38 @@ def train_model_xai(config: dict, fun_control: dict, timestamp: bool = True) ->
     if "IntegratedGradients" in fun_control["xai_methods"]:
         attr_ig = IntegratedGradients(model)
         attribution_ig = attr_ig.attribute(X_val_tensor, baselines=baseline)
-        ig_attr_test_sum = attribution_ig.detach().numpy().sum(0)
-        row_sum_ig = np.sum(ig_attr_test_sum, axis=0)
-        if row_sum_ig == 0:
-            row_sum_ig += 1e-10
-        scaled_attribution_ig = ig_attr_test_sum / row_sum_ig
-        attributions_dict["IntegratedGradients"] = scaled_attribution_ig
+        ig_attr_test_sum = attribution_ig.detach().numpy().sum(axis=0)
+        l2_norm = np.linalg.norm(ig_attr_test_sum)
+        l2_normalized_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
+        attributions_dict["IntegratedGradients"] = l2_normalized_ig
 
     if "KernelShap" in fun_control["xai_methods"]:
         attr_ks = KernelShap(model)
         attribution_ks = attr_ks.attribute(X_val_tensor, baselines=baseline)
-        ks_attr_test_sum = attribution_ks.detach().numpy().sum(0)
-        row_sum_ks = np.sum(ks_attr_test_sum, axis=0)
-        if row_sum_ks == 0:
-            row_sum_ks += 1e-10
-        scaled_attribution_ks = ks_attr_test_sum / row_sum_ks
-        attributions_dict["KernelShap"] = scaled_attribution_ks
+        ks_attr_test_sum = attribution_ks.detach().numpy().sum(axis=0)
+        l2_norm = np.linalg.norm(ks_attr_test_sum)
+        l2_normalized_ks = ks_attr_test_sum / l2_norm if l2_norm != 0 else ks_attr_test_sum
+        attributions_dict["KernelShap"] = l2_normalized_ks
 
     if "DeepLift" in fun_control["xai_methods"]:
         attr_dl = DeepLift(model)
         attribution_dl = attr_dl.attribute(X_val_tensor, baselines=baseline)
-        dl_attr_test_sum = attribution_dl.detach().numpy().sum(0)
-        row_sum_dl = np.sum(dl_attr_test_sum, axis=0)
-        if row_sum_dl == 0:
-            row_sum_dl += 1e-10
-        scaled_attribution_dl = dl_attr_test_sum / row_sum_dl
-        attributions_dict["DeepLift"] = scaled_attribution_dl
+        dl_attr_test_sum = attribution_dl.detach().numpy().sum(axis=0)
+        l2_norm = np.linalg.norm(dl_attr_test_sum)
+        l2_normalized_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
+        attributions_dict["DeepLift"] = l2_normalized_dl
 
     attributions_list = [attributions_dict[method] for method in fun_control["xai_methods"]]
     attributions = np.stack(attributions_list, axis=0)
 
-    result_xai = calculate_xai_consistency(attributions)
-
-    # -------------------------------------------------------------------------------------------------------------------
+    # Calculate corr:
+    if fun_control["xai_metric"] not in ["corr", "cosine", "euclidean"]:
+        raise ValueError(f"Invalid xai_metric: {fun_control['xai_metric']}. Valid metrics are: 'corr', 'cosine', 'euclidean'")
+    if fun_control["xai_metric"] == "corr":
+        result_xai = calculate_xai_consistency_corr(attributions)
+    elif fun_control["xai_metric"] == "cosine":
+        result_xai = calculate_xai_consistency_cosine(attributions)
+    elif fun_control["xai_metric"] == "euclidean":
+        result_xai = calculate_xai_consistency_euclidean(attributions)
 
     return result["val_loss"], result_xai

src/spotpython/utils/metrics.py

@@ -26,6 +26,7 @@
 
 import numpy as np
 from spotpython.utils.convert import series_to_array
+from sklearn.metrics.pairwise import euclidean_distances
 
 
 def apk(actual, predicted, k=10):
@@ -198,43 +199,17 @@ def get_metric_sign(metric_name):
         raise ValueError(f"Metric '{metric_name}' not found.")
 
 
-def calculate_xai_consistency(attributions) -> float:
-    """Calculate the consistency between different XAI methods.
-    Computes the pairwise correlation between different XAI methods' attributions
-    and returns their mean correlation as a measure of consistency. A higher value
-    indicates greater agreement between different XAI methods.
+def calculate_xai_consistency_corr(attributions):
+    """
+    Calculates the consistency of XAI methods by computing the mean of the upper triangle
+    of the correlation matrix of the provided attributions.
 
     Args:
         attributions (np.ndarray): Array of shape (n_methods, n_features) containing
-            feature importance scores from different XAI methods. Each row represents
-            a different XAI method's attributions, and each column represents a feature.
+                                   the attributions from different XAI methods.
 
     Returns:
-        float: Mean correlation between XAI methods, ranging from -1 to 1.
-            - 1: Perfect consistency between methods
-            - 0: No consistency between methods
-            - -1: Perfect negative consistency between methods
-
-    Examples:
-        >>> import numpy as np
-        >>> # Three XAI methods' attributions for four features
-        >>> attributions = np.array([
-        ...     [0.1, 0.2, 0.3, 0.4],  # Method 1
-        ...     [0.2, 0.3, 0.4, 0.5],  # Method 2
-        ...     [0.0, 0.1, 0.2, 0.3]   # Method 3
-        ... ])
-        >>> consistency = calculate_xai_consistency(attributions)
-        >>> print(f"XAI Consistency: {consistency:.2f}")
-        Attribution Correlation Matrix:
-        [[ 1.    0.97  0.98]
-         [ 0.97  1.    0.99]
-         [ 0.98  0.99  1.  ]]
-        XAI Consistency: 0.98
-
-    Note:
-        The correlation matrix is computed using numpy's corrcoef function, which
-        calculates Pearson correlation coefficients. Only the upper triangle of
-        the correlation matrix is used to avoid counting correlations twice.
+        float: Mean value of the upper triangle of the correlation matrix.
     """
     global_attr_np = np.array(attributions)
     corr_matrix = np.corrcoef(global_attr_np)
@@ -248,3 +223,55 @@ def calculate_xai_consistency(attributions) -> float:
     print("XAI Consistency (mean of upper triangle of correlation matrix):")
     print(result_xai)
     return result_xai
+
+
+def calculate_xai_consistency_cosine(attributions):
+    """
+    Calculates the consistency of XAI methods by computing the mean of the upper triangle
+    of the cosine similarity matrix of the provided attributions.
+
+    Args:
+        attributions (np.ndarray): Array of shape (n_methods, n_features) containing
+                                   the attributions from different XAI methods.
+
+    Returns:
+        float: Mean value of the upper triangle of the cosine similarity matrix.
+    """
+    global_attr_np = np.array(attributions)
+    cosine_sim_matrix = np.array([[np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) for b in global_attr_np] for a in global_attr_np])
+    print("Attribution Cosine Similarity Matrix:")
+    print(cosine_sim_matrix)
+
+    # Calculate the mean of the upper triangle of the cosine similarity matrix
+    upper_triangle_indices = np.triu_indices_from(cosine_sim_matrix, k=1)
+    upper_triangle_values = cosine_sim_matrix[upper_triangle_indices]
+    result_xai = upper_triangle_values.mean()
+    print("XAI Consistency (mean of upper triangle of cosine similarity matrix):")
+    print(result_xai)
+    return result_xai
+
+
+def calculate_xai_consistency_euclidean(attributions):
+    """
+    Calculates the consistency of XAI methods by computing the mean of the upper triangle
+    of the Euclidean distance matrix of the provided attributions.
+
+    Args:
+        attributions (np.ndarray): Array of shape (n_methods, n_features) containing
+                                   the attributions from different XAI methods.
+
+    Returns:
+        float: Mean value of the upper triangle of the Euclidean distance matrix.
+    """
+    global_attr_np = np.array(attributions)
+    euclidean_dist_matrix = euclidean_distances(global_attr_np)
+    print("Attribution Euclidean Distance Matrix:")
+    print(euclidean_dist_matrix)
+
+    # Calculate the mean of the upper triangle of the Euclidean distance matrix
+    upper_triangle_indices = np.triu_indices_from(euclidean_dist_matrix, k=1)
+    upper_triangle_values = euclidean_dist_matrix[upper_triangle_indices]
+    result_xai = upper_triangle_values.mean()
+    print("XAI Consistency (mean of upper triangle of Euclidean distance matrix):")
+    print(result_xai)
+    return result_xai

test/test_xai_consistency.py

@@ -1,50 +1,117 @@
 import numpy as np
-from spotpython.utils.metrics import calculate_xai_consistency
+from spotpython.utils.metrics import calculate_xai_consistency_corr, calculate_xai_consistency_cosine, calculate_xai_consistency_euclidean
 
 
-def test_xai_consistency():
+def test_xai_consistency_corr():
     # Mock data for testing
 
     dl_attr_test_sum = [1, 2, 3, 4, 5]
-    row_sum_dl = np.sum(dl_attr_test_sum, axis=0)
-    if row_sum_dl == 0:
-        row_sum_dl += 1e-10
-    scaled_attribution_dl = dl_attr_test_sum / row_sum_dl
+    l2_norm = np.linalg.norm(dl_attr_test_sum)
+    scaled_attribution_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
 
     ig_attr_test_sum = [1, 2, 3, 4, 5]
-    row_sum_ig = np.sum(ig_attr_test_sum, axis=0)
-    if row_sum_ig == 0:
-        row_sum_ig += 1e-10
-    scaled_attribution_ig = ig_attr_test_sum / row_sum_ig
+    l2_norm = np.linalg.norm(ig_attr_test_sum)
+    scaled_attribution_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
 
     attributions = [scaled_attribution_dl, scaled_attribution_ig]
-    result = calculate_xai_consistency(attributions)
+    result = calculate_xai_consistency_corr(attributions)
     print("XAI Consistency Result:")
     print(result)
 
     # Assert that the result is 1
     assert abs(result - 1) < 1e-10
 
 
-def test_xai_consistency_negative():
+def test_xai_consistency_negative_corr():
     # Mock data for testing negative consistency
 
     dl_attr_test_sum = [1, 2, 3, 4, 5]
-    row_sum_dl = np.sum(dl_attr_test_sum, axis=0)
-    if row_sum_dl == 0:
-        row_sum_dl += 1e-10
-    scaled_attribution_dl = dl_attr_test_sum / row_sum_dl
+    l2_norm = np.linalg.norm(dl_attr_test_sum)
+    scaled_attribution_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
 
-    ig_attr_test_sum = [-1, -2, -3, -4, -5]
-    row_sum_ig = np.sum(np.abs(ig_attr_test_sum), axis=0)
-    if row_sum_ig == 0:
-        row_sum_ig += 1e-10
-    scaled_attribution_ig = ig_attr_test_sum / row_sum_ig
+    ig_attr_test_sum = [-2, -3, -4, -5, -6]
+    l2_norm = np.linalg.norm(ig_attr_test_sum)
+    scaled_attribution_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
 
     attributions = [scaled_attribution_dl, scaled_attribution_ig]
-    result = calculate_xai_consistency(attributions)
+    result = calculate_xai_consistency_corr(attributions)
     print("XAI Consistency Result (Negative):")
     print(result)
 
     # Assert that the result is -1
-    assert abs(result + 1) < 1e-10
+    assert abs(result + 1) < 1e-10
+
+
+def test_xai_consistency_cosine():
+    # Mock data for testing cosine consistency
+
+    dl_attr_test_sum = [1, 2, 3, 4, 5]
+    l2_norm = np.linalg.norm(dl_attr_test_sum)
+    scaled_attribution_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
+    ig_attr_test_sum = [1, 2, 3, 4, 5]
+    l2_norm = np.linalg.norm(ig_attr_test_sum)
+    scaled_attribution_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
+
+    attributions = [scaled_attribution_dl, scaled_attribution_ig]
+    result = calculate_xai_consistency_cosine(attributions)
+    print("XAI Consistency Cosine Result:")
+    print(result)
+    # Assert that the result is 1
+    assert abs(result - 1) < 1e-10
+
+
+def test_xai_consistency_negative_cosine():
+    # Mock data for testing negative cosine consistency
+
+    dl_attr_test_sum = [1, 2, 3, 4, 5]
+    l2_norm = np.linalg.norm(dl_attr_test_sum)
+    scaled_attribution_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
+    ig_attr_test_sum = [-1, -2, -3, -4, -5]
+    l2_norm = np.linalg.norm(ig_attr_test_sum)
+    scaled_attribution_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
+
+    attributions = [scaled_attribution_dl, scaled_attribution_ig]
+    result = calculate_xai_consistency_cosine(attributions)
+    print("XAI Consistency Cosine Result (Negative):")
+    print(result)
+
+    # Assert that the result is -1
+    assert abs(result + 1) < 1e-10
+
+
+def test_xai_consistency_euclidean():
+    # Mock data for testing Euclidean consistency
+
+    dl_attr_test_sum = [1, 2, 3, 4, 5]
+    l2_norm = np.linalg.norm(dl_attr_test_sum)
+    scaled_attribution_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
+    ig_attr_test_sum = [1, 2, 3, 4, 5]
+    l2_norm = np.linalg.norm(ig_attr_test_sum)
+    scaled_attribution_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
+
+    attributions = [scaled_attribution_dl, scaled_attribution_ig]
+    result = calculate_xai_consistency_euclidean(attributions)
+    print("XAI Consistency Euclidean Result:")
+    print(result)
+
+    # Assert that the result is close to zero
+    assert abs(result) < 1e-10
+
+
+def test_xai_consistency_negative_euclidean():
+    # Mock data for testing negative Euclidean consistency
+
+    dl_attr_test_sum = [1, 2, 3, 4, 5]
+    l2_norm = np.linalg.norm(dl_attr_test_sum)
+    scaled_attribution_dl = dl_attr_test_sum / l2_norm if l2_norm != 0 else dl_attr_test_sum
+    ig_attr_test_sum = [-1, -2, -3, -4, -5]
+    l2_norm = np.linalg.norm(ig_attr_test_sum)
+    scaled_attribution_ig = ig_attr_test_sum / l2_norm if l2_norm != 0 else ig_attr_test_sum
+
+    attributions = [scaled_attribution_dl, scaled_attribution_ig]
+    result = calculate_xai_consistency_euclidean(attributions)
+    print("XAI Consistency Euclidean Result (Negative):")
+    print(result)
+
+    # Assert that the result is close to two
+    assert abs(result - 2) < 1e-10