DataPoisoningCounterfactuals/Implementation/experiments_labelflipping.py at main · andreArtelt/DataPoisoningCounterfactuals · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
import warnings
warnings.filterwarnings("ignore")

import os
import random
import numpy as np
from joblib import Parallel, delayed
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score, confusion_matrix
from imblearn.under_sampling import RandomUnderSampler

from datasets import load_benchmarkdata
from cf_dice import DiceExplainer
from cf_memory import MemoryExplainer
from cf_proto import ProtoExplainer
from label_flipping_attack import label_flipping_attack
from utils import SvcWrapper


n_folds = 5
pos_class = 1
neg_class = 0


def get_model(model_desc, cf_desc):
    if model_desc == "svc":
        return LinearSVC()
    elif model_desc == "randomforest":
        return RandomForestClassifier(n_estimators=10, max_depth=7)
    elif model_desc == "dnn":
        return MLPClassifier(hidden_layer_sizes=(128, 32))


def run_exp(data_desc, model_desc, cf_desc, apply_data_poisoning, consider_fairness_in_poisoning,
            percent_data_poisoning=.5, out_path="my-exp-results-labelflipping"):
    print(cf_desc, data_desc, model_desc, apply_data_poisoning, consider_fairness_in_poisoning, percent_data_poisoning)

    np.random.seed(42)  # Fix random numbers as much as possible!
    random.seed(42)

    X, y, y_sensitive, _ = load_benchmarkdata(data_desc)

    kf = KFold(n_splits=n_folds, shuffle=True, random_state=42)

    X_orig = []
    X_cf = []
    Y_cf = []
    Y_orig_sensitive = []
    Y_test_pred = []
    Y_test = []
    accuracies = []

    for train_index, test_index in kf.split(X):
        try:
            X_train, X_test = X[train_index], X[test_index]
            y_train, y_test = y[train_index], y[test_index]
            y_sensitive_train, y_sensitive_test = y_sensitive[train_index], y_sensitive[test_index]

            # Deal with imbalanced data
            sampling = RandomUnderSampler() # Undersample majority class
            X_train, y_train = sampling.fit_resample(np.concatenate((X_train, y_sensitive_train.reshape(-1, 1)), axis=1), y_train)
            y_sensitive_train = X_train[:,-1].flatten()
            X_train = X_train[:,:-1]
            print(f"Training samples: {X_train.shape}")

            scaler = StandardScaler()
            X_train = scaler.fit_transform(X_train)
            X_test = scaler.transform(X_test)

            # Apply data poisoning
            if apply_data_poisoning is True:
                clf = get_model(model_desc, cf_desc)     # Assume access to the model (prediciton interface only)
                if isinstance(clf, LinearSVC):
                    clf = SvcWrapper(clf)
                clf.fit(X_train, y_train)

                n_samples = min(int(percent_data_poisoning * X_train.shape[0]), 400)    # For performance reasons do not poison more than some maximum numbe rof samples!
                print(f"n_samples for poisoning: {n_samples}")

                X_train, y_train = label_flipping_attack(X_train, y_train, n_samples)   # Label flipping attack

            # Fit model
            clf = get_model(model_desc, cf_desc)
            if isinstance(clf, LinearSVC):
                clf = SvcWrapper(clf)
            clf.fit(X_train, y_train)

            y_train_pred = clf.predict(X_train)
            y_test_pred = clf.predict(X_test)

            print(f"Train: {f1_score(y_train, clf.predict(X_train))}   Test: {f1_score(y_test, y_test_pred)}")
            print(confusion_matrix(y_test, y_test_pred))
            accuracies.append(f1_score(y_test, y_test_pred))

            # Compute counterfactuals
            if cf_desc == "dice":
                exp = DiceExplainer(clf, X_train, y_train)
            elif cf_desc == "mem":
                exp = MemoryExplainer(clf, X_train, y_train)
            elif cf_desc == "proto":
                exp = ProtoExplainer(clf, X_train, y_train)

            for i in range(X_test.shape[0]):
                x_orig = X_test[i,:]
                y_orig = y_test[i]
                y_orig_sensitive = y_sensitive_test[i]
                y_orig_pred = y_test_pred[i]
                y_target = 1 if y_orig_pred == 0 else 0  # ATTENTION: Assume binary classification problem -- determine target label based on prediction (i.e. assume no ground truth is available)

                if y_orig_pred != y_orig:   # Ignore some missclassified samples
                    if not((y_orig_pred == neg_class and y_orig == pos_class) or (y_orig_pred == neg_class and y_orig == neg_class)):   # Consider TNs and FNs -- recourse: neg -> pos
                        continue

                try:
                    xcf = exp.compute_counterfactual(x_orig, y_target)
                    for xcf_ in xcf:        # DiCE: We compute multiple diverse counterfactuals
                        X_orig.append(x_orig)
                        X_cf.append(xcf_)
                        Y_cf.append(y_target)
                        Y_orig_sensitive.append(y_orig_sensitive)
                        Y_test_pred.append(y_test_pred[i])
                        Y_test.append(y_test[i])
                except Exception as ex:
                    print(ex)
        except Exception as ex:
            print(ex)

    # Store results
    np.savez(os.path.join(out_path, f"{cf_desc}-{data_desc}_{model_desc}_datapoisoning={str(apply_data_poisoning)}_fairness={consider_fairness_in_poisoning}_n-samples={percent_data_poisoning}.npz"), X_orig=X_orig, X_cf=X_cf, Y_cf=Y_cf, Y_test_pred=Y_test_pred, Y_test=Y_test, Y_orig_sensitive=Y_orig_sensitive, accuracies=accuracies)


if __name__ == "__main__":
    config_sets = []
    out_path = "my-exp-results-labelflipping"

    for data_desc in ["german", "diabetes", "communitiescrimes"]:
        for model_desc in ["svc", "randomforest", "dnn"]:
            for cf_desc in ["mem", "dice", "proto"]:
                for apply_data_poisoning in [True, False]:
                    for percent_data_poisoning in [.05, .1, .2, .3, .4, .5, .6, .7]:
                        consider_fairness_choices = [True, False]
                        if apply_data_poisoning is False:
                            consider_fairness_choices = [False]
                        for consider_fairness in consider_fairness_choices:
                            config_sets.append({"data_desc": data_desc, "model_desc": model_desc, "cf_desc": cf_desc,
                                                "apply_data_poisoning": apply_data_poisoning,
                                                "consider_fairness_in_poisoning": consider_fairness, "percent_data_poisoning": percent_data_poisoning, "out_path": out_path})

    Parallel(n_jobs=8)(delayed(run_exp)(**param_config) for param_config in config_sets)