Skip to content
GitLab
Commit 62f52680 authored by ALVES Guilherme's avatar ALVES Guilherme
Browse files

Removed unecessary files. Updated datasets parameters file.

parent 35ce914a
Hide whitespace changes
Inline Side-by-side
adult.py deleted 100644 → 0
View file @ 35ce914a
import datetime
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble._forest import RandomForestClassifier
from sklearn.ensemble._gb import GradientBoostingClassifier
from sklearn.linear_model._logistic import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.neural_network._multilayer_perceptron import MLPClassifier
from sklearn.svm._classes import SVC
from sklearn.tree import DecisionTreeClassifier
from core import load_data, Model, evaluation, \
find_threshold, remove, ensemble_out, train_classifier
from lime_global import fairness_eval
import os
import numpy as np
os.environ['KMP_DUPLICATE_LIB_OK']='True'
np.random.seed(10)
def main(seed):
train_size = 0.8
max_features = 4
sample_size = 500
algo = MLPClassifier
print(algo.__name__)
source_name = 'adult.data'
to_drop = [5,8,9] # sex, race, marital status
all_categorical_features = [1,3,5,6,7,8,9,13]
data, labels, class_names, feature_names, categorical_names = load_data(source_name, all_categorical_features, delimiter=',')
train, test, labels_train, labels_test = train_test_split(data, labels, train_size=train_size, test_size=1-train_size, random_state=seed)
print("Features")
for i in range(len(feature_names)):
f_name = feature_names[i]
print(i,f_name,end=' ')
if i in all_categorical_features:
print("[c]",end=' ')
if i in to_drop:
print("[s]",end=' ')
print('')
model, encoder = train_classifier(algo, train, test, labels_train, [], all_categorical_features)
original_model = Model([model],[encoder],[[]])
threshold_1 = find_threshold(original_model.prob(train), labels_train)
print("Original model OK")
fairness_eval(original_model, train, max_features, to_drop, feature_names, class_names, all_categorical_features, categorical_names, sample_size)
ensemble = ensemble_out(algo,to_drop,train, test, labels_train, all_categorical_features)
threshold_2 = find_threshold(ensemble.prob(train), labels_train)
print("ExpOut ensemble's model OK")
accuracy_original = evaluation(original_model.prob(test), labels_test, threshold_1)
print("accuracy_original", accuracy_original)
accuracy_ensemble = evaluation(ensemble.prob(test), labels_test, threshold_2)
print("accuracy_ensemble", accuracy_ensemble)
# debug
for i in range(len(ensemble.models)):
m = Model([ensemble.models[i]],[ensemble.encoders[i]],[ensemble.features_to_remove[i]])
print("accuracy_m",i,' ', evaluation(m.prob(test), labels_test, threshold_2), sep='')
fairness_eval(ensemble, train, max_features, to_drop, feature_names, class_names, all_categorical_features, categorical_names, sample_size)
if __name__ == "__main__":
now = datetime.datetime.now()
print('adult mlp (100,100,)\n',now.year,'-', now.month,'-', now.day,',', now.hour,':', now.minute,':', now.second, sep='')
for i in range(5):
print("experiment i=",i)
main(i)
\ No newline at end of file
bank.py deleted 100644 → 0
View file @ 35ce914a
import datetime
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble._forest import RandomForestClassifier
from sklearn.linear_model._logistic import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.neural_network._multilayer_perceptron import MLPClassifier
from sklearn.svm._classes import SVC
from sklearn.tree import DecisionTreeClassifier
from anchor_global import fairness_eval
from core import load_data, Model, evaluation, \
find_threshold, remove, ensemble_out, train_classifier
def main(seed):
train_size = 0.8
max_features = 10
algo = SVC
print(algo.__name__)
source_name = 'bank.data'
to_drop = [0,2] # age, marital
all_categorical_features = [1,2,3,4,6,7,8,10,15]
data, labels, class_names, feature_names, categorical_names = load_data(source_name, all_categorical_features,delimiter=";")
train, test, labels_train, labels_test = train_test_split(data, labels, train_size=train_size, test_size=1-train_size, random_state=seed)
model, encoder = train_classifier(algo, train, test, labels_train, [], all_categorical_features)
original_model = Model([model],[encoder],[[]])
# threshold_1 = find_threshold(original_model.prob(train), labels_train)
print("Original model OK")
fairness_eval(original_model, train, max_features, to_drop, feature_names, class_names, all_categorical_features, categorical_names)
ensemble = ensemble_out(algo,to_drop,train, test, labels_train, all_categorical_features)
# threshold_2 = find_threshold(ensemble.prob(train), labels_train)
print("ExpOut ensemble's model OK")
# accuracy_original = evaluation(original_model.prob(test), labels_test, threshold_1)
# print("accuracy_original", accuracy_original)
# accuracy_ensemble = evaluation(ensemble.prob(test), labels_test, threshold_2)
# print("accuracy_ensemble", accuracy_ensemble)
fairness_eval(ensemble, train, max_features, to_drop, feature_names, class_names, all_categorical_features, categorical_names)
if __name__ == "__main__":
now = datetime.datetime.now()
print('bank\n',now.year,'-', now.month,'-', now.day,',', now.hour,':', now.minute,':', now.second, sep='')
for i in range(1):
print("experiment i=",i)
main(i)
\ No newline at end of file
batch_experiments.py deleted 100644 → 0
View file @ 35ce914a
import datetime
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble._forest import RandomForestClassifier
from sklearn.linear_model._logistic import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.neural_network._multilayer_perceptron import MLPClassifier
from sklearn.tree import DecisionTreeClassifier
from anchor_global import features_contributions
from core import load_data, train_classifier, Model, evaluation, \
find_threshold, remove, ensemble_out, random_state
def main():
train_size = 0.8
test_size = 0.2
algo = MLPClassifier
print(algo.__name__)
source_name = 'german.data'
feature_names = ['existingchecking', 'duration', 'credithistory', 'purpose', 'creditamount', 'savings', 'employmentsince', 'installmentrate', 'statussex', 'otherdebtors', 'residencesince', 'property', 'age', 'otherinstallmentplans', 'housing', 'existingcredits', 'job', 'peopleliable', 'telephone', 'foreignworker', 'classification']
to_drop = [8,18,19]
all_categorical_features = [0,2,3,5,6,8,9,11,13,14,16,18,19]
data, labels, class_names, feature_names, categorical_names = load_data(source_name, feature_names, all_categorical_features)
for i in range(30):
train, test, labels_train, labels_test = train_test_split(data, labels, train_size=train_size, test_size=test_size, random_state=i)
model, encoder = train_classifier(algo, train, labels_train, [], all_categorical_features)
original_model = Model([model],[encoder],[[]])
threshold_1 = find_threshold(original_model.prob(train), labels_train)
accuracy1 = evaluation(original_model.prob(test), labels_test, threshold_1)
ensemble = ensemble_out(algo,to_drop,train,labels_train, all_categorical_features)
threshold_2 = find_threshold(ensemble.prob(train), labels_train)
accuracy2 = evaluation(ensemble.prob(test), labels_test, threshold_2)
print(i,accuracy1,accuracy2)
if __name__ == "__main__":
now = datetime.datetime.now()
print(now.year,'-', now.month,'-', now.day,',', now.hour,':', now.minute,':', now.second)
main()
\ No newline at end of file
core.py
View file @ 62f52680
......@@ -57,11 +57,12 @@ class FixOut:
model, encoder = train_classifier(self.algo, self.train, self.test, self.labels_train, [], self.all_categorical_f, self.seed)
self.original_model = Model([model],[encoder],[[]])
accuracy, _ = evaluation(self.original_model.prob(self.test), self.labels_test)
# accuracy = evaluation_fixed_threshold(self.original_model.prob(self.test), self.labels_test,0.5)
accuracy, threshold = evaluation(self.original_model.prob(self.test), self.labels_test)
actual_sensitive, is_fair_flag ,ans_data, explainer = self.exp(self.original_model, self.train, self.max_features, self.sensitive_f, self.feature_names, self.class_names, self.all_categorical_f, self.categorical_names, self.sampling_size, self.threshold)
return actual_sensitive, is_fair_flag, ans_data, accuracy
return actual_sensitive, is_fair_flag, ans_data, accuracy, threshold
def ensemble_out(self, actual_sensitive):
......@@ -95,12 +96,13 @@ class FixOut:
self.ensemble = Model(models,encoders,features_to_remove)
accuracy, _ = evaluation(self.ensemble.prob(self.test), self.labels_test)
accuracy, threshold = evaluation(self.ensemble.prob(self.test), self.labels_test)
# accuracy = evaluation_fixed_threshold(self.ensemble.prob(self.test), self.labels_test, 0.5)
# _, is_fair_flag ,ans_data, explainer = self.exp(self.ensemble, self.train, self.max_features, actual_sensitive, self.feature_names, self.class_names, self.all_categorical_f, self.categorical_names, self.sampling_size, self.threshold)
#
# return is_fair_flag, ans_data, accuracy
return False, None, accuracy
_, is_fair_flag ,ans_data, explainer = self.exp(self.ensemble, self.train, self.max_features, actual_sensitive, self.feature_names, self.class_names, self.all_categorical_f, self.categorical_names, self.sampling_size, self.threshold)
return is_fair_flag, ans_data, accuracy, threshold
# return False, None, accuracy
class Model:
......@@ -253,6 +255,11 @@ def evaluation(probs, true_labels):
accuracy = accuracy_score(true_labels, to_labels(probs, thresholds[ix]))
return accuracy, thresholds[ix]
def evaluation_fixed_threshold(probs, true_labels, threshold):
probs = probs[:, 1]
return accuracy_score(true_labels, to_labels(probs, threshold))
def fairness_metrics_eval(probs, true_labels, protected_attr, p_group, p_label):
_probs = probs[:, 1]
......
datasets_param.py
View file @ 62f52680
......@@ -44,8 +44,11 @@ hdma = {
# "all_categorical_features" : [0,1,2,3,5,10,12,13,14,19,20], # 15,16
"sensitive" : [1,2,3], # "derived_race","derived_sex","derived_ethnicity"
"priv_group" : {
"derived_sex" : 1, # Male ('Female', 'Male')
"derived_race" : 6, # White ('Black or African American', 'White', 'Asian', 'Native Hawaiian or Other Pacific Islander', 'American Indian or Alaska Native', '2 or more minority races', 'Race Not Available')
"derived_ethnicity" : 0 # Ethnicity Not Available ('Ethnicity Not Available', 'Hispanic or Latino', 'Joint', 'Not Hispanic or Latino')
},
"pos_label" : 0
"pos_label" : 1 # high-priced
}
nypd_sqf = {
......@@ -61,8 +64,10 @@ lsac = {
"all_categorical_features" : [3,4,5],
"sensitive" : [4,5],
"priv_group" : {
"sex" : 1, # "Male" (0 means Female)
"race" : 2 # "White" (0 means Black and 1 means "Other")
},
"pos_label" : 0
"pos_label" : 1 # Passed (0 is Failed_or_not_attempted)
}
default = {
......@@ -71,8 +76,10 @@ default = {
"all_categorical_features" : [1,2,3],
"sensitive" : [1,3], # sex, marriage,
"priv_group" : {
"SEX" : 0, # "1" male (X2: Gender (1 = male; 2 = female).)
"MARRIAGE" : 2 # "2" single (X4: Marital status (1 = married; 2 = single; 3 = others).)
},
"pos_label" : 0
"pos_label" : 1 # This research employed a binary variable, default payment (Yes = 1, No = 0), as the response variable
}
comcrime = {
......
experimenter.py
View file @ 62f52680
import datetime
from core import FixOut, evaluation, Model, fairness_metrics_eval
from core import FixOut, evaluation, Model, fairness_metrics_eval, \
evaluation_fixed_threshold
from datasets_param import *
from runner import algo_parser, exp_parser
import numpy as np
import pandas as pd
from runner import algo_parser, exp_parser
def one_experiment(source_name, sep, train_size, to_drop, all_categorical_features, max_features, algo, exp, sampling_size, threshold_exp, p_groups, p_label):
original_f_metrics = {}
ensemble_f_metrics = {}
for i in range(0,2):
for i in range(0,20):
fixout = FixOut(source_name, sep, train_size, to_drop, all_categorical_features, algo, exp, max_features, sampling_size, seed=i, threshold=threshold_exp)
actual_sensitive, is_fair1, contributions_original, accuracy_original = fixout.is_fair()
actual_sensitive, is_fair1, contributions_original, accuracy_original, threshold_orig = fixout.is_fair()
print("~~~~~~~~~\nOriginal model DONE\n~~~~~~~~~")
if not is_fair1:
is_fair2, contributions_ensemble, accuracy_ensemble = fixout.ensemble_out(actual_sensitive)
is_fair2, contributions_ensemble, accuracy_ensemble, threshold_ens = fixout.ensemble_out(actual_sensitive)
print("Ensemble model DONE\n~~~~~~~~~")
else:
print("Original model is FAIR")
print("~~~~~~~~~")
print("#########\nRESULTS",i,"BEGIN\n#########")
print("Original model accuracy", accuracy_original)
print("Original model accuracy", accuracy_original, threshold_orig)
print(contributions_original)
if is_fair1:
continue
if not is_fair1:
print("Ensemble model accuracy", accuracy_ensemble)
print("Ensemble model accuracy", accuracy_ensemble, threshold_ens)
ensemble = fixout.ensemble
for f in range(len(actual_sensitive)):
submodel = Model([ensemble.models[f]],[ensemble.encoders[f]],[actual_sensitive[f]])
accuracy, threshold = evaluation(submodel.prob(fixout.test), fixout.labels_test)
print("Ensemble Submodel accuracy", accuracy, threshold)
# accuracy, threshold = evaluation(submodel.prob(fixout.test), fixout.labels_test)
accuracy = evaluation_fixed_threshold(submodel.prob(fixout.test), fixout.labels_test, threshold_ens)
print("Ensemble Submodel accuracy", accuracy, threshold_ens)
submodel = Model([ensemble.models[len(actual_sensitive)]],[ensemble.encoders[len(actual_sensitive)]],[actual_sensitive])
# accuracy = evaluation_fixed_threshold(submodel.prob(fixout.test), fixout.labels_test,threshold_orig)
accuracy, threshold = evaluation(submodel.prob(fixout.test), fixout.labels_test)
print("Ensemble Submodel accuracy", accuracy, threshold)
......
fariness_metrics.py deleted 100644 → 0
View file @ 35ce914a
from aif360.datasets.adult_dataset import AdultDataset
from aif360.sklearn.metrics.metrics import disparate_impact_ratio, \
equal_opportunity_difference, statistical_parity_difference, theil_index, \
average_odds_difference, difference, generalized_fpr, specificity_score
from aif360.sklearn.utils import check_groups
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, f1_score
from sklearn.metrics import confusion_matrix
import numpy as np
import pandas as pd
def main():
ex = np.matrix([[1,1],[1,1],[0,0],[0,0]])
y_true = pd.DataFrame(ex,columns=['y_true','x1'])
y_true.set_index(['x1'], inplace=True)
y_pred = np.array([1,1,0,0])
probs = np.array([[0.2,0.8],[0.1,0.9],[0.6,0.4],[0.7,0.3]])
p_class=1
print(statistical_parity_difference(y_true, y_pred, prot_attr='x1',priv_group=1, pos_label=p_class)) # Demographic Parity
print(equal_opportunity_difference(y_true, y_pred, prot_attr='x1')) # Equal Opportunity
print(average_odds_difference(y_true, y_pred, prot_attr='x1')) # Equal Odds ???
print(disparate_impact_ratio(y_true, y_pred, prot_attr='x1')) # Disparate Impact
print(difference(accuracy_score, y_true, y_pred, prot_attr='x1'))
theil_index(np.array([1,1,0,0])) # Theil Index
__difference_fpr(y_true, y_pred, prot_attr='x1',priv_group=1, pos_label=p_class)
# print(difference(generalized_fpr, np.array([1,1,0,0]), probs[:,p_class], prot_attr='x1', pos_label=p_class))
# print(difference(generalized_fpr, y_true, probs[:,p_class], prot_attr='x1'))
# generalized_fpr(y_true, probs, pos_label=1), true_labels, probs)
def __difference_fpr(y_true, y_pred, prot_attr=None, priv_group=1, pos_label=1):
# y_true
print("true")
groups, _ = check_groups(y_true, prot_attr)
idx = (groups == priv_group)
n_y_true = y_true['y_true'].to_numpy()
priv_true = n_y_true[idx]
unpriv_true = n_y_true[~idx]
priv_pred = y_pred[idx]
unpriv_pred = y_pred[~idx]
tn, fp, fn, tp = confusion_matrix(priv_true, priv_pred).ravel()
tn, fp, fn, tp = confusion_matrix(unpriv_true, unpriv_pred).ravel()
#
# u = generalized_fpr(unpriv, probs, pos_label)
# p = generalized_fpr(priv, probs, pos_label)
if __name__ == "__main__":
main()
# data = AdultDataset()
# # data = datasets.CompasDataset()
#
# X,y = data.features, data.labels
#
# res = np.concatenate((y,data.protected_attributes),axis=1)
#
# y_true = pd.DataFrame(res,columns=['y_true']+data.protected_attribute_names)
# y_true.set_index(data.protected_attribute_names, inplace=True)
#
# # Equal Accuracy
# # Equal Odds
#
# rf = RandomForestClassifier()
# rf.fit(X, y)
#
# y_pred = rf.predict(X)
#
# print(statistical_parity_difference(y_true, y_pred, prot_attr='sex')) # Demographic Parity
# print(equal_opportunity_difference(y_true, y_pred, prot_attr='sex')) # Equal Opportunity
# print(average_odds_difference(y_true, y_pred, prot_attr='sex')) # Equal Odds ???
# # theil_index() # Theil Index
#
# print("###########")
#
# print(statistical_parity_difference(y_true, y_pred, prot_attr='race')) # Demographic Parity
# print(equal_opportunity_difference(y_true, y_pred, prot_attr='race')) # Equal Opportunity
# print(average_odds_difference(y_true, y_pred, prot_attr='race')) # Equal Odds ???
# print(disparate_impact_ratio(y_true, y_pred, prot_attr='race')) # Disparate Impact
german.py deleted 100644 → 0
View file @ 35ce914a
import datetime
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble._forest import RandomForestClassifier
from sklearn.ensemble._gb import GradientBoostingClassifier
from sklearn.linear_model._logistic import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.neural_network._multilayer_perceptron import MLPClassifier
from sklearn.svm._classes import SVC
from sklearn.tree import DecisionTreeClassifier
from core import load_data, Model, evaluation, \
find_threshold, remove, train_classifier
from lime_global import fairness_eval
from decimal import Decimal
global_seed = 2
# categorical_features1 = [0,2,3,5,6,8,10,12,13,15,17,18]
# categorical_features2 = [0,2,3,5,6,8,9,11,13,14,16,18]
# categorical_features3 = [0,2,3,5,6,8,9,11,13,14,16,18]
# categorical_features4 = [0,2,3,5,6,8,10,12,13,15]
def main(seed):
train_size = 0.7
test_size = 0.3
max_features = 10
sample_size = 500
algo = MLPClassifier # hidden_layer_sizes=(100,100,)
print(algo.__name__)
source_name = 'german.data'
to_drop = [8,18,19]
all_categorical_features = [0,2,3,5,6,8,9,11,13,14,16,18,19]
data, labels, class_names, feature_names, categorical_names = load_data(source_name, all_categorical_features)
train, test, labels_train, labels_test = train_test_split(data, labels, train_size=train_size, test_size=test_size, random_state=global_seed)
model, encoder = train_classifier(algo, train, test, labels_train, [], all_categorical_features)
original_model = Model([model],[encoder],[[]])
threshold_1 = find_threshold(original_model.prob(test), labels_test)
print("Original model OK")
# fairness_eval(original_model, train, max_features, to_drop, feature_names, class_names, all_categorical_features, categorical_names, sample_size)
ensemble = ensemble_out(algo,to_drop,train, test, labels_train, all_categorical_features)
threshold_2 = find_threshold(ensemble.prob(test), labels_test)
print("ExpOut ensemble's model OK")
accuracy_original = evaluation(original_model.prob(test), labels_test, threshold_1)
print("accuracy_original", accuracy_original)
accuracy_ensemble = evaluation(ensemble.prob(test), labels_test, threshold_2)
print("accuracy_ensemble", accuracy_ensemble)
# fairness_eval(ensemble, train, max_features, to_drop, feature_names, class_names, all_categorical_features, categorical_names, sample_size)
if __name__ == "__main__":
now = datetime.datetime.now()
print('german MLP(100,100,) solver=lbfgs \n',now.year,'-', now.month,'-', now.day,',', now.hour,':', now.minute,':', now.second, sep='')
for i in range(1):
print("experiment i=",i)
main(i)
\ No newline at end of file
preprocess_data.py deleted 100644 → 0
View file @ 35ce914a
import pickle
from sklearn.model_selection._split import train_test_split
from core import load_data, pre_process, remove
def main(params):
to_drop = params['remove_features']
all_categorical_f = params['all_categorical_features']
data, labels, class_names, feature_names, categorical_names = load_data(params['source_name'], all_categorical_f, delimiter=params['delimiter'])
train, test, labels_train, labels_test = train_test_split(data, labels, train_size=params['train_size'], test_size=1-params['train_size'], random_state=2)
batchs = []
train_res, labels_train_res, encoder = pre_process(train, test, labels_train, [], all_categorical_f)
batchs.append((train_res, labels_train_res, encoder))
for feature in to_drop:
categorical_features = remove([feature], all_categorical_f)
train_res, labels_train_res, encoder = pre_process(train, test, labels_train, [feature], categorical_features)
batchs.append((train_res, labels_train_res, encoder))
categorical_features = remove(to_drop, all_categorical_f)
train_res, labels_train_res, encoder = pre_process(train, test, labels_train, to_drop, categorical_features)
batchs.append((train_res, labels_train_res, encoder))
res = {
"batchs" : batchs,
"labels_test" : labels_test,
"class_names" : class_names,
"to_drop" : to_drop,
"all_categorical_f" : all_categorical_f,
"feature_names" : feature_names,
"categorical_names" : categorical_names
}
pickle.dump(res, open( params['source_name']+".p", "wb" ) )
if __name__ == "__main__":
adult = {
"source_name" : "adult.data",
"all_categorical_features" : [1, 3, 5, 6, 7, 8, 9, 13],
"delimiter" : ",",
"remove_features" : [5, 8, 9],
"train_size" : 0.8
}
main(adult)
#--data adult.data
#--trainsize 0.8
#--algo logreg
#--max_features 10
#--cat_features 1 3 5 6 7 8 9 13
#--drop 5 8 9
#--exp lime
#--samplesize 50
#--sep ,
\ No newline at end of file
runner.py
View file @ 62f52680
......@@ -14,7 +14,7 @@ import anchor_global
from core import FixOut
import lime_global
import shap_global
from sklearn.tree.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeClassifier
def main(source_name, sep, train_size, to_drop, all_categorical_features, max_features, algo, exp, sampling_size):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment