Attention une mise à jour du service Gitlab va être effectuée le mardi 30 novembre entre 17h30 et 18h00. Cette mise à jour va générer une interruption du service dont nous ne maîtrisons pas complètement la durée mais qui ne devrait pas excéder quelques minutes. Cette mise à jour intermédiaire en version 14.0.12 nous permettra de rapidement pouvoir mettre à votre disposition une version plus récente.

Commit 553773c2 authored by BERNIER Fabien's avatar BERNIER Fabien
Browse files

[+] Hate speech model test

[~] Corrected EnsembleOut bugs on textual data
parent 1e20cf6c
......@@ -39,7 +39,7 @@ class FixOutText:
self.class_names = le.classes_
self.feature_names = vocabulary
self.train, self.test, self.train_labels, self.labels_test = train_test_split(self.data, self.labels, train_size=train_size, random_state=self.seed)
self.train, self.test, self.train_labels, self.test_labels = train_test_split(self.data, self.labels, train_size=train_size, random_state=self.seed)
self.sensitive_f = to_drop
self.max_features = max_features
......@@ -54,7 +54,7 @@ class FixOutText:
model = train_classifier(self.algo, self.train, self.train_labels)
self.original_model = Model([model], [None])
accuracy, threshold = evaluation(self.original_model.prob(self.test), self.labels_test)
accuracy, threshold = evaluation(self.original_model.prob(self.test), self.test_labels)
actual_sensitive, is_fair_flag, ans_data, explainer = self.exp(self.original_model, self.train, self.max_features, self.sensitive_f, self.feature_names, self.class_names, self.sampling_size, self.threshold)
return actual_sensitive, is_fair_flag, ans_data, accuracy, threshold
......@@ -75,12 +75,11 @@ class FixOutText:
"""
models, removers = [], []
words_to_drop = [self.feature_names[i] for i in actual_sensitive]
words_to_drop = actual_sensitive
flags = re.IGNORECASE if ignorecase else 0
for i in actual_sensitive:
word = self.feature_names[i] # getting the word to drop
for word in words_to_drop: # getting the word to drop
rep = re.compile(token_pattern.replace("\\w", '('+word+')'), flags=flags) # compiling a regex to find the word
train = list(map(lambda text: rep.sub("", text), self.train)) # removing the word
model = train_classifier(self.algo, train, self.train_labels)
......@@ -95,7 +94,7 @@ class FixOutText:
removers.append(rep_all)
self.ensemble = Model(models, removers)
accuracy, threshold = evaluation(self.ensemble.prob(self.test), self.labels_test)
accuracy, threshold = evaluation(self.ensemble.prob(self.test), self.test_labels)
_, is_fair_flag, ans_data, explainer = self.exp(self.ensemble, self.train, self.max_features, actual_sensitive, self.feature_names, self.class_names, self.sampling_size, self.threshold)
return is_fair_flag, ans_data, accuracy, threshold
......@@ -115,7 +114,7 @@ class Model:
self.removers = removers
def prob(self,X):
def prob(self, X):
"""
Returns probability for each class label.
"""
......@@ -127,7 +126,7 @@ class Model:
model = self.models[i]
rep = self.removers[i]
if rep is not None :
X = rep.sub("", X)
X = list(map(lambda s: rep.sub("", s), X))
comp = model.predict_proba(X).astype(float)
probs.append(comp)
......
This diff is collapsed.
import numpy as np
import pandas as pd
from nltk.corpus import stopwords
from nltk.stem.porter import *
from nltk import pos_tag
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
hsdata = pd.read_csv("datasets/hate_speech.csv")
class_names = ["hate speech", "offensive", "neither"]
stopwords = stopwords.words("english")
stopwords.extend(["#ff", "ff", "rt"])
stemmer = PorterStemmer()
def preprocess(text_string):
"""
Accepts a text string and replaces:
1) urls with URLHERE
2) lots of whitespace with one instance
3) mentions with MENTIONHERE
This allows us to get standardized counts of urls and mentions
Without caring about specific people mentioned
"""
space_pattern = '\s+'
giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
'[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
mention_regex = '@[\w\-]+'
parsed_text = re.sub(space_pattern, ' ', text_string)
parsed_text = re.sub(giant_url_regex, '', parsed_text)
parsed_text = re.sub(mention_regex, '', parsed_text)
return parsed_text
def tokenize(tweet):
"""Removes punctuation & excess whitespace, sets to lowercase,
and stems tweets. Returns a list of stemmed tokens."""
tweet = " ".join(re.split("[^a-zA-Z]*", tweet.lower())).strip()
tokens = [stemmer.stem(t) for t in tweet.split()]
return tokens
def basic_tokenize(tweet):
"""Same as tokenize but without the stemming"""
tweet = " ".join(re.split("[^a-zA-Z.,!?]*", tweet.lower())).strip()
return tweet.split()
tweet_tags = []
for tweet in hsdata.tweet:
tokens = basic_tokenize(preprocess(tweet))
tags = pos_tag(tokens)
tag_list = [x[1] for x in tags]
tag_str = " ".join(tag_list)
tweet_tags.append(tag_str)
X = hsdata.tweet
y = (hsdata["class"] != 2).astype(np.uint8)
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8)
vectorizer = TfidfVectorizer(
tokenizer=tokenize,
preprocessor=preprocess,
ngram_range=(1, 3),
stop_words=stopwords,
use_idf=True,
smooth_idf=False,
norm=None,
decode_error='replace',
max_features=10000,
min_df=5,
max_df=0.75
)
rf = RandomForestClassifier(n_estimators=500)
model = make_pipeline(vectorizer, rf)
model.fit(X_train, y_train)
print(model.predict(["piece of cake", "piece of shit"]))
\ No newline at end of file
......@@ -17,6 +17,7 @@ clusters = 50
def features_contributions(predict_fn, train, class_names, sample_size, kernel_width=3):
explainer = LimeTextExplainer(class_names=class_names, kernel_width=kernel_width)
# TODO: temporarily replace submodular_pick by explanation on a random subset of samples
sp_obj = submodular_pick.SubmodularPick(explainer, train, predict_fn, sample_size=sample_size, num_features=1000, clusters=clusters)
return explainer, sp_obj
......@@ -43,25 +44,21 @@ def fairness_eval(model, train, max_features, sensitive_features, feature_names,
def fairness_valid_top(contributions, feature_names, sensitive_features, max_features):
actual_sensitive = []
counter_top = 0
ans_data = []
sorted_dict = sorted(contributions.items(), key=lambda x: abs(x[1]), reverse=True)
for key,value in sorted_dict:
ans_data1 = [key,feature_names[key],value]
ans_data.append(ans_data1)
if key in sensitive_features:
actual_sensitive.append(key)
for i in range(max_features):
key, value = sorted_dict[i]
feature = feature_names[key]
ans_data.append([key, feature, value])
counter_top += 1
if counter_top >= max_features:
break
if feature in sensitive_features:
actual_sensitive.append(feature)
df = pd.DataFrame(ans_data, columns = ["Index", "Feature", "Contribution"])
return actual_sensitive, len(actual_sensitive) < 2, df
def fairness_valid_threshold(contributions, feature_names, sensitive_features, threshold):
actual_sensitive = []
......
......@@ -27,4 +27,11 @@ X = newsgroups_train.data + newsgroups_test.data
y = np.concatenate([newsgroups_train.target, newsgroups_test.target])
vocab = list(model[0].vocabulary_.keys())
fixout = FixOutText(X, y, vocab, to_drop=["com", "nigel", "of", "host", "library", "canrem", "symposium", "desks"], algo=model)
actual_sensitive, is_fair_flag, ans_data, accuracy, threshold = fixout.is_fair()
\ No newline at end of file
actual_sensitive, is_fair_flag, ans_data, accuracy, threshold = fixout.is_fair()
# correcting fairness if necessary
# if is_fair_flag :
# print("The model is fair ! \o/")
# else :
# print("Model not fair, " + " ".join(actual_sensitive) + " in the main features...")
# is_fair2, contributions_ensemble, accuracy_ensemble, threshold_ens = fixout.ensemble_out(actual_sensitive)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment