Machine Learning Interpretability

!pip  install   matplotlib 
!pip install eli5
#pdpbox has some problems in their pypi version
!pip install git+https://github.com/SauceCat/PDPbox
!pip install eli5
!pip install pycebox
!pip install ml_insights
!pip install scikit-optimize
!pip install alepython

#or
#!conda install -c conda-forge shap
!pip install shap
!pip install plotly_express
!pip install seaborn
!pip install category_encoders
!conda install -c conda-forge category_encoders
!pip install --upgrade git+https://github.com/scikit-learn-contrib/categorical-encoding
! pip install lime
!pip install graphviz  
!pip install skll

# Load All Libraries
from __future__ import print_function
# future allows compatitbility between python two and three modules
print(__doc__)
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.base import TransformerMixin
from sklearn.metrics import classification_report, f1_score, accuracy_score, precision_score, confusion_matrix
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_validate
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from lime import lime_text
from sklearn.pipeline import make_pipeline
from sklearn.pipeline import Pipeline
import sklearn
import sklearn.datasets
import sklearn.ensemble
import sklearn
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.datasets import load_boston
import xgboost as xgb
import numpy as np
import statsmodels.api as sm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import *
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn import preprocessing
import plotly_express as px
import datetime
from datetime import *
import seaborn as sns; sns.set()
import shap
import warnings
from eli5.sklearn import PermutationImportance
from sklearn.preprocessing import Imputer
from sklearn.model_selection import  cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.metrics import make_scorer
from sklearn.ensemble import RandomForestRegressor
#from alepython  import *

from skll.metrics import spearman

from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer


warnings.filterwarnings("ignore", category=FutureWarning)
#from __future__ import print_function

import numpy as np
import pandas as pd
from google.colab import files
import io

uploaded = files.upload()

for fn in uploaded.keys():
  print('User uploaded file "{name}" with length {length} bytes'.format(
      name=fn, length=len(uploaded[fn])))

#uploaded

heart_data=pd.read_csv(io.StringIO(uploaded['heart.csv'].decode('utf-8')))
heart_data.head()

heart_data.info()

heart_data.describe()

d=pd.DataFrame(heart_data.target.value_counts())

d=d.reset_index()

d.columns=['Prediction','Frequency']

d['Prediction']=d['Prediction'].astype('category')
d['Prediction']=d['Prediction'].astype(str)
d
d['Heart_Diseae']=['Present','Absent']
#d.dtypes()
d.info()

#px.bar(d, x="Heart_Diseae", y="Frequency",orientation="v", color='Heart_Diseae')

bg_color = (0.5, 0.5, 0.5)

sns.set(rc={"font.style":"normal",
            "axes.facecolor":bg_color,
            "axes.titlesize":30,
            "figure.facecolor":bg_color,
            "text.color":"black",
            "xtick.color":"black",
            "ytick.color":"black",
            "axes.labelcolor":"black",
            "axes.grid":False,
            'axes.labelsize':30,
            'figure.figsize':(10.0, 10.0),
            'xtick.labelsize':25,
            'ytick.labelsize':20})

#plt.rcParams.update(params)
flatui = ["#9b59b6", "#3498db", "#95a5a6", "#e74c3c", "#34495e", "#2ecc71"]
sns.countplot(heart_data.target,palette=flatui)
plt.xlabel('Heart_Disease ,Present(1), Absent(0)')
plt.title('frequency of Heart_Diseae  ')
plt.show()

X=heart_data.drop('target',axis=1)
Y=heart_data.target
scaler = StandardScaler()

#X = scaler.fit_transform(X)


train_X, val_X, train_y, val_y = train_test_split(X, Y,test_size=0.3, random_state=148)

from category_encoders import *
import pandas as pd



enc = TargetEncoder(cols=['sex', 'cp','fbs','restecg','exang','slope','thal']).fit(train_X,train_y)

# transform the datasets
training_numeric_dataset = enc.transform(train_X,train_y)
testing_numeric_dataset = enc.transform(val_X)


print(training_numeric_dataset.thal.value_counts())

training_numeric_dataset.head()

cat_cols=['sex', 'cp','fbs','restecg','exang','slope','thal']

heart_data[cat_cols] = heart_data[cat_cols].apply(lambda x: x.astype('category'))


heart_data.info()

cat_cols=['sex', 'cp','fbs','restecg','exang','slope','thal']

for col_name in cat_cols:
    #if(heart_data[col_name].dtype == 'object'):
        heart_data[col_name]= heart_data[col_name].astype('category')
        #heart_data[col_name] = heart_data[col_name].cat.codes
        

#heart_data[cat_cols] = heart_data[cat_cols].apply(lambda x: x.astype('category'))



heart_data.head(3)
heart_data.info()

train_X.columns.tolist()

train_data=lgb.Dataset(train_X ,label=train_y,feature_name=train_X.columns.tolist(), categorical_feature=cat_cols)


lgb_train = lgb.Dataset(train_X ,label=train_y
 ,feature_name =train_X.columns.tolist()
 , categorical_feature = cat_cols
)


#params = {
# 'task': 'train'
# , 'boosting_type': 'gbdt'
# , 'objective': 'regression'
#   ,'objective': 'binary'    
# , 'num_class': num_of_classes
# , 'metric': 'rmsle' 
# , 'min_data': 1
# , 'verbose': -1
#}
 
#lgbm = lgb.train(params, lgb_train, num_boost_round=50)



params = {'num_leaves':150, 
         'objective':'binary',
         'max_depth':7,
         'learning_rate':.05,
         'is_unbalance':True,
         'max_bin':200}
params['metric'] = ['auc', 'binary_logloss']


cv_results = lgb.cv(
        params,
        train_data,
        num_boost_round=100,
        nfold=3,
        #metrics='mae',
        early_stopping_rounds=10
        )

# Display results
print('Current parameters:\n', params)
print('\nBest num_boost_round:', len(cv_results['auc-mean']))
print('Best CV score:', max(cv_results['auc-mean']))

#cv_results.best_params_

#setting parameters for lightgbm

#train_data=lgb.Dataset(train_X ,label=train_y)

param = {'num_leaves':150, 
         'objective':'binary',
         'max_depth':7,
         'learning_rate':.05,
         'is_unbalance':True,
         'max_bin':200}
param['metric'] = ['auc', 'binary_logloss']


#training our model using light gbm
num_round=50
start=datetime.now()
lgbm=lgb.train(param,train_data,num_round,feature_name=list(train_X.columns), categorical_feature=cat_cols)
stop=datetime.now()


#lgb_estimator = lgb.LGBMClassifier(boosting_type='gbdt')
#Equivalent way of fitting light gbm

lgb_estimator = lgb.LGBMClassifier(boosting_type='gbdt',
                        # class_weight='balanced', #used only in multiclass training
                         #is_unbalance ='True'   #used in binary class training    
                         objective='binary',
                         n_jobs=-1, 
                         verbose=0)

lgb_model = lgb_estimator.fit(X=train_X, y=train_y)

#Execution time of the model
execution_time_lgbm = stop-start
execution_time_lgbm

#predicting the target label on test set
ypred=lgbm.predict(val_X)
print(ypred[0:5])  # showing first 5 predictions

#predicting the target probability on test set
#yprob=lgbm.predict_proba(val_X)
#print(yprob[0:2])  # showing first 5 predictions

#predicting the target label on test set
ypred=lgb_model.predict(val_X)
print(ypred[0:5])  # showing first 5 predictions

#predicting the target probability on test set
yprob=lgb_model.predict_proba(val_X)
print(yprob[0:2])  # showing first 5 predictions

predictions = lgb_model.predict(val_X)
print("Confusion Matrix:")

cm=confusion_matrix(val_y, predictions)

print(cm)

print()
print("Classification Report")
print(classification_report(val_y, predictions))

# ROC curve and Area-Under-Curve (AUC)
#calculating accuracy
accuracy_lgbm = accuracy_score(predictions,val_y)

print('accuracy score : {:0.3f}'.format( accuracy_lgbm))

roc_auc_lgbm = roc_auc_score(val_y,predictions)

print('roc score : {:0.3f}'.format( roc_auc_lgbm))

import seaborn as sn
rcParams['figure.figsize'] = 8,6


df_cm = pd.DataFrame(cm, range(2),
                  range(2))
#plt.figure(figsize = (10,7))
sn.set(font_scale=1.4)#for label size
# Show confusion matrix in a separate window
#plt.matshow(cm)
sn.heatmap(df_cm, annot=True,annot_kws={"size": 16},cmap=plt.cm.Blues)# font size
plt.title('Confusion matrix')
#plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

rcParams['figure.figsize'] = 10,8
lgb.plot_importance(lgb_model,title='Variable Importance')
plt.show()

#plt.figure(figsize=(10,10))
lgb.plot_tree(lgb_model)
plt.show()

lgb.create_tree_digraph(lgb_model)

from eli5.sklearn import PermutationImportance
from eli5.sklearn import *
import eli5
from eli5.permutation_importance import get_score_importances


perm = PermutationImportance(lgb_model, random_state=1).fit(train_X ,train_y)
eli5.show_weights(perm, feature_names=train_X.columns.tolist())

from eli5.sklearn import PermutationImportance


# set up the met-estimator to calculate permutation importance on our training
# data
perm_train = PermutationImportance(lgb_model, scoring='accuracy',
                                   n_iter=100, random_state=1)
# fit and see the permuation importances
perm_train.fit(train_X, train_y)
eli5.explain_weights_df(perm_train, feature_names=train_X.columns.tolist()).head()

# plot the distributions
from matplotlib import rcParams

# figure size in inches
rcParams['figure.figsize'] = 15,9

perm_train_df = pd.DataFrame(data=perm.results_,
                                      columns=X.columns)



(sns.boxplot(data=perm_train_df)
        .set(title='Permutation Importance Distributions (training data)',
             ylabel='Importance'));

plt.show()

# create our dataframe of feature importances
feat_imp_df = eli5.explain_weights_df(estimator =lgb_model, feature_names=train_X.columns.tolist())
feat_imp_df.head()

estimator =lgb_model


estimator.feature_importances_

features=train_X.columns.tolist()

# simple exmaple of a patient with feature values below
example = np.array([70,	1,	3,	100,	200,	1,	0,	150,	0,	2.3,	0,	0,	1])
eli5.explain_prediction_df(lgb_model, example, feature_names=features)
lgb_model.predict(example.reshape(1,-1))

from matplotlib import pyplot as plt
from pdpbox import pdp, get_dataset, info_plots
from sklearn.ensemble.partial_dependence import partial_dependence, plot_partial_dependence
# import machine learning algorithms
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc

bg_color = (0.5, 0.5, 0.5)

sns.set(rc={"font.style":"normal",
            "axes.facecolor":bg_color,
            "axes.titlesize":20,
            "figure.facecolor":bg_color,
            "text.color":"black",
            "xtick.color":"black",
            "ytick.color":"black",
            "axes.labelcolor":"black",
            "axes.grid":False,
            'axes.labelsize':10,
            'figure.figsize':(10.0, 10.0),
            'xtick.labelsize':10,
            'ytick.labelsize':10})
sns.set_style("whitegrid")
my_model = GradientBoostingClassifier()
my_model.fit(X=train_X, y=train_y)
# Here we make the plot
my_plots = plot_partial_dependence(my_model ,       
                                   features=[0,1, 2,3,4,5], # column numbers of plots we want to show
                                   X=train_X,            # raw predictors data.
                                   feature_names=['age','cp', 'thalach','ca','chol','sex'], # labels on graphs
                                   grid_resolution=10) # number of values to plot on x axis
plt.show()

from pycebox.ice import ice, ice_plot

# pcyebox likes the data to be in a DataFrame so let's create one with our imputed data
# we first need to impute the missing data
train_X_df = pd.DataFrame(train_X, columns=X.columns)

thalach_ice_df = ice(data=train_X_df, column='thalach', 
                   predict=lgbm.predict)

rcParams['figure.figsize'] = 8,6

ice_plot(thalach_ice_df , c='dimgray', linewidth=0.3)
plt.ylabel('Pred. AV %ile')
plt.xlabel('thalach');
plt.show()

#ice_plot(thalach_ice_df, linewidth=0.5,color_by='data_thalach', cmap=cmap2)
#wt_vals
#thalach_ice_df
#plt.show()

# figure size in inches
rcParams['figure.figsize'] = 10,8
# new colormap for ICE plot
cmap2 = plt.get_cmap('OrRd')
# set color_by to Wt, in order to color each curve by that player's weight
ice_plot(thalach_ice_df, linewidth=0.5, color_by='data_thalach', cmap=cmap2)
# ice_plot doesn't return a colorbar so we have to add one
# hack to add in colorbar taken from here:
# https://stackoverflow.com/questions/8342549/matplotlib-add-colorbar-to-a-sequence-of-line-plots/11558629#11558629
wt_vals = thalach_ice_df.columns.get_level_values('data_thalach').values
sm = plt.cm.ScalarMappable(cmap=cmap2, 
                           norm=plt.Normalize(vmin=wt_vals.min(), 
                                              vmax=wt_vals.max()))
# need to create fake array for the scalar mappable or else we get an error
sm._A = []
plt.colorbar(sm, label='thalach')
plt.ylabel('Pred. AV %ile')
plt.xlabel('thalach');
plt.show()

# figure size in inches
rcParams['figure.figsize'] = 10,8
ice_plot(thalach_ice_df, linewidth=.5, color_by='data_thalach', cmap=cmap2, plot_pdp=True, 
         pdp_kwargs={'c': 'k', 'linewidth': 5})
plt.colorbar(sm, label='thalach')
plt.ylabel('Pred. AV %ile')
plt.xlabel('thalach');
plt.show()

age_ice_df = ice(data=train_X_df, column='age', 
                   predict=lgbm.predict)

# figure size in inches
rcParams['figure.figsize'] = 10,8
ice_plot(age_ice_df, linewidth=.5, color_by='data_age', cmap=cmap2, plot_pdp=True, 
         pdp_kwargs={'c': 'k', 'linewidth': 5})
plt.colorbar(sm, label='age')
plt.ylabel('Pred. AV %ile')
plt.xlabel('age');
plt.show()

ca_ice_df = ice(data=train_X_df, column='ca'
                   ,predict=lgbm.predict)

# figure size in inches
rcParams['figure.figsize'] = 10,8
ice_plot(ca_ice_df, linewidth=.5, color_by='data_ca', cmap=cmap2, plot_pdp=True, 
         pdp_kwargs={'c': 'k', 'linewidth': 5})
plt.colorbar(sm, label='ca')
plt.ylabel('Pred. AV %ile')
plt.xlabel('ca');
plt.show()

from matplotlib import pyplot as plt
from pdpbox import pdp, get_dataset, info_plots

# Create the data that we will plot
pdp_goals = pdp.pdp_isolate(model=lgb_model, dataset=train_X,
                            model_features=val_X.columns.tolist(), feature='age')

# plot it
pdp.pdp_plot(pdp_goals, 'age')
plt.show()

# Create the data that we will plot
pdp_goals = pdp.pdp_isolate(model=lgb_model, dataset=train_X,
                            model_features=val_X.columns.tolist(), feature='ca')

# plot it
pdp.pdp_plot(pdp_goals, 'ca')
plt.show()

# Create the data that we will plot
pdp_goals = pdp.pdp_isolate(model=lgb_model, dataset=train_X,
                            model_features=val_X.columns.tolist(), feature='thalach')

# plot it
pdp.pdp_plot(pdp_goals, 'thalach')
plt.show()

# Create the data that we will plot
pdp_goals = pdp.pdp_isolate(model=lgb_model, dataset=train_X,
                            model_features=val_X.columns.tolist(), feature='cp')

# plot it
pdp.pdp_plot(pdp_goals, 'cp')
plt.show()

# Create the data that we will plot
pdp_goals = pdp.pdp_isolate(model=lgb_model, dataset=train_X,
                            model_features=val_X.columns.tolist(), feature='sex')

# plot it
pdp.pdp_plot(pdp_goals, 'sex')
plt.show()

# Create the data that we will plot
sns.set_style("whitegrid")
pdp_goals = pdp.pdp_isolate(model=lgb_model, dataset=val_X,
                            model_features=val_X.columns.tolist(), feature='chol')

# plot it
pdp.pdp_plot(pdp_goals, 'chol')
plt.show()

# figure size in inches
rcParams['figure.figsize'] = 10,13
sns.set_style("whitegrid")
pdp_interaction = pdp.pdp_interact(model=lgb_model, dataset=val_X, model_features=val_X.columns.tolist(), features=['thalach', 'oldpeak'])
pdp.pdp_interact_plot(pdp_interact_out=pdp_interaction, feature_names=['thalach', 'oldpeak'], plot_type='contour')
plt.show()

from matplotlib import pyplot as plt
plt.figure(figsize=(15,10))
sns.set_style("whitegrid")
pdp_interaction = pdp.pdp_interact(model=lgb_model, dataset=val_X, model_features=val_X.columns.tolist(), features=['age', 'ca'])
pdp.pdp_interact_plot(pdp_interact_out=pdp_interaction, feature_names=['age', 'ca'], plot_type='contour')
plt.show()

from matplotlib import pyplot as plt
plt.figure(figsize=(15,10))
sns.set_style("whitegrid")
pdp_interaction = pdp.pdp_interact(model=lgb_model, dataset=val_X, model_features=val_X.columns.tolist(), features=['age', 'sex'])
pdp.pdp_interact_plot(pdp_interact_out=pdp_interaction, feature_names=['age', 'sex'], plot_type='contour')
plt.show()

import xgboost
import shap

# load JS visualization code to notebook
shap.initjs()
#select a random row between 0 and 80
row_to_show = random.randint(0,val_X.shape[0])
#row_to_show = 10
data_for_prediction = val_X.iloc[row_to_show]  # use 1 row of data here. Could use multiple rows if desired
data_for_prediction_array = data_for_prediction.values.reshape(1, -1)


# explain the model's predictions using SHAP values
# (same syntax works for LightGBM, CatBoost, and scikit-learn models)
explainer = shap.TreeExplainer(lgb_model)
#shap_values = explainer.shap_values(train_X)

# visualize the first prediction's explanation (use matplotlib=True to avoid Javascript)
#shap.force_plot(explainer.expected_value, shap_values[0,:], train_X.iloc[0,:])
shap.initjs()
#shap.force_plot(explainer.expected_value, shap_values[row_to_show], data_for_prediction)
shap.force_plot(explainer.expected_value[0], shap_values[1], data_for_prediction)
#shap.force_plot(explainer.expected_value, shap_values[0,:], val_X.iloc[0,:])

import random


lgb_model.predict_proba(data_for_prediction_array)

data_for_prediction.as_matrix()
data_for_prediction.values
explainer.expected_value
shap_values[row_to_show]

import shap  # package used to calculate Shap values

# Create object that can calculate shap values
explainer = shap.TreeExplainer(lgb_model)

# Calculate Shap values
shap_values = explainer.shap_values(val_X)


# Create object that can calculate shap values
#explainer = shap.TreeExplainer(my_model)

# Calculate Shap values
#shap_values = explainer.shap_values(data_for_prediction)

# use Kernel SHAP to explain test set predictions
shap.initjs()
k_explainer = shap.KernelExplainer(my_model.predict_proba, train_X)
k_shap_values = k_explainer.shap_values(data_for_prediction)
shap.force_plot(k_explainer.expected_value[1], k_shap_values[1], data_for_prediction)

# visualize the training set predictions
# load JS visualization code to notebook
shap.initjs()
shap.force_plot(explainer.expected_value, shap_values, train_X)

# create a SHAP dependence plot to show the effect of a single feature across the whole dataset
sns.set_style("whitegrid")
shap.dependence_plot("ca", shap_values, train_X)

import shap  # package used to calculate Shap values

# Create object that can calculate shap values
explainer = shap.TreeExplainer(my_model)

# calculate shap values. This is what we will plot.
shap_values = explainer.shap_values(X)

# make plot.
shap.dependence_plot('Ball Possession %', shap_values[1], X, interaction_index="Goal Scored")

# summarize the effects of all the features
sns.set_style("whitegrid")
shap.summary_plot(shap_values, train_X)

sns.set_style("whitegrid")
plt.title("Variable Importance Plot")
shap.summary_plot(shap_values, train_X, plot_type="bar")

from lime import lime_text
from lime import *
import lime
from sklearn.pipeline import make_pipeline

sns.set_style("whitegrid")
predict_fn_lgbm = lambda x: lgb_model.predict_proba(x).astype(float)

X_val=val_X.as_matrix()

#target_names= ['1','0']
target_names= [str(i) for  i in train_y.unique()]

explainer = lime.lime_tabular.LimeTabularExplainer(X_val, feature_names=train_X.columns.tolist(), class_names=target_names, discretize_continuous=True)

i = np.random.randint(0, val_X.shape[0])
exp = explainer.explain_instance(X_val[i], lgb_model.predict_proba, num_features=10, top_labels=1)

#We now explain a single instance:

exp.show_in_notebook(show_table=True, show_all=False)

# a plot of the weights for each feature
exp.as_pyplot_figure();

from sklearn.datasets import load_iris
from sklearn import tree
iris = load_iris()
clf = tree.DecisionTreeClassifier()
clf = clf.fit(iris.data, iris.target)
#tree.plot_tree(clf.fit(iris.data, iris.target)) 


import graphviz 
dot_data = tree.export_graphviz(clf, out_file=None) 
graph = graphviz.Source(dot_data) 
graph.render("iris") 

dot_data = tree.export_graphviz(clf, out_file=None, 
                      feature_names=iris.feature_names,  
                      class_names=iris.target_names,  
                      filled=True, rounded=True,  
                     special_characters=True)  
graph = graphviz.Source(dot_data)  
graph