Using Scikit-Learn Pipelines and Converting Them To PMML

#!pip install category-encoders
#!pip install scipy
#!pip install --upgrade pip --user
#!pip install scipy==1.2 --upgrade --user
#!pip install sklearn2pmml
#!pip install numpy
#!pip install sklearn
#!pip install pandas

from pandas_profiling import ProfileReport
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from xgboost import XGBClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, classification_report, recall_score
import pandas_profiling
from sklearn.impute import SimpleImputer
Imputer = SimpleImputer(fill_value=-9999999, strategy='constant')

import pandas as pd
from sklearn2pmml.decoration import ContinuousDomain
from sklearn2pmml.pipeline import PMMLPipeline
from sklearn2pmml import make_pmml_pipeline, sklearn2pmml
import xgboost as xgb
from joblib import dump, load
from sklearn.metrics import classification_report, f1_score, accuracy_score, precision_score, confusion_matrix
from sklearn.metrics import roc_auc_score
from sklearn.metrics import auc
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import plot_confusion_matrix
import numpy as np
from sklearn.metrics import classification_report, accuracy_score
from ml_things import plot_dict, plot_confusion_matrix, fix_text
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt
%matplotlib inline
pd.set_option('display.max_columns', None)  # or 1000
pd.set_option('display.max_rows', None)  # or 1000
pd.set_option('display.max_colwidth', None)  # or 19

% %autosave 5

import numpy as np
import tensorflow as tf
from tensorflow import keras
#from tensorflow.keras.preprocessing.image import image_dataset_from_directory
from keras import  layers
import matplotlib.pyplot as plt
import matplotlib as mpl
import os
mpl.rcParams['figure.figsize'] = (12, 10)
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
%autosave 5
print(tf.__version__)

from google.colab import drive
import glob
drive.mount('/content/drive', force_remount=True)

header=['age',
        'workclas',
        'fnlwgt',
        'education',
        'education-num',
        'marital-status',
        'occupation',
        'relationship',
        'race',
        'sex',
        'capital-gain',
        'capital-loss',
        'hours-per-week',
        'native-country',
        'Income']

len(header)

data= pd.read_csv('/content/drive/My Drive/Data/adult.data',header=0,names=header)

data.shape

data.head()

#data['income']  = [1 if  i ==' >50K' else 0  for i in data.Income]

#from sklearn.preprocessing import LabelEncoder 
#labelencoder= LabelEncoder() #initializing an object of class LabelEncoder
#data['Income'] = labelencoder.fit_transform(data['Income']) #fitting and transforming the desired categorical column.

print(data.Income.value_counts())
print(data.Income.value_counts(normalize=True)
)

#def do_the_job(x):
#    ref=[]
#    if (x ==' <=50K'):
#       ret = 0
#    elif (x == ' >50K'):
#       ret =1

#    return ret

#data['income'] = data['Income'].apply(do_the_job)
#data['native-country'].value_counts()
#data[data[data['native-country']==' Holand-Netherlands']['native-country']=='Germany']
#data[data['native-country']==' Holand-Netherlands']['native-country']
#data[data['native-country']==' Germany']['native-country'].tolist()
#data['native-country'] = data['native-country'].str.replace(' Holand-Netherlands' ,' Germany')

data.info()

data.describe()

neg, pos = data.Income.value_counts()
total = neg + pos
print('Examples:\n    Total: {}\n    >50K: {} ({:.2f}% of total)\n  '.format(
    total, pos, 100 * pos / total,100 * neg / total))

print('Total: {}\n    <=50K: {} ({:.2f}% of total)\n  '.format(
    total, neg, 100 * neg / total))

data.isnull().sum(axis=0)
# let df be your dataframe and x be the value you want to fill it with
#heartdata.fillna(-9999.01)

data = data.loc[:,~data.columns.duplicated()]

data.drop_duplicates(keep=False, inplace=True)

from tqdm import tqdm
from scipy import stats
from scipy.stats import chi2_contingency
import sys #threshold=0.95
#threshold=0.95



def Remove_correlation(data, threshold):
    ''' 
    Find correlation among columns of a dataframe
    return names of columns removed  
    return  reduced dataframe with correlated columns removed
    '''
# Create correlation matrix
    corr_matrix = data.corr().abs()

# Select upper triangle of correlation matrix
    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))

# Find features with correlation greater than 0.95
    drop_columns = [column for column in upper.columns if any(upper[column] > threshold)]

# Drop features 
    red_data=data.drop(drop_columns, axis=1)
    return  drop_columns, red_data

print('Features Removed : {}'.format(Remove_correlation(data, threshold=0.95)[0]))
data = Remove_correlation(data, threshold=0.95)[1]
print('shape of dataframe {}'.format(data.shape))

import seaborn as sns
sns.set_theme(style="darkgrid")
#titanic = sns.load_dataset("titanic")
ax = sns.countplot(x="Income", data=data,palette="Set3")
plt.xlabel('Income')
plt.ylabel('Frequency')
plt.title('frequency of Income  ')
plt.show()

# get the target, separate target and time from predictors
y = data.Income             
X= data.drop(['Income'], axis=1, inplace=False)
#X= heartdata.drop(['DEATH_EVENT', 'time'], axis=1, inplace=True)
X.info()

# Break off validation set from training data
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.75, test_size=0.25,random_state=0)
# Display results
print ("Shapes:")
print ("X: {}".format(X.shape))
print ("y: {}".format(y.shape))
print()
print ("X_train: {}".format(X_train.shape))
print ("X_valid: {}".format(X_valid.shape))
print ("y_train: {}".format(y_train.shape))
print ("y_valid: {}\n".format(y_valid.shape))

from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer, make_column_transformer

##
#numerical_features=heartdata.select_dtypes(exclude=['object','category']).drop('DEATH_EVENT',axis=1).columns.tolist()
# Select numeric columns
numeric_features = [col for col in X_train.columns if X_train[col].dtype in ['int64', 'float64']]
numeric_features

categorical_features=X.select_dtypes(include=['object','category']).columns.tolist()
categorical_features

#R_test = pd.read_csv('C:\\Users\\admin1\\Desktop\\WORK\\R_test_M_LIN.csv',delimiter="\t")
#R_train = pd.read_csv('C:\\Users\\admin1\\Desktop\\WORK\\R_train_M_LIN.csv',delimiter="\t")        

categorical_transformer = Pipeline(steps=[
    #('encoding',LabelEncoder()),
    ('onehot', OneHotEncoder(handle_unknown='ignore')),
    #('encoding',MyLabelEncoder()),
    #('encoding',MultiColumnLabelEncoder())
    ('imputer', SimpleImputer(fill_value= -9999.01, strategy='constant'))
    #('encoding',OrdinalEncoder(categories='auto')),
   # 
])

numeric_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(fill_value=-9999999, strategy='constant')),
    ('scaler', StandardScaler()) 
    ])

xgb_classifier = xgb.XGBClassifier(
              n_jobs = -1,
              max_depth = 6,
              #learning_rate=  0.1,
              min_child_weight= 2, 
              #min_samples_split= 0.9,
              n_estimators= 100,
              eta = 0.1, 
              verbose = 1, 
              gamma=0.05,
              #nrounds = 100
              objective = "binary:logistic", 
              eval_metric = "auc",       #"aucpr",    # "aucpr",  #aucpr, auc
              subsample = 0.7,
              colsample_bytree =0.8,
              max_delta_step=1,
              verbosity=1,
              tree_method='approx')

preprocessor = ColumnTransformer(
    transformers=[
        ('num', numeric_transformer, numeric_features),
        ('cat', categorical_transformer, categorical_features)])

# Append classifier to preprocessing pipeline.

pipeline_f = Pipeline(steps=[('preprocessor', preprocessor),
                      #('dropfeature',UniqueDropper()),
                      #('anova', SelectPercentile(chi2)),
                      # ('feature_selection', SelectFromModel(LinearSVC(penalty="l1"))),
                      ('classifier', xgb_classifier)])


xgb_model = pipeline_f.fit(X_train,  y_train)

print("model score: %.3f" % pipeline_f.score(X_valid, y_valid))

# Get predictions
preds = pipeline_f.predict(X_valid)

# Evaluate the model
score = accuracy_score(y_valid,preds)

# Display the result
print("Accuracy Score: {}".format(score))


pred_prob = pipeline_f.predict_proba(X_valid)
print(pred_prob)

#pipe.predict(X_valid)
#y_valid
y_val  = [1 if  i ==' >50K' else 0  for i in y_valid]
y_pred  = [1 if  i ==' >50K' else 0  for i in preds]

from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_recall_fscore_support
from numpy import trapz
from scipy.integrate import simps
from sklearn.metrics import f1_score

def Evaluate(y_val, y_pred):
      ''' A function that calculates the confusion matrix, accuracy, precision, recall, and f1_score.
        Accuracy, precision, recall, and f1_score.'''
      CM = confusion_matrix(y_val, y_pred)
      TN = CM[0][0]
      FN = CM[1][0]
      TP = CM[1][1]
      FP = CM[0][1]
      print(' (True Negatives): {}'.format(TN))
      print(' (False Negatives):  {}'.format(FN))
      print(' (True Positives): {}'.format(TP))
      print(' (False Positives):{}'.format(FP))
      print('Total Number from positive class  from Confusion Matrix : ', np.sum(CM[1]))
      auc = roc_auc_score(y_val, y_pred)
      prec=precision_score(y_val, y_pred)   
      rec=recall_score(y_val, y_pred)
     # calculate F1 score
      f1 = f1_score(y_val, y_pred)
      print('auc :{}'.format(auc))
      print('precision :{}'.format(prec))
      print('recall :{}'.format(rec))
      print('f1 :{}'.format(f1))
    # Compute Precision-Recall and plot curve
      precision, recall, thresholds = precision_recall_curve(y_val, y_pred)
    #use the trapezoidal rule to calculate the area under the precion-recall curve
      area =  trapz(recall, precision)
   
    #area =  simps(recall, precision)
      print("Area Under Precision Recall  Curve(AP): %0.4f" % area)  

    

Evaluate(y_val, y_pred)

# Create the evaluation report.
#evaluation_report = classification_report(y_val, y_pred, labels=['<=50K','>50K'], target_names=['<=50K','>50K'])
evaluation_report = classification_report(y_valid, preds )
# Show the evaluation report.
print(evaluation_report)

from ml_things import plot_dict, plot_confusion_matrix, fix_text

# Plot confusion matrix.
# <=50K    24719
# >50K      7841
# Plot confusion matrix.

plot_confusion_matrix(y_true=y_val, y_pred=y_pred, 
                      classes=['<=50K','>50K'], normalize=False, 
                      magnify=1,
                      );


plot_confusion_matrix(y_true=y_val, y_pred=y_pred, 
                      classes=['<=50K','>50K'], normalize=True, 
                      magnify=1,
                      );
 #y_true=y_val, y_pred=y_pred,                     

#xgb_no_weights_pred = xgb_no_weights.predict_proba(test_x)
#xgb_pred = xgb_model.predict_proba(test_set)[:,1]
#joblib.dump(xgb_model, 'C:\\Users\\admin1\\Desktop\\WORK\\xgbmodel.joblib')
#xgb_model =  joblib.load( '/content/drive/My Drive/ImbalancedData/xgb_no_weights.joblib')
#print(pipe.steps[1])
print(pipeline_f.steps[1][1].feature_importances_.shape)
print(pipeline_f.named_steps.classifier)
print(pipeline_f.named_steps.preprocessor)

preprocessor.fit(X_train)
#preprocessor.get_feature_names()

from sklearn.compose import ColumnTransformer
#preprocessor.get_feature_names()
#pipeline_f.get_feature_names()
#pipeline_f["pipeline_f"]["pipeline_f"][-1]
preprocessor.transform(X).toarray()

#!pip install scikit-learn==0.23.2
from sklearn.utils import estimator_html_repr

with open('/content/drive/My Drive/Data/pipeline.html','w') as f:
     f.write(estimator_html_repr(pipeline_f))

#plot pipeline
from sklearn import set_config


set_config(display='diagram')   
# diplays HTML representation in a jupyter context
pipeline_f 

from sklearn2pmml import make_pmml_pipeline, sklearn2pmml

pipeline = make_pmml_pipeline(
pipeline_f
#active_fields = ["x1", "x2", ...], optional Feature name
#target_fields = ["y"]  optional target name
)
sklearn2pmml(pipeline, "/content/drive/My Drive/Data/xgbmodel.pmml")

from sklearn2pmml import PMMLPipeline
from sklearn2pmml.decoration import CategoricalDomain, ContinuousDomain
from sklearn_pandas import DataFrameMapper


# it takes a list of tuples as parameter
pipeline = PMMLPipeline(
    [('preprocessor', preprocessor),
                      #('dropfeature',UniqueDropper()),
                      #('anova', SelectPercentile(chi2))
                     # ("feat_sel", SelectKBest(10)),

                      ('classifier', xgb_classifier)]
)

#  the pipeline object is  used similar to how a regular classifier is used in
#scikit-learn.
pipeline.fit(X=X_train,  y=y_train)

from sklearn2pmml import sklearn2pmml

#pipeline = sklearn2pmml.make_pmml_pipeline(pipeline)

sklearn2pmml(pipeline, "/content/drive/My Drive/Data/xgbmodel.pmml", with_repr = True)

#help(make_pmml_pipeline)
#help(sklearn2pmml)

from sklearn.model_selection import GridSearchCV
from sklearn2pmml import PMMLPipeline
from sklearn2pmml import sklearn2pmml




#pipeline = PMMLPipeline([...])


#tuner = RandomizedSearchCV(pipeline, param_grid = {...})
#tuner.fit(X_train,  y_train)
# GridSearchCV.best_estimator_ is of type PMMLPipeline
#sklearn2pmml(tuner.best_estimator_, "/content/drive/My Drive/Data/xgbmodel.pmml")

pipeline.fit(X=X_train,  y=y_train)

# Define model parameters for grid search
search_spaces = {'classifier__learning_rate': [0.1],
              'classifier__n_estimators': [100],
              'classifier__max_depth': [ 5, 6],
              'classifier__min_child_weight': [1, 2],
              'classifier__gamma': [0.05],
              'classifier__subsample': [ 0.70, 0.80],
              'classifier__colsample_bytree': [ 0.70, 0.80, 0.90],
              'classifier__random_state' : [10]}

#RandomizedSearchCV.best_estimator_ is of type PMMLPipeline
tuner  = RandomizedSearchCV(pipeline_f, search_spaces, cv=5, n_jobs=-1,scoring='roc_auc')
tuner.fit(X=X_train,  y=y_train)
sklearn2pmml(make_pmml_pipeline(tuner.best_estimator_),
 "/content/drive/My Drive/Data/xgbmodel.pmml")

tuner.best_estimator_.get_params

#import matplotlib.pyplot as plt
#from sklearn2pmml import make_pmml_pipeline, sklearn2pmml


#tuner = RandomizedSearchCV(pipeline, search_spaces)
#tuner.fit(X=X_train,  y=y_train)
# GridSearchCV.best_estimator_ is of type Pipeline
#sklearn2pmml(make_pmml_pipeline(tuner.best_estimator_),
                  #active_fields =
#[...], target_fields = [...]),
# "/content/drive/My Drive/Data/xgbmodel.pmml"
#)
print("Best parameter (CV score=%0.3f):" % tuner.best_score_)
print('Best Hyperparameter values :{}'.format(tuner.best_params_))

#fig, ax = plt.subplots(figsize=(12,18))
#xgb.plot_importance(xgb_model, max_num_features=50, height=0.8, ax=ax)
#plt.show()

sklearn2pmml(pmml_pipeline, "C:/Users/admin1/Desktop/WORK/xgboost.pmml")

#!pip install pypmml
!ls '/content/drive/My Drive/Data/xgbmodel.pmml'

"""
import pmml file into python
"""

from pypmml import Model
import pandas as pd



model = Model.fromFile("/content/drive/My Drive/Data/xgbmodel.pmml") 

#
test_data = preprocessor.transform(X_valid).toarray()
#Prediction
new_pred = model.predict(test_data)