Bayesian Optimization

#!pip install kaggledatasets
#import kaggledatasets as kd
#!pip install -q kaggle

#dataset = kd.CreditCardFraudDetection(download=True)
#dataset.head()
cols=\
['age','workclass','fnlwgt','education','education-num','marital-status','occupation','relationship','race','sex','capital-gain','capital-loss','hours-per-week','native-country','Income']

%%capture
# Install helper functions.
!pip install catboost -U
!pip install talos
!pip install bayesian-optimization==1.0.1
!pip install -q git+https://github.com/gmihaila/ml_things.git
!pip install skll
!pip install imblearn
!pip install hyperopt

!pip install -Iv scikit-learn==0.22.1
!pip install  scikit-optimize==0.7.4

import xgboost as xgb
from bayes_opt import BayesianOptimization
from sklearn.metrics import mean_squared_error
import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from xgboost import XGBClassifier
from sklearn.metrics import classification_report, confusion_matrix
% autosave 5

from google.colab import drive
drive.mount('/drive')

import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from xgboost import XGBClassifier

#Make directory named kaggle and copy kaggle.json file there.
train_data = pd.read_csv("/Data/adult.data.zip",header=None,names=cols)

train_data.head()

test_data = pd.read_csv("/Data/adult.test",skiprows=1,header=None,names=cols)

test_data.head()

categorical_columns  = train_data.select_dtypes(include=['object']).columns

le = LabelEncoder()

#y=  le.fit_transform(df.label)

train_data[categorical_columns]  = train_data[categorical_columns].apply(le.fit_transform)
test_data[categorical_columns]  =  test_data[categorical_columns].apply(le.fit_transform)
train_data.head()

import xgboost as xgb
from bayes_opt import BayesianOptimization
from sklearn.metrics import mean_squared_error
import pandas as pd
import numpy as np
from xgboost import XGBClassifier

from sklearn.model_selection import train_test_split

#X_train, X_test, y_train, y_test = train_test_split(df.drop('fare_amount', axis=1),df['Income'], test_size=0.25)
#del(df)

#Converting the dataframe into XGBoost’s Dmatrix object
dtrain = xgb.DMatrix(train_data.drop('Income', axis=1),label=train_data.Income)
#del(X_train)
dtest = xgb.DMatrix(test_data.drop('Income', axis=1),label=test_data.Income)
#del(X_test)

import gc
import warnings

from bayes_opt import BayesianOptimization
from sklearn.model_selection  import cross_val_score, StratifiedKFold, StratifiedShuffleSplit
from sklearn.metrics import log_loss, matthews_corrcoef, roc_auc_score
from sklearn.preprocessing import MinMaxScaler
import xgboost as xgb
import contextlib


#The cross-va;lidation  parameters  is saved into the log file
log_file = open('xgb_opt.log', 'a')


# Comment out any parameter you don't want to test
def xgb_evaluate(
          max_depth,
          gamma,
          min_child_weight,
          max_delta_step,
          subsample,
          colsample_bytree,
          num_boost_round
         ):

  

#
# Define all XGboost parameters
#
#

    params = {
              'booster' : 'gbtree',
              'max_depth' : int(max_depth),
              'gamma' : gamma,
              'eta' : 0.1,
              'objective' : 'binary:logistic',
              'num_boost_round':num_boost_round,
              'nthread' : -1,
              'silent' : True,
              'eval_metric': 'auc',
              'subsample' : max(min(subsample, 1), 0),
              'colsample_bytree' : max(min(colsample_bytree, 1), 0),
              'min_child_weight' : min_child_weight,
              'max_delta_step' : int(max_delta_step),
              'seed' : 1001
              }

    folds = 5
    cv_score = 0

    print("\n Search parameters (%d-fold validation):\n %s" % (folds, params), file=log_file )
    log_file.flush()

     #Cross validating with the specified parameters in 5 folds and 70 iterations
    cv_result = xgb.cv(
                    params,
                    dtrain,
                    #num_boost_round = 70,
                    stratified = True,
                    nfold = folds,
#                    verbose_eval = 10,
                    early_stopping_rounds = 10,
                    metrics = 'auc',
                    show_stdv = True
               )
    
    val_score =  cv_result['test-auc-mean'].iloc[-1]
    train_score =  cv_result['train-auc-mean'].iloc[-1]
    print(' Stopped after %d iterations with train-auc = %f val-auc = %f ( diff = %f ) train-gini = %f val-gini = %f' % ( len(cv_result), train_score, val_score, (train_score - val_score), (train_score*2-1),
(val_score*2-1)) )
  # Bayesian optimization only knows how to maximize, not minimize, so return the negative RMSE
  #in case you are tuning a regression model
    #return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
    return (val_score*2) - 1

xgb_bo = BayesianOptimization(xgb_evaluate, {
                                     'max_depth': (2, 12),
                                     'gamma': (0.001, 10.0),
                                     'min_child_weight': (0, 20),
                                     'max_delta_step': (0, 10),
                                     'subsample': (0.4, 1.0),
                                     'num_boost_round':(20,100),
                                     'colsample_bytree' :(0.2, 1.0)
                                    })




print('-'*130)
print('-'*130, file=log_file)
#The flush() method cleans out the internal buffer.
log_file.flush()

with warnings.catch_warnings():
    warnings.filterwarnings('ignore')
#performing Bayesian optimization for 5 iterations with 8 steps of random 
#exploration with an #acquisition function of expected improvement    
    xgb_bo.maximize(init_points=2, n_iter=5, acq='ei', xi=0.0)

results =  pd.DataFrame(xgb_bo.res)
f"maximum Gini {max(results.target)}"
print('-'*130)
print('Final Results')
best_params = results[results.target == max(pd.DataFrame(xgb_bo.res).target)]['params']

print('best_params {}'.format(best_params.values))

#log_file.flush()
#log_file.close()



f"best AUC {( max(results.target) + 1 ) / 2}"

#Extracting the best parameters
#type(best_params.values)
params = dict(best_params)
print(params)
param = {'colsample_bytree': 1.0,
  'gamma': 0.001,
  'max_delta_step': 0.0,
  'max_depth': 12,
  'objective': 'binary:logistic',
  'min_child_weight': 0.0,
  #'num_boost_round': 83,
  'n_estimators': 83,
  'subsample': 0.4}



#Converting the max_depth and n_estimator values from float to int
#params['max_depth']= int(params['max_depth'])
#params['n_estimators']= int(params['n_estimators'])

#Initialize an XGBClassifier with the tuned parameters and fit the training data
from xgboost import XGBClassifier
classifier = XGBClassifier( max_depth=12, learning_rate=0.1, n_estimators=83,
                              silent=True, objective='binary:logistic',
                             booster='gbtree')

#final_model = classifier.fit(train_data.drop('Income', axis=1),train_data.Income)

classifier2 = XGBClassifier(**param).fit(train_data.drop('Income', axis=1),train_data.Income)

#predicting for training set
pred = classifier2.predict(test_data.drop('Income', axis=1))

#Looking at the classification report
print(classification_report(pred, test_data.Income))

#Attained prediction accuracy on the training set
cm = confusion_matrix(pred, test_data.Income)
acc = cm.diagonal().sum()/cm.sum()
print(acc)

# Predict on holdout set
test = transform(test)
dtest = xgb.DMatrix(test)
y_pred_test = model2.predict(dtest)

#train_data = lgb.Dataset(data, label=label, weight=w)
import lightgbm as lgb
lgbdtrain =   lgb.Dataset(train_data.drop('Income', axis=1), label=train_data.Income)

folds =5

def lgb_evaluate(max_depth,  colsample_bytree,scale_pos_weight,num_boost_round):
    params = {'metric': 'auc',
              'max_depth': int(max_depth),
              'subsample': 0.8,
              'objective' : 'binary',
             'learning_rate': 0.1,
             # ' importance_type'='gain',
              'colsample_bytree': colsample_bytree,
             'scale_pos_weight' : scale_pos_weight,
             'num_boost_round': int(num_boost_round)}
    # Used around 1000 boosting rounds in the full model
    cv_result = lgb.cv(params,
                       lgbdtrain,
                       #num_boost_round=70, #same as n_estimators
                         stratified = True,
                    nfold = folds,
#                    verbose_eval = 10,
                    early_stopping_rounds = 10,
                    metrics = 'auc',
                    show_stdv = True
                       )    
    
    # Bayesian optimization only knows how to maximize, not minimize, so return the negative RMSE
    #when building a regression model.
    return cv_result['auc-mean'][-1]  
  

lgb_bo = BayesianOptimization(lgb_evaluate, {'max_depth': (3, 10), 
                                             #'gamma': (0, 1),
                                             'colsample_bytree': (0.3, 0.9),
                                            'scale_pos_weight' : (1,100),
                                            'num_boost_round': (50,150)},
                             verbose=5 # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
                             )
# Use the expected improvement acquisition function to handle negative numbers
# Optimally needs quite a few more initiation points and number of iterations
lgb_bo.maximize(init_points=10, n_iter=5, acq='ei')

results =  pd.DataFrame(lgb_bo.res)
f"maximum AUC {max(results.target)}"
print('-'*130)
print('Final Results')
best_params = results[results.target == max(pd.DataFrame(lgb_bo.res).target)]['params']

print('best_params {}'.format(best_params.values))

import pandas as pd
import numpy as np
from tqdm import tqdm
from scipy import stats
from scipy.stats import chi2_contingency
import sys 
import pandas as pd
import io
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns; sns.set(style="ticks", color_codes=True)
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.feature_selection import chi2
from sklearn.linear_model import LogisticRegression
from sklearn import feature_selection
from sklearn.impute import SimpleImputer
from sklearn.model_selection import  cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.metrics import make_scorer
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_validate
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV
from xgboost import XGBClassifier
from sklearn.impute import SimpleImputer
import xgboost as xgb
import lightgbm as lgb
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from skopt.space import Real, Categorical, Integer
#from skll.metrics import spearman
from skopt import BayesSearchCV
from sklearn.model_selection import  cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.metrics import make_scorer
from skopt.space import Real, Categorical, Integer
from sklearn.metrics import classification_report
from sklearn.base import TransformerMixin
from sklearn.metrics import classification_report, f1_score, accuracy_score, precision_score, confusion_matrix
from sklearn.metrics import roc_auc_score
from sklearn.metrics import precision_recall_curve
pd.set_option('display.max_rows', 600)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
import warnings
import pandas_profiling
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.pipeline import make_pipeline, FeatureUnion, Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.svm import SVC
import random
import matplotlib.pyplot as plt
import seaborn as sns
import re
from sklearn.preprocessing import StandardScaler, OneHotEncoder
#from imblearn.over_sampling import RandomOverSampler
#from imblearn.under_sampling import RandomUnderSampler
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import *
from sklearn.utils import resample
#from imblearn.over_sampling import SMOTE
#import smote
import os
from sklearn.tree import DecisionTreeClassifier
#from imblearn.metrics import geometric_mean_score as gmean
#from imblearn.metrics import make_index_balanced_accuracy as iba
#from imblearn.metrics import *
#import rus
# Skopt functions
from skopt import BayesSearchCV
from skopt import gp_minimize # Bayesian optimization using Gaussian Processes
from skopt.space import Real, Categorical, Integer
from skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments
from skopt.callbacks import DeadlineStopper # Stop the optimization before running out of a fixed budget of time.
from skopt.callbacks import VerboseCallback # Callback to control the verbosity
from skopt.callbacks import DeltaXStopper # Stop the optimization If the last two positions at which the objective has been evaluated are less than delta
from joblib import dump, load
from prettytable import PrettyTable
from collections import Counter
from sklearn.datasets import make_classification
from hyperopt import fmin, tpe, space_eval,hp
from bayes_opt import BayesianOptimization
import skopt
import sklearn
import gc
warnings.filterwarnings("ignore")
%matplotlib inline
%autosave 2

print('The scikit-optimize  version is {}.'.format(skopt.__version__))
print('The scikit-learn version is {}.'.format(sklearn.__version__))
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')

# Setting a 5-fold stratified cross-validation (note: shuffle=True)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)


clf = xgb.XGBClassifier(
        n_jobs = -1,
        objective = 'binary:logistic',
        silent=1,
        tree_method='approx')
search_spaces = {'learning_rate': Real(0.1, 1.0, 'log-uniform'),
                 'min_child_weight': Integer(0, 10),
                 'max_depth': Integer(1,12),
                 'max_delta_step': Integer(0, 10), # Maximum delta step we allow each leaf output
                 'subsample': Real(0.5, 1.0, 'uniform'),
                 'colsample_bytree': Real(0.1, 1.0, 'uniform'), # subsample ratio of columns by tree
                 'colsample_bylevel': Real(0.1, 1.0, 'uniform'), # subsample ratio by level in trees
                 #'reg_lambda': Real(1e-9, 1000, 'log-uniform'), # L2 regularization
                 #'reg_alpha': Real(1e-9, 1.0, 'log-uniform'), # L1 regularization
                 'gamma': Real(1e-1, 0.5, 'log-uniform'), # Minimum loss reduction for partition
                 'n_estimators': Integer(50, 200),
                 #'scale_pos_weight': Real(1e-6, 500, 'log-uniform'), this parameter allows to build a
                 #cost-sensitive model to handle target class imbalance.
                 'scale_pos_weight': Real(1, 1000)}

bayessearch = BayesSearchCV(clf,
                    search_spaces,
                    scoring='f1',
                    cv=skf,
                    n_iter=40,
                    n_jobs=-1,
                    return_train_score=True,
                    refit=True,
                    optimizer_kwargs={'base_estimator': 'GP'},
                    random_state=22)
    

xgbm_model = bayessearch.fit(X=train_data.drop('Income', axis=1),y=train_data.Income)



print("val. score: %s" % bayessearch.best_score_)
print("test score: %s" % bayessearch.score(test_data.drop('Income', axis=1), test_data.Income))
print("best params: %s" % str(bayessearch.best_params_))

dump(xgbm_model, '/xgb_model.joblib') 

xgbm_model = load('/xgb_model.joblib') 

# Create the evaluation report.
evaluation_report = classification_report(true_labels, predictions_labels, labels=list(labels_ids.values()), target_names=list(labels_ids.keys()))
# Show the evaluation report.
print("Classification Report")
print(evaluation_report)

# Plot confusion matrix.
plot_confusion_matrix(y_true=true_labels, y_pred=predictions_labels, 
                      classes=list(labels_ids.keys()), normalize=True, 
                      magnify=1,
                      );

#train_data=pd.concat([cat_enc_df,num_df],axis=1)
#categorical_feat=[i for i in  x_train_imp.columns.tolist() if i in catdf.columns.tolist()]

#lgb_model_ros = lgb_estimator.fit(X=X_ros, y=y_ros,feature_name=list(x_train_imp.columns),
#                                         categorical_feature=categorical_feat)

RANDOM_STATE = 148

clf=lgb.LGBMClassifier( random_state=RANDOM_STATE,
                      importance_type='gain')

# the modeling pipeline
pipe = Pipeline([("imputer", SimpleImputer()),
                 ("scaler" ,MinMaxScaler()),
                 ("estimator", clf)])




# Setting a 5-fold stratified cross-validation (note: shuffle=True)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)


search_spaces = {
         'imputer__strategy': Categorical(['median']),
        'estimator__learning_rate': Real(0.01, 1.0, 'log-uniform'),
        'estimator__num_leaves': Integer(2, 100),
        'estimator__max_depth': Integer(0, 500),
        'estimator__min_child_samples': Integer(0, 200), # minimal number of data in one leaf
        'estimator__max_bin': Integer(100, 1000), # max number of bins that feature values will be bucketed
        'estimator__subsample': Real(0.01, 1.0, 'uniform'),
        'estimator__subsample_freq': Integer(0, 10), # bagging fraction
        'estimator__colsample_bytree': Real(0.01, 1.0, 'uniform'), # enabler of bagging fraction
        'estimator__min_child_weight': Integer(1, 10), # minimal number of data in one leaf.
        'estimator__subsample_for_bin': Integer(100, 500), # number of data that sampled for histogram bins
        'estimator__reg_lambda': Real(1e-9, 1000, 'log-uniform'), # L2 regularization
        'estimator__reg_alpha': Real(1e-9, 1.0, 'log-uniform'), # L1 regularization
        'estimator__scale_pos_weight': Real(1e-6, 500, 'log-uniform'), 
        'estimator__n_estimators': Integer(100,400) ,
        'estimator__scale_pos_weight':Integer(1,1000)       
        }


# create our search object
opt = BayesSearchCV(pipe, 
                      search_spaces, 
                      cv=5,
                      n_jobs=-1, 
                      scoring='f1', #roc_auc, f1, f1_micro/macro,balaced_accuracy,recall
                   
                     #is_unbalance ='True' ,  #used in binary class training  
                      verbose=0, 
                      error_score=-9999, 
                      #scoring='average_precision', 
                      random_state=RANDOM_STATE,
                      return_train_score=True, 
                      n_iter=75)

lgb_model= opt.fit(X=train_data.drop('Income', axis=1),y=train_data.Income)

print("val. score: %s" % opt.best_score_)
print("test score: %s" % opt.score(test_data.drop('Income', axis=1), test_data.Income))
print("best params: %s" % str(opt.best_params_))

!pip install optuna
!pip install neptune-client
!pip install neptune-contrib

import lightgbm as lgb
import neptune.new as neptune
#import neptune
import neptunecontrib.monitoring.optuna as opt_utils
import optuna
import pandas as pd
from sklearn.model_selection import train_test_split
import neptunecontrib.monitoring.optuna as opt_utils #


!set NEPTUNE_API_TOKEN="eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiIxNzcwN2E0Yy05M2NkLTRjZTUtOGQ1MC01NTE2MGUyZjE2ODkifQ=="
NEPTUNE_API_TOKEN ="eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiIxNzcwN2E0Yy05M2NkLTRjZTUtOGQ1MC01NTE2MGUyZjE2ODkifQ=="

NUM_BOOST_ROUND = 100
EARLY_STOPPING_ROUNDS = 30
STATIC_PARAMS = {'boosting': 'gbdt',
                 'objective': 'binary',
                 'metric': 'auc',
                 }
N_TRIALS = 100

X = train_data.drop('Income', axis=1)
y =  train_data.Income


def train_evaluate(X, y, params):
   
    train_x, valid_x, train_y, valid_y = train_test_split( X, y, test_size=0.2, random_state=42)

  
    train_data = lgb.Dataset(train_x, label = train_y)
    valid_data = lgb.Dataset(valid_x, label= valid_y, reference=train_data)

    model = lgb.train(params, train_data,
                      num_boost_round=NUM_BOOST_ROUND,
                      early_stopping_rounds=EARLY_STOPPING_ROUNDS,
                      valid_sets=[valid_data],
                      valid_names=['valid'])
    score = model.best_score['valid']['auc']
    return score


def objective(trial):
    params = {'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),
              'max_depth': trial.suggest_int('max_depth', 1, 30),
              'num_leaves': trial.suggest_int('num_leaves', 2, 100),
              'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 1000),
              'feature_fraction': trial.suggest_uniform('feature_fraction', 0.1, 1.0),
              'subsample': trial.suggest_uniform('subsample', 0.1, 1.0)}
    all_params = {**params, **STATIC_PARAMS}

    return train_evaluate(X, y, all_params)



#neptune.init(project='unltd148/bayeshyper')
neptune.init(project='unltd148/bayeshyper',
                   api_token='eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiIxNzcwN2E0Yy05M2NkLTRjZTUtOGQ1MC01NTE2MGUyZjE2ODkifQ==') # your credentials

#neptune.create_experiment(name='bayes hpo')

#monitor = opt_utils.NeptuneMonitor()
#we want to maximize auc which is the value of the objective function
study = optuna.create_study(direction='maximize', study_name="BayeshpoExperiment", storage='sqlite:///bayeshpo.db')
study.optimize(objective, n_trials=N_TRIALS)
#study.optimize(objective, n_trials=N_TRIALS, callbacks=[monitor])
#opt_utils.log_study(study)

#neptune.stop()

trial = study.best_trial
print('AUC: {}'.format(trial.value))
print("Best hyperparameters: {}".format(trial.params))

import lightgbm as lgb
import neptune.new as neptune
import neptune
import neptunecontrib.monitoring.optuna as opt_utils
import optuna
import pandas as pd
from sklearn.model_selection import train_test_split
import neptunecontrib.monitoring.optuna as opt_utils #


NUM_BOOST_ROUND = 100
EARLY_STOPPING_ROUNDS = 30
STATIC_PARAMS = {'boosting': 'gbdt',
                 'objective': 'binary',
                 'metric': 'auc',
                 }
N_TRIALS = 100
#data = pd.read_csv(TRAIN_PATH, nrows=N_ROWS)

X = train_data.drop('Income', axis=1)
y =  train_data.Income


def train_evaluate(X, y, params):
   
    train_x, valid_x, train_y, valid_y = train_test_split( X, y, test_size=0.2, random_state=42)

    
    train_data = lgb.Dataset(train_x, label = train_y)
    valid_data = lgb.Dataset(valid_x, label= valid_y, reference=train_data)

    model = lgb.train(params, train_data,
                      num_boost_round=NUM_BOOST_ROUND,
                      early_stopping_rounds=EARLY_STOPPING_ROUNDS,
                      valid_sets=[valid_data],
                      valid_names=['valid'])
    score = model.best_score['valid']['auc']
    return score


def objective(trial):
    params = {'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),
              'max_depth': trial.suggest_int('max_depth', 1, 15),
              'num_leaves': trial.suggest_int('num_leaves', 2, 100),
              'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 1000),
              'n_estimators' : trial.suggest_int('n_estimators', 20, 100),
              'feature_fraction': trial.suggest_uniform('feature_fraction', 0.1, 1.0),
              'subsample': trial.suggest_uniform('subsample', 0.1, 1.0)}
    all_params = {**params, **STATIC_PARAMS}

    return train_evaluate(X, y, all_params)



study = optuna.create_study(direction='maximize', study_name="bayes_hpo_experiment", storage='sqlite:///bayeshpo.db')
study.optimize(objective, n_trials=N_TRIALS)

trial = study.best_trial
print('AUC: {}'.format(trial.value))
print("Best hyperparameters: {}".format(trial.params))

optuna.visualization.plot_optimization_history(study)

optuna.visualization.plot_slice(study)

# Run an XGBoost model with hyperparmaters that are optimized using hyperopt
# The output of the script are the best hyperparmaters
# The optimization part using hyperopt is partly inspired from the following script: 
# https://github.com/bamine/Kaggle-stuff/blob/master/otto/hyperopt_xgboost.py
import pandas as pd
import numpy as np
import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from hyperopt import STATUS_OK, Trials, fmin, hp, tpe



#-------------------------------------------------#
X = train_data.drop('Income', axis=1)
y =  train_data.Income

train_x, valid_x, train_y, valid_y = train_test_split( X, y, test_size=0.2, random_state=42)

print('The training set is of length: ', len(train_y))
print('The validation set is of length: ', len(valid_y))

#-------------------------------------------------#

# Utility functions

def intersect(l_1, l_2):
    return list(set(l_1) & set(l_2))


def get_features(train, test):
    intersecting_features = intersect(train.columns, test.columns)
    intersecting_features.remove('people_id')
    intersecting_features.remove('activity_id')
    return sorted(intersecting_features)

#-------------------------------------------------#

# Scoring and optimization functions


def score(params):
    print("Training with params: ")
    print(params)
    num_round = int(params['n_estimators'])
    del params['n_estimators']
    dtrain = xgb.DMatrix(train_x, label=train_y)
    dvalid = xgb.DMatrix(valid_x, label=valid_y)
    watchlist = [(dvalid, 'eval'), (dtrain, 'train')]
    gbm_model = xgb.train(params, dtrain, num_round,
                          evals=watchlist,
                          verbose_eval=True)
    predictions = gbm_model.predict(dvalid,
                                    ntree_limit=gbm_model.best_iteration + 1)
    score = roc_auc_score(valid_y, predictions)
    # TODO: Add the importance for the selected features
    print("\tScore {0}\n\n".format(score))
    # The score function should return the loss (1-score)
    # since the optimize function looks for the minimum
    loss = 1 - score
    return {'loss': loss, 'status': STATUS_OK}


def optimize(
             #trials, 
             random_state=SEED):
    """
    This is the optimization function that given a space (space here) of 
    hyperparameters and a scoring function (score here), finds the best hyperparameters.
    """
    # To learn more about XGBoost parameters, head to this page: 
    # https://github.com/dmlc/xgboost/blob/master/doc/parameter.md
    space = {
        'n_estimators': hp.quniform('n_estimators', 100, 1000, 1),
        'eta': hp.quniform('eta', 0.025, 0.5, 0.025),
        # A problem with max_depth casted to float instead of int with
        # the hp.quniform method.
        'max_depth':  hp.choice('max_depth', np.arange(1, 14, dtype=int)),
        'min_child_weight': hp.quniform('min_child_weight', 1, 6, 1),
        'subsample': hp.quniform('subsample', 0.5, 1, 0.05),
        'gamma': hp.quniform('gamma', 0.5, 1, 0.05),
        'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05),
        'eval_metric': 'auc',
        'objective': 'binary:logistic',
        # Increase this number if you have more cores. Otherwise, remove it and it will default 
        # to the maxium number. 
        'nthread': 4,
        'booster': 'gbtree',
        'tree_method': 'exact',
        'silent': 1,
        'seed': random_state
    }
    # Use the fmin function from Hyperopt to find the best hyperparameters
    best = fmin(score, space, algo=tpe.suggest, 
                # trials=trials, 
                max_evals=250)
    return best

#-------------------------------------------------#



#-------------------------------------------------#



# Run the optimization

# Trials object where the history of search will be stored
# For the time being, there is a bug with the following version of hyperopt.
# You can read the error messag on the log file.
# For the curious, you can read more about it here: https://github.com/hyperopt/hyperopt/issues/234
# => So I am commenting it.
# trials = Trials()

best_hyperparams = optimize(
                            #trials
                            )
print("The best hyperparameters are: ", "\n")
print(best_hyperparams)

"best hyperparameters : {}".format(best_hyperparams)