複数の機械学習アルゴリズムを比較する

# data analysis and wrangling
import pandas as pd
import numpy as np
import random as rnd

# visualization
import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
color = sns.color_palette()
sns.set_style('darkgrid')

import warnings
def ignore_warn(*args, **kwargs):
  pass
warnings.warn = ignore_warn # Ignore annoying warning (from sklearn and seaborn)

# machine learning
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron, SGDClassifier
from sklearn.tree import DecisionTreeClassifier

from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve

from pathlib import Path

TRAIN_DATA_PATH = str(Path.home()) + '/vscode/syosetu/train_data.pkl'

train = pd.read_pickle(TRAIN_DATA_PATH)

from sklearn.model_selection import train_test_split

X = train['vectors']
y = train['female']

X_array = np.array([np.array(v) for v in X])
y_array = np.array([i for i in y])

X_train, X_test, y_train, y_test = train_test_split(X_array, y_array, random_state=0)

kfold = StratifiedKFold(n_splits=10)
cls_weight = (y_train.shape[0] - np.sum(y_train)) / np.sum(y_train)
random_state = 0
classifiers = []
classifiers.append(LogisticRegression(random_state=random_state))
classifiers.append(SVC(random_state=random_state))
classifiers.append(KNeighborsClassifier(n_neighbors=3))
classifiers.append(GaussianNB())
classifiers.append(Perceptron(random_state=random_state))
classifiers.append(LinearSVC(random_state=random_state))
classifiers.append(SGDClassifier(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(n_estimators=100, random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state), random_state=random_state, learning_rate=0.1))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))

cv_results = []
for classifier in classifiers :
  cv_results.append(cross_val_score(classifier, X_train, y=y_train, scoring='accuracy', cv=kfold, n_jobs=4))

cv_means = []
cv_std = []
for cv_result in cv_results:
  cv_means.append(cv_result.mean())
  cv_std.append(cv_result.std())

cv_res = pd.DataFrame({'CrossVal Means': cv_means, 'CrossVal Errors': cv_std,
                       'Algorithm': ['LogisticRegression', 'SVC', 'KNeighborsClassifier',
                                     'GaussianNB', 'Perceptron', 'LinearSVC', 'SGDClassifier',
                                     'DecisionTreeClassifier', 'RandomForestClassifier',
                                     'AdaBoostClassifier', 'GradientBoostingClassifier',
                                     'ExtraTreesClassifier']})

cv_res.sort_values(by=['CrossVal Means'], ascending=False, inplace=True)

cv_res

g = sns.barplot('CrossVal Means', 'Algorithm', data=cv_res, palette='Set3', orient='h', **{'xerr': cv_std})
g.set_xlabel('Mean Accuracy')
g = g.set_title('Cross validation scores')

### META MODELING WITH LR, RF and SGD

# Logistic Regression
LR = LogisticRegression(random_state=0)

## Search grid for optimal parameters
lr_param_grid = {'penalty': ['l1','l2']}

gsLR = GridSearchCV(LR, param_grid=lr_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)

gsLR.fit(X_train, y_train)

LR_best = gsLR.best_estimator_

# Best score
gsLR.best_score_

gsLR.best_params_

# Random Forest Classifier
RF = RandomForestClassifier(random_state=0)

## Search grid for optimal parameters
'''rf_param_grid = {'criterion': ['gini', 'entropy'],
                 'max_features': [1, 3, 10],
                 'min_samples_split': [2, 3, 10],
                 'min_samples_leaf': [1, 3, 10],
                 'n_estimators':[50, 100, 200]}'''
rf_param_grid = {'criterion': ['gini', 'entropy'],
                 'max_features': [40, 50, 60],
                 'min_samples_split': [2, 3, 10],
                 'min_samples_leaf': [1, 3, 10],
                 'n_estimators':[20, 30, 40]}

gsRF = GridSearchCV(RF, param_grid=rf_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)

gsRF.fit(X_train, y_train)

RF_best = gsRF.best_estimator_

# Best score
gsRF.best_score_

gsRF.best_params_

# SGD Classifier
SGD = SGDClassifier(random_state=0)

## Search grid for optimal parameters
sgd_param_grid = {'alpha'  : [0.0001, 0.001, 0.01, 0.1],
                  'loss'   : ['log', 'modified_huber'],
                  'penalty': ['l2', 'l1', 'none']}

gsSGD = GridSearchCV(SGD, param_grid=sgd_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)

gsSGD.fit(X_train, y_train)

SGD_best = gsSGD.best_estimator_

# Best score
gsSGD.best_score_

gsSGD.best_params_

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5)):
  '''Generate a simple plot of the test and training learning curve'''
  plt.figure()
  plt.title(title)
  if ylim is not None:
    plt.ylim(*ylim)
  plt.xlabel('Training examples')
  plt.ylabel('Score')
  train_sizes, train_scores, test_scores = learning_curve(
      estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
  train_scores_mean = np.mean(train_scores, axis=1)
  train_scores_std = np.std(train_scores, axis=1)
  test_scores_mean = np.mean(test_scores, axis=1)
  test_scores_std = np.std(test_scores, axis=1)
  plt.grid()

  plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                   train_scores_mean + train_scores_std, alpha=0.1,
                   color='r')
  plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                   test_scores_mean + test_scores_std, alpha=0.1,
                   color='g')
  plt.plot(train_sizes, train_scores_mean, 'o-', color='r',
           label='Training score')
  plt.plot(train_sizes, test_scores_mean, 'o-', color='g',
           label='Cross-validation score')

  plt.legend(loc='best')
  return plt

g = plot_learning_curve(gsLR.best_estimator_, 'LR mearning curves', X_train, y_train, cv=kfold)
g = plot_learning_curve(gsRF.best_estimator_, 'RF learning curves', X_train, y_train, cv=kfold)
g = plot_learning_curve(gsSGD.best_estimator_, 'SGD learning curves', X_train, y_train, cv=kfold)