タグ: 小説を読もう

from gensim.models.doc2vec import Doc2Vec
from pathlib import Path

MODEL_DATA_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novel_doc2vec_100.model'

m = Doc2Vec.load(MODEL_DATA_PATH)
vectors_list = [m.dv[n] for n in range(len(m.dv))]

import pandas as pd
import numpy as np

MODEL_TITLES_CSV_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novel_titles.csv'

df = pd.read_csv(MODEL_TITLES_CSV_PATH, thousands=',')
df = df.drop(columns=['URL'])
df['VECTORS'] = vectors_list

df['EVALUATION'] = df['EVALUATION'].str.replace(',', '')
df['EVALUATION'] = df['EVALUATION'].str.replace('評価受付停止中', '').astype(float)

df['EVALUATION'].describe()

count      2000.000000
mean      32735.061500
std       33540.729876
min        4019.000000
25%       14118.000000
50%       21271.000000
75%       38219.250000
max      323929.000000
Name: EVALUATION, dtype: float64

df['EVALUATION_BAND'] = pd.cut(df['EVALUATION'], 5)
df[['EVALUATION_BAND', 'EVALUATION']].groupby(['EVALUATION_BAND'], as_index=False).count().sort_values(by='EVALUATION_BAND', ascending=True)

EVALUATION_BAND				EVALUATION
0	(3699.09,   68001.0]	1801
1	(68001.0,  131983.0]	149
2	(131983.0, 195965.0]	38
3	(195965.0, 259947.0]	6
4	(259947.0, 323929.0]	6

df.loc[ df['EVALUATION'] <= 68001, 'EVALUATION'] = 0
df.loc[(df['EVALUATION'] >  68001) & (df['EVALUATION'] <= 131983), 'EVALUATION'] = 1
df.loc[ df['EVALUATION'] > 131983, 'EVALUATION'] = 2

df = df.drop(['EVALUATION_BAND'], axis=1)
df['EVALUATION'].value_counts()

0.0    1801
1.0     149
2.0      50
Name: EVALUATION, dtype: int64

df_0 = df[df['EVALUATION'] == 0].sample(n=50, random_state=0)
df_1 = df[df['EVALUATION'] == 1].sample(n=50, random_state=0)
df_2 = df[df['EVALUATION'] == 2]

df = pd.concat([df_0, df_1, df_2])

from sklearn.model_selection import train_test_split

X = pd.DataFrame(df['VECTORS'].tolist(), index=df.index)
X.columns = [f'No{i+1}' for i in range(len(X.columns))]
y = df['EVALUATION']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

from sklearn.ensemble import RandomForestClassifier

random_forest = RandomForestClassifier(random_state=0)
random_forest.fit(X_train, y_train)

random_forest.score(X_test, y_test)

0.5263157894736842

random_forest.predict(X_test)

array([2., 1., 2., 2., 2., 1., 0., 1., 1., 0., 2., 1., 2., 2., 1., 0., 1.,
       1., 2., 1., 0., 2., 2., 0., 0., 1., 2., 1., 1., 0., 2., 2., 1., 2.,
       2., 2., 2., 1.])

import requests
import socket

from bs4 import BeautifulSoup
from pathlib import Path
from urllib.error import HTTPError, URLError
from time import sleep
from tqdm import tqdm

REQUEST_HEADERS = {
    'User-Agent':
    'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
  }
REQUEST_TIMEOUT = 20

SEARCH_URL = 'https://yomou.syosetu.com/search.php'
NOVEL_SEARCH_RESULT_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novels.txt'

# 小説を検索してダウンロード
def novel_search_dler(max_search_page=100, max_search_result=20):
  with open(NOVEL_SEARCH_RESULT_PATH, 'w') as f:
    for page in tqdm(range(max_search_page)):
      #search_params = {'word': '', 'type': 're', 'order_former': 'search', 'order': 'new', 'notnizi': '1', 'minlen': '5000', 'min_globalpoint': '1000', 'p': page + 1}
      search_params = {'word': '', 'type': 're', 'order_former': 'search', 'order': 'hyoka', 'notnizi': '1', 'minlen': '5000', 'min_globalpoint': '1000', 'p': page + 1}
      r = requests.get(SEARCH_URL, headers=REQUEST_HEADERS, timeout=REQUEST_TIMEOUT, params=search_params)
      r.encoding = r.apparent_encoding
      soup = BeautifulSoup(r.text, 'html.parser')
      sleep(1)

      search_result = soup.find_all('div', class_='searchkekka_box')

      for r_count in range(max_search_result):
        a_list = search_result[r_count].find_all('a', limit=3)
        url = a_list[0].get('href')
        info_url = a_list[2].get('href')
        title = a_list[0].get_text()
        # 小説のURL、小説情報のURLと小説のタイトルを設定
        f.write('{0}\t{1}\t{2}\n'.format(url, info_url, title))
        print('url: {0} info_url: {1} title: {2}'.format(url, info_url, title))

novel_search_dler()

import morphologicalAnalysis as ma

import requests
import socket

import pandas as pd

from bs4 import BeautifulSoup
from pathlib import Path
from urllib.error import HTTPError, URLError
from time import sleep
from tqdm import tqdm

REQUEST_HEADERS = {
    'User-Agent':
    'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
  }
REQUEST_TIMEOUT = 60

SCRAP_WORD = 5000
SCRAP_CHAPTER = 40

COL_NAMES = ['URL', 'INFO_URL', 'TITLE']
NOVEL_SEARCH_RESULT_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novels.txt'
NOVEL_SEARCH_RESULT_DATA_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novel_datas.txt'

# 小説情報をダウンロード
def novel_info_dler(url):
  KEYWORD = 2
  EVALUATION = 6

  r = requests.get(url, headers=REQUEST_HEADERS, timeout=REQUEST_TIMEOUT)
  r.encoding = r.apparent_encoding
  soup = BeautifulSoup(r.text, 'html.parser')
  sleep(1)

  keyword = soup.find(id='noveltable1').find_all('td')[KEYWORD].get_text(strip=True)
  evaluation = soup.find(id='noveltable2').find_all('td')[EVALUATION].get_text(strip=True).replace('pt', '')
  return keyword,evaluation

# 本文をダウンロード
def novel_text_dler(url):
  r = requests.get(url, headers=REQUEST_HEADERS, timeout=REQUEST_TIMEOUT)
  r.encoding = r.apparent_encoding
  soup = BeautifulSoup(r.text, 'html.parser')
  sleep(1)

  novel_text = ''
  # 章のタイトルに「設定」、「登場人物」が含まれる場合、戻り値は''とする
  subtitle = soup.find('p', class_='novel_subtitle')
  if subtitle is None or '設定' in subtitle or '登場人物' in subtitle:
    return novel_text

  honbun = soup.find_all('div', class_='novel_view')

  for text in honbun:
    novel_text += text.text.replace('\n', ' ') # 改行コードをスペースに変換
  return novel_text

# 小説の各話をダウンロード
def novel_chapter_dler(url, scrap_word=SCRAP_WORD, scrap_chapter=SCRAP_CHAPTER):
  chapter = 0
  word_count = 0
  total_words = ''

  while word_count < scrap_word and chapter < scrap_chapter:
    # 単語がscrap_word未満かつ、scrap_chapter話以下の間繰り返し取得する
    try:
      words = novel_text_dler('{0}{1}/'.format(url, chapter + 1))
    except (HTTPError, URLError) as e:
      print(e)
      break
    except socket.timeout as e:
      print(e)
      continue
    else:
      #print('chapter:{0}'.format(chapter + 1))
      word_count += len(ma.analysis(words).split())
      total_words += words
    chapter += 1
  return total_words

# 小説の検索結果をもとにダウンロード
def novel_search_data_dler():
  df = pd.read_table(NOVEL_SEARCH_RESULT_PATH, header=None, names=COL_NAMES)

  with open(NOVEL_SEARCH_RESULT_DATA_PATH, 'w') as f:
    for index,item in tqdm(df.iterrows()):
      url = item[COL_NAMES[0]]
      info_url = item[COL_NAMES[1]]
      title = item[COL_NAMES[2]]
      keyword,evaluation = novel_info_dler(info_url)
      # URL、小説のタイトル、キーワードと評価ポイントを設定
      f.write('{0}\t{1}\t{2}\t{3}\t'.format(url, title, keyword, evaluation))
      #print('url:{0} title:{1} keywords:{2} evaluation:{3}'.format(url, title, keyword, evaluation))
      # 小説の各話を設定
      f.write(novel_chapter_dler(url))
      f.write('\n')

novel_search_data_dler()

from gensim.models.doc2vec import Doc2Vec
from pathlib import Path

MODEL_DATA_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novel_doc2vec_100.model'

m = Doc2Vec.load(MODEL_DATA_PATH)
vectors_list = [m.dv[n] for n in range(len(m.dv))]

import pandas as pd
import numpy as np

MODEL_TITLES_CSV_PATH = str(Path.home()) + '/vscode/syosetu/data/search_novel_titles.csv'

df = pd.read_csv(MODEL_TITLES_CSV_PATH, thousands=',')
df = df.drop(columns=['URL'])
df['VECTORS'] = vectors_list

from sklearn.decomposition import PCA

def dimension_reduction(data, pca_dimension=20):
  pca_data = data.copy()
  pca = PCA(n_components=pca_dimension)
  vector = np.array([np.array(v) for v in pca_data['VECTORS']])
  pca_vectors = pca.fit_transform(vector)
  pca_data['PCA_VECTORS'] = [v for v in pca_vectors]

  return pca_data

df = dimension_reduction(data=df)
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2000 entries, 0 to 1999
Data columns (total 5 columns):
 #   Column       Non-Null Count  Dtype 
---  ------       --------------  ----- 
 0   TITLE        2000 non-null   object
 1   KEYWORDS     2000 non-null   object
 2   EVALUATION   2000 non-null   object
 3   VECTORS      2000 non-null   object
 4   PCA_VECTORS  2000 non-null   object
dtypes: object(5)
memory usage: 78.2+ KB

df['EVALUATION'] = df['EVALUATION'].str.replace(',', '')
df['EVALUATION'] = df['EVALUATION'].str.replace('評価受付停止中', '').astype(float)

df['EVALUATION'].describe()

count      2000.000000
mean      32735.061500
std       33540.729876
min        4019.000000
25%       14118.000000
50%       21271.000000
75%       38219.250000
max      323929.000000
Name: EVALUATION, dtype: float64

from sklearn.model_selection import train_test_split

X = pd.DataFrame(df['PCA_VECTORS'].tolist(), index=df.index)
X.columns = [f'No{i+1}' for i in range(len(X.columns))]
y = df['EVALUATION']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

from scipy import stats
from scipy.stats import norm, skew # for some statistics

# visualization
import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
color = sns.color_palette()
sns.set_style('darkgrid')

import warnings
def ignore_warn(*args, **kwargs):
  pass
warnings.warn = ignore_warn # Ignore annoying warning (from sklearn and seaborn)

sns.distplot(y_train, fit=norm)

# Get the fitted parameters used by the function
(mu, sigma) = norm.fit(y_train)
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))

# Now plot the distribution
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
            loc='best')
plt.ylabel('Frequency')
plt.title('Evaluation distribution')

# Get also the QQ-plot
fig = plt.figure()
res = stats.probplot(y_train, plot=plt)
plt.show()

# We use the numpy fuction log1p which applies log(1+x) to all elements of the column
y_train = np.log1p(y_train)

# Check the new distribution 
sns.distplot(y_train, fit=norm)

# Get the fitted parameters used by the function
(mu, sigma) = norm.fit(y_train)
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))

# Now plot the distribution
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
            loc='best')
plt.ylabel('Frequency')
plt.title('Evaluation distribution')

# Get also the QQ-plot
fig = plt.figure()
res = stats.probplot(y_train, plot=plt)
plt.show()

from sklearn.model_selection import cross_val_score, KFold

kf = KFold(5, shuffle=True, random_state=0).get_n_splits(X_train)

# Validation function
def rmsle_cv(classifier):
  return np.sqrt(-cross_val_score(classifier, X_train.values, y_train.values, scoring="neg_mean_squared_error", cv=kf))

from sklearn.linear_model import Lasso
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler

lasso = make_pipeline(RobustScaler(), Lasso(random_state=0))

score = rmsle_cv(lasso)
print("Lasso score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))

Lasso score: 0.7408 (0.0095)

# data analysis and wrangling
import pandas as pd
import numpy as np
import random as rnd

# visualization
import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
color = sns.color_palette()
sns.set_style('darkgrid')

import warnings
def ignore_warn(*args, **kwargs):
  pass
warnings.warn = ignore_warn # Ignore annoying warning (from sklearn and seaborn)

# machine learning
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron, SGDClassifier
from sklearn.tree import DecisionTreeClassifier

from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve

from pathlib import Path

TRAIN_DATA_PATH = str(Path.home()) + '/vscode/syosetu/train_data.pkl'

train = pd.read_pickle(TRAIN_DATA_PATH)

from sklearn.model_selection import train_test_split

X = train['vectors']
y = train['female']

X_array = np.array([np.array(v) for v in X])
y_array = np.array([i for i in y])

X_train, X_test, y_train, y_test = train_test_split(X_array, y_array, random_state=0)

kfold = StratifiedKFold(n_splits=10)
cls_weight = (y_train.shape[0] - np.sum(y_train)) / np.sum(y_train)
random_state = 0
classifiers = []
classifiers.append(LogisticRegression(random_state=random_state))
classifiers.append(SVC(random_state=random_state))
classifiers.append(KNeighborsClassifier(n_neighbors=3))
classifiers.append(GaussianNB())
classifiers.append(Perceptron(random_state=random_state))
classifiers.append(LinearSVC(random_state=random_state))
classifiers.append(SGDClassifier(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(n_estimators=100, random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state), random_state=random_state, learning_rate=0.1))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))

cv_results = []
for classifier in classifiers :
  cv_results.append(cross_val_score(classifier, X_train, y=y_train, scoring='accuracy', cv=kfold, n_jobs=4))

cv_means = []
cv_std = []
for cv_result in cv_results:
  cv_means.append(cv_result.mean())
  cv_std.append(cv_result.std())

cv_res = pd.DataFrame({'CrossVal Means': cv_means, 'CrossVal Errors': cv_std,
                       'Algorithm': ['LogisticRegression', 'SVC', 'KNeighborsClassifier',
                                     'GaussianNB', 'Perceptron', 'LinearSVC', 'SGDClassifier',
                                     'DecisionTreeClassifier', 'RandomForestClassifier',
                                     'AdaBoostClassifier', 'GradientBoostingClassifier',
                                     'ExtraTreesClassifier']})

cv_res.sort_values(by=['CrossVal Means'], ascending=False, inplace=True)

cv_res

g = sns.barplot('CrossVal Means', 'Algorithm', data=cv_res, palette='Set3', orient='h', **{'xerr': cv_std})
g.set_xlabel('Mean Accuracy')
g = g.set_title('Cross validation scores')

### META MODELING WITH LR, RF and SGD

# Logistic Regression
LR = LogisticRegression(random_state=0)

## Search grid for optimal parameters
lr_param_grid = {'penalty': ['l1','l2']}

gsLR = GridSearchCV(LR, param_grid=lr_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)

gsLR.fit(X_train, y_train)

LR_best = gsLR.best_estimator_

# Best score
gsLR.best_score_

gsLR.best_params_

# Random Forest Classifier
RF = RandomForestClassifier(random_state=0)

## Search grid for optimal parameters
'''rf_param_grid = {'criterion': ['gini', 'entropy'],
                 'max_features': [1, 3, 10],
                 'min_samples_split': [2, 3, 10],
                 'min_samples_leaf': [1, 3, 10],
                 'n_estimators':[50, 100, 200]}'''
rf_param_grid = {'criterion': ['gini', 'entropy'],
                 'max_features': [40, 50, 60],
                 'min_samples_split': [2, 3, 10],
                 'min_samples_leaf': [1, 3, 10],
                 'n_estimators':[20, 30, 40]}

gsRF = GridSearchCV(RF, param_grid=rf_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)

gsRF.fit(X_train, y_train)

RF_best = gsRF.best_estimator_

# Best score
gsRF.best_score_

gsRF.best_params_

# SGD Classifier
SGD = SGDClassifier(random_state=0)

## Search grid for optimal parameters
sgd_param_grid = {'alpha'  : [0.0001, 0.001, 0.01, 0.1],
                  'loss'   : ['log', 'modified_huber'],
                  'penalty': ['l2', 'l1', 'none']}

gsSGD = GridSearchCV(SGD, param_grid=sgd_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)

gsSGD.fit(X_train, y_train)

SGD_best = gsSGD.best_estimator_

# Best score
gsSGD.best_score_

gsSGD.best_params_

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5)):
  '''Generate a simple plot of the test and training learning curve'''
  plt.figure()
  plt.title(title)
  if ylim is not None:
    plt.ylim(*ylim)
  plt.xlabel('Training examples')
  plt.ylabel('Score')
  train_sizes, train_scores, test_scores = learning_curve(
      estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
  train_scores_mean = np.mean(train_scores, axis=1)
  train_scores_std = np.std(train_scores, axis=1)
  test_scores_mean = np.mean(test_scores, axis=1)
  test_scores_std = np.std(test_scores, axis=1)
  plt.grid()

  plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                   train_scores_mean + train_scores_std, alpha=0.1,
                   color='r')
  plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                   test_scores_mean + test_scores_std, alpha=0.1,
                   color='g')
  plt.plot(train_sizes, train_scores_mean, 'o-', color='r',
           label='Training score')
  plt.plot(train_sizes, test_scores_mean, 'o-', color='g',
           label='Cross-validation score')

  plt.legend(loc='best')
  return plt

g = plot_learning_curve(gsLR.best_estimator_, 'LR mearning curves', X_train, y_train, cv=kfold)
g = plot_learning_curve(gsRF.best_estimator_, 'RF learning curves', X_train, y_train, cv=kfold)
g = plot_learning_curve(gsSGD.best_estimator_, 'SGD learning curves', X_train, y_train, cv=kfold)

from gensim.models.doc2vec import Doc2Vec

MODEL_DATA_PATH = 'drive/My Drive/Colab Notebooks/syosetu/doc2vec_100.model'

m = Doc2Vec.load(MODEL_DATA_PATH)
vectors_list = [m.docvecs[n] for n in range(len(m.docvecs))]

import pandas as pd

MODEL_TITLES_CSV_PATH = 'drive/My Drive/Colab Notebooks/syosetu/novel_titles.csv'

df = pd.read_csv(MODEL_TITLES_CSV_PATH, header=None, names=['url', 'title', 'tag'])
df = df.drop(columns=['url'])
df['vectors'] = vectors_list

df.head(2)

import collections
import itertools

tag = collections.Counter(list(itertools.chain.from_iterable(df['tag'].str.split().tolist())))
sorted(tag.items(), key=lambda pair: pair[1], reverse=True)[:20]

[('R15', 233),
 ('残酷な描写あり', 215),
 ('異世界転生', 98),
 ('異世界', 87),
 ('異世界転移', 86),
 ('チート', 84),
 ('ファンタジー', 80),
 ('魔法', 73),
 ('男主人公', 65),
 ('冒険', 63),
 ('主人公最強', 59),
 ('書籍化', 53),
 ('ハーレム', 48),
 ('成り上がり', 43),
 ('ほのぼの', 33),
 ('ハッピーエンド', 32),
 ('女主人公', 31),
 ('恋愛', 30),
 ('オリジナル戦記', 30),
 ('転生', 28)]

df_m = df[df['tag'].str.contains('男主人公')]
df_m['female'] = 0

df_f = df[df['tag'].str.contains('女主人公|女性視点|悪役令嬢|後宮|少女マンガ')]
df_f['female'] = 1

df_f[df_f['tag'].str.contains('男主人公')]

df_f = df_f.drop([47, 74])

df = pd.concat([df_m, df_f])
df['female'].value_counts()

#男主人公の一覧
df_0 = df[df['female'] == 0]
#女主人公の一覧
df_1 = df[df['female'] == 1]

df_0 = df_0.sample(n=len(df_1), random_state=0)

df = pd.concat([df_0, df_1])

X = df['vectors']
y = df['female']

X_array = np.array([np.array(v) for v in X])
y_array = np.array([i for i in y])

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X_array, y_array, random_state=0)

from sklearn.ensemble import RandomForestClassifier

random_forest = RandomForestClassifier(random_state=0)
random_forest.fit(X_train, y_train)

random_forest.score(X_test, y_test)

random_forest.predict(X_test)

from sklearn.linear_model import LogisticRegression

logreg = LogisticRegression()
logreg.fit(X_train, y_train)

logreg.score(X_test, y_test)

logreg.predict(X_test)