小説を読もうの累計ランキングをDoc2Vecで解析する その２

# 累計ランキングをTop100まで取得
def novel_total():
  headers = {
    'User-Agent':
    'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
  }
  r = requests.get('http://yomou.syosetu.com/rank/list/type/total_total/', headers=headers, timeout=20)
  r.encoding = r.apparent_encoding
  soup =  BeautifulSoup(r.text)
  rank_index = soup.find_all('div', class_='rank_h')
  sleep(1)
  with open('drive/My Drive/Colab Notebooks/syosetu/novel_datas.txt', 'w') as f:
    for rank in range(100):
      link = rank_index[rank].find('a')
      url = link.get('href')
      title = link.get_text()
      # 後でわかりやすいようにURLと小説のタイトルを設定
      f.write('{0}\t{1}\t'.format(url, title))
      print('rank:{0} title:{1}'.format(rank + 1, title))
      chapter = 0
      word_count = 0
      while word_count < 5000 and chapter < 40:
        # 単語が5000未満かつ、40話以下の間繰り返し取得する
        try:
          words = keitaiso(novel_text_dler('{0}{1}/'.format(url, chapter + 1)))
        except (HTTPError, URLError) as e:
          print(e)
          break
        except socket.timeout as e:
          print(e)
          continue
        else:
          f.write(words)
          print('chapter:{0}'.format(chapter + 1))
          word_count += len(words.split())
        chapter += 1
      f.write('\n')

# 学習の実行
m = Doc2Vec(documents=trainings, dm=1, vector_size=400, min_count=4, workers=4, epochs=40)

m = Doc2Vec.load('drive/My Drive/Colab Notebooks/syosetu/doc2vec.model')
# 0番目の小説に似ている小説は？（0番目の小説のタイトルは、「転生したらスライムだった件」）
print(m.docvecs.most_similar(0))

import numpy as np
import matplotlib.pyplot as plt
from gensim.models.doc2vec import Doc2Vec
from scipy.cluster.hierarchy import linkage, fcluster, dendrogram
from matplotlib.font_manager import FontProperties

# 階層型クラスタリングの実施
def hierarchical_clustering(emb, threshold):
  # ウォード法 x ユークリッド距離
  linkage_result = linkage(emb, method='ward', metric='euclidean')
  # クラスタ分けするしきい値を決める
  threshold_distance = threshold * np.max(linkage_result[:, 2])
  # クラスタリング結果の値を取得
  clustered = fcluster(linkage_result, threshold_distance, criterion='distance')
  print("end clustering.")
  return linkage_result, threshold_distance, clustered

# 階層型クラスタリングの可視化
def plot_dendrogram(linkage_result, doc_labels, threshold):
  fp = FontProperties(fname=r'drive/My Drive/Colab Notebooks/IPAexfont00301/ipaexg.ttf')
  plt.figure(figsize=(16, 8), facecolor='w', edgecolor='k')
  dendrogram(linkage_result, labels=doc_labels, color_threshold=threshold)
  plt.title('Dendrogram', fontproperties=fp)
  plt.xticks(fontsize=10)
  print('end plot.')
  plt.savefig('drive/My Drive/Colab Notebooks/syosetu/novel_hierarchy.png')

# 階層型クラスタリング結果の保存
def save_cluster(doc_index, clustered):
  doc_cluster = np.array([doc_index, clustered])
  doc_cluster = doc_cluster.T
  doc_cluster = doc_cluster.astype(np.dtype(int).type)
  doc_cluster = doc_cluster[np.argsort(doc_cluster[:, 1])]
  np.savetxt('drive/My Drive/Colab Notebooks/syosetu/novel_cluster.csv', doc_cluster, delimiter=',', fmt='%.0f')
  print('save cluster.')

m = Doc2Vec.load('drive/My Drive/Colab Notebooks/syosetu/doc2vec.model')
vectors_list = [m.docvecs[n] for n in range(len(m.docvecs))]

threshold = 0.8
linkage_result, threshold, clustered = hierarchical_clustering(emb=vectors_list, threshold=threshold)
plot_dendrogram(linkage_result=linkage_result, doc_labels=list(range(100)), threshold=threshold)
save_cluster(list(range(100)), clustered)