Day 24 bert 文字情感分类-3

安装繁简转换函式库
pip install hanziconv

在昨天的分类中，把简体评论的改成繁体。

import pandas as pd
import os
from hanziconv import HanziConv
all_df = pd.read_csv("ChnSentiCorp_htl_all.csv")

shuffled = all_df.sample(frac=1).reset_index(drop=True)

train_df = shuffled.iloc[:int(len(shuffled)*0.8)]
test_df = shuffled.iloc[int(len(shuffled)*0.8):]

mypaths = ["chinese/train/neg", "chinese/train/pos", "chinese/test/neg", "chinese/test/pos"]
for i in mypaths:
  os.makedirs(i, exist_ok=True)

for i, row in train_df.iterrows():
  if row["label"] == 1:
    with open("chinese/train/pos/" + str(i) + ".txt", "w", encoding="UTF-8") as f:
      f.write(HanziConv.toTraditional(str(row["review"])))
  if row["label"] == 0:
    with open("chinese/train/neg/" + str(i) + ".txt", "w", encoding="UTF-8") as f:
      f.write(HanziConv.toTraditional(str(row["review"])))


for i, row in test_df.iterrows():
  if row["label"] == 1:
    with open("chinese/test/pos/" + str(i) + ".txt", "w", encoding="UTF-8") as f:
      f.write(HanziConv.toTraditional(str(row["review"])))
  if row["label"] == 0:
    with open("chinese/test/neg/" + str(i) + ".txt", "w", encoding="UTF-8") as f:
      f.write(HanziConv.toTraditional(str(row["review"])))

将 tf.keras.preprocessing.text_dataset_from_directory 读取的资料夹从 aclImdb 改为 我们刚才分好的 chinese

AUTOTUNE = tf.data.AUTOTUNE
batch_size = 32
seed = 42

raw_train_ds = tf.keras.preprocessing.text_dataset_from_directory(
    'chinese/train',
    batch_size=batch_size,
    validation_split=0.2,
    subset='training',
    seed=seed)

class_names = raw_train_ds.class_names
train_ds = raw_train_ds.cache().prefetch(buffer_size=AUTOTUNE)

val_ds = tf.keras.preprocessing.text_dataset_from_directory(
    'chinese/train',
    batch_size=batch_size,
    validation_split=0.2,
    subset='validation',
    seed=seed)

val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)

test_ds = tf.keras.preprocessing.text_dataset_from_directory(
    'chinese/test',
    batch_size=batch_size)

test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)

最後，将使用的 bert 模型，从 en (英文)

转为 multi_cased (多语言)