{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint #導入tensorflow\n\nfrom kaggle_datasets import KaggleDatasets #採用Kaggle資料集\nimport transformers\n\nfrom tokenizers import BertWordPieceTokenizer #分詞器\nfrom tqdm import tqdm #進度條顯示\nimport numpy as np\n\n!pip install wandb\n\n#基本模型導入\nimport os, time\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import KaggleDatasets\n\n!pip install bert-tensorflow\nimport bert.tokenization\n\nprint(tf.version.VERSION) #tensorflow版本輸出","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TPU 檢測. \ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu) #TPU的連接\nelse:\n    \n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n#在TPU上針對Kaggle用戶運行Bert模型","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEQUENCE_LENGTH = 128 #一個輸入字串長度為128的list\n\n#設置Kaggle數據的訪問路徑\nDATA_PATH =  KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\nBERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')\nBERT_PATH_SAVEDMODEL = BERT_PATH + \"/bert_multi_from_tfhub\"\n\nOUTPUT_PATH = \"/kaggle/working\"","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsub2 = pd.read_csv('../input/ensemble/submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BERT Tokenizer","metadata":{}},{"cell_type":"code","source":"#把文字切割並轉成BERT所需要的編碼\n\ndef get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):\n    bert_layer = tf.saved_model.load(bert_path)\n    bert_layer = hub.KerasLayer(bert_layer, trainable=False)\n    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy() \n    cased = bert_layer.resolved_object.do_lower_case.numpy()\n    tf.gfile = tf.io.gfile  \n    tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)\n  \n    return tokenizer\n\ntokenizer = get_tokenizer()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"#編碼器，用於將文本編碼為整數序列，以進行BERT輸入\n\ndef fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):#批次上傳256，最長序列512\n    \n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(length=maxlen) #最大長度為512，不足會自動補0\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist() #將數據轉換為最接近Python的類型\n        encs = tokenizer.encode_batch(text_chunk) \n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#用於配置的IMP數據\n\nAUTO = tf.data.experimental.AUTOTUNE\n\n\n# 配置\nEPOCHS = 5 #定義訓練過程數據輪5次\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync  #資料集大小\nMAX_LEN = 192","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')# 使用分詞器加載DistilBERT\n\ntokenizer.save_pretrained('.') #儲存\n\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\nfast_tokenizer  #利用 huggingface tokenizers庫 重新加載詞向量，lowercase=False:詞向量皆為大寫","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#快速編碼\nx_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n\ny_train = train1.toxic.values\ny_valid = valid.toxic.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#轉化成數據集 生成對應的Dataset\n\ntrain_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat() #重複數據集count次數\n    .shuffle(2048) #隨機混洗數據集多元素\n    .batch(BATCH_SIZE) #將數據集多連續元素合成批次\n    .prefetch(AUTO)#將一部分內存加載到cache裡面\n)\nvalid_dataset =(\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#訓練BERT模型\n\ndef build_model(transformer, max_len=512):  #建立模型，輸入句子最大長度512\n    \n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\") #dtype=tf.int 返回數據元素的數據類型int\n    sequence_output = transformer(input_word_ids)[0] #BERT模型的輸出 \n    cls_token = sequence_output[:, 0, :]\n    \n    #激活函數\n    out = tf.keras.layers.Dense(300, activation='relu')(cls_token)\n    out = tf.keras.layers.Dense(128, activation='relu')(out)\n    out = tf.keras.layers.Dense(128, activation='relu')(out)\n    out = Dense(1, activation='sigmoid')(out) #relu線性函數激活 sigmoid非線性激活函數\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy']) #損失函數的用法，Adam是優化器，loss：計算損失\n    \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope(): #表明分散式執行的程式碼區塊\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary() #輸出各層的輸出情況","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE #讀取矩陣第一維度的長度\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n) #使用model.fit()執行訓練過程","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS*2\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.predict()返回值是數值,表示樣本屬於toxic類別的概率\n\nsub['toxic'] = model.predict(test_dataset, verbose=1)\n\nsub1 = sub[['id', 'toxic']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sub1)\nprint(sub2)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1.rename(columns={'toxic':'toxic1'}, inplace=True)\nsub2.rename(columns={'toxic':'toxic2'}, inplace=True) #命名文件或目錄\nsub3 = pd.merge(sub1, sub2, how='left', on='id')\n\nsub3['toxic'] = (sub3['toxic1'] * 0.1) + (sub3['toxic2'] * 0.9) \nsub3['toxic'] = (sub3['toxic2'] * 0.39) + (sub3['toxic'] * 0.61) \n\nsub3[['id', 'toxic']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}