{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint #導入tensorflow\nfrom tensorflow import keras\n\nfrom sklearn.model_selection import StratifiedKFold, KFold\n\nfrom kaggle_datasets import KaggleDatasets #採用Kaggle資料集\nimport transformers\n\nfrom tokenizers import BertWordPieceTokenizer #分詞器\nfrom tqdm import tqdm #進度條顯示\nimport numpy as np\n\n#!pip install wandb\n\n#基本模型導入\nimport os, time\nimport gc\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import KaggleDatasets\n\n!pip install bert-tensorflow\nimport bert.tokenization\n\nprint(tf.version.VERSION) #tensorflow版本輸出","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-23T20:39:14.714330Z","iopub.execute_input":"2021-09-23T20:39:14.714712Z","iopub.status.idle":"2021-09-23T20:39:22.831005Z","shell.execute_reply.started":"2021-09-23T20:39:14.714601Z","shell.execute_reply":"2021-09-23T20:39:22.830179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(transformers.__version__) #tensorflow版本輸出","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:22.832737Z","iopub.execute_input":"2021-09-23T20:39:22.833024Z","iopub.status.idle":"2021-09-23T20:39:22.838556Z","shell.execute_reply.started":"2021-09-23T20:39:22.832985Z","shell.execute_reply":"2021-09-23T20:39:22.837649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TPU 檢測. \ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu) #TPU的連接\nelse:\n    \n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n#在TPU上針對Kaggle用戶運行Bert模型","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:22.840324Z","iopub.execute_input":"2021-09-23T20:39:22.840701Z","iopub.status.idle":"2021-09-23T20:39:28.682037Z","shell.execute_reply.started":"2021-09-23T20:39:22.840668Z","shell.execute_reply":"2021-09-23T20:39:28.681146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEQUENCE_LENGTH = 128 #一個輸入字串長度為128的list\nseed = 7\nn_splits = 5\n\n#設置Kaggle數據的訪問路徑\nDATA_PATH =  KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\nBERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')\nBERT_PATH_SAVEDMODEL = BERT_PATH + \"/bert_multi_from_tfhub\"\nWEIGHTS_PATH = '../input/jigsaw-weights'\n\n\nOUTPUT_PATH = \"/kaggle/working\"","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-09-23T20:39:28.683474Z","iopub.execute_input":"2021-09-23T20:39:28.683705Z","iopub.status.idle":"2021-09-23T20:39:29.604288Z","shell.execute_reply.started":"2021-09-23T20:39:28.683677Z","shell.execute_reply":"2021-09-23T20:39:29.603612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsawch/train-ch.csv\")\ntrain3 = pd.read_csv(\"/kaggle/input/pttcommentch/123u.csv\")\ntrain4 = pd.read_csv(\"/kaggle/input/jigsawch/666.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsub2 = pd.read_csv('../input/ensemble/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:29.606244Z","iopub.execute_input":"2021-09-23T20:39:29.606607Z","iopub.status.idle":"2021-09-23T20:39:34.069826Z","shell.execute_reply.started":"2021-09-23T20:39:29.606579Z","shell.execute_reply":"2021-09-23T20:39:34.069173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train4.info()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:50.873639Z","iopub.execute_input":"2021-09-23T20:39:50.873931Z","iopub.status.idle":"2021-09-23T20:39:51.143852Z","shell.execute_reply.started":"2021-09-23T20:39:50.873903Z","shell.execute_reply":"2021-09-23T20:39:51.143055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = [\"id\",\"comment_text\",\"toxic\"]\ntrain4 = train4[col]\ntrain4['toxic'] = pd.to_numeric(train4['toxic'], errors='coerce')\nprint(train4.head())\ntrain4 = train4.dropna(how='any')\nprint(train4.info())","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:51.145616Z","iopub.execute_input":"2021-09-23T20:39:51.145834Z","iopub.status.idle":"2021-09-23T20:39:51.588278Z","shell.execute_reply.started":"2021-09-23T20:39:51.145809Z","shell.execute_reply":"2021-09-23T20:39:51.587469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = train1.sample(n=15000,random_state = seed)\ntrain2 = train2.sample(n=15000,random_state = seed)\ntrain3 = train3.sample(n=15000,random_state = seed)\ntrain4 = train4.sample(n=15000,random_state = seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:51.589433Z","iopub.execute_input":"2021-09-23T20:39:51.589652Z","iopub.status.idle":"2021-09-23T20:39:51.670265Z","shell.execute_reply.started":"2021-09-23T20:39:51.589628Z","shell.execute_reply":"2021-09-23T20:39:51.669364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train1.info())\nprint(train2.info())\nprint(train3.info())\nprint(train4.info())","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:51.671945Z","iopub.execute_input":"2021-09-23T20:39:51.672760Z","iopub.status.idle":"2021-09-23T20:39:51.727280Z","shell.execute_reply.started":"2021-09-23T20:39:51.672718Z","shell.execute_reply":"2021-09-23T20:39:51.726365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train1.isnull().any())\nprint(train2.isnull().any())\nprint(train3.isnull().any())\nprint(train4.isnull().any())","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:51.729876Z","iopub.execute_input":"2021-09-23T20:39:51.730447Z","iopub.status.idle":"2021-09-23T20:39:51.809827Z","shell.execute_reply.started":"2021-09-23T20:39:51.730388Z","shell.execute_reply":"2021-09-23T20:39:51.808948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"new=pd.DataFrame({'id':['63812'],\n                  'content':['你是白癡'],\n                  'lang':['zh']}) \ntest = test.append(new,ignore_index = True)\nprint(test.tail())","metadata":{"execution":{"iopub.status.busy":"2021-08-09T08:26:01.180014Z","iopub.execute_input":"2021-08-09T08:26:01.180343Z","iopub.status.idle":"2021-08-09T08:26:01.198418Z","shell.execute_reply.started":"2021-08-09T08:26:01.180314Z","shell.execute_reply":"2021-08-09T08:26:01.197677Z"}}},{"cell_type":"markdown","source":"# BERT Tokenizer","metadata":{}},{"cell_type":"code","source":"#把文字切割並轉成BERT所需要的編碼\n\ndef get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):\n    bert_layer = tf.saved_model.load(bert_path)\n    bert_layer = hub.KerasLayer(bert_layer, trainable=False)\n    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy() \n    cased = bert_layer.resolved_object.do_lower_case.numpy()\n    tf.gfile = tf.io.gfile  \n    tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)\n  \n    return tokenizer\n\ntokenizer = get_tokenizer()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:39:51.811256Z","iopub.execute_input":"2021-09-23T20:39:51.811577Z","iopub.status.idle":"2021-09-23T20:40:00.261328Z","shell.execute_reply.started":"2021-09-23T20:39:51.811536Z","shell.execute_reply":"2021-09-23T20:40:00.260426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"#編碼器，用於將文本編碼為整數序列，以進行BERT輸入\n\ndef fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):#批次上傳256，最長序列512\n    \n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(length=maxlen) #最大長度為512，不足會自動補0\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist() #將數據轉換為最接近Python的類型\n        encs = tokenizer.encode_batch(text_chunk)\n        #print(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n        \n    \n    return np.array(all_ids)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:00.268694Z","iopub.execute_input":"2021-09-23T20:40:00.269317Z","iopub.status.idle":"2021-09-23T20:40:00.277197Z","shell.execute_reply.started":"2021-09-23T20:40:00.269272Z","shell.execute_reply":"2021-09-23T20:40:00.276115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#用於配置的IMP數據\n\nAUTO = tf.data.experimental.AUTOTUNE\n\n\n# 配置\nEPOCHS = 5 #定義訓練過程數據輪5次\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync  #資料集大小\nMAX_LEN = 192","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:00.278689Z","iopub.execute_input":"2021-09-23T20:40:00.278929Z","iopub.status.idle":"2021-09-23T20:40:00.291174Z","shell.execute_reply.started":"2021-09-23T20:40:00.278905Z","shell.execute_reply":"2021-09-23T20:40:00.290026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')# 使用分詞器加載DistilBERT\n\ntokenizer.save_pretrained('.') #儲存\n\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\nfast_tokenizer  #利用 huggingface tokenizers庫 重新加載詞向量，lowercase=False:詞向量皆為大寫","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:00.292445Z","iopub.execute_input":"2021-09-23T20:40:00.292653Z","iopub.status.idle":"2021-09-23T20:40:01.939363Z","shell.execute_reply.started":"2021-09-23T20:40:00.292629Z","shell.execute_reply":"2021-09-23T20:40:01.938523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#快速編碼\nx_train = fast_encode(train4.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n\ny_train = train4.toxic.values\ny_valid = valid.toxic.values","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:01.940581Z","iopub.execute_input":"2021-09-23T20:40:01.940799Z","iopub.status.idle":"2021-09-23T20:40:20.615252Z","shell.execute_reply.started":"2021-09-23T20:40:01.940775Z","shell.execute_reply":"2021-09-23T20:40:20.614258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#kkfold = KFold(n_splits).split(x_train)\nkfold = StratifiedKFold(n_splits, shuffle=True, random_state=seed)\ncvscores = []","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:20.620478Z","iopub.execute_input":"2021-09-23T20:40:20.620769Z","iopub.status.idle":"2021-09-23T20:40:20.626909Z","shell.execute_reply.started":"2021-09-23T20:40:20.620736Z","shell.execute_reply":"2021-09-23T20:40:20.626226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#訓練BERT模型\n\ndef build_model(transformer, max_len=512):  #建立模型，輸入句子最大長度512\n    \n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\") #dtype=tf.int 返回數據元素的數據類型int\n    sequence_output = transformer(input_word_ids)[0] #BERT模型的輸出 \n    cls_token = sequence_output[:, 0, :]\n    \n    #激活函數\n    out = tf.keras.layers.Dense(300, activation='relu')(cls_token)\n    out = tf.keras.layers.Dense(128, activation='relu')(out)\n    out = tf.keras.layers.Dense(128, activation='relu')(out)\n    out = Dense(1, activation='sigmoid')(out) #relu線性函數激活 sigmoid非線性激活函數\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy']) #損失函數的用法，Adam是優化器，loss：計算損失\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:20.627917Z","iopub.execute_input":"2021-09-23T20:40:20.628185Z","iopub.status.idle":"2021-09-23T20:40:20.643881Z","shell.execute_reply.started":"2021-09-23T20:40:20.628156Z","shell.execute_reply":"2021-09-23T20:40:20.642637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#轉化成數據集 生成對應的Dataset\n\n# train_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((x_train, y_train))\n#     .repeat() #重複數據集count次數\n#     .shuffle(2048) #隨機混洗數據集多元素\n#     .batch(BATCH_SIZE) #將數據集多連續元素合成批次\n#     .prefetch(AUTO)#將一部分內存加載到cache裡面\n# )\nvalid_dataset =(\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:20.646058Z","iopub.execute_input":"2021-09-23T20:40:20.646943Z","iopub.status.idle":"2021-09-23T20:40:21.183435Z","shell.execute_reply.started":"2021-09-23T20:40:20.646893Z","shell.execute_reply":"2021-09-23T20:40:21.182511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"for x_s_train, y_s_train in kfold.split(x_train, y_train):\n    # print(x_train[x_s_train])\n    print(y_train[x_s_train])\nprint(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:21.186886Z","iopub.execute_input":"2021-09-23T20:40:21.187135Z","iopub.status.idle":"2021-09-23T20:40:21.203476Z","shell.execute_reply.started":"2021-09-23T20:40:21.187109Z","shell.execute_reply":"2021-09-23T20:40:21.202512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:21.204547Z","iopub.execute_input":"2021-09-23T20:40:21.204777Z","iopub.status.idle":"2021-09-23T20:40:21.210207Z","shell.execute_reply.started":"2021-09-23T20:40:21.204750Z","shell.execute_reply":"2021-09-23T20:40:21.209067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope(): #表明分散式執行的程式碼區塊\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:21.211599Z","iopub.execute_input":"2021-09-23T20:40:21.211824Z","iopub.status.idle":"2021-09-23T20:40:38.084475Z","shell.execute_reply.started":"2021-09-23T20:40:21.211799Z","shell.execute_reply":"2021-09-23T20:40:38.083854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history_list = []","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:38.085758Z","iopub.execute_input":"2021-09-23T20:40:38.086159Z","iopub.status.idle":"2021-09-23T20:40:38.090036Z","shell.execute_reply.started":"2021-09-23T20:40:38.086114Z","shell.execute_reply":"2021-09-23T20:40:38.089173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor x_l_train, y_l_train in kfold.split(x_train, y_train):\n    x_s_train = x_train[x_l_train]\n    y_s_train = y_train[x_l_train]\n    train_dataset = (\n        tf.data.Dataset\n        .from_tensor_slices((x_s_train, y_s_train))\n        .repeat() #重複數據集count次數\n        .shuffle(2048) #隨機混洗數據集多元素\n        .batch(BATCH_SIZE) #將數據集多連續元素合成批次\n        .prefetch(AUTO)#將一部分內存加載到cache裡面\n    )\n    x_s_valid = x_train[y_l_train]\n    y_s_valid = y_train[y_l_train]\n    valid_dataset =(\n        tf.data.Dataset\n        .from_tensor_slices((x_s_valid, y_s_valid))\n        .batch(BATCH_SIZE)\n        .cache()\n        .prefetch(AUTO)\n    )\n    n_steps = x_s_train.shape[0] // BATCH_SIZE #讀取矩陣第一維度的長度\n    train_history = model.fit(\n        train_dataset,\n        steps_per_epoch=n_steps,\n        validation_data=valid_dataset,\n        epochs=EPOCHS,\n    ) # 使用model.fit()執行訓練過程\n    train_history_list.append(train_history)\n    print(\"-----------------------------------------------------------\")","metadata":{"execution":{"iopub.status.busy":"2021-09-23T20:40:38.091408Z","iopub.execute_input":"2021-09-23T20:40:38.091722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary() #輸出各層的輸出情況","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import statistics\n\nhis_val_loss = []\nhis_val_accuracy = []\n\nfor history in train_history_list:\n    his_val_loss.append(statistics.mean(history.history['val_loss']))\n    his_val_accuracy.append(statistics.mean(history.history['val_accuracy']))\nhis_val_loss.append(statistics.mean(his_val_loss))\nhis_val_accuracy.append(statistics.mean(his_val_accuracy))\n\nprint(his_val_loss[-1])\nprint(his_val_accuracy[-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights(WEIGHTS_PATH+\"/weights.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"n_steps = x_train.shape[0] // BATCH_SIZE #讀取矩陣第一維度的長度\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS,\n) # 使用model.fit()執行訓練過程","metadata":{"execution":{"iopub.status.busy":"2021-09-12T04:31:01.874005Z","iopub.status.idle":"2021-09-12T04:31:01.874928Z","shell.execute_reply.started":"2021-09-12T04:31:01.874701Z","shell.execute_reply":"2021-09-12T04:31:01.874729Z"}}},{"cell_type":"markdown","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS*2,\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T04:31:01.875859Z","iopub.status.idle":"2021-09-12T04:31:01.87621Z","shell.execute_reply.started":"2021-09-12T04:31:01.876032Z","shell.execute_reply":"2021-09-12T04:31:01.876054Z"}}},{"cell_type":"markdown","source":"model.save_weights(\"weights.h5\")","metadata":{"execution":{"iopub.status.busy":"2021-09-12T04:31:01.877069Z","iopub.status.idle":"2021-09-12T04:31:01.877421Z","shell.execute_reply.started":"2021-09-12T04:31:01.877241Z","shell.execute_reply":"2021-09-12T04:31:01.877265Z"}}},{"cell_type":"code","source":"# print(train_history)\n# print(train_history_2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.predict()返回值是數值,表示樣本屬於toxic類別的概率\n\n'''test_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)'''\n\nsub['toxic'] = model.predict(test_dataset, verbose=1)\n\nsub1 = sub[['id', 'toxic']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights(\"weights.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 保存架構\n#config = model.get_config()\n#new_model = keras.Model.from_config(config)\n\n#json_config = model.to_json()\n#new_model = keras.models.model_from_json(json_config)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# New in Tensorflow 2.4: models can be save locally from TPUs in Tensorflow's SavedModel format\n\n# TPUs need this extra setting to save to local disk, otherwise, they can only save models to GCS (Google Cloud Storage).\n# The setting instructs Tensorflow to retrieve all parameters from the TPU then do the saving from the local VM, not the TPU.\n# This setting does nothing on GPUs.\n#save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n#model.save('./model', options=save_locally) # saving in Tensorflow's \"saved model\" format","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.save('saved_model', save_format='tf')\n#with strategy.scope():\n#    load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\n#    model = tf.keras.models.load_model(OUTPUT_PATH+\"/model\", options=load_locally) # loading in Tensorflow's \"SavedModel\" format","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #checkpoint_1 = ModelCheckpoint(OUTPUT_PATH, monitor='train_history_2', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=1)\n# checkpoint = ModelCheckpoint(OUTPUT_PATH, monitor='history = model.fit()', verbose=1, save_best_only=True, mode='max')\n# print(checkpoint)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(sub1)\n# print(sub2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub1.rename(columns={'toxic':'toxic1'}, inplace=True)\n# sub2.rename(columns={'toxic':'toxic2'}, inplace=True) #命名文件或目錄\n# sub3 = pd.merge(sub1, sub2, how='left', on='id')\n\n# sub3['toxic'] = (sub3['toxic1'] * 0.1) + (sub3['toxic2'] * 0.9)\n# sub3['toxic'] = (sub3['toxic2'] * 0.39) + (sub3['toxic'] * 0.61)\n\n# sub3[['id', 'toxic']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}