{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint #導入tensorflow\nfrom tensorflow import keras\n\nfrom sklearn.model_selection import StratifiedKFold, KFold\n\nfrom kaggle_datasets import KaggleDatasets #採用Kaggle資料集\nimport transformers\n\nfrom tokenizers import BertWordPieceTokenizer #分詞器\nfrom tqdm import tqdm #進度條顯示\nimport numpy as np\n\n#!pip install wandb\n\n#基本模型導入\nimport os, time\nimport gc\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import KaggleDatasets\n\n!pip install bert-tensorflow\nimport bert.tokenization\n\nprint(tf.version.VERSION) #tensorflow版本輸出","metadata":{"_uuid":"5d5439b8-06af-4f21-89ce-9607b8f8d5c0","_cell_guid":"82449d2e-988b-4db9-8dec-a9ec972f39a7","collapsed":false,"papermill":{"duration":17.633435,"end_time":"2021-09-12T17:47:34.735304","exception":false,"start_time":"2021-09-12T17:47:17.101869","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:27:35.859099Z","iopub.execute_input":"2021-11-09T18:27:35.859654Z","iopub.status.idle":"2021-11-09T18:27:53.449190Z","shell.execute_reply.started":"2021-11-09T18:27:35.859621Z","shell.execute_reply":"2021-11-09T18:27:53.447786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 7\nn_splits = 5\n\n#kkfold = KFold(n_splits).split(x_train)\nkfold = StratifiedKFold(n_splits, shuffle=True, random_state=seed)\ncvscores = []","metadata":{"execution":{"iopub.status.busy":"2021-11-09T18:27:53.453396Z","iopub.execute_input":"2021-11-09T18:27:53.454152Z","iopub.status.idle":"2021-11-09T18:27:53.461359Z","shell.execute_reply.started":"2021-11-09T18:27:53.454074Z","shell.execute_reply":"2021-11-09T18:27:53.460269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(transformers.__version__) #tensorflow版本輸出","metadata":{"_uuid":"ce55a467-6310-4762-8c18-a666a7f2c5c0","_cell_guid":"9d7fc820-36ed-43bb-8f2f-1a4e0b8abc38","collapsed":false,"papermill":{"duration":0.038624,"end_time":"2021-09-12T17:47:34.804223","exception":false,"start_time":"2021-09-12T17:47:34.765599","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:27:53.462921Z","iopub.execute_input":"2021-11-09T18:27:53.463203Z","iopub.status.idle":"2021-11-09T18:27:53.476022Z","shell.execute_reply.started":"2021-11-09T18:27:53.463167Z","shell.execute_reply":"2021-11-09T18:27:53.475202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TPU 檢測. \ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu) #TPU的連接\nelse:\n    \n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n#在TPU上針對Kaggle用戶運行Bert模型","metadata":{"_uuid":"9960ac93-6ac4-4b3f-8c3f-bda392d196f4","_cell_guid":"74bc698f-2f16-4aa2-83e2-1230314ee01e","collapsed":false,"papermill":{"duration":5.781799,"end_time":"2021-09-12T17:47:40.617597","exception":false,"start_time":"2021-09-12T17:47:34.835798","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:27:53.477973Z","iopub.execute_input":"2021-11-09T18:27:53.478361Z","iopub.status.idle":"2021-11-09T18:27:59.136161Z","shell.execute_reply.started":"2021-11-09T18:27:53.478317Z","shell.execute_reply":"2021-11-09T18:27:59.134860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEQUENCE_LENGTH = 128 #一個輸入字串長度為128的list\n\n#設置Kaggle數據的訪問路徑\nDATA_PATH =  KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n#BERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')\n#BERT_PATH_SAVEDMODEL = BERT_PATH + \"/bert_multi_from_tfhub\"\nWEIGHTS_PATH = '../input/jigsaw-weights'\n\n\nOUTPUT_PATH = \"/kaggle/working\"","metadata":{"_uuid":"0dae9095-7ba0-427c-916b-451bb7b8ce25","_cell_guid":"54f3fada-7f08-4541-ae41-eaa4c9e27ff8","collapsed":false,"papermill":{"duration":0.981464,"end_time":"2021-09-12T17:47:41.630351","exception":false,"start_time":"2021-09-12T17:47:40.648887","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:27:59.139592Z","iopub.execute_input":"2021-11-09T18:27:59.139934Z","iopub.status.idle":"2021-11-09T18:27:59.577719Z","shell.execute_reply.started":"2021-11-09T18:27:59.139899Z","shell.execute_reply":"2021-11-09T18:27:59.576571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain1 = pd.read_csv(\"/kaggle/input/jigsawch/666666.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n# sub2 = pd.read_csv('../input/ensemble/submission.csv')","metadata":{"_uuid":"290c2303-7283-4426-b9b6-360a27351cbf","_cell_guid":"8ba129da-455d-475e-a813-b45230784433","collapsed":false,"papermill":{"duration":4.522124,"end_time":"2021-09-12T17:47:46.183581","exception":false,"start_time":"2021-09-12T17:47:41.661457","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:27:59.579737Z","iopub.execute_input":"2021-11-09T18:27:59.580013Z","iopub.status.idle":"2021-11-09T18:28:04.047424Z","shell.execute_reply.started":"2021-11-09T18:27:59.579982Z","shell.execute_reply":"2021-11-09T18:28:04.046362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = train1.dropna(how='any')\ntrain1 = train1.sample(n=100000,random_state = seed)","metadata":{"_uuid":"ab79331c-c8cb-41b8-9b45-df7ba83a72b7","_cell_guid":"060779e8-2dc5-4c77-b3f4-ca429d5f6f68","collapsed":false,"papermill":{"duration":0.031657,"end_time":"2021-09-12T17:47:46.248188","exception":false,"start_time":"2021-09-12T17:47:46.216531","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:04.049040Z","iopub.execute_input":"2021-11-09T18:28:04.049390Z","iopub.status.idle":"2021-11-09T18:28:04.265519Z","shell.execute_reply.started":"2021-11-09T18:28:04.049349Z","shell.execute_reply":"2021-11-09T18:28:04.264705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train1.head())","metadata":{"execution":{"iopub.status.busy":"2021-11-09T18:28:04.267043Z","iopub.execute_input":"2021-11-09T18:28:04.267280Z","iopub.status.idle":"2021-11-09T18:28:04.282836Z","shell.execute_reply.started":"2021-11-09T18:28:04.267254Z","shell.execute_reply":"2021-11-09T18:28:04.281860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"new=pd.DataFrame({'id':['63812'],\n                  'content':['你是白癡'],\n                  'lang':['zh']}) \ntest = test.append(new,ignore_index = True)\nprint(test.tail())","metadata":{"_uuid":"506c82be-f4d6-46f4-a0b9-91921e4ab4fe","_cell_guid":"42f58f9b-e796-45db-87f9-67c14bba449f","execution":{"iopub.execute_input":"2021-08-09T08:26:01.180343Z","iopub.status.busy":"2021-08-09T08:26:01.180014Z","iopub.status.idle":"2021-08-09T08:26:01.198418Z","shell.execute_reply":"2021-08-09T08:26:01.197677Z","shell.execute_reply.started":"2021-08-09T08:26:01.180314Z"},"papermill":{"duration":0.030677,"end_time":"2021-09-12T17:47:46.309632","exception":false,"start_time":"2021-09-12T17:47:46.278955","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"markdown","source":"# BERT Tokenizer","metadata":{"_uuid":"bcddbbf9-f83a-48bc-8959-6686360cf240","_cell_guid":"a8eef6a9-832e-4b41-a227-8accb34ad5ec","papermill":{"duration":0.030513,"end_time":"2021-09-12T17:47:46.371194","exception":false,"start_time":"2021-09-12T17:47:46.340681","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"#把文字切割並轉成BERT所需要的編碼\n\n# def get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):\n#     bert_layer = tf.saved_model.load(bert_path)\n#     bert_layer = hub.KerasLayer(bert_layer, trainable=False)\n#     vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy() \n#     cased = bert_layer.resolved_object.do_lower_case.numpy()\n#     tf.gfile = tf.io.gfile  \n#     tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)\n  \n#     return tokenizer\n\n# tokenizer = get_tokenizer()","metadata":{"_uuid":"4ff0511f-98e6-4a55-88ea-2bdebf630b51","_cell_guid":"f728543b-ee96-41b9-8b82-3d486c967c3e","collapsed":false,"papermill":{"duration":19.591119,"end_time":"2021-09-12T17:48:05.993334","exception":false,"start_time":"2021-09-12T17:47:46.402215","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:04.283955Z","iopub.execute_input":"2021-11-09T18:28:04.284187Z","iopub.status.idle":"2021-11-09T18:28:04.288790Z","shell.execute_reply.started":"2021-11-09T18:28:04.284160Z","shell.execute_reply":"2021-11-09T18:28:04.287850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"_uuid":"1758a7ff-213a-4c66-ad8d-99a2b123049a","_cell_guid":"e0cabfee-7df9-4867-a647-c41a0255f009","papermill":{"duration":0.032776,"end_time":"2021-09-12T17:48:06.059802","exception":false,"start_time":"2021-09-12T17:48:06.027026","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"#編碼器，用於將文本編碼為整數序列，以進行BERT輸入\n\ndef fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):#批次上傳256，最長序列512\n    \n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(length=maxlen) #最大長度為512，不足會自動補0\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist() #將數據轉換為最接近Python的類型\n        encs = tokenizer.encode_batch(text_chunk)\n        #print(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n        \n    \n    return np.array(all_ids)","metadata":{"_uuid":"bebef34f-e587-42b7-b5b2-d3fb5afef9d2","_cell_guid":"777cde54-7112-4ad0-9aee-4affcc801f4d","collapsed":false,"papermill":{"duration":0.042877,"end_time":"2021-09-12T17:48:06.13501","exception":false,"start_time":"2021-09-12T17:48:06.092133","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:04.290396Z","iopub.execute_input":"2021-11-09T18:28:04.290710Z","iopub.status.idle":"2021-11-09T18:28:04.301253Z","shell.execute_reply.started":"2021-11-09T18:28:04.290669Z","shell.execute_reply":"2021-11-09T18:28:04.300534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#用於配置的IMP數據\n\nAUTO = tf.data.experimental.AUTOTUNE\n\n\n# 配置\nEPOCHS = 5 #定義訓練過程數據輪5次\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync  #資料集大小\nMAX_LEN = 192","metadata":{"_uuid":"f2a74416-cb2a-41c1-8a6b-ff61d9def53f","_cell_guid":"b75e18f5-51d0-4e9a-83fe-3f417be57748","collapsed":false,"papermill":{"duration":0.040588,"end_time":"2021-09-12T17:48:06.208295","exception":false,"start_time":"2021-09-12T17:48:06.167707","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:04.302777Z","iopub.execute_input":"2021-11-09T18:28:04.303329Z","iopub.status.idle":"2021-11-09T18:28:04.313014Z","shell.execute_reply.started":"2021-11-09T18:28:04.303295Z","shell.execute_reply":"2021-11-09T18:28:04.311952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')# 使用分詞器加載DistilBERT\n\ntokenizer.save_pretrained('.') #儲存\n\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\nfast_tokenizer  #利用 huggingface tokenizers庫 重新加載詞向量，lowercase=False:詞向量皆為大寫","metadata":{"_uuid":"8935b452-b130-4d13-bb81-908456721db7","_cell_guid":"bbdf2691-486e-49d0-b2d8-226ba49cdb7c","collapsed":false,"papermill":{"duration":3.299386,"end_time":"2021-09-12T17:48:09.540365","exception":false,"start_time":"2021-09-12T17:48:06.240979","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:04.314499Z","iopub.execute_input":"2021-11-09T18:28:04.314772Z","iopub.status.idle":"2021-11-09T18:28:07.221536Z","shell.execute_reply.started":"2021-11-09T18:28:04.314743Z","shell.execute_reply":"2021-11-09T18:28:07.220632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#快速編碼\nx_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n\ny_train = train1.toxic.values\ny_valid = valid.toxic.values","metadata":{"_uuid":"c36bb531-e9cd-4cdb-8ea1-e3f67b6f6382","_cell_guid":"df87aefe-57e1-434f-8e88-31c457d8f74e","collapsed":false,"papermill":{"duration":59.187406,"end_time":"2021-09-12T17:49:08.763519","exception":false,"start_time":"2021-09-12T17:48:09.576113","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:07.222865Z","iopub.execute_input":"2021-11-09T18:28:07.223096Z","iopub.status.idle":"2021-11-09T18:28:44.270388Z","shell.execute_reply.started":"2021-11-09T18:28:07.223069Z","shell.execute_reply":"2021-11-09T18:28:44.269261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#訓練BERT模型\n\ndef build_model(transformer, max_len=512):  #建立模型，輸入句子最大長度512\n    \n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\") #dtype=tf.int 返回數據元素的數據類型int\n    sequence_output = transformer(input_word_ids)[0] #BERT模型的輸出 \n    cls_token = sequence_output[:, 0, :]\n    \n    #激活函數\n    out = tf.keras.layers.Dense(300, activation='relu')(cls_token)\n    out = tf.keras.layers.Dense(128, activation='relu')(out)\n    out = tf.keras.layers.Dense(128, activation='relu')(out)\n    out = Dense(1, activation='sigmoid')(out) #relu線性函數激活 sigmoid非線性激活函數\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy']) #損失函數的用法，Adam是優化器，loss：計算損失\n    \n    return model","metadata":{"_uuid":"e8c52005-5d28-4b78-93a8-52e17eac2209","_cell_guid":"76138b89-e4e9-4f67-b904-9b1c1e3b0705","collapsed":false,"papermill":{"duration":0.164378,"end_time":"2021-09-12T17:49:09.395707","exception":false,"start_time":"2021-09-12T17:49:09.231329","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:28:44.273132Z","iopub.execute_input":"2021-11-09T18:28:44.273425Z","iopub.status.idle":"2021-11-09T18:28:44.285798Z","shell.execute_reply.started":"2021-11-09T18:28:44.273391Z","shell.execute_reply":"2021-11-09T18:28:44.284797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# 轉化成數據集 生成對應的Dataset\n\ntrain_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat() #重複數據集count次數\n    .shuffle(2048) #隨機混洗數據集多元素\n    .batch(BATCH_SIZE) #將數據集多連續元素合成批次\n    .prefetch(AUTO)#將一部分內存加載到cache裡面\n)\nvalid_dataset =(\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","metadata":{"_uuid":"11b6db5f-5fe8-4006-806c-ca8aa3eff22c","_cell_guid":"9b9ef98d-4970-4e41-b3bd-059319ad964f","execution":{"iopub.execute_input":"2021-09-12T17:49:09.729989Z","iopub.status.busy":"2021-09-12T17:49:09.729301Z","iopub.status.idle":"2021-09-12T17:49:10.201756Z","shell.execute_reply":"2021-09-12T17:49:10.201195Z","shell.execute_reply.started":"2021-09-12T13:27:31.14555Z"},"papermill":{"duration":0.655175,"end_time":"2021-09-12T17:49:10.201907","exception":false,"start_time":"2021-09-12T17:49:09.546732","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"alldatalen=[]\nfor i in range(100000):\n    alldatalen.append(i)\n# print(alldatalen)","metadata":{"execution":{"iopub.status.busy":"2021-11-09T18:28:44.287285Z","iopub.execute_input":"2021-11-09T18:28:44.287625Z","iopub.status.idle":"2021-11-09T18:28:44.314679Z","shell.execute_reply.started":"2021-11-09T18:28:44.287579Z","shell.execute_reply":"2021-11-09T18:28:44.313156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"foldlist = []\nnewfoldtrain = []\nnewfoldval = []\nfor x_s_train, y_s_train in kfold.split(x_train, y_train):\n#     print(x_train[x_s_train])\n#     print(y_train[x_s_train])\n#     print(x_s_train)\n    foldlist.append(y_s_train)\n    print(len(x_s_train))\nfor i in range(5):\n    newfoldtrain.append(np.concatenate([foldlist[i],foldlist[(i-1)]]))\n    newfoldval.append(np.setdiff1d(alldatalen,newfoldtrain[i]))\n    print(len(newfoldtrain[i]))\n    print(len(newfoldval[i]))","metadata":{"_uuid":"97b3bfd9-e7f2-4d54-83d7-081904a90bd6","_cell_guid":"4af3bf1c-e3e6-444c-ad96-1f8fff700f53","collapsed":false,"papermill":{"duration":0.207867,"end_time":"2021-09-12T17:49:10.868581","exception":false,"start_time":"2021-09-12T17:49:10.660714","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-09T18:30:56.930771Z","iopub.execute_input":"2021-11-09T18:30:56.931143Z","iopub.status.idle":"2021-11-09T18:30:57.102267Z","shell.execute_reply.started":"2021-11-09T18:30:56.931092Z","shell.execute_reply":"2021-11-09T18:30:57.101126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope(): #表明分散式執行的程式碼區塊\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)","metadata":{"_uuid":"a74d046e-b17e-4631-baec-76339289b452","_cell_guid":"42e20ff4-67c8-4308-aebd-906c4a3a07ff","collapsed":false,"papermill":{"duration":45.086611,"end_time":"2021-09-12T17:49:56.418238","exception":false,"start_time":"2021-09-12T17:49:11.331627","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T15:47:11.566793Z","iopub.execute_input":"2021-11-07T15:47:11.567085Z","iopub.status.idle":"2021-11-07T15:47:27.684991Z","shell.execute_reply.started":"2021-11-07T15:47:11.567018Z","shell.execute_reply":"2021-11-07T15:47:27.684047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history_list = []","metadata":{"_uuid":"1a69eeee-e4d7-4300-93c6-0546ac781364","_cell_guid":"a9fad273-cf58-4b72-8ace-516f402ed1b1","collapsed":false,"papermill":{"duration":0.181199,"end_time":"2021-09-12T17:49:56.760183","exception":false,"start_time":"2021-09-12T17:49:56.578984","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T15:47:27.686469Z","iopub.execute_input":"2021-11-07T15:47:27.686745Z","iopub.status.idle":"2021-11-07T15:47:27.691385Z","shell.execute_reply.started":"2021-11-07T15:47:27.686713Z","shell.execute_reply":"2021-11-07T15:47:27.690313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for a,b in kfold.split(x_train, y_train):\n    print(a,b)\n    print(len(a),len(b))\n    print(type(a),type(b))","metadata":{"execution":{"iopub.status.busy":"2021-11-07T15:47:27.692727Z","iopub.execute_input":"2021-11-07T15:47:27.693058Z","iopub.status.idle":"2021-11-07T15:47:27.72572Z","shell.execute_reply.started":"2021-11-07T15:47:27.693Z","shell.execute_reply":"2021-11-07T15:47:27.724755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# for i in range(5):\n#     x_s_train = x_train[newfoldtrain[i]]\n#     y_s_train = y_train[newfoldtrain[i]]\n#     train_dataset = (\n#         tf.data.Dataset\n#         .from_tensor_slices((x_s_train, y_s_train))\n#         .repeat() #重複數據集count次數\n#         .shuffle(2048) #隨機混洗數據集多元素\n#         .batch(BATCH_SIZE) #將數據集多連續元素合成批次\n#         .prefetch(AUTO)#將一部分內存加載到cache裡面\n#     )\n#     x_s_valid = x_train[newfoldval[i]]\n#     y_s_valid = y_train[newfoldval[i]]\n#     valid_dataset =(\n#         tf.data.Dataset\n#         .from_tensor_slices((x_s_valid, y_s_valid))\n#         .batch(BATCH_SIZE)\n#         .cache()\n#         .prefetch(AUTO)\n#     )\n#     n_steps = x_s_train.shape[0] // BATCH_SIZE #讀取矩陣第一維度的長度\n#     train_history = model.fit(\n#         train_dataset,\n#         steps_per_epoch=n_steps,\n#         validation_data=valid_dataset,\n#         epochs=EPOCHS,\n#     ) # 使用model.fit()執行訓練過程\n#     train_history_list.append(train_history)\n#     print(\"-----------------------------------------------------------\")","metadata":{"execution":{"iopub.status.busy":"2021-11-07T15:47:27.727154Z","iopub.execute_input":"2021-11-07T15:47:27.72764Z","iopub.status.idle":"2021-11-07T16:02:35.397311Z","shell.execute_reply.started":"2021-11-07T15:47:27.727588Z","shell.execute_reply":"2021-11-07T16:02:35.396229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor x_l_train, y_l_train in kfold.split(x_train, y_train):\n    x_s_train = x_train[y_l_train]\n    y_s_train = y_train[y_l_train]\n    train_dataset = (\n        tf.data.Dataset\n        .from_tensor_slices((x_s_train, y_s_train))\n        .repeat() #重複數據集count次數\n        .shuffle(2048) #隨機混洗數據集多元素\n        .batch(BATCH_SIZE) #將數據集多連續元素合成批次\n        .prefetch(AUTO)#將一部分內存加載到cache裡面\n    )\n    x_s_valid = x_train[x_l_train]\n    y_s_valid = y_train[x_l_train]\n    valid_dataset =(\n        tf.data.Dataset\n        .from_tensor_slices((x_s_valid, y_s_valid))\n        .batch(BATCH_SIZE)\n        .cache()\n        .prefetch(AUTO)\n    )\n    n_steps = x_s_train.shape[0] // BATCH_SIZE #讀取矩陣第一維度的長度\n    train_history = model.fit(\n        train_dataset,\n        steps_per_epoch=n_steps,\n        validation_data=valid_dataset,\n        epochs=EPOCHS,\n    ) # 使用model.fit()執行訓練過程\n    train_history_list.append(train_history)\n    print(\"-----------------------------------------------------------\")","metadata":{"_uuid":"20850112-813e-47ce-8880-be4f7d2655f1","_cell_guid":"9bc735ab-b9e9-4503-bd26-c30b5a2e71c9","collapsed":false,"papermill":{"duration":2892.82231,"end_time":"2021-09-12T18:38:09.746815","exception":false,"start_time":"2021-09-12T17:49:56.924505","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.398974Z","iopub.execute_input":"2021-11-07T16:02:35.399299Z","iopub.status.idle":"2021-11-07T16:02:35.404046Z","shell.execute_reply.started":"2021-11-07T16:02:35.399266Z","shell.execute_reply":"2021-11-07T16:02:35.403373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary() #輸出各層的輸出情況","metadata":{"_uuid":"04971ffc-563d-4973-bea9-04f03b771afc","_cell_guid":"7846ec56-b050-4cca-8e7c-eae1db2068cf","collapsed":false,"papermill":{"duration":10.871232,"end_time":"2021-09-12T18:38:31.568194","exception":false,"start_time":"2021-09-12T18:38:20.696962","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.405084Z","iopub.execute_input":"2021-11-07T16:02:35.40576Z","iopub.status.idle":"2021-11-07T16:02:35.436586Z","shell.execute_reply.started":"2021-11-07T16:02:35.405711Z","shell.execute_reply":"2021-11-07T16:02:35.435477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for history in train_history_list:\n    print(history.history)","metadata":{"execution":{"iopub.status.busy":"2021-11-07T16:02:35.43806Z","iopub.execute_input":"2021-11-07T16:02:35.438364Z","iopub.status.idle":"2021-11-07T16:02:35.448521Z","shell.execute_reply.started":"2021-11-07T16:02:35.438333Z","shell.execute_reply":"2021-11-07T16:02:35.447626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import statistics\n\nhis_val_loss = []\nhis_val_accuracy = []\n\nfor history in train_history_list:\n    his_val_loss.append(statistics.mean(history.history['val_loss']))\n    his_val_accuracy.append(statistics.mean(history.history['val_accuracy']))\nhis_val_loss.append(statistics.mean(his_val_loss))\nhis_val_accuracy.append(statistics.mean(his_val_accuracy))\n\nprint(his_val_loss[-1])\nprint(his_val_accuracy[-1])","metadata":{"_uuid":"c0923e79-b9e7-4a96-9662-276a3a7e5b35","_cell_guid":"f6f76a9f-93bb-484b-bd1d-871f772f914b","collapsed":false,"papermill":{"duration":11.566027,"end_time":"2021-09-12T18:38:54.696291","exception":false,"start_time":"2021-09-12T18:38:43.130264","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.449939Z","iopub.execute_input":"2021-11-07T16:02:35.450373Z","iopub.status.idle":"2021-11-07T16:02:35.466088Z","shell.execute_reply.started":"2021-11-07T16:02:35.450339Z","shell.execute_reply":"2021-11-07T16:02:35.465125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights(WEIGHTS_PATH+\"/weights.h5\")","metadata":{"_uuid":"c244d463-830b-402b-b386-abea440c5650","_cell_guid":"97707052-de47-4323-8f6e-9939577a7115","collapsed":false,"papermill":{"duration":10.900534,"end_time":"2021-09-12T18:39:16.553204","exception":false,"start_time":"2021-09-12T18:39:05.65267","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.467428Z","iopub.execute_input":"2021-11-07T16:02:35.467968Z","iopub.status.idle":"2021-11-07T16:02:35.471856Z","shell.execute_reply.started":"2021-11-07T16:02:35.467933Z","shell.execute_reply":"2021-11-07T16:02:35.471112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"n_steps = x_train.shape[0] // BATCH_SIZE #讀取矩陣第一維度的長度\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS,\n) # 使用model.fit()執行訓練過程","metadata":{"_uuid":"fc81050b-acc4-4f77-8caa-e9c5905f1d0b","_cell_guid":"33a9066a-a6ab-47f0-9e5e-a81cb5281c98","execution":{"iopub.status.busy":"2021-09-12T04:31:01.874005Z","iopub.status.idle":"2021-09-12T04:31:01.874928Z","shell.execute_reply":"2021-09-12T04:31:01.874729Z","shell.execute_reply.started":"2021-09-12T04:31:01.874701Z"},"papermill":{"duration":10.901888,"end_time":"2021-09-12T18:39:38.249963","exception":false,"start_time":"2021-09-12T18:39:27.348075","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"markdown","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS*2,\n)","metadata":{"_uuid":"6de52d84-a03b-4630-8778-dda56070d03b","_cell_guid":"85b4afcd-d0b8-43c2-aa2d-2983ac99286c","execution":{"iopub.status.busy":"2021-09-12T04:31:01.875859Z","iopub.status.idle":"2021-09-12T04:31:01.87621Z","shell.execute_reply":"2021-09-12T04:31:01.876054Z","shell.execute_reply.started":"2021-09-12T04:31:01.876032Z"},"papermill":{"duration":10.885346,"end_time":"2021-09-12T18:39:59.971338","exception":false,"start_time":"2021-09-12T18:39:49.085992","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"markdown","source":"model.save_weights(\"weights.h5\")","metadata":{"_uuid":"b98bd5e3-8018-4d0c-92a9-dfe5d4343ec7","_cell_guid":"106b7024-a251-42b3-b0ab-65d94e92c441","execution":{"iopub.status.busy":"2021-09-12T04:31:01.877069Z","iopub.status.idle":"2021-09-12T04:31:01.877421Z","shell.execute_reply":"2021-09-12T04:31:01.877265Z","shell.execute_reply.started":"2021-09-12T04:31:01.877241Z"},"papermill":{"duration":10.924558,"end_time":"2021-09-12T18:40:21.724099","exception":false,"start_time":"2021-09-12T18:40:10.799541","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# print(train_history)\n# print(train_history_2)","metadata":{"_uuid":"7e9b4dab-34e2-4f0e-b23c-19bded1712c8","_cell_guid":"6f49d7d7-9947-4645-9dd8-0a9004c89648","collapsed":false,"papermill":{"duration":10.881731,"end_time":"2021-09-12T18:40:43.551723","exception":false,"start_time":"2021-09-12T18:40:32.669992","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.472935Z","iopub.execute_input":"2021-11-07T16:02:35.473248Z","iopub.status.idle":"2021-11-07T16:02:35.483268Z","shell.execute_reply.started":"2021-11-07T16:02:35.473219Z","shell.execute_reply":"2021-11-07T16:02:35.482347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.predict()返回值是數值,表示樣本屬於toxic類別的概率\n\n'''test_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)'''\n\n# sub['toxic'] = model.predict(test_dataset, verbose=1)\n\n# sub1 = sub[['id', 'toxic']]","metadata":{"_uuid":"db2b8f0c-1265-47ce-a294-71b983e07071","_cell_guid":"b8846cee-3803-4a4c-980c-cdd39c6de970","collapsed":false,"papermill":{"duration":30.538256,"end_time":"2021-09-12T18:41:24.935275","exception":false,"start_time":"2021-09-12T18:40:54.397019","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.484591Z","iopub.execute_input":"2021-11-07T16:02:35.484904Z","iopub.status.idle":"2021-11-07T16:02:35.493976Z","shell.execute_reply.started":"2021-11-07T16:02:35.484873Z","shell.execute_reply":"2021-11-07T16:02:35.493118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights(\"weights.h5\")","metadata":{"_uuid":"47893a4d-1352-46d0-851d-b78d5fccb009","_cell_guid":"9ff1d92c-b711-4635-9aa4-00733d44d129","collapsed":false,"papermill":{"duration":10.860563,"end_time":"2021-09-12T18:41:46.753761","exception":false,"start_time":"2021-09-12T18:41:35.893198","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-07T16:02:35.494989Z","iopub.execute_input":"2021-11-07T16:02:35.49523Z","iopub.status.idle":"2021-11-07T16:02:35.507095Z","shell.execute_reply.started":"2021-11-07T16:02:35.495204Z","shell.execute_reply":"2021-11-07T16:02:35.506204Z"},"trusted":true},"execution_count":null,"outputs":[]}]}