{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-28T09:48:32.773627Z","iopub.execute_input":"2022-07-28T09:48:32.774029Z","iopub.status.idle":"2022-07-28T09:48:32.805434Z","shell.execute_reply.started":"2022-07-28T09:48:32.773952Z","shell.execute_reply":"2022-07-28T09:48:32.804440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ライブラリのインストール","metadata":{}},{"cell_type":"code","source":"!pip install japanize-matplotlib","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:32.807510Z","iopub.execute_input":"2022-07-28T09:48:32.808009Z","iopub.status.idle":"2022-07-28T09:48:46.588769Z","shell.execute_reply.started":"2022-07-28T09:48:32.807968Z","shell.execute_reply":"2022-07-28T09:48:46.587692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"import transformers\nfrom transformers import AutoTokenizer\nfrom tokenizers import BertWordPieceTokenizer\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport japanize_matplotlib\nimport seaborn as sns\n\nimport re\nimport string\n\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import layers\nfrom keras import regularizers\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom sklearn.metrics import f1_score","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:46.590531Z","iopub.execute_input":"2022-07-28T09:48:46.590895Z","iopub.status.idle":"2022-07-28T09:48:53.194126Z","shell.execute_reply.started":"2022-07-28T09:48:46.590865Z","shell.execute_reply":"2022-07-28T09:48:53.193148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 訓練データの読み込み","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/nlp-getting-started/train.csv')\ntest = pd.read_csv('../input/nlp-getting-started/test.csv')\nsubmission = pd.read_csv('../input/nlp-getting-started/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.198307Z","iopub.execute_input":"2022-07-28T09:48:53.200456Z","iopub.status.idle":"2022-07-28T09:48:53.263724Z","shell.execute_reply.started":"2022-07-28T09:48:53.200427Z","shell.execute_reply":"2022-07-28T09:48:53.262775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 特徴量の確認","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.265088Z","iopub.execute_input":"2022-07-28T09:48:53.265431Z","iopub.status.idle":"2022-07-28T09:48:53.304557Z","shell.execute_reply.started":"2022-07-28T09:48:53.265396Z","shell.execute_reply":"2022-07-28T09:48:53.303580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.306265Z","iopub.execute_input":"2022-07-28T09:48:53.306648Z","iopub.status.idle":"2022-07-28T09:48:53.320362Z","shell.execute_reply.started":"2022-07-28T09:48:53.306599Z","shell.execute_reply":"2022-07-28T09:48:53.319269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(100)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.321698Z","iopub.execute_input":"2022-07-28T09:48:53.322631Z","iopub.status.idle":"2022-07-28T09:48:53.339562Z","shell.execute_reply.started":"2022-07-28T09:48:53.322595Z","shell.execute_reply":"2022-07-28T09:48:53.338461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.340827Z","iopub.execute_input":"2022-07-28T09:48:53.341101Z","iopub.status.idle":"2022-07-28T09:48:53.354761Z","shell.execute_reply.started":"2022-07-28T09:48:53.341077Z","shell.execute_reply":"2022-07-28T09:48:53.353766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.356063Z","iopub.execute_input":"2022-07-28T09:48:53.356889Z","iopub.status.idle":"2022-07-28T09:48:53.365946Z","shell.execute_reply.started":"2022-07-28T09:48:53.356853Z","shell.execute_reply":"2022-07-28T09:48:53.364929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 災害ツイートは1、そうでないものは0\ntrain_BC=train['target'].value_counts().to_frame()\n\n## 棒グラフ\nsns.set(font='IPAexGothic')\naaa = sns.barplot(data=train_BC,x=train_BC.index,y=\"target\")\naaa.set_title('train.csvのtargetの分布')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.370783Z","iopub.execute_input":"2022-07-28T09:48:53.371419Z","iopub.status.idle":"2022-07-28T09:48:53.552141Z","shell.execute_reply.started":"2022-07-28T09:48:53.371382Z","shell.execute_reply":"2022-07-28T09:48:53.551107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keywordの検出件数上位10ワードを表示（0は欠損値）\ntrain.keyword.value_counts()[:10].to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.553645Z","iopub.execute_input":"2022-07-28T09:48:53.554480Z","iopub.status.idle":"2022-07-28T09:48:53.567816Z","shell.execute_reply.started":"2022-07-28T09:48:53.554439Z","shell.execute_reply":"2022-07-28T09:48:53.566912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## テストデータの確認","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.569720Z","iopub.execute_input":"2022-07-28T09:48:53.570428Z","iopub.status.idle":"2022-07-28T09:48:53.583271Z","shell.execute_reply.started":"2022-07-28T09:48:53.570389Z","shell.execute_reply":"2022-07-28T09:48:53.581710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 関数の定義","metadata":{}},{"cell_type":"code","source":"def encode_data(tokenizer, text_list, max_length):\n    tokenizer.enable_truncation(max_length)\n    tokenizer.enable_padding(length=max_length)\n    \n    encoded = tokenizer.encode_batch(text_list)\n    \n    id_list = []\n    \n    for item in encoded:\n        id_list.append(item.ids)\n        \n    return np.array(id_list)\n\ndef load_data(train_data_path, test_data_path):\n    train_data = pd.read_csv(train_data_path)\n    test_data = pd.read_csv(test_data_path)\n    \n    return train_data, test_data\n\ndef impute(dataset : pd.DataFrame):\n    dataset_copy = dataset.copy()\n    dataset_copy['keyword'].fillna('0', inplace=True)\n    dataset_copy['keyword'] = dataset_copy['keyword'].str.split('%20').str.join(',')\n    \n    return dataset_copy\n\ndef recall(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall_keras = true_positives / (possible_positives + K.epsilon())\n    return recall_keras\n\ndef precision(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision_keras = true_positives / (predicted_positives + K.epsilon())\n    return precision_keras\n\ndef f1(y_true, y_pred):\n    p = precision(y_true, y_pred)\n    r = recall(y_true, y_pred)\n    return 2 * ((p * r) / (p + r + K.epsilon()))\n\ndef convert_probabilities_to_predictions(probabilities):\n    predictions = [np.rint(x) for x in probabilities]\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.584596Z","iopub.execute_input":"2022-07-28T09:48:53.585477Z","iopub.status.idle":"2022-07-28T09:48:53.598988Z","shell.execute_reply.started":"2022-07-28T09:48:53.585440Z","shell.execute_reply":"2022-07-28T09:48:53.597612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 交差検証\n訓練データを80:20に分割し、80%を訓練データ、20%を検証データとすることで訓練データの汎用性を確かめる","metadata":{}},{"cell_type":"code","source":"train = impute(train) \ntest = impute(test)\n\nX_train, X_valid, y_train, y_valid = train_test_split(train[['text', 'keyword']],\n                                                      train.target,   \n                                                      train_size=0.8, \n                                                      random_state=1,\n                                                      shuffle=True)\n\n#DataFrameからnumpyに変換\ny_train = y_train.to_numpy()\ny_valid = y_valid.to_numpy()\n\ntrain_text = X_train['text'].to_numpy()\nvalid_text = X_valid['text'].to_numpy()\ntest_text = test['text'].to_numpy()\n\ntrain_keyword = X_train['keyword'].to_numpy()\nvalid_keyword = X_valid['keyword'].to_numpy()\ntest_keyword = test['keyword'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.600753Z","iopub.execute_input":"2022-07-28T09:48:53.601376Z","iopub.status.idle":"2022-07-28T09:48:53.631695Z","shell.execute_reply.started":"2022-07-28T09:48:53.601336Z","shell.execute_reply":"2022-07-28T09:48:53.630805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 単語の解析と変換\nBERTを使用する","metadata":{}},{"cell_type":"code","source":"#tokenizerの作成\n#tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\ntokenizer = transformers.BertTokenizer.from_pretrained('bert-base-cased')\ntokenizer.save_pretrained('.')\nbertWordPieceTokenizer = BertWordPieceTokenizer('vocab.txt', strip_accents=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:53.633045Z","iopub.execute_input":"2022-07-28T09:48:53.633559Z","iopub.status.idle":"2022-07-28T09:48:56.773018Z","shell.execute_reply.started":"2022-07-28T09:48:53.633526Z","shell.execute_reply":"2022-07-28T09:48:56.771855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#全データ(train, valid, test)のエンコード\ntrain_text_encoded = encode_data(bertWordPieceTokenizer, train_text, 200)\nvalid_text_encoded = encode_data(bertWordPieceTokenizer, valid_text, 200)\ntest_text_encoded = encode_data(bertWordPieceTokenizer, test_text, 200)\n\ntrain_keyword_encoded = encode_data(bertWordPieceTokenizer, train_keyword, 20)\nvalid_keyword_encoded = encode_data(bertWordPieceTokenizer, valid_keyword, 20)\ntest_keyword_encoded = encode_data(bertWordPieceTokenizer, test_keyword, 20)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:56.774988Z","iopub.execute_input":"2022-07-28T09:48:56.775264Z","iopub.status.idle":"2022-07-28T09:48:58.261375Z","shell.execute_reply.started":"2022-07-28T09:48:56.775238Z","shell.execute_reply":"2022-07-28T09:48:58.260433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_tranformer_layer = transformers.TFBertForSequenceClassification.from_pretrained('bert-base-cased')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:48:58.264553Z","iopub.execute_input":"2022-07-28T09:48:58.265156Z","iopub.status.idle":"2022-07-28T09:49:19.369671Z","shell.execute_reply.started":"2022-07-28T09:48:58.265119Z","shell.execute_reply":"2022-07-28T09:49:19.365247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = layers.Input(shape=(None,), dtype=tf.int32)\noutputs = text_tranformer_layer(inputs)[0]\noutputs = layers.Dense(1,activation='sigmoid')(outputs)\n\nmodel = Model(inputs=inputs, outputs=outputs)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:49:19.371743Z","iopub.execute_input":"2022-07-28T09:49:19.372334Z","iopub.status.idle":"2022-07-28T09:49:26.499788Z","shell.execute_reply.started":"2022-07-28T09:49:19.372294Z","shell.execute_reply":"2022-07-28T09:49:26.498817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:49:26.501409Z","iopub.execute_input":"2022-07-28T09:49:26.502176Z","iopub.status.idle":"2022-07-28T09:49:26.524057Z","shell.execute_reply.started":"2022-07-28T09:49:26.502133Z","shell.execute_reply":"2022-07-28T09:49:26.523037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 学習","metadata":{}},{"cell_type":"code","source":"learning_rate_scheduler = tf.keras.optimizers.schedules.InverseTimeDecay(\n    0.0001,\n    100,\n    0.0001,\n    staircase=False, \n    name=None\n)\n\ncallback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=4, restore_best_weights=True)\n\n\nAdam = tf.keras.optimizers.Adam(\n    learning_rate=learning_rate_scheduler,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=0.1,\n    amsgrad=False,\n    name=\"Adam\"\n)\n\nmodel.compile(optimizer=Adam, loss='binary_crossentropy', \n                  metrics=['accuracy',f1]\n                 )\nmodel.fit(train_text_encoded, y_train,\n          validation_data=(valid_text_encoded, y_valid),\n          callbacks=callback,\n          batch_size=32,\n          epochs=10)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T09:49:26.525681Z","iopub.execute_input":"2022-07-28T09:49:26.526351Z","iopub.status.idle":"2022-07-28T10:15:20.338253Z","shell.execute_reply.started":"2022-07-28T09:49:26.526315Z","shell.execute_reply":"2022-07-28T10:15:20.337300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## F1 scoreの算出","metadata":{}},{"cell_type":"code","source":"probabilities = model.predict(valid_text_encoded)\npredictions = convert_probabilities_to_predictions(probabilities)\n\nscore = f1_score(y_valid, predictions)\nscore","metadata":{"execution":{"iopub.status.busy":"2022-07-28T10:15:20.339504Z","iopub.execute_input":"2022-07-28T10:15:20.339889Z","iopub.status.idle":"2022-07-28T10:15:34.772533Z","shell.execute_reply.started":"2022-07-28T10:15:20.339852Z","shell.execute_reply":"2022-07-28T10:15:34.771476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 提出する形式に整理","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['id'] = test['id']\n\nprobabilities = model.predict(test_text_encoded)\npredictions = convert_probabilities_to_predictions(probabilities)\n\nsubmission['target'] = predictions\nsubmission['target'] = submission['target'].astype(int)\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T10:15:34.773873Z","iopub.execute_input":"2022-07-28T10:15:34.774811Z","iopub.status.idle":"2022-07-28T10:16:15.779208Z","shell.execute_reply.started":"2022-07-28T10:15:34.774773Z","shell.execute_reply":"2022-07-28T10:16:15.778255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 予測結果の確認","metadata":{}},{"cell_type":"code","source":"subm = pd.read_csv('submission.csv')\nsubm","metadata":{"execution":{"iopub.status.busy":"2022-07-28T10:16:15.780861Z","iopub.execute_input":"2022-07-28T10:16:15.781235Z","iopub.status.idle":"2022-07-28T10:16:15.797929Z","shell.execute_reply.started":"2022-07-28T10:16:15.781197Z","shell.execute_reply":"2022-07-28T10:16:15.796661Z"},"trusted":true},"execution_count":null,"outputs":[]}]}