{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-23T02:11:35.258308Z","iopub.execute_input":"2022-07-23T02:11:35.258690Z","iopub.status.idle":"2022-07-23T02:11:35.267758Z","shell.execute_reply.started":"2022-07-23T02:11:35.258659Z","shell.execute_reply":"2022-07-23T02:11:35.266610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ライブラリのインストール","metadata":{}},{"cell_type":"code","source":"!pip install japanize-matplotlib","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:35.290224Z","iopub.execute_input":"2022-07-23T02:11:35.290516Z","iopub.status.idle":"2022-07-23T02:11:44.635633Z","shell.execute_reply.started":"2022-07-23T02:11:35.290490Z","shell.execute_reply":"2022-07-23T02:11:44.634456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"import transformers\nfrom transformers import AutoTokenizer\nfrom tokenizers import BertWordPieceTokenizer\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport japanize_matplotlib\nimport seaborn as sns\n\nimport re\nimport string\n\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import layers\nfrom keras import regularizers\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom sklearn.metrics import f1_score","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.639824Z","iopub.execute_input":"2022-07-23T02:11:44.640151Z","iopub.status.idle":"2022-07-23T02:11:44.650001Z","shell.execute_reply.started":"2022-07-23T02:11:44.640122Z","shell.execute_reply":"2022-07-23T02:11:44.649049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 訓練データの読み込み","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/nlp-getting-started/train.csv')\ntest = pd.read_csv('../input/nlp-getting-started/test.csv')\nsubmission = pd.read_csv('../input/nlp-getting-started/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.652077Z","iopub.execute_input":"2022-07-23T02:11:44.652441Z","iopub.status.idle":"2022-07-23T02:11:44.696220Z","shell.execute_reply.started":"2022-07-23T02:11:44.652393Z","shell.execute_reply":"2022-07-23T02:11:44.695254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 特徴量の確認","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.699191Z","iopub.execute_input":"2022-07-23T02:11:44.699910Z","iopub.status.idle":"2022-07-23T02:11:44.718651Z","shell.execute_reply.started":"2022-07-23T02:11:44.699865Z","shell.execute_reply":"2022-07-23T02:11:44.717661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.720210Z","iopub.execute_input":"2022-07-23T02:11:44.720802Z","iopub.status.idle":"2022-07-23T02:11:44.731531Z","shell.execute_reply.started":"2022-07-23T02:11:44.720766Z","shell.execute_reply":"2022-07-23T02:11:44.730432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(100)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:54:19.312334Z","iopub.execute_input":"2022-07-23T02:54:19.312889Z","iopub.status.idle":"2022-07-23T02:54:19.329321Z","shell.execute_reply.started":"2022-07-23T02:54:19.312844Z","shell.execute_reply":"2022-07-23T02:54:19.328290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.750911Z","iopub.execute_input":"2022-07-23T02:11:44.751509Z","iopub.status.idle":"2022-07-23T02:11:44.762580Z","shell.execute_reply.started":"2022-07-23T02:11:44.751473Z","shell.execute_reply":"2022-07-23T02:11:44.761479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.764075Z","iopub.execute_input":"2022-07-23T02:11:44.764660Z","iopub.status.idle":"2022-07-23T02:11:44.771988Z","shell.execute_reply.started":"2022-07-23T02:11:44.764625Z","shell.execute_reply":"2022-07-23T02:11:44.770986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 災害ツイートは1、そうでないものは0\ntrain_BC=train['target'].value_counts().to_frame()\n\n## 棒グラフ\nsns.set(font='IPAexGothic')\naaa = sns.barplot(data=train_BC,x=train_BC.index,y=\"target\")\naaa.set_title('train.csvのtargetの分布')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.773642Z","iopub.execute_input":"2022-07-23T02:11:44.774417Z","iopub.status.idle":"2022-07-23T02:11:44.924783Z","shell.execute_reply.started":"2022-07-23T02:11:44.774380Z","shell.execute_reply":"2022-07-23T02:11:44.923622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keywordの検出件数上位10ワードを表示（0は欠損値）\ntrain.keyword.value_counts()[:10].to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.929549Z","iopub.execute_input":"2022-07-23T02:11:44.930242Z","iopub.status.idle":"2022-07-23T02:11:44.944157Z","shell.execute_reply.started":"2022-07-23T02:11:44.930201Z","shell.execute_reply":"2022-07-23T02:11:44.943208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## テストデータの確認","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.945605Z","iopub.execute_input":"2022-07-23T02:11:44.946316Z","iopub.status.idle":"2022-07-23T02:11:44.958944Z","shell.execute_reply.started":"2022-07-23T02:11:44.946277Z","shell.execute_reply":"2022-07-23T02:11:44.957826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 関数の定義","metadata":{}},{"cell_type":"code","source":"def encode_data(tokenizer, text_list, max_length):\n    tokenizer.enable_truncation(max_length)\n    tokenizer.enable_padding(length=max_length)\n    \n    encoded = tokenizer.encode_batch(text_list)\n    \n    id_list = []\n    \n    for item in encoded:\n        id_list.append(item.ids)\n        \n    return np.array(id_list)\n\ndef load_data(train_data_path, test_data_path):\n    train_data = pd.read_csv(train_data_path)\n    test_data = pd.read_csv(test_data_path)\n    \n    return train_data, test_data\n\ndef impute(dataset : pd.DataFrame):\n    dataset_copy = dataset.copy()\n    dataset_copy['keyword'].fillna('0', inplace=True)\n    dataset_copy['keyword'] = dataset_copy['keyword'].str.split('%20').str.join(',')\n    \n    return dataset_copy\n\ndef recall(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall_keras = true_positives / (possible_positives + K.epsilon())\n    return recall_keras\n\ndef precision(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision_keras = true_positives / (predicted_positives + K.epsilon())\n    return precision_keras\n\ndef f1(y_true, y_pred):\n    p = precision(y_true, y_pred)\n    r = recall(y_true, y_pred)\n    return 2 * ((p * r) / (p + r + K.epsilon()))\n\ndef convert_probabilities_to_predictions(probabilities):\n    predictions = [np.rint(x) for x in probabilities]\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.960795Z","iopub.execute_input":"2022-07-23T02:11:44.961468Z","iopub.status.idle":"2022-07-23T02:11:44.974759Z","shell.execute_reply.started":"2022-07-23T02:11:44.961431Z","shell.execute_reply":"2022-07-23T02:11:44.973813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 交差検証\n訓練データを80:20に分割し、80%を訓練データ、20%を検証データとすることで訓練データの汎用性を確かめる","metadata":{}},{"cell_type":"code","source":"train = impute(train) \ntest = impute(test)\n\nX_train, X_valid, y_train, y_valid = train_test_split(train[['text', 'keyword']],\n                                                      train.target,   \n                                                      train_size=0.8, \n                                                      random_state=1,\n                                                      shuffle=True)\n\n#DataFrameからnumpyに変換\ny_train = y_train.to_numpy()\ny_valid = y_valid.to_numpy()\n\ntrain_text = X_train['text'].to_numpy()\nvalid_text = X_valid['text'].to_numpy()\ntest_text = test['text'].to_numpy()\n\ntrain_keyword = X_train['keyword'].to_numpy()\nvalid_keyword = X_valid['keyword'].to_numpy()\ntest_keyword = test['keyword'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:44.975859Z","iopub.execute_input":"2022-07-23T02:11:44.976301Z","iopub.status.idle":"2022-07-23T02:11:45.007636Z","shell.execute_reply.started":"2022-07-23T02:11:44.976262Z","shell.execute_reply":"2022-07-23T02:11:45.006749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 単語の解析と変換\nBERTを使用する","metadata":{}},{"cell_type":"code","source":"#tokenizerの作成\n#tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\ntokenizer = transformers.BertTokenizer.from_pretrained('bert-base-cased')\ntokenizer.save_pretrained('.')\nbertWordPieceTokenizer = BertWordPieceTokenizer('vocab.txt', strip_accents=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:45.009228Z","iopub.execute_input":"2022-07-23T02:11:45.009891Z","iopub.status.idle":"2022-07-23T02:11:46.812866Z","shell.execute_reply.started":"2022-07-23T02:11:45.009853Z","shell.execute_reply":"2022-07-23T02:11:46.811900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#全データ(train, valid, test)のエンコード\ntrain_text_encoded = encode_data(bertWordPieceTokenizer, train_text, 200)\nvalid_text_encoded = encode_data(bertWordPieceTokenizer, valid_text, 200)\ntest_text_encoded = encode_data(bertWordPieceTokenizer, test_text, 200)\n\ntrain_keyword_encoded = encode_data(bertWordPieceTokenizer, train_keyword, 20)\nvalid_keyword_encoded = encode_data(bertWordPieceTokenizer, valid_keyword, 20)\ntest_keyword_encoded = encode_data(bertWordPieceTokenizer, test_keyword, 20)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:46.814560Z","iopub.execute_input":"2022-07-23T02:11:46.814898Z","iopub.status.idle":"2022-07-23T02:11:48.260718Z","shell.execute_reply.started":"2022-07-23T02:11:46.814871Z","shell.execute_reply":"2022-07-23T02:11:48.259747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_tranformer_layer = transformers.TFBertForSequenceClassification.from_pretrained('bert-base-cased')","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:48.262143Z","iopub.execute_input":"2022-07-23T02:11:48.262515Z","iopub.status.idle":"2022-07-23T02:11:49.998470Z","shell.execute_reply.started":"2022-07-23T02:11:48.262479Z","shell.execute_reply":"2022-07-23T02:11:49.997579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = layers.Input(shape=(None,), dtype=tf.int32)\noutputs = text_tranformer_layer(inputs)[0]\noutputs = layers.Dense(1,activation='sigmoid')(outputs)\n\nmodel = Model(inputs=inputs, outputs=outputs)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:50.002650Z","iopub.execute_input":"2022-07-23T02:11:50.003683Z","iopub.status.idle":"2022-07-23T02:11:51.963852Z","shell.execute_reply.started":"2022-07-23T02:11:50.003640Z","shell.execute_reply":"2022-07-23T02:11:51.962943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:51.965270Z","iopub.execute_input":"2022-07-23T02:11:51.965848Z","iopub.status.idle":"2022-07-23T02:11:51.986250Z","shell.execute_reply.started":"2022-07-23T02:11:51.965808Z","shell.execute_reply":"2022-07-23T02:11:51.985244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 学習","metadata":{}},{"cell_type":"code","source":"learning_rate_scheduler = tf.keras.optimizers.schedules.InverseTimeDecay(\n    0.0001,\n    100,\n    0.0001,\n    staircase=False, \n    name=None\n)\n\ncallback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=4, restore_best_weights=True)\n\n\nAdam = tf.keras.optimizers.Adam(\n    learning_rate=learning_rate_scheduler,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=0.1,\n    amsgrad=False,\n    name=\"Adam\"\n)\n\nmodel.compile(optimizer=Adam, loss='binary_crossentropy', \n                  metrics=['accuracy',f1]\n                 )\nmodel.fit(train_text_encoded, y_train,\n          validation_data=(valid_text_encoded, y_valid),\n          callbacks=callback,\n          batch_size=32,\n          epochs=10)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:11:51.987519Z","iopub.execute_input":"2022-07-23T02:11:51.987963Z","iopub.status.idle":"2022-07-23T02:20:28.579691Z","shell.execute_reply.started":"2022-07-23T02:11:51.987908Z","shell.execute_reply":"2022-07-23T02:20:28.578560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## F1 scoreの算出","metadata":{}},{"cell_type":"code","source":"probabilities = model.predict(valid_text_encoded)\npredictions = convert_probabilities_to_predictions(probabilities)\n\nscore = f1_score(y_valid, predictions)\nscore","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:20:28.581393Z","iopub.execute_input":"2022-07-23T02:20:28.581805Z","iopub.status.idle":"2022-07-23T02:20:51.953622Z","shell.execute_reply.started":"2022-07-23T02:20:28.581767Z","shell.execute_reply":"2022-07-23T02:20:51.952612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 提出する形式に整理","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['id'] = test['id']\n\nprobabilities = model.predict(test_text_encoded)\npredictions = convert_probabilities_to_predictions(probabilities)\n\nsubmission['target'] = predictions\nsubmission['target'] = submission['target'].astype(int)\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:20:51.954990Z","iopub.execute_input":"2022-07-23T02:20:51.955441Z","iopub.status.idle":"2022-07-23T02:21:17.521906Z","shell.execute_reply.started":"2022-07-23T02:20:51.955400Z","shell.execute_reply":"2022-07-23T02:21:17.520946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 予測結果の確認","metadata":{}},{"cell_type":"code","source":"subm = pd.read_csv('submission.csv')\nsubm","metadata":{"execution":{"iopub.status.busy":"2022-07-23T02:21:17.523192Z","iopub.execute_input":"2022-07-23T02:21:17.523536Z","iopub.status.idle":"2022-07-23T02:21:17.539104Z","shell.execute_reply.started":"2022-07-23T02:21:17.523502Z","shell.execute_reply":"2022-07-23T02:21:17.538136Z"},"trusted":true},"execution_count":null,"outputs":[]}]}