{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T07:50:21.248634Z","iopub.execute_input":"2022-07-22T07:50:21.249351Z","iopub.status.idle":"2022-07-22T07:50:21.395960Z","shell.execute_reply.started":"2022-07-22T07:50:21.249314Z","shell.execute_reply":"2022-07-22T07:50:21.394881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn import feature_extraction, linear_model, model_selection, preprocessing\nimport numpy as np\nimport transformers\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import BertTokenizer, BertModel\nfrom torch import optim\nfrom torch import cuda\nimport time\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.398287Z","iopub.execute_input":"2022-07-22T07:50:21.398899Z","iopub.status.idle":"2022-07-22T07:50:21.407086Z","shell.execute_reply.started":"2022-07-22T07:50:21.398853Z","shell.execute_reply":"2022-07-22T07:50:21.406042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練・評価データの読込\ndf = pd.read_csv(\"/kaggle/input/nlp-getting-started/train.csv\").loc[:, [\"text\", \"target\"]]\n\n# 訓練・評価データの分割\ntrain_df, valid_df = train_test_split(df, test_size=0.2, shuffle=True, random_state=123, stratify=df['target'])\n\n# train_df, valid_test_df = train_test_split(df, test_size=0.2, shuffle=True, random_state=123, stratify=df['target'])\n# valid_df, test_df = train_test_split(valid_test_df, test_size=0.5, shuffle=True, random_state=123, stratify=valid_test_df['target'])\n\ntrain_df.reset_index(drop=True, inplace=True)\nvalid_df.reset_index(drop=True, inplace=True)\n# test_df.reset_index(drop=True, inplace=True)\n\n# print(test_df.head())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.409135Z","iopub.execute_input":"2022-07-22T07:50:21.410025Z","iopub.status.idle":"2022-07-22T07:50:21.453590Z","shell.execute_reply.started":"2022-07-22T07:50:21.409980Z","shell.execute_reply":"2022-07-22T07:50:21.452425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 事例数の確認\nprint('【学習データ】')\nprint(train_df['target'].value_counts())\nprint('【検証データ】')\nprint(valid_df['target'].value_counts())\n# print('【テストデータ】')\n# print(test_df['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.456938Z","iopub.execute_input":"2022-07-22T07:50:21.457647Z","iopub.status.idle":"2022-07-22T07:50:21.467939Z","shell.execute_reply.started":"2022-07-22T07:50:21.457601Z","shell.execute_reply":"2022-07-22T07:50:21.466390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasetの定義\nclass CreateDataset(Dataset):\n    def __init__(self, X, y, tokenizer, max_len):\n        self.X = X\n        self.y = y\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n\n    def __len__(self):  # len(Dataset)で返す値を指定\n        return len(self.y)\n\n    def __getitem__(self, index):  # Dataset[index]で返す値を指定\n        text = self.X[index]\n        inputs = self.tokenizer.encode_plus(\n            text,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            pad_to_max_length=True\n        )\n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n\n        return {\n            'ids': torch.LongTensor(ids),\n            'mask': torch.LongTensor(mask),\n            'labels': torch.Tensor(self.y[index])\n        }\n# 正解ラベルのone-hot化\ny_train = pd.get_dummies(train_df, columns=['target'])[['target_0', 'target_1']].values\ny_valid = pd.get_dummies(valid_df, columns=['target'])[['target_0', 'target_1']].values\n# y_test = pd.get_dummies(test_df, columns=['target'])[['target_0', 'target_1']].values\n\n# Datasetの作成\nmax_len = 56\ntokenizer = BertTokenizer.from_pretrained('/kaggle/input/huggingface-bert/bert-base-cased')\ndataset_train = CreateDataset(train_df['text'], y_train, tokenizer, max_len)\ndataset_valid = CreateDataset(valid_df['text'], y_valid, tokenizer, max_len)\n# dataset_test = CreateDataset(test_df['text'], y_test, tokenizer, max_len)\n\nfor var in dataset_train[0]:\n    print(f'{var}: {dataset_train[0][var]}')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.470339Z","iopub.execute_input":"2022-07-22T07:50:21.471805Z","iopub.status.idle":"2022-07-22T07:50:21.557278Z","shell.execute_reply.started":"2022-07-22T07:50:21.471575Z","shell.execute_reply":"2022-07-22T07:50:21.556038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BERT分類モデルの定義\nclass BERTClass(torch.nn.Module):\n    def __init__(self, drop_rate, otuput_size):\n        super().__init__()\n        self.bert = BertModel.from_pretrained(\n            '/kaggle/input/huggingface-bert/bert-base-cased',\n            return_dict=False)\n        self.drop = torch.nn.Dropout(drop_rate)\n        self.fc = torch.nn.Linear(768, otuput_size)  # BERTの出力に合わせて768次元を指定\n\n    def forward(self, ids, mask):\n        _, out = self.bert(ids, attention_mask=mask)\n        out = self.fc(self.drop(out))\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.559196Z","iopub.execute_input":"2022-07-22T07:50:21.560260Z","iopub.status.idle":"2022-07-22T07:50:21.570688Z","shell.execute_reply.started":"2022-07-22T07:50:21.560210Z","shell.execute_reply":"2022-07-22T07:50:21.569114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_loss_and_accuracy(model, criterion, loader, device):\n    \"\"\" 損失・正解率を計算\"\"\"\n    model.eval()\n    loss = 0.0\n    total = 0\n    correct = 0\n    with torch.no_grad():\n        for data in loader:\n            # デバイスの指定\n            ids = data['ids'].to(device)\n            mask = data['mask'].to(device)\n            labels = data['labels'].to(device)\n\n            # 順伝播\n            outputs = model(ids, mask)\n\n            # 損失計算\n            loss += criterion(outputs, labels).item()\n\n            # 正解率計算\n            pred = torch.argmax(outputs, dim=-1).cpu().numpy() # バッチサイズの長さの予測ラベル配列\n            labels = torch.argmax(labels, dim=-1).cpu().numpy()  # バッチサイズの長さの正解ラベル配列\n            total += len(labels)\n            correct += (pred == labels).sum().item()\n\n    return loss / len(loader), correct / total\n\n\ndef train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, device=None):\n    \"\"\"モデルの学習を実行し、損失・正解率のログを返す\"\"\"\n    # デバイスの指定\n    model.to(device)\n\n    # dataloaderの作成\n    dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)\n    dataloader_valid = DataLoader(dataset_valid, batch_size=len(dataset_valid), shuffle=False)\n\n    # 学習\n    log_train = []\n    log_valid = []\n    for epoch in range(num_epochs):\n        # 開始時刻の記録\n        s_time = time.time()\n\n        # 訓練モードに設定\n        model.train()\n        for data in dataloader_train:\n            # デバイスの指定\n            ids = data['ids'].to(device)\n            mask = data['mask'].to(device)\n            labels = data['labels'].to(device)\n\n            # 勾配をゼロで初期化\n            optimizer.zero_grad()\n\n            # 順伝播 + 誤差逆伝播 + 重み更新\n            outputs = model(ids, mask)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n        # 損失と正解率の算出\n        loss_train, acc_train = calculate_loss_and_accuracy(model, criterion, dataloader_train, device)\n        loss_valid, acc_valid = calculate_loss_and_accuracy(model, criterion, dataloader_valid, device)\n        log_train.append([loss_train, acc_train])\n        log_valid.append([loss_valid, acc_valid])\n\n        # チェックポイントの保存\n        torch.save({'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict()}, f'checkpoint{epoch + 1}.pt')\n\n        # 終了時刻の記録\n        e_time = time.time()\n\n        # ログを出力\n        print(f'epoch: {epoch + 1}, loss_train: {loss_train:.4f}, accuracy_train: {acc_train:.4f}, loss_valid: {loss_valid:.4f}, accuracy_valid: {acc_valid:.4f}, {(e_time - s_time):.4f}sec') \n\n    torch.save(model, '/kaggle/working/mymodel.pth')\n    \n    return {'train': log_train, 'valid': log_valid}","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.572865Z","iopub.execute_input":"2022-07-22T07:50:21.573393Z","iopub.status.idle":"2022-07-22T07:50:21.597121Z","shell.execute_reply.started":"2022-07-22T07:50:21.573347Z","shell.execute_reply":"2022-07-22T07:50:21.595965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# パラメータの設定\nDROP_RATE = 0.4\nOUTPUT_SIZE = 2\nBATCH_SIZE = 32\nNUM_EPOCHS = 4\nLEARNING_RATE = 2e-5\n\n# モデルの定義\nmodel = BERTClass(DROP_RATE, OUTPUT_SIZE)\n\n# 損失関数の定義\ncriterion = torch.nn.BCEWithLogitsLoss()\n\n# オプティマイザの定義\noptimizer = torch.optim.AdamW(params=model.parameters(), lr=LEARNING_RATE)\n\n# デバイスの指定\ndevice = 'cuda' if cuda.is_available() else 'cpu'\n\n# モデルの学習\nlog = train_model(dataset_train, dataset_valid, BATCH_SIZE, model, criterion, optimizer, NUM_EPOCHS, device=device)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:50:21.600949Z","iopub.execute_input":"2022-07-22T07:50:21.601264Z","iopub.status.idle":"2022-07-22T07:54:35.593881Z","shell.execute_reply.started":"2022-07-22T07:50:21.601237Z","shell.execute_reply":"2022-07-22T07:54:35.592782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ログの可視化\nx_axis = [x for x in range(1, len(log['train']) + 1)]\nfig, ax = plt.subplots(1, 2, figsize=(15, 5))\nax[0].plot(x_axis, np.array(log['train']).T[0], label='train')\nax[0].plot(x_axis, np.array(log['valid']).T[0], label='valid')\nax[0].set_xlabel('epoch')\nax[0].set_ylabel('loss')\nax[0].legend()\nax[1].plot(x_axis, np.array(log['train']).T[1], label='train')\nax[1].plot(x_axis, np.array(log['valid']).T[1], label='valid')\nax[1].set_xlabel('epoch')\nax[1].set_ylabel('accuracy')\nax[1].legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:54:35.596577Z","iopub.execute_input":"2022-07-22T07:54:35.597015Z","iopub.status.idle":"2022-07-22T07:54:36.040915Z","shell.execute_reply.started":"2022-07-22T07:54:35.596970Z","shell.execute_reply":"2022-07-22T07:54:36.039765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 正解率の算出\ndef calculate_accuracy(model, dataset, device):\n    \n    # Dataloaderの作成\n    loader = DataLoader(dataset, batch_size=len(dataset), shuffle=False)\n\n    model.eval()\n    total = 0\n    correct = 0\n    with torch.no_grad():\n        for data in loader:\n            # デバイスの指定\n            ids = data['ids'].to(device)\n            mask = data['mask'].to(device)\n            labels = data['labels'].to(device)\n\n            # 順伝播 + 予測値の取得 + 正解数のカウント\n            outputs = model.forward(ids, mask)\n            pred = torch.argmax(outputs, dim=-1).cpu().numpy()\n            labels = torch.argmax(labels, dim=-1).cpu().numpy()\n            total += len(labels)\n            correct += (pred == labels).sum().item()\n\n    return correct / total","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:54:36.044812Z","iopub.execute_input":"2022-07-22T07:54:36.045875Z","iopub.status.idle":"2022-07-22T07:54:36.670087Z","shell.execute_reply.started":"2022-07-22T07:54:36.045830Z","shell.execute_reply":"2022-07-22T07:54:36.668772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if cuda.is_available() else 'cpu'\nprint('正解率（学習データ）：'+str(calculate_accuracy(model, dataset_train, device)))\nprint('正解率（検証データ）：'+str(calculate_accuracy(model, dataset_valid, device)))\n# print('正解率（テストデータ）：'+str(calculate_accuracy(model, dataset_test, device)))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:54:36.671596Z","iopub.execute_input":"2022-07-22T07:54:36.672236Z","iopub.status.idle":"2022-07-22T07:54:56.409469Z","shell.execute_reply.started":"2022-07-22T07:54:36.672193Z","shell.execute_reply":"2022-07-22T07:54:56.408406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Judge(model, dataset, device):\n    \n    # Dataloaderの作成\n    loader = DataLoader(dataset, batch_size=len(dataset), shuffle=False)\n\n    model.eval()\n    result = None\n    with torch.no_grad():\n        for data in loader:\n            # デバイスの指定\n            ids = data['ids'].to(device)\n            mask = data['mask'].to(device)\n            labels = data['labels'].to(device)\n\n            # 順伝播 + 予測値の取得 + 正解数のカウント\n            outputs = model.forward(ids, mask)\n            pred = torch.argmax(outputs, dim=-1).cpu().numpy()\n            result=pred\n\n    return result","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:54:56.414199Z","iopub.execute_input":"2022-07-22T07:54:56.418031Z","iopub.status.idle":"2022-07-22T07:54:56.429820Z","shell.execute_reply.started":"2022-07-22T07:54:56.417964Z","shell.execute_reply":"2022-07-22T07:54:56.428776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/nlp-getting-started/test.csv\").loc[:, [\"id\", \"text\"]]\ntest_df = pd.DataFrame([[text, 0] for text in df[\"text\"].tolist()], columns=[\"text\", \"target\"])\ny_test = pd.get_dummies(test_df, columns=['target'])[['target_0']].values\ndataset_test = CreateDataset(test_df['text'], y_test, tokenizer, max_len)\n\nresult=Judge(model, dataset_test, device)\n\nprint(len(result))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:54:56.434670Z","iopub.execute_input":"2022-07-22T07:54:56.437665Z","iopub.status.idle":"2022-07-22T07:55:04.702820Z","shell.execute_reply.started":"2022-07-22T07:54:56.437605Z","shell.execute_reply":"2022-07-22T07:55:04.701362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"/kaggle/input/nlp-getting-started/sample_submission.csv\").loc[:, [\"id\"]]\nids = sample_df[\"id\"].tolist()\nsubmission_df = pd.DataFrame([[ids[i], result[i]] for i in range(len(ids))], columns=[\"id\", \"target\"])\n\nprint(submission_df)\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T07:55:04.704632Z","iopub.execute_input":"2022-07-22T07:55:04.705263Z","iopub.status.idle":"2022-07-22T07:55:04.743360Z","shell.execute_reply.started":"2022-07-22T07:55:04.705219Z","shell.execute_reply":"2022-07-22T07:55:04.741958Z"},"trusted":true},"execution_count":null,"outputs":[]}]}