{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#before import process\nimport sys\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\n\n#imports\nimport os, warnings, random, time\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\n\n#評価指標(ROCスコア)を算出してくれるモジュール\nfrom sklearn.metrics import accuracy_score, roc_auc_score\n\n#trainingデータ分割を工夫するモジュール\nfrom sklearn.model_selection import StratifiedKFold\n\n#プログレスバーを表示してくれる\nfrom tqdm import tqdm_notebook as tqdm\n\n#pytorchのライブラリ\nimport torch\nfrom torch import nn, optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader\n\n#Pytorch用EfficientNetを読み込むためのモジュール\nfrom efficientnet_pytorch import model as enet\n\n#Data Augmentation用ライブラリ\nimport albumentations as A\n\n#モデルの保存\nimport pickle\n\n#描画設定\n%matplotlib inline\n\n#警告文を全て無視\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 再現性の担保\nランダムな値を固定しておくことで、2回同じコードを回したときに全く同じ結果が得られるようにします。  \nこれをシード値の固定といいます  "},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 32 #69\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# deviceの指定  \npytorchではgpuを使用したい場合にはdeviceを指定する必要があります。  \ngpuで計算を行うプロセスには.to(device)関数を用いることによってgpuに計算を回します。  "},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# パラメータの設定  \n変数に必要なパラメータを格納しておくことにより、より効率的に検証を進めることが出来ます。  "},{"metadata":{"trusted":true},"cell_type":"code","source":"#params\nenet_type = 'efficientnet-b1' #b0~b8まであります。パラメータ数が大きくなります。\nmodel_name = 'v1'\nn_epochs = 2 if DEBUG else 15\ncosine_t = 15\nn_fold = 5\n\nbatch_size = 64\nimage_size = 224\n\nnum_workers = 2 #サーバ用gpu特有の概念です。並行処理をいくつ行うかを指定します。\n\ninit_lr = 1e-3\nTTA = 1\n\nProgress_Bar = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# csvデータの読み込み  \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\ntest_df = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ntrain_images = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma'\ntest_images = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test'\n\nif DEBUG:\n    train_df = train_df[:100]\n    test_df = test_df[:50]\nelse:\n    train_df = train_df[:10000]\n    test_df = test_df[:]\n# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_model = {\n        'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n        'efficientnet-b1': '../input/efficientnet-pytorch/efficientnet-b1-dbc7070a.pth',\n        'efficientnet-b2': '../input/efficientnet-pytorch/efficientnet-b2-27687264.pth',\n        'efficientnet-b3': '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth',\n        'efficientnet-b4': '../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth',\n        'efficientnet-b5': '../input/efficientnet-pytorch/efficientnet-b5-586e6cc6.pth',\n        \n    }\nmodel_save_path = False\n\n\nclass enetv2(nn.Module):\n    def __init__(self, backbone, out_dim=1):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n        self.sigmoid = nn.Sigmoid()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        #x = self.sigmoid(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = A.Compose([\n    A.Transpose(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightness(limit=0.2, p=0.75),\n    A.RandomContrast(limit=0.2, p=0.75),\n    A.OneOf([\n        A.MotionBlur(blur_limit=5),\n        A.MedianBlur(blur_limit=5),\n        A.GaussianBlur(blur_limit=5),\n        A.GaussNoise(var_limit=(5.0, 30.0)),\n    ], p=0.7),\n    A.Resize(image_size, image_size),\n    A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1, p=0.7),\n    #A.Normalize()\n])\n\ntransforms_val = A.Compose([\n    A.Resize(image_size, image_size),\n    #A.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None):\n\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        \n    def __getitem__(self, index):\n        if self.train:\n            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_id'] + '.jpg')\n        else:\n            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n\n        x = cv2.imread(im_path)\n\n        if self.transforms:\n            x = self.transforms(image = x) #albumentationsに画像を投げます\n            x = x['image'].astype(np.float32) #帰ってきたデータから画像データを取り出します(SSDのような場合には矩形領域データも合わせて帰ってきたりするため、このような仕様になっています)\n            \n        x = x.transpose(2, 0, 1) #channel first\n        \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return x, y\n        else:\n            return x\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# データの確認"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_show = MelanomaDataset(train_df, train_images, train=True,transforms = transforms_train)\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(2):\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        idx = np.random.randint(0, len(dataset_show))\n        img, label = dataset_show[idx]\n        img = np.asarray(img)\n        img = img.transpose(1,2,0)\n        img = img.astype(np.uint8)\n        #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) #rgb→bgr\n        axarr[p].imshow(img) \n        axarr[p].set_title(str(label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    \n    #プログレスバーを表示するか否か\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    for (x, y) in bar:\n        x = torch.tensor(x, device=device, dtype=torch.float32)\n        y = torch.tensor(y, device=device, dtype=torch.float32)\n        \n        optimizer.zero_grad()\n        y_pred = model(x)\n        loss = criterion(y_pred, y.unsqueeze(1)) #ここで、yにy.unsqueeze()次元拡張を挟む必要がある？(BCEだったら、だと思われる)\n        loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        \n        if Progress_Bar:\n            bar.set_description('Training loss: %.5f' % (loss_np))\n        \n    return epoch_loss/len(iterator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(model, iterator, criterion, device):\n    \n    epoch_loss = 0\n    preds = np.array([])\n    targets = np.array([])\n    model.eval()\n    \n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad(): #validation時には学習を行いません\n        \n        for (x, y) in bar:\n        \n            x = torch.tensor(x, device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n            \n            y_pred = model(x)\n            loss = criterion(y_pred, y.unsqueeze(1))\n            loss_np = loss.detach().cpu().numpy()\n            epoch_loss += loss_np\n            y_pred = torch.sigmoid(y_pred)\n            preds = np.append(preds, y_pred.detach().cpu().numpy())\n            targets = np.append(targets, y.detach().cpu().numpy())\n            \n            if Progress_Bar:\n                bar.set_description('Validation loss: %.5f' % (loss_np))\n    \n    val_acc = accuracy_score(targets, np.round(preds))\n    \n    try:\n       val_roc = roc_auc_score(targets, preds)\n    except ValueError:\n       val_roc = -1\n    \n            \n    return epoch_loss/len(iterator), val_acc,val_roc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, loss_criterion, device, epochs):\n    \"\"\" Fits a dataset to model\"\"\"\n    best_valid_score = float('inf')\n    \n    train_losses = []\n    valid_losses = []\n    valid_roc_scores = []\n    \n    for epoch in range(epochs):\n        scheduler.step(epoch)\n        start_time = time.time()\n    \n        train_loss = train(model, train_iterator, optimizer, loss_criterion, device)\n        valid_loss, valid_acc_score, valid_roc_score = evaluate(model, valid_iterator, loss_criterion, device)\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_roc_scores.append(valid_roc_score)\n\n        if valid_roc_score < best_valid_score:\n            best_valid_score = valid_roc_score\n            if model_save_path:\n                torch.save(model.state_dict(), os.path.join(model_save_path,f'{model_name}.pt'))\n            else:\n                torch.save(model.state_dict(), f'{model_name}_best.pt')\n        \n        #schedulerの処理 cosineannealingは別\n        #if scheduler != None:\n        #    scheduler.step(valid_loss)\n        end_time = time.time()\n\n        epoch_mins, epoch_secs = (end_time-start_time)//60,round((end_time-start_time)%60)\n    \n        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n        print(f'Train Loss: {train_loss:.3f}')\n        print(f'Val. Loss: {valid_loss:.3f} | Val. ACC Score: {valid_acc_score:.3f} | Val. Metric Score: {valid_roc_score:.4f}')\n        print(f'lr:{optimizer.param_groups[0][\"lr\"]:.7f}')\n        \n        torch.save(model.state_dict(), f'{model_name}_final.pt')\n        #pickle\n        with open(f'{model_name}_final.pickle', mode = \"wb\") as fp:\n            pickle.dump(model,fp)\n        \n    return train_losses, valid_losses, valid_roc_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_loss=[]\nval_loss=[]\nval_roc=[]\nmodels = []\nfor fold in range(1):\n    print(f\"Fitting on Fold {fold+1}\")\n    #Make Train and Valid DataFrame from fold\n    train_df_fold = train_df[train_df['fold'] != fold].reset_index(drop=True)\n    valid_df_fold = train_df[train_df['fold'] == fold].reset_index(drop=True)\n    \n    #Build and load Dataset\n    train_data = MelanomaDataset(train_df_fold, train_images, train=True, transforms = transforms_train) \n    valid_data = MelanomaDataset(valid_df_fold, train_images, train=True, transforms = transforms_val) \n    \n    train_iterator = DataLoader(train_data, shuffle=True, batch_size=batch_size, num_workers=num_workers)\n    valid_iterator = DataLoader(valid_data, shuffle=False, batch_size=16, num_workers=num_workers)\n    \n    #モデルの呼び出し(設計図からインスタンスへ)\n    model = enetv2(enet_type, out_dim=1).to(device) #to(device)：gpuで計算を行うことを宣言\n    \n    #損失関数の定義(BCEはBinary Cross Entropyの略で、通常のCross Entropy Lossをちょっと工夫したものです)\n    loss_criterion = nn.BCEWithLogitsLoss()\n    \n    #最適化手法の定義(現在ではAdam1強です)\n    opt= optim.Adam(model.parameters(), lr=init_lr)\n    \n    #学習率を徐々に下げていくためのスケジューラーを定義します。\n    scheduler = CosineAnnealingLR(opt, n_epochs)\n    \n    name = model_name + \"_\" + enet_type + \"_f\" + str(fold)\n    \n    #全ての情報をfit_modelに入れて、学習を開始します\n    temp_tr_loss, temp_val_loss, temp_val_roc = fit_model(model, name, train_iterator, valid_iterator, opt, loss_criterion, device, epochs=n_epochs)\n    \n    #lossと評価指標に対するスコアを記録します\n    tr_loss.append(temp_tr_loss)\n    val_loss.append(temp_val_loss)\n    val_roc.append(temp_val_roc)\n    \n    #foldごとにモデルを定義する為、学習し終わったモデルはリストに保持しておきます\n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(tr_loss)):\n    fig,ax = plt.subplots(nrows=1, ncols=2, figsize=(20,5))\n    ax[0].plot(tr_loss[i])\n    ax[0].set_title('Training and Validation Loss')\n    ax[0].plot(val_loss[i])\n    ax[0].set_xlabel('Epoch')\n\n    ax[1].plot(val_roc[i])\n    ax[1].set_title('Val ROC Score')\n    ax[1].set_xlabel('Epoch')\n\n\n    ax[0].legend();\n    ax[1].legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# テストデータに対する予測を行い、提出します  "},{"metadata":{"trusted":true},"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder=test_images, \n                       train=False,\n                       transforms=transforms_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#(model数*TTA数)回すので注意\ndef get_predictions(model, iterator, device):\n    \n    preds = np.array([0.]*len(test_df))\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        for tta in range(TTA):\n            res = np.array([])\n            for x in bar:\n                x = torch.tensor(x, device=device, dtype=torch.float32)\n                y_pred = model(x)\n                y_pred = torch.sigmoid(y_pred)\n                res = np.append(res, y_pred.detach().cpu().numpy())\n            preds += res\n    preds /= TTA\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = np.array([0.]*len(test_df))\nfor i in range(len(models)):\n    test_iterator = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=num_workers)\n    preds = get_predictions(models[i], test_iterator, device)\n    prediction += preds\nprediction /= len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsub_df = sub_df[:50] if DEBUG else sub_df\nsub_df['target'] = prediction\n\nsub_df.to_csv('submission.csv', index=False) #indexをfalseにしないと、先頭列にindex情報が付加されたcsvファイルが出力されるので注意\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}