{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Melanoma Classification SIIM-ISIC 2020 fast.ai with EfficientNetB0 and additional data (images only)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install iterative-stratification\n!pip install efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import random\nimport numpy as np\nimport pandas as pd\nimport torch\nimport PIL.Image as pil\nimport matplotlib.pyplot as plt\n\nfrom fastai.vision import *\nfrom efficientnet_pytorch import EfficientNet\nfrom sklearn.model_selection import StratifiedKFold\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's ignore embarrasing error output for updated version of pytorch and downgrade it to ver. 1.4.0","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch.nn.functional\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip uninstall torch torchvision -y\n!pip install torch==1.4.0 torchvision==0.5.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 42\n\ndef random_seed(seed_value):\n     \n    random.seed(seed_value) \n    np.random.seed(seed_value) \n    torch.manual_seed(seed_value) \n    os.environ['PYTHONHASHSEED'] = str(seed_value)\n\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) \n        torch.backends.cudnn.deterministic = True \n        torch.backends.cudnn.benchmark = False\n\nrandom_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/siim-isic-melanoma-classification'\npath","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's use preprocessed 512x512 .jpg images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg'\nimg_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(img_path + '/folds_13062020.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(path + '/test.csv')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv(path + '/sample_submission.csv')\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Augmentation transformations here","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = get_transforms( flip_vert=True, max_rotate=15, max_zoom=1.2, max_lighting=0.3, max_warp=0, p_affine=0, p_lighting=0.8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some сustom Loss functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2., reduction='mean'):\n        super().__init__()\n        self.gamma = gamma\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        CE_loss = nn.CrossEntropyLoss(reduction='none')(inputs, targets)\n        pt = torch.exp(-CE_loss)\n        F_loss = ((1 - pt)**self.gamma) * CE_loss\n        if self.reduction == 'sum':\n            return F_loss.sum()\n        elif self.reduction == 'mean':\n            return F_loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Settings dashboard \n\n#########################\n# GENERAL\n\nsubmission_ver = '0002'\n\n#########################\n# ARCHITECTURE\n\n# [models.densenet121, models.resnet50, models.resnet152, EfficientNet.from_pretrained('efficientnet-b7', num_classes=4)]\narch = [EfficientNet.from_pretrained('efficientnet-b0', num_classes=2)]\n\n# fc layer size for effnet: b7 -> 2560, b6 -> 2304, b5 -> 2048, b4 -> 1792,  \n# ------------------------- b3 -> 1536, b2 -> 1408, b1 -> 1280, b0 -> 1280\nfc_size = 1280\n\n# linear layer size for effnet: 1000 / 500\nlin_size = 1000\n\n#########################\n# FOLDS\nn_folds = 5\n\n#########################\n# DATA\n\n# [224, 352, 499]\nsize = [256]\n\n#########################\n# TRAIN\n\nbs = 32\nstage_1_epochs = 3\n\nlr1       = [1e-1]\nlr_eff_1  = [1e-3]\n\nis_stage_2 = False\nstage_2_epochs = 4\n\nlr2       = [slice(1e-7, 1e-4)]\nlr_eff_2  = [slice(1e-4, 1e-3)]\n\ncustom_loss = True\nloss_func = FocalLoss()\n\nw_decay = 0.01\n\n#########################\n# DEVICE \n\nuse_fp16 = True\nnum_wkrs = os.cpu_count()\n\n#########################\n# OTHER\n\nuse_tta = True\n\nsubmit_after_train = False\n\noversampling_flag = True\noversampling_size = 5 # 'auto' or number like: 2-10\n\npreds_to_int = False\nsmooth_preds = False\nsmooth_alpha = 0.01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(np.unique(train_df['target']))\nnum_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['image_name'] = '512x512-test/512x512-test/' + test_df['image_name'] + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = ImageList.from_df(test_df, img_path)\ntest_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df = train_df[['image_id', 'target']].copy()\nlabels_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[['image_id', 'target', 'fold']].copy()\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def k_fold(df, num_fld, seed = seed):\n    #df['fold'] = -1\n    #strat_kfold = StratifiedKFold(n_splits=num_fld, random_state=seed, shuffle=True)\n    #for i, (_, test_index) in enumerate(strat_kfold.split(df.image_name.values, df.target.values)):\n    #    df.iloc[test_index, -1] = i\n        \n    #df['fold'] = df['fold'].astype('int')\n\n    for fold in range(num_fld):\n        df.loc[df.fold == fold, f'is_valid_{fold}'] = True\n        df.loc[df.fold != fold, f'is_valid_{fold}'] = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k_fold(train_df, n_folds, seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def oversample(fld, df, os_size, num_fld=5):\n    # Let's get Fold train data\n    train_df_fld = df.loc[df['fold'] != fld]\n    valid_df_fld = df.loc[df['fold'] == fld]\n    \n    # Now let's save as separate df only \"multiple_diseases\" images for exact fold train data\n    train_df_md = train_df_fld.loc[train_df_fld['target'] == 1]\n    \n    # Oversample to \"malignant\" class size\n    if os_size == 'auto':\n        os_size = int(np.floor(train_df_fld.loc[train_df_fld['target'] == 0]['target'].value_counts()[0]/train_df_fld.loc[train_df_fld['target'] == 1]['target'].value_counts()[1]))\n    \n    train_df_md = train_df_md.append([train_df_md] * (os_size - 1))\n    \n    # Finally add \"multiple_diseases\" images to whole data, so this class gets x2 images \n    full_df_fld = pd.concat([train_df_fld, train_df_md, valid_df_fld]).sample(frac=1.0, random_state=seed).reset_index(drop=True)\n    \n    return full_df_fld\n\nfor x in range(n_folds):\n    fold_df = train_df.copy()\n    \n    if oversampling_flag:\n        fold_df = oversample(x, train_df, oversampling_size)\n        \n    globals()['src_%s' %x] = (ImageList.from_df(fold_df, img_path + '/512x512-dataset-melanoma', folder='512x512-dataset-melanoma', suffix='.jpg').split_from_df(col = (3 + x)))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(fold, size, bs, padding_mode='reflection'):\n    return (globals()['src_%s' %fold].label_from_df(cols='target')\n                                        .add_test(test_data)\n                                        .transform(tfms, size=size, padding_mode=padding_mode)\n                                        .databunch(bs=bs, num_workers = num_wkrs).normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preds_smoothing(encodings , alpha):\n    K = encodings.shape[1]\n    y_ls = (1 - alpha) * encodings + alpha / K\n    return y_ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_metrics(val_preds, val_labels):\n    targs, preds = LongTensor([]), Tensor([])  \n    \n    # val_preds = val_preds[:,1]\n    val_preds = F.softmax(val_preds, dim=1)[:,-1]\n\n    preds = torch.cat((preds, val_preds.cpu()))\n    targs = torch.cat((targs, val_labels.cpu().long()))\n\n    print('AUCROC = ' + str(auc_roc_score(preds, targs).item()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model in arch:\n    \n    ############ DEFINING VARS & SETTINGS ############\n    if hasattr(model, '__name__'):\n        model_name = model.__name__\n    else:\n        model_name = \"EfficientNet\"\n    \n    globals()[model_name + '___val_preds']  = []\n    globals()[model_name + '___val_labels'] = []\n    globals()[model_name + '___test_preds'] = []\n    \n    print(f'/////////////////////////////////////////////////////')\n    print(f'//////////////// MODEL: {model_name} ////////////////')\n    print(f'/////////////////////////////////////////////////////\\n')\n\n    for fld in range(n_folds):\n        \n        print(f'\\n//////////////// FOLD {fld} ////////////////\\n')\n        \n        for sz in size:\n            \n            print(f'-------- Size: {sz} --------\\n')\n            \n            ############ STAGE_1 ############\n            \n            print(\"Preparing data & applying settings...\\n\")\n            \n            data = get_data(fld, sz, bs)\n            \n            gc.collect()\n            \n            if sz == size[0]:\n                if model_name != \"EfficientNet\":\n                    learn = cnn_learner(data, model, metrics=[AUROC()], bn_final=True)\n                else:\n                    model._fc = nn.Sequential(nn.Linear(fc_size, lin_size, bias=True),\n                    nn.ReLU(),\n                    nn.Dropout(p=0.5),\n                    nn.Linear(lin_size, num_classes, bias = True))\n                    #\n                    learn = Learner(data, model, metrics=[AUROC()])\n                    learn = learn.split([learn.model._conv_stem,learn.model._blocks,learn.model._conv_head])\n            else:\n                learn.data = data\n\n            if custom_loss:\n                learn.loss_func = loss_func\n            \n            if use_fp16:\n                learn = learn.to_fp16() \n            \n            if model_name != \"EfficientNet\":\n                lr = lr1[size.index(sz)]\n            else:\n                lr = lr_eff_1[size.index(sz)]\n                \n            print(\"Data is ready. Learning - Stage 1...\")\n            \n            #learn.freeze()\n            learn.fit_one_cycle(stage_1_epochs, slice(lr), wd=w_decay)\n                \n            ############ STAGE_2 ############\n            \n            if is_stage_2:\n                \n                print(\"Stage 1 complete. Stage 2...\")\n                \n                if model_name != \"EfficientNet\":\n                    lr = lr2[size.index(sz)]\n                else:\n                    lr = lr_eff_2[size.index(sz)]\n                \n                learn.unfreeze()\n                learn.fit_one_cycle(stage_2_epochs, lr, wd=w_decay)\n\n            ############ RESULTS ############\n            print(f\"Final learning is over for size {sz}\\n\")\n            \n            val_preds, val_labels = learn.get_preds()\n            print_metrics(val_preds, val_labels)\n\n            # learn.save('arch-' + str(model_name) + '_fold-' + str(fld) + '_size-' + str(sz))\n\n            #---------- END OF SIZE ----------\n        \n        ############ SAVE ############\n        \n        globals()[model_name + '___val_preds'].append(val_preds)\n        globals()[model_name + '___val_labels'].append(val_labels)\n        \n        if use_tta == False:\n            print(f'\\nSaving test results for fold {fld}...')\n            test_preds, _ = learn.get_preds(DatasetType.Test)\n            globals()[model_name + '___test_preds'].append(test_preds[:, 1])\n        else:\n            print(f'\\nSaving test TTA results for fold {fld}...')\n            test_preds, _ = learn.TTA(ds_type=DatasetType.Test)\n            globals()[model_name + '___test_preds'].append(test_preds[:, 1])\n        \n        print(\"Done!\")\n        \n        gc.collect()\n        \n        #---------- END OF FOLD ----------\n    \n    print(\"All folds are trained successfully\\n\")\n    \n    print_metrics(torch.cat(globals()[model_name + '___val_preds']), torch.cat(globals()[model_name + '___val_labels']))\n    \n    print(\"\\nWriting submission file...\")\n    \n    test_df_output = pd.concat([test_df, pd.DataFrame(np.mean(np.stack(globals()[model_name + '___test_preds']), axis=0), columns=['target'])], axis=1)\n    sample_df.iloc[:,1:] = test_df_output.iloc[:,5]\n    sample_df.to_csv(f'submission_v{submission_ver}.csv', index=False)\n    \n    print(f'File is ready to submit\\n')\n    \n    if submit_after_train:\n        print(\"Submitting to Kaggle\\n\")\n        !kaggle competitions submit -c siim-isic-melanoma-classification -f 'submission_v{submission_ver}.csv' -m 'Md: {model_name}, Fd: {n_folds}, Bs: {bs}, Sz: {size[0]}, Os: {oversampling_flag}, TTa: {use_tta}'\n        print(' \\n\\n\\n\\n')\n    \n    #---------- END OF MODEL ----------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}