{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importings & Configurations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip -q install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git\n!pip -q install geffnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport time\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nimport cv2\nimport PIL.Image\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.metrics import roc_auc_score, roc_curve, auc\n\n# PyTorch elements\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset # Using to construct Customed Datasets\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.utils.data import random_split\n\nimport torchvision\nfrom torchvision import models\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data.sampler import SubsetRandomSampler, RandomSampler, SequentialSampler\n\n# Scheduler\nfrom torch.optim import lr_scheduler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom warmup_scheduler import GradualWarmupScheduler  # https://github.com/ildoonet/pytorch-gradual-warmup-lr\n\n# Augmentation\nimport albumentations as A\nimport geffnet\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG         = True\n# Config\nkernel_type   = 'efficientNet_Lite_256_meta_ext_9classes_5epochs'\n\norg_data_dir  = '../input/jpeg-melanoma-256x256'\next_data_dir  = '../input/jpeg-isic2019-256x256'\n\nimage_size    = 256\nnum_workers   = 4  # Using while loading data\n\nbatch_size    = 64\nout_dim       = 9  # Output dims of CNN models\nn_epochs      = 2\nlearning_rate = 3e-5\n\nn_TTA         = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading CSV ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Training ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Original Training Set ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(org_data_dir, 'train.csv'))\ndf_train.sample(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Duplicates are rows with 'tfrecord' == -1\ndf_train = df_train[df_train['tfrecord'] != -1].reset_index(drop=True)\n\n# 'jpg_path'\ndf_train['jpg_path'] = df_train['image_name'].apply(lambda image_name: os.path.join(org_data_dir, 'train', f'{image_name}.jpg'))\n\n# 'diagnosis': matching 2020 (original) diagnosis to 2019(external) diagnosis\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda string : string.replace('seborrheic keratosis', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda string : string.replace('lichenoid keratosis', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda string : string.replace('solar lentigo', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda string : string.replace('lentigo NOS', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda string : string.replace('cafe-au-lait macule', 'unknown'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda string : string.replace('atypical melanocytic proliferation', 'unknown'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Original + External Training Sets ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_ext = pd.read_csv(os.path.join(ext_data_dir, 'train.csv'))\ndf_train_ext","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Duplicates are rows with 'tfrecord' == -1\ndf_train_ext = df_train_ext[df_train_ext['tfrecord'] != -1].reset_index(drop=True)\n\n# 'jpg_path'\ndf_train_ext['jpg_path'] = df_train_ext['image_name'].apply(lambda image_name: os.path.join(ext_data_dir, 'train', f'{image_name}.jpg'))\n\n# 'diagnosis': matching 2020 (original) diagnosis to 2019(external) diagnosis\ndf_train_ext['diagnosis'] = df_train_ext['diagnosis'].apply(lambda string : string.replace('NV', 'nevus'))\ndf_train_ext['diagnosis'] = df_train_ext['diagnosis'].apply(lambda string: string.replace('MEL', 'melanoma'))\n\ndf_train = pd.concat([df_train, df_train_ext]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CHANGE: using 'diagnosis' as target ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Label encoding MANUALLY\n# to assure what 'melanoma' index is\ndiagnosis_to_idx = {diag : idx for idx, diag in enumerate(np.sort(df_train.diagnosis.unique()))}\ndf_train['target'] = df_train['diagnosis'].map(diagnosis_to_idx)\n\nmelanoma_idx = diagnosis_to_idx['melanoma'] # We would use mel_idx latter\n\nprint(\"'diagnosis' encoding: \\n\",diagnosis_to_idx)\nprint('\\n melanoma_idx: ', melanoma_idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['target'].value_counts().plot(kind ='bar', \n                                       title='Counts: Target by Diagnosis')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing: Meta Data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# 'anatom_site_general_challenge': One-hot encode\ntmp_dummies = pd.get_dummies(df_train['anatom_site_general_challenge'], prefix='site_', dummy_na=True)\ndf_train = pd.concat([df_train, tmp_dummies], axis=1)\n\n# 'sex': male = 1, female = 0\ndf_train['sex'] = df_train['sex'].map({'male':   0,\n                                       'female': 1})\ndf_train['sex'].fillna(-1, inplace=True)\n\n# 'age_approx': max normalize\nmax_age = np.max(df_train['age_approx'])\ndf_train['age_approx'] /= max_age\ndf_train['age_approx'].fillna(0, inplace=True)\n\ndf_train['patient_id'].fillna(0, inplace=True)\n\n# 'n_img' per user\nmap_PatientID_to_N_image_name = df_train.groupby(['patient_id']).image_name.count()\ndf_train['n_images'] = df_train['patient_id'].map(map_PatientID_to_N_image_name)\ndf_train.loc[df_train['patient_id']==-1, 'n_images'] = 1\ndf_train['n_images'] = np.log1p(df_train.n_images.values)\n\n# 'image_size'\ntrain_images = df_train['jpg_path'].values\ntmp_train_sizes = np.zeros(train_images.shape[0])\nfor i, img_path in enumerate(tqdm(train_images)):\n    tmp_train_sizes[i] = os.path.getsize(img_path)\n\ndf_train['image_size'] = np.log(tmp_train_sizes) # Logarit normalize\n\n# Sum up\nmeta_features = ['sex', 'age_approx', 'n_images', 'image_size'] + [col for col in df_train.columns if col.startswith('site_')]\nn_meta_features = len(meta_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.sample(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_meta_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_debug = df_train.sample(batch_size * 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define: Datasets\n\nOur custom Dataset subclass returns:\n* Training data points with their Target values (which are Diagnosis), all in torch.tensor type.\n* Augmentation images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class img_meta_Dataset(torch.utils.data.Dataset):\n    def __init__(self, df_data=df_train, transformer=None):\n        self.df_data = df_data\n        self.transformer = transformer\n        \n    def __len__(self):\n        return self.df_data.shape[0]\n    \n    def __getitem__(self, idx):\n        row = self.df_data.iloc[idx]\n        img = cv2.imread(row['jpg_path'])\n        meta = self.df_data.iloc[idx][meta_features]\n        # Augmentation\n        if self.transformer is not None:\n            img = self.transformer(image=img)  ## Transformer returns: {'images': np.array[list of Images]}\n            img = img['image'].astype(np.float32)\n        else:\n            img = img.astype(np.float32)\n        # Covert to 3 Channels * Hight * Weight format\n        img = img.transpose(2, 0, 1)\n        \n        ## To torch.tensor.float32\n        tensor_img = torch.tensor(img).float()\n        tensor_metadata = torch.tensor(meta).float()\n        x = (tensor_img, tensor_metadata)\n        target = torch.tensor(row['target']).long()\n        \n        return (x, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# img_meta_Dataset works?\n'''\ndf_debug = df_train.sample(5)\ndf_debug = img_meta_Dataset(df_debug, transformer=None)\ndataloader = DataLoader(df_debug, batch_size=1)\n\nfor x, target in dataloader:\n    x_img, x_meta = x\n    print('x_img.shape: ',x_img.shape)\n    print('x_meta.shape: ',x_meta.shape)\n\n    print('target: {} \\n'.format(target))\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define: Augmentation Transformer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentation = A.Compose([\n    A.Transpose(p=0.5),\n    A.HorizontalFlip(p=05.),\n    A.VerticalFlip(p=0.5),\n    A.RandomBrightness(p=0.75, limit=0.2),\n    A.RandomContrast(p=0.75, limit=0.2),   \n    A.OneOf([\n        A.MotionBlur(blur_limit=5),\n        A.MedianBlur(blur_limit=5),\n        A.GaussianBlur(blur_limit=5),\n        A.GaussNoise(var_limit=(0.5, 30))],\n        p=0.7),\n    A.OneOf([\n        A.OpticalDistortion(distort_limit=1.0),\n        A.GridDistortion(num_steps=5, distort_limit=1),\n        A.ElasticTransform(alpha=3)],\n        p=0.7),\n    \n    A.CLAHE(clip_limit=4.0, p=0.7),\n    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n    A.Resize(image_size, image_size),\n    A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1, p=0.7),    \n    A.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_resize_norm = A.Compose([\n    A.Resize(image_size, image_size),\n    A.Normalize(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augmentation works?\ndf_debug = df_train.sample(batch_size * 3)\ndf_debug = img_meta_Dataset(df_debug, transformer=augmentation)\n\nfor plots in range(2):\n    f, axis = plt.subplots(1, 5, figsize=(20, 20))\n    for subplot in range(5):\n        idx = np.random.randint(0, len(df_debug))\n        x, target = df_debug[idx]\n        x_img, _ = x\n        axis[subplot].imshow(x_img.transpose(0, 1).transpose(1, 2).squeeze())\n        axis[subplot].set_title(str(target))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define: Models\n\nMy model predicts based on both images and patients metadata with two sections:\n* Section img_model: **EfficientNet_Lite**\n* Section meta_data_model: **a CNN**.\n\nThen, aboved sections outputs are concatenated to make final predictions.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predicting with BOTH images and metadata\nclass blood_sweat_tears(nn.Module):\n    def __init__(self, out_dim=out_dim, n_meta_features=n_meta_features):\n        super().__init__()\n        self.n_meta_features = n_meta_features\n        \n        self.effnet = geffnet.create_model('efficientnet_lite0', pretrained=True)\n        \n        in_ch = self.effnet.classifier.in_features + 128 # 128 is below meta_data_model's out_dim \n        self.final_fc = nn.Linear(in_ch, out_dim)\n        \n        self.effnet.classifier = nn.Identity()\n        self.dropout = nn.Dropout(0.5) # Using while concatenating predictions, dropout = 0.5\n    \n    def img_model(self, x_img):       # To predict basedon IMAGES only\n        x_img = self.effnet(x_img)\n        return x_img\n        \n    def meta_data_model(self, x_meta): # To predict basedon META only\n        self.meta_model = nn.Sequential(\n                    nn.Linear(self.n_meta_features, 512), # 74 x 512\n                    nn.BatchNorm1d(512),\n                    nn.ReLU(),\n                    nn.Dropout(p=0.3),\n                    nn.Linear(512, 128),\n                    nn.BatchNorm1d(128),\n                    nn.ReLU(),)\n        self.meta_model = self.meta_model.to(device)\n        x_meta = self.meta_model(x_meta)\n        return x_meta\n    \n    def forward(self, x_img, x_meta):  # Torch's mandatory\n        x_img  = self.img_model(x_img).squeeze(-1).squeeze(-1) ####\n        x_meta = self.meta_data_model(x_meta)\n        # Concatenate BOTH predictions from images & meta\n        x      = torch.cat((x_img, x_meta), dim=1)\n        x      = self.dropout(x)\n        x      = self.final_fc(x)\n        # x      = x.softmax(1)  \n        # NOT an activation exists here, as CrossEntropyLoss may prefer a logit input.\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check: if MODELS works?\n'''\ndf_debug = df_train.sample(batch_size * 3)\ndf_debug = img_meta_Dataset(df_debug, transformer=augmentation)\ndataloader = DataLoader(df_debug, batch_size=3)\n\nmodel = blood_sweat_tears(out_dim=out_dim)\nmodel = model.to(device)\n\nloss = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\ntrain_loss = []\n\nmodel.train()\n\nbar = tqdm(dataloader)\nfor (x, target) in bar:\n    optimizer.zero_grad()\n    # print(f'target {target}')\n    x_img, x_meta = x\n    x_img, x_meta, target = x_img.to(device), x_meta.to(device), target.to(device)\n    \n    print('x_img.shape: ',x_img.shape)\n    print('x_meta.shape: ',x_meta.shape)\n    print('target: {} \\n'.format(target))\n    \n    logits = model(x_img, x_meta)\n    print('Logits shape: ', logits.shape)\n    print(f'Logits {logits}')\n    \n    loss = criterion(logits, target)\n    print('loss: ',loss)\n\n    print(' \\n ----------------------- \\n')\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Test Splitting ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_set, df_test_set = train_test_split(df_train, test_size=0.2, random_state=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data\nif DEBUG:\n    train_df_debug = df_train_set.sample(batch_size * 3)\n    valid_df_debug = df_test_set.sample(batch_size * 3)\n    \n    train_Dataset = img_meta_Dataset(train_df_debug, transformer=augmentation)\n    valid_Dataset = img_meta_Dataset(valid_df_debug, transformer=augmentation)\n\n    train_DataLoader = DataLoader(train_Dataset, batch_size=3)\n    valid_DataLoader = DataLoader(valid_Dataset, batch_size=3)\n\nelse:\n    train_Dataset = img_meta_Dataset(df_train_set, transformer=augmentation)\n    valid_Dataset = img_meta_Dataset(df_test_set, transformer=augmentation)\n    \n    train_DataLoader = DataLoader(train_Dataset, batch_size=batch_size)\n    valid_DataLoader = DataLoader(valid_Dataset, batch_size=batch_size)\n\n# Initiating\nmodel = blood_sweat_tears(out_dim=out_dim)\nmodel = model.to(device)\n\nloss = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\ntrain_loss = []\nauc_max = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training\nfor epoch in range(1, n_epochs + 1):\n    print('---------Epoch {}----------'.format(epoch))\n    \n    PREDS = []\n    TARGETS = []\n    \n    model.train()\n    bar = tqdm(train_DataLoader)\n    for (x, target) in bar:\n        optimizer.zero_grad()\n        x_img, x_meta = x\n        x_img, x_meta, target = x_img.to(device), x_meta.to(device), target.to(device)  \n\n        logits = model(x_img, x_meta)\n        preds = logits.softmax(1)\n\n        loss = criterion(logits, target)\n        loss.backward()\n        optimizer.step()\n\n        PREDS.append(preds.detach().cpu())\n        TARGETS.append(target.detach().cpu())\n\n        # Screen output\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n\n    PREDS = torch.cat(PREDS).numpy()\n    y_pred =  PREDS[:, melanoma_idx]\n    \n    TARGETS = torch.cat(TARGETS).numpy()\n    y_true = (TARGETS==melanoma_idx).astype(float)\n    \n    acc = (PREDS.argmax(1) == TARGETS).mean() * 100.\n    auc = roc_auc_score(y_true, y_pred)\n    gini_score = 2 * auc - 1\n    \n    print('loss: %.2f' % (train_loss[-1]))\n    print('Accuracy: {} -- Gini score: {}'.format(acc, gini_score))\n    \n    best_gini_model_file = f'{kernel_type}_{epoch}_gini_{gini_score}.pth'\n    torch.save(model.state_dict(), best_gini_model_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluate WITHOUT TTA\nPREDS = []\nTARGETS = []\nauc_max = 0\ngini_max = 0\n\nmodel.eval()  \n\n# Evaluating\nwith torch.no_grad():\n    bar = tqdm(valid_DataLoader)\n    for (x, target) in bar:\n        optimizer.zero_grad()\n        x_img, x_meta = x\n        x_img, x_meta, target = x_img.to(device), x_meta.to(device), target.to(device)  \n\n        logits = model(x_img, x_meta)\n        preds = logits.softmax(1)\n\n        loss = criterion(logits, target)\n\n        PREDS.append(preds.detach().cpu())\n        TARGETS.append(target.detach().cpu())\n\n        # Screen output\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n\n# Gini score\nPREDS = torch.cat(PREDS).numpy()\nTARGETS = torch.cat(TARGETS).numpy()\n        \nacc = (PREDS.argmax(1) == TARGETS).mean() * 100.\nauc = roc_auc_score((TARGETS==melanoma_idx).astype(float), PREDS[:, melanoma_idx])\ngini_score = 2 * auc - 1\n\nif gini_score > gini_max:\n    print('gini_max ({:.6f} --> {:.6f}). Saving model ...'.format(gini_max, gini_score))\n    best_gini_model_file = f'{kernel_type}_{epoch}_gini_{gini_score}.pth'\n    torch.save(model.state_dict(), best_gini_model_file)\n    gini_max = gini_score\n\nprint('Last Gini score: ', gini_score)\nprint('Best Gini score: {}'.format(gini_max))\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def TTA_transformer(x_img, n_TTA): # x_img in torch.tensor -- not jpg anymore\n    if n_TTA >= 4:\n        x_img = x_img.transpose(2,3)\n    if n_TTA % 4 == 0:\n        return x_img\n    elif n_TTA % 4 == 1:\n        return x_img.flip(2)\n    elif n_TTA % 4 == 2:\n        return x_img.flip(3)\n    elif n_TTA % 4 == 3:\n        return x_img.flip(2).flip(3)\n    return x_img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluate WITH TTA\nmodel.eval()\n\nval_loss = []\nPREDICTIONS = []\nTARGETS = []\n\n# TTA predictions\nwith torch.no_grad():\n    for (x, target) in tqdm(valid_DataLoader):\n\n        x_img, x_meta = x\n        x_img, x_meta, target = x_img.to(device), x_meta.to(device), target.to(device)\n\n        # TTA - Test Time Augmentation\n        predictions = torch.zeros((x_img.shape[0], out_dim)).to(device)   \n        for n in range(n_TTA):  # TTA - For each data point, predict n_TTA times\n            x_img_transformed = TTA_transformer(x_img, n)\n            logits = model(x_img_transformed, x_meta)\n            preds  = logits.softmax(1)  \n        predictions /= n_TTA  # TTA - final result = mean of predictions\n        \n        loss = criterion(logits, target)\n        loss_np = loss.detach().cpu().numpy()\n        val_loss.append(loss_np)\n        \n        PREDICTIONS.append(predictions.detach().cpu())\n        TARGETS.append(target.detach().cpu())     \n        \nval_loss = np.mean(val_loss)\nPREDICTIONS = torch.cat(PREDICTIONS).numpy()\nTARGETS = torch.cat(TARGETS).numpy()\n\nauc = roc_auc_score((TARGETS==melanoma_idx).astype(float), PREDICTIONS[:, melanoma_idx])\ngini_score = 2 * auc - 1\n\nprint('loss: %.5f' % (val_loss))\nprint('Validation Gini score: {}'.format(gini_max))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}