\n
\n\n### Libraries 📚⬇\n\n> [Youtube videos for 2018 Melanoma Winner](https://www.youtube.com/watch?v=meg8I1GdtUg)","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch # Convolutional Neural Net from Google Research","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# System\nimport cv2\nimport os, os.path\nfrom PIL import Image # from RBG to YCbCr\nimport gc\nimport time\nimport datetime\n\n# Basics\nimport pandas as pd\nimport numpy as np\nimport random\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg # to check images\n# %matplotlib inline\nfrom tqdm.notebook import tqdm # beautiful progression bar\n\n# SKlearn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn import preprocessing\n\n# PyTorch\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import FloatTensor, LongTensor\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Data Augmentation for Image Preprocessing\nfrom albumentations import (ToFloat, Normalize, VerticalFlip, HorizontalFlip, Compose, Resize,\n RandomBrightnessContrast, HueSaturationValue, Blur, GaussNoise,\n Rotate, RandomResizedCrop, Cutout, ShiftScaleRotate)\nfrom albumentations.pytorch import ToTensorV2, ToTensor\n\nfrom efficientnet_pytorch import EfficientNet\nfrom torchvision.models import resnet34, resnet50\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Set the Seeds 🌱\n> For reproducibility","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 1234):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device available now:', device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data Preparation 📁📂\n\n> The data we'll work on has a Train and Test .csv files with coresponding .jpg images. Number of data points also increased with other [external sources.](https://www.kaggle.com/nroman/melanoma-external-malignant-256)\n
\n\n## 2.1 Read in the data 🔎","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\noutput_size = 1\n# -------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# My Train: with imputed missing values + OHE\nmy_train = pd.read_csv('../input/siim-melanoma-prep-data/train_clean.csv')\n\n# Drop path columns and Diagnosis (it won't be available during TEST)\n# We'll rewrite them once the data is concatenated\nto_drop = ['path_dicom','path_jpeg', 'diagnosis']\nfor drop in to_drop:\n if drop in my_train.columns :\n my_train.drop([drop], axis=1, inplace=True)\n\n# Roman's Train: with added data for Malignant category\nroman_train = pd.read_csv('../input/../input/melanoma-external-malignant-256/train_concat.csv')\n\n\n# --- Before concatenatenating both together, let's preprocess roman_train ---\n# Replace NAN with 0 for patient_id\nroman_train['patient_id'] = roman_train['patient_id'].fillna(0)\n\n# OHE\nto_encode = ['sex', 'anatom_site_general_challenge']\nencoded_all = []\n\nroman_train[to_encode[0]] = roman_train[to_encode[0]].astype(str)\nroman_train[to_encode[1]] = roman_train[to_encode[1]].astype(str)\n\nlabel_encoder = LabelEncoder()\n\nfor column in to_encode:\n encoded = label_encoder.fit_transform(roman_train[column])\n encoded_all.append(encoded)\n \nroman_train[to_encode[0]] = encoded_all[0]\nroman_train[to_encode[1]] = encoded_all[1]\n\n# Give all columns the same name\nroman_train.columns = my_train.columns\n\n\n# --- Concatenate info which is not available in my_train ---\ncommon_images = my_train['dcm_name'].unique()\nnew_data = roman_train[~roman_train['dcm_name'].isin(common_images)]\n\n# Merge all together\ntrain_df = pd.concat([my_train, new_data], axis=0)\n\n\n\n# --- Read in Test data (also cleaned, imputed, OHE) ---\ntest_df = pd.read_csv('../input/siim-melanoma-prep-data/test_clean.csv')\n\n# Drop columns\nfor drop in to_drop:\n if drop in test_df.columns :\n test_df.drop([drop], axis=1, inplace=True)\n\n# Create path column to image folder for both Train and Test\npath_train = '../input/melanoma-external-malignant-256/train/train/'\npath_test = '../input/melanoma-external-malignant-256/test/test/'\n\ntrain_df['path_jpg'] = path_train + train_df['dcm_name'] + '.jpg'\ntest_df['path_jpg'] = path_test + test_df['dcm_name'] + '.jpg'\n\n\n# --- Last final thing: NORMALIZE! ---\ntrain_df['age'] = train_df['age'].fillna(-1)\n\nnormalized_train = preprocessing.normalize(train_df[['sex', 'age', 'anatomy']])\nnormalized_test = preprocessing.normalize(test_df[['sex', 'age', 'anatomy']])\n\ntrain_df['sex'] = normalized_train[:, 0]\ntrain_df['age'] = normalized_train[:, 1]\ntrain_df['anatomy'] = normalized_train[:, 2]\n\ntest_df['sex'] = normalized_test[:, 0]\ntest_df['age'] = normalized_test[:, 1]\ntest_df['anatomy'] = normalized_test[:, 2]\n\n\nprint('Len Train: {:,}'.format(len(train_df)), '\\n' +\n 'Len Test: {:,}'.format(len(test_df)))\n\n# Yay!","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2.2 PyTorch Dataset 💾\n\nThis class retrieves the data from the `train_df` or `test_df` and reads the corresponding information from the image folders.\n\n> Note: when reading the images, custom `augmentations` are applied. Train will have a complex transformation, while valid data will have no augmentations. Test WILL have `augmentations` like Train because we're doing **Test Time Augmentations**, meaning that we'll transform the test images, predict and **average** the result.\n
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\nvertical_flip = 0.5\nhorizontal_flip = 0.5\n\ncsv_columns = ['sex', 'age', 'anatomy']\nno_columns = 3\n# ------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Example of csv_data at index=0\nnp.array(train_df.iloc[0][csv_columns].values,dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n \n def __init__(self, dataframe, vertical_flip, horizontal_flip,\n is_train=True, is_valid=False, is_test=False):\n self.dataframe, self.is_train, self.is_valid = dataframe, is_train, is_valid\n self.vertical_flip, self.horizontal_flip = vertical_flip, horizontal_flip\n \n # Data Augmentation (custom for each dataset type)\n if is_train or is_test:\n self.transform = Compose([RandomResizedCrop(height=224, width=224, scale=(0.4, 1.0)),\n ShiftScaleRotate(rotate_limit=90, scale_limit = [0.8, 1.2]),\n HorizontalFlip(p = self.horizontal_flip),\n VerticalFlip(p = self.vertical_flip),\n HueSaturationValue(sat_shift_limit=[0.7, 1.3], \n hue_shift_limit=[-0.1, 0.1]),\n RandomBrightnessContrast(brightness_limit=[0.7, 1.3],\n contrast_limit= [0.7, 1.3]),\n Normalize(),\n ToTensor()])\n else:\n self.transform = Compose([Normalize(),\n ToTensor()])\n \n def __len__(self):\n return len(self.dataframe)\n \n def __getitem__(self, index):\n # Select path and read image\n image_path = self.dataframe['path_jpg'][index]\n image = cv2.imread(image_path)\n # For this image also import .csv information (sex, age, anatomy)\n csv_data = np.array(self.dataframe.iloc[index][['sex', 'age', 'anatomy']].values, \n dtype=np.float32)\n \n # Apply transforms\n image = self.transform(image=image)\n # Extract image from dictionary\n image = image['image']\n \n # If train/valid: image + class | If test: only image\n if self.is_train or self.is_valid:\n return (image, csv_data), self.dataframe['target'][index]\n else:\n return (image, csv_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Neural Networks 🎇\n\n## 3.1 ResNet50 💾\n> [For more information about ResNet Architecture, check this link.](https://iq.opengenus.org/resnet50-architecture/#:~:text=ResNet50%20is%20a%20variant%20of,have%20explored%20this%20in%20depth.)\n\n> `Batch Normalization`: [Accelerating Deep Network Training by Reducing Internal Covariate Shift](https://arxiv.org/abs/1502.03167)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet50Network(nn.Module):\n def __init__(self, output_size, no_columns):\n super().__init__()\n self.no_columns, self.output_size = no_columns, output_size\n \n # Define Feature part (IMAGE)\n self.features = resnet50(pretrained=True) # 1000 neurons out\n # (CSV data)\n self.csv = nn.Sequential(nn.Linear(self.no_columns, 500),\n nn.BatchNorm1d(500),\n nn.ReLU(),\n nn.Dropout(p=0.2))\n \n # Define Classification part\n self.classification = nn.Linear(1000 + 500, output_size)\n \n \n def forward(self, image, csv_data, prints=False):\n \n if prints: print('Input Image shape:', image.shape, '\\n'+\n 'Input csv_data shape:', csv_data.shape)\n \n # Image CNN\n image = self.features(image)\n if prints: print('Features Image shape:', image.shape)\n \n # CSV FNN\n csv_data = self.csv(csv_data)\n if prints: print('CSV Data:', csv_data.shape)\n \n # Concatenate layers from image with layers from csv_data\n image_csv_data = torch.cat((image, csv_data), dim=1)\n \n # CLASSIF\n out = self.classification(image_csv_data)\n if prints: print('Out shape:', out.shape)\n \n return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### How is ResNet50() working?\n> A schema of the example below:\n
","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model_example = ResNet50Network(output_size=output_size, no_columns=no_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data object and Loader\nexample_data = MelanomaDataset(train_df, vertical_flip=0.5, horizontal_flip=0.5, \n is_train=True, is_valid=False, is_test=False)\nexample_loader = torch.utils.data.DataLoader(example_data, batch_size = 3, shuffle=True)\n\n# Get a sample\nfor (image, csv_data), labels in example_loader:\n image_example, csv_data_example = image, csv_data\n labels_example = torch.tensor(labels, dtype=torch.float32)\n break\nprint('Data shape:', image_example.shape, '| \\n' , csv_data_example)\nprint('Label:', labels_example, '\\n')\n\n# Outputs\nout = model_example(image_example, csv_data_example, prints=True)\n\n# Criterion example\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] to shape=[3, 1]\nloss = criterion_example(out, labels_example.unsqueeze(1)) \nprint('Loss:', loss.item())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.2 EfficientNet 💾\n\n> There are multiple EffNet Architectures that can be used, but at the expense of computing more and more parameters.\n
\n\n\n*Note: B7 not working due to low memory*","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetwork(nn.Module):\n def __init__(self, output_size, no_columns, b4=False, b2=False):\n super().__init__()\n self.b4, self.b2, self.no_columns = b4, b2, no_columns\n \n # Define Feature part (IMAGE)\n if b4:\n self.features = EfficientNet.from_pretrained('efficientnet-b4')\n elif b2:\n self.features = EfficientNet.from_pretrained('efficientnet-b2')\n else:\n self.features = EfficientNet.from_pretrained('efficientnet-b7')\n \n # (CSV)\n self.csv = nn.Sequential(nn.Linear(self.no_columns, 250),\n nn.BatchNorm1d(250),\n nn.ReLU(),\n nn.Dropout(p=0.2),\n \n nn.Linear(250, 250),\n nn.BatchNorm1d(250),\n nn.ReLU(),\n nn.Dropout(p=0.2))\n \n # Define Classification part\n if b4:\n self.classification = nn.Sequential(nn.Linear(1792 + 250, output_size))\n elif b2:\n self.classification = nn.Sequential(nn.Linear(1408 + 250, output_size))\n else:\n self.classification = nn.Sequential(nn.Linear(2560 + 250, output_size))\n \n \n def forward(self, image, csv_data, prints=False): \n \n if prints: print('Input Image shape:', image.shape, '\\n'+\n 'Input csv_data shape:', csv_data.shape)\n \n # IMAGE CNN\n image = self.features.extract_features(image)\n if prints: print('Features Image shape:', image.shape)\n \n if self.b4:\n image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1792)\n elif self.b2:\n image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1408)\n else:\n image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 2560)\n if prints: print('Image Reshaped shape:', image.shape)\n \n # CSV FNN\n csv_data = self.csv(csv_data)\n if prints: print('CSV Data:', csv_data.shape)\n \n # Concatenate\n image_csv_data = torch.cat((image, csv_data), dim=1)\n \n # CLASSIF\n out = self.classification(image_csv_data)\n if prints: print('Out shape:', out.shape)\n \n return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### How is EfficientNet working?\n\n> A schema of the below example:\n
","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# Create an example model - Effnet\nmodel_example = EfficientNetwork(output_size=output_size, no_columns=no_columns,\n b4=False, b2=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data object and Loader\nexample_data = MelanomaDataset(train_df, vertical_flip=0.5, horizontal_flip=0.5, \n is_train=True, is_valid=False, is_test=False)\nexample_loader = torch.utils.data.DataLoader(example_data, batch_size = 3, shuffle=True)\n\n# Get a sample\nfor (image, csv_data), labels in example_loader:\n image_example, csv_data_example = image, csv_data\n labels_example = torch.tensor(labels, dtype=torch.float32)\n break\nprint('Data shape:', image_example.shape, '| \\n' , csv_data_example)\nprint('Label:', labels_example, '\\n')\n\n# Outputs\nout = model_example(image_example, csv_data_example, prints=True)\n\n# Criterion example\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] to shape=[3, 1]\nloss = criterion_example(out, labels_example.unsqueeze(1)) \nprint('Loss:', loss.item())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Training ... 💻\n\n### Prepare OOF and Predictions Matrixes\n> `OOF` will be used to assess the overall ROC value of the entire Train data (Train + Valid)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\ntrain_len = len(train_df)\ntest_len = len(test_df)\n# -------------------\n\n\n# Out of Fold Predictions\noof = np.zeros(shape = (train_len, 1))\n\n# Predictions\npreds_submission = torch.zeros(size = (test_len, 1), dtype=torch.float32, device=device)\n\nprint('oof shape:', oof.shape, '\\n' +\n 'predictions shape:', preds_submission.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### GroupKFold() 🦗🦗🦗\n> K-fold iterator variant with non-overlapping groups. The same group **will not appear** in two different folds (the number of distinct groups has to be at least equal to the number of folds).\n\n> We're using `patient_id` for our grouping column: there are multiple patients with *multiple images taken*, so we need to be careful with that.\n
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\nk = 6 # number of folds in Group K Fold\n# -------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Object\ngroup_fold = GroupKFold(n_splits = k)\n\n# Generate indices to split data into training and test set.\nfolds = group_fold.split(X = np.zeros(train_len), \n y = train_df['target'], \n groups = train_df['ID'].tolist())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.1 Training Loop... 🔋\n\nNext step would be to also incorporate the data from the .csv file
\n
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\nepochs = 15\npatience = 3\nTTA = 3\nnum_workers = 8\nlearning_rate = 0.0005\nweight_decay = 0.0\nlr_patience = 1 # 1 model not improving until lr is decreasing\nlr_factor = 0.4 # by how much the lr is decreasing\n\nbatch_size1 = 32\nbatch_size2 = 16\n\nversion = 'v6' # to keep tabs on versions\n# -------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_folds(preds_submission, model, version = 'v1'):\n # Creates a .txt file that will contain the logs\n f = open(f\"logs_{version}.txt\", \"w+\")\n \n \n for fold, (train_index, valid_index) in enumerate(folds):\n # Append to .txt\n with open(f\"logs_{version}.txt\", 'a+') as f:\n print('-'*10, 'Fold:', fold+1, '-'*10, file=f)\n print('-'*10, 'Fold:', fold+1, '-'*10)\n\n\n # --- Create Instances ---\n # Best ROC score in this fold\n best_roc = None\n # Reset patience before every fold\n patience_f = patience\n \n # Initiate the model\n model = model\n\n optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)\n scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', \n patience=lr_patience, verbose=True, factor=lr_factor)\n criterion = nn.BCEWithLogitsLoss()\n\n\n # --- Read in Data ---\n train_data = train_df.iloc[train_index].reset_index(drop=True)\n valid_data = train_df.iloc[valid_index].reset_index(drop=True)\n\n # Create Data instances\n train = MelanomaDataset(train_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n is_train=True, is_valid=False, is_test=False)\n valid = MelanomaDataset(valid_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n is_train=False, is_valid=True, is_test=False)\n # Read in test data | Remember! We're using data augmentation like we use for Train data.\n test = MelanomaDataset(test_df, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip,\n is_train=False, is_valid=False, is_test=True)\n\n # Dataloaders\n train_loader = DataLoader(train, batch_size=batch_size1, shuffle=True, num_workers=num_workers)\n # shuffle=False! Otherwise function won't work!!!\n # how do I know? ^^\n valid_loader = DataLoader(valid, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n test_loader = DataLoader(test, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n\n\n # === EPOCHS ===\n for epoch in range(epochs):\n start_time = time.time()\n correct = 0\n train_losses = 0\n\n # === TRAIN ===\n # Sets the module in training mode.\n model.train()\n\n for (images, csv_data), labels in train_loader:\n # Save them to device\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n # Clear gradients first; very important, usually done BEFORE prediction\n optimizer.zero_grad()\n\n # Log Probabilities & Backpropagation\n out = model(images, csv_data)\n loss = criterion(out, labels.unsqueeze(1))\n loss.backward()\n optimizer.step()\n\n # --- Save information after this batch ---\n # Save loss\n train_losses += loss.item()\n # From log probabilities to actual probabilities\n train_preds = torch.round(torch.sigmoid(out)) # 0 and 1\n # Number of correct predictions\n correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n\n # Compute Train Accuracy\n train_acc = correct / len(train_index)\n\n\n # === EVAL ===\n # Sets the model in evaluation mode\n model.eval()\n\n # Create matrix to store evaluation predictions (for accuracy)\n valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n\n # Disables gradients (we need to be sure no optimization happens)\n with torch.no_grad():\n for k, ((images, csv_data), labels) in enumerate(valid_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n pred = torch.sigmoid(out)\n valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n # Compute accuracy\n valid_acc = accuracy_score(valid_data['target'].values, \n torch.round(valid_preds.cpu()))\n # Compute ROC\n valid_roc = roc_auc_score(valid_data['target'].values, \n valid_preds.cpu())\n\n # Compute time on Train + Eval\n duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n\n # PRINT INFO\n # Append to .txt file\n with open(f\"logs_{version}.txt\", 'a+') as f:\n print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc), file=f)\n # Print to console\n print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n\n\n # === SAVE MODEL ===\n\n # Update scheduler (for learning_rate)\n scheduler.step(valid_roc)\n\n # Update best_roc\n if not best_roc: # If best_roc = None\n best_roc = valid_roc\n torch.save(model.state_dict(),\n f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n continue\n\n if valid_roc > best_roc:\n best_roc = valid_roc\n # Reset patience (because we have improvement)\n patience_f = patience\n torch.save(model.state_dict(),\n f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n else:\n # Decrease patience (no improvement in ROC)\n patience_f = patience_f - 1\n if patience_f == 0:\n with open(f\"logs_{version}.txt\", 'a+') as f:\n print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n format(best_roc), file=f)\n print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n format(best_roc))\n break\n\n\n # === INFERENCE ===\n # Choose model with best_roc in this fold\n best_model_path = '../working/' + [file for file in os.listdir('../working') if str(round(best_roc, 3)) in file and 'Fold'+str(fold+1) in file][0]\n # Using best model from Epoch Train\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n model = EfficientNetwork(output_size = output_size, no_columns=no_columns,\n b4=False, b2=True).to(device)\n model.load_state_dict(torch.load(best_model_path))\n # Set the model in evaluation mode\n model.eval()\n\n\n with torch.no_grad():\n # --- EVAL ---\n # Predicting again on Validation data to get preds for OOF\n valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n for k, ((images, csv_data), _) in enumerate(valid_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n pred = torch.sigmoid(out)\n valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n # Save info to OOF\n oof[valid_index] = valid_preds.cpu().numpy()\n\n\n # --- TEST ---\n # Now (Finally) prediction for our TEST data\n for i in range(TTA):\n for k, (images, csv_data) in enumerate(test_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n # Covert to probablities\n out = torch.sigmoid(out)\n\n # ADDS! the prediction to the matrix we already created\n preds_submission[k*images.shape[0] : k*images.shape[0] + images.shape[0]] += out\n\n\n # Divide Predictions by TTA (to average the results during TTA)\n preds_submission /= TTA\n\n\n # === CLEANING ===\n # Clear memory\n del train, valid, train_loader, valid_loader, images, labels\n # Garbage collector\n gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### #Training in progress... 🛫 Please do not disturb","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# --- EffNet B2 ---\nmodel = EfficientNetwork(output_size = output_size, no_columns=no_columns,\n b4=False, b2=True).to(device)\n\n# # ===== Uncomment and Train =====\n# train_folds(preds_submission = preds_submission, model = model, version = version)\n\n# # Save OOF values\n# save_oof = pd.DataFrame(data = oof, columns=['oof'])\n# save_oof.to_csv(f'oof_{version}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# Print the logs during training\nf = open('../input/siim-melanoma-prep-data/logs_v7.txt', \"r\")\ncontents = f.read()\nprint(contents)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # --- EffNet B4 ---\n# model = EfficientNetwork(output_size = output_size, no_columns=no_columns,\n# b4=True, b2=False).to(device)\n\n# # Uncomment and Train\n# train_folds(preds_submission = preds_submission, model = model, version = version)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # --- ResNet50 ---\n# model = ResNet50Network(output_size=output_size, no_columns=no_columns).to(device)\n\n# # Uncomment and Train\n# train_folds(preds_submission = preds_submission, model = model, version = version)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.2 Making the Submission 📩\n\n### #1. ROC for OOF","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import OOF (pretrained)\noof = pd.read_csv('../input/siim-melanoma-prep-data/oof_v7.csv')\n\n# ROC on full Training data\nprint('OOF ROC: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### #2. Confusion Matrix","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make OOF Binary\noof.loc[oof.oof >= 0.5, 'oof'] = 1\noof.loc[oof.oof < 0.5, 'oof'] = 0\n\n# Create Confusion Matrix\ncf_matrix = confusion_matrix(train_df['target'], oof)\n\n# Pretty CM:\ngroup_names = ['True Neg','False Pos','False Neg','True Pos']\n# Format of the absolute numbers\ngroup_counts = ['{:,}'.format(value) for value in cf_matrix.flatten()]\n# Format for relative numbers\ngroup_percentages = ['{0:.1%}'.format(value) for value in cf_matrix.flatten()/np.sum(cf_matrix)]\n\nlabels = [f'{v1}\\n{v2}\\n{v3}' for v1, v2, v3 in zip(group_names,group_counts,group_percentages)]\nlabels = np.asarray(labels).reshape(2,2)\n\n# --- The figure ---\nplt.figure(figsize=(16, 5))\nsns.heatmap(cf_matrix, annot=labels, fmt='', cmap='Oranges',xticklabels=['benign', 'malignant'], \n yticklabels=['benign', 'malignant'], cbar=False)\n\nmatplotlib.rcParams.update({'font.size': 15})\nplt.tick_params(axis='both', labelsize=15)\nplt.title('Confusion Matrix: OOF Data', fontsize=20);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### #3. Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Divide predictions by the number of folds\npreds_submission /= k\npreds_submission = preds_submission.cpu().numpy().reshape(-1,)\n\n# Import submission file\nss = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n\nss['target'] = preds_submission\nss.to_csv(f'submission_{version}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bonus: Best Single Model Function ⏩\n> Function that predicts solely on a pretrained model (uses TTA).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def best_single_model(model, preds_submission, TTA=3):\n '''Function that takes an input model (trained) and makes the prediction for submission.'''\n \n test = MelanomaDataset(test_df, vertical_flip=0.5, horizontal_flip=0.5,\n is_train=False, is_valid=False, is_test=True)\n test_loader = DataLoader(test, batch_size=16, shuffle=False, num_workers=8)\n \n model.eval()\n\n with torch.no_grad():\n for i in range(TTA):\n for k, (images, csv_data) in enumerate(test_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n # Covert to probablities\n out = torch.sigmoid(out)\n\n # ADDS! the prediction to the matrix we already created\n preds_submission[k*images.shape[0] : k*images.shape[0] + images.shape[0]] += out\n\n\n # Divide Predictions by TTA (to average the results during TTA)\n preds_submission /= TTA\n \n return preds_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# path = '../input/siim-melanoma-prep-data/Fold6_Epoch2_ValidAcc_0.981_ROC_0.986.pth'\n# best_model = EfficientNetwork(output_size = output_size, no_columns=no_columns,\n# b4=False, b2=True).to(device)\n# best_model.load_state_dict(torch.load(path, map_location=torch.device(device)))\n\n# # Submission Preds Vector\n# preds_submission = torch.zeros(size = (test_len, 1), dtype=torch.float32, device=device)\n# x = best_single_model(model=best_model, preds_submission=preds_submission)\n# preds_submission = preds_submission.cpu().numpy().reshape(-1,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # --- Submission ---\n\n# # Import submission file\n# ss = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n\n# ss['target'] = preds_submission\n# ss.to_csv(f'submission_v7.2_SingleModel.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# To Be Continued... ⏳\n> **Work in progress...**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# References: 📇\n* [Melanoma. Pytorch starter. EfficientNet](https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet/output)\n* [melanoma external malignant 256](https://www.kaggle.com/nroman/melanoma-external-malignant-256)\n* [Understanding ResNet50 architecture](https://iq.opengenus.org/resnet50-architecture/#:~:text=ResNet50%20is%20a%20variant%20of,have%20explored%20this%20in%20depth.)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}