{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import torch","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:14.991749Z","iopub.execute_input":"2022-12-09T19:33:14.992430Z","iopub.status.idle":"2022-12-09T19:33:18.406814Z","shell.execute_reply.started":"2022-12-09T19:33:14.992394Z","shell.execute_reply":"2022-12-09T19:33:18.405670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet_pytorch torchtoolbox","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:18.408812Z","iopub.execute_input":"2022-12-09T19:33:18.409243Z","iopub.status.idle":"2022-12-09T19:33:33.071098Z","shell.execute_reply.started":"2022-12-09T19:33:18.409195Z","shell.execute_reply":"2022-12-09T19:33:33.069958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torchtoolbox.transform as transforms\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os \nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:33.072795Z","iopub.execute_input":"2022-12-09T19:33:33.073181Z","iopub.status.idle":"2022-12-09T19:33:34.598039Z","shell.execute_reply.started":"2022-12-09T19:33:33.073138Z","shell.execute_reply":"2022-12-09T19:33:34.597082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 42\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:34.600438Z","iopub.execute_input":"2022-12-09T19:33:34.600712Z","iopub.status.idle":"2022-12-09T19:33:34.610232Z","shell.execute_reply.started":"2022-12-09T19:33:34.600687Z","shell.execute_reply":"2022-12-09T19:33:34.609348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:34.611693Z","iopub.execute_input":"2022-12-09T19:33:34.612078Z","iopub.status.idle":"2022-12-09T19:33:34.737065Z","shell.execute_reply.started":"2022-12-09T19:33:34.612045Z","shell.execute_reply":"2022-12-09T19:33:34.735965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.over_sampling import RandomOverSampler","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:34.738882Z","iopub.execute_input":"2022-12-09T19:33:34.739371Z","iopub.status.idle":"2022-12-09T19:33:35.398975Z","shell.execute_reply.started":"2022-12-09T19:33:34.739333Z","shell.execute_reply":"2022-12-09T19:33:35.398018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, meta_features = None, augment=0.05, log=False, train_transform=None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        # Don't augment if we're not training (validation or test) or if we actively disabled augment,\n        # the augment label lets us know whether it is one of the augmented images, \n        # which will mean it was duplicated by the RandomOverSampler\n        if train and augment!=0:\n            ros = RandomOverSampler(sampling_strategy=augment, random_state=SEED)\n            X, y = ros.fit_resample(df.drop(columns='target'), df[['target']])\n            df = X.join(y)\n        df['augment'] = df.duplicated()\n  \n            \n        self.df = df\n        self.imfolder = imfolder\n        if train_transform is None:\n            train_transform = transforms.Compose([\n                transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomVerticalFlip()\n            ])\n        test_transform = transforms.Compose([\n            transforms.ToTensor(),\n            # Normalize to imagenet means and stds\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n        ])\n        self.transforms = {'Augment':train_transform, 'Normal':test_transform}\n        self.train = train\n        self.meta_features = meta_features\n        self.log=log\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n\n        if self.log:\n            print(im_path)\n        x = cv2.imread(im_path)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n        augment = self.df.iloc[index]['augment']\n        if train and augment:\n            x = self.transforms['Augment'](x)\n        x = self.transforms['Normal'](x)\n            \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n        \n    def __img_index__(self, image, disp:bool=False):\n        sub_df = self.df[self.df.image_name==image]\n        if disp:\n            print(sub_df)\n        return sub_df.index\n        \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass Net(nn.Module):\n    def __init__(self, arch, n_meta_features: int):\n        super(Net, self).__init__()\n        self.arch = arch\n        if 'ResNet' in str(arch.__class__):\n            self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n        if 'EfficientNet' in str(arch.__class__):\n            self.arch._fc = nn.Linear(in_features=1792, out_features=500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.ouput = nn.Linear(500 + 250, 1)\n        \n    def forward(self, inputs):\n        \"\"\"\n        No sigmoid in forward because we are going to use BCEWithLogitsLoss\n        Which applies sigmoid for us when calculating a loss\n        \"\"\"\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:35.400362Z","iopub.execute_input":"2022-12-09T19:33:35.400701Z","iopub.status.idle":"2022-12-09T19:33:35.421155Z","shell.execute_reply.started":"2022-12-09T19:33:35.400664Z","shell.execute_reply":"2022-12-09T19:33:35.420179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arch = EfficientNet.from_pretrained('efficientnet-b4')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:35.422751Z","iopub.execute_input":"2022-12-09T19:33:35.423098Z","iopub.status.idle":"2022-12-09T19:33:36.601124Z","shell.execute_reply.started":"2022-12-09T19:33:35.423065Z","shell.execute_reply":"2022-12-09T19:33:36.599932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')\ntest_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.602997Z","iopub.execute_input":"2022-12-09T19:33:36.603429Z","iopub.status.idle":"2022-12-09T19:33:36.729456Z","shell.execute_reply.started":"2022-12-09T19:33:36.603390Z","shell.execute_reply":"2022-12-09T19:33:36.728476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = train_df.join(pd.get_dummies(train_df.diagnosis).iloc[:,:-1])\n# train_df['dum_male'] = pd.get_dummies(train_df.sex)['male']\n# train_df = train_df.drop(columns=['anatom_site_general_challenge', 'sex', 'benign_malignant', 'diagnosis'])","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.733316Z","iopub.execute_input":"2022-12-09T19:33:36.733619Z","iopub.status.idle":"2022-12-09T19:33:36.737566Z","shell.execute_reply.started":"2022-12-09T19:33:36.733592Z","shell.execute_reply":"2022-12-09T19:33:36.736536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.739046Z","iopub.execute_input":"2022-12-09T19:33:36.739647Z","iopub.status.idle":"2022-12-09T19:33:36.792113Z","shell.execute_reply.started":"2022-12-09T19:33:36.739611Z","shell.execute_reply":"2022-12-09T19:33:36.791251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.794988Z","iopub.execute_input":"2022-12-09T19:33:36.795283Z","iopub.status.idle":"2022-12-09T19:33:36.801919Z","shell.execute_reply.started":"2022-12-09T19:33:36.795249Z","shell.execute_reply":"2022-12-09T19:33:36.800952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation ","metadata":{}},{"cell_type":"code","source":"skf = GroupKFold(n_splits=3)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.804983Z","iopub.execute_input":"2022-12-09T19:33:36.805281Z","iopub.status.idle":"2022-12-09T19:33:36.812032Z","shell.execute_reply.started":"2022-12-09T19:33:36.805237Z","shell.execute_reply":"2022-12-09T19:33:36.811075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tt=(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1)[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.813408Z","iopub.execute_input":"2022-12-09T19:33:36.813993Z","iopub.status.idle":"2022-12-09T19:33:36.825824Z","shell.execute_reply.started":"2022-12-09T19:33:36.813881Z","shell.execute_reply":"2022-12-09T19:33:36.824711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in tt:\n    train_idx=t[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.827343Z","iopub.execute_input":"2022-12-09T19:33:36.827763Z","iopub.status.idle":"2022-12-09T19:33:36.866712Z","shell.execute_reply.started":"2022-12-09T19:33:36.827728Z","shell.execute_reply":"2022-12-09T19:33:36.865756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(), \n    transforms.RandomApply([transforms.RandomSPNoise(p=0.3, prob=0.01), transforms.RandomGaussianNoise(p=0.3)], p=0.33),\n    transforms.RandomApply([transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0)], p=0.75)\n    \n])","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.867928Z","iopub.execute_input":"2022-12-09T19:33:36.868293Z","iopub.status.idle":"2022-12-09T19:33:36.875591Z","shell.execute_reply.started":"2022-12-09T19:33:36.868251Z","shell.execute_reply":"2022-12-09T19:33:36.874437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n                        imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                        train=True, train_transform=train_transform,\n                        meta_features=meta_features, augment=0.05)\ntrain_loader = DataLoader(dataset=train, batch_size=16, shuffle=True, num_workers=2)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.877155Z","iopub.execute_input":"2022-12-09T19:33:36.877530Z","iopub.status.idle":"2022-12-09T19:33:36.960250Z","shell.execute_reply.started":"2022-12-09T19:33:36.877496Z","shell.execute_reply":"2022-12-09T19:33:36.959276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we observe the transformations being applied","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8,8))\nimg=cv2.imread(f'/kaggle/input/melanoma-external-malignant-256/train/train/{train.df[train.df.augment].iloc[0].image_name}.jpg')\nimg = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\nprint(train.df[train.df.augment].iloc[0].image_name)\nax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:36.961701Z","iopub.execute_input":"2022-12-09T19:33:36.962070Z","iopub.status.idle":"2022-12-09T19:33:37.448599Z","shell.execute_reply.started":"2022-12-09T19:33:36.962032Z","shell.execute_reply":"2022-12-09T19:33:37.447713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices = train.__img_index__(train.df[train.df.augment].iloc[0].image_name)\nfig, ax = plt.subplots(len(indices), figsize=(5,5*len(indices)))\nfor i, index in enumerate(indices):\n    image = train[index]\n    img = cv2.cvtColor(image[0][0].numpy().transpose(1, 2, 0), cv2.COLOR_RGB2BGR)\n    ax[i].imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:33:58.467947Z","iopub.execute_input":"2022-12-09T19:33:58.468341Z","iopub.status.idle":"2022-12-09T19:33:59.619335Z","shell.execute_reply.started":"2022-12-09T19:33:58.468306Z","shell.execute_reply":"2022-12-09T19:33:59.618267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/', \n                       train=False,\n                       train_transform=train_transform,  # For TTA\n                       meta_features=meta_features)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:34:21.890747Z","iopub.execute_input":"2022-12-09T19:34:21.891096Z","iopub.status.idle":"2022-12-09T19:34:21.904481Z","shell.execute_reply.started":"2022-12-09T19:34:21.891066Z","shell.execute_reply":"2022-12-09T19:34:21.903468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS=3","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:34:22.960813Z","iopub.execute_input":"2022-12-09T19:34:22.961168Z","iopub.status.idle":"2022-12-09T19:34:22.965607Z","shell.execute_reply.started":"2022-12-09T19:34:22.961138Z","shell.execute_reply":"2022-12-09T19:34:22.964632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = GroupKFold(n_splits=FOLDS)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T19:34:23.217080Z","iopub.execute_input":"2022-12-09T19:34:23.217958Z","iopub.status.idle":"2022-12-09T19:34:23.223892Z","shell.execute_reply.started":"2022-12-09T19:34:23.217916Z","shell.execute_reply":"2022-12-09T19:34:23.222617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter\nfrom sklearn.metrics import confusion_matrix, average_precision_score, precision_recall_curve, PrecisionRecallDisplay\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T02:25:05.867852Z","iopub.execute_input":"2022-12-10T02:25:05.868261Z","iopub.status.idle":"2022-12-10T02:25:05.874460Z","shell.execute_reply.started":"2022-12-10T02:25:05.868205Z","shell.execute_reply":"2022-12-10T02:25:05.873394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_writer(experiment_name: str, \n                  model_name: str, \n                  extra: str=None) -> torch.utils.tensorboard.writer.SummaryWriter():\n    \"\"\"Creates a torch.utils.tensorboard.writer.SummaryWriter() instance saving to a specific log_dir.\n\n    log_dir is a combination of runs/timestamp/experiment_name/model_name/extra.\n\n    Where timestamp is the current date in YYYY-MM-DD format.\n\n    Args:\n        experiment_name (str): Name of experiment.\n        model_name (str): Name of model.\n        extra (str, optional): Anything extra to add to the directory. Defaults to None.\n\n    Returns:\n        torch.utils.tensorboard.writer.SummaryWriter(): Instance of a writer saving to log_dir.\n\n    Example usage:\n        # Create a writer saving to \"runs/2022-06-04/data_10_percent/effnetb2/5_epochs/\"\n        writer = create_writer(experiment_name=\"data_10_percent\",\n                               model_name=\"effnetb2\",\n                               extra=\"5_epochs\")\n        # The above is the same as:\n        writer = SummaryWriter(log_dir=\"runs/2022-06-04/data_10_percent/effnetb2/5_epochs/\")\n    \"\"\"\n    from datetime import datetime\n    import os\n\n    # Get timestamp of current date (all experiments on certain day live in same folder)\n    timestamp = datetime.now().strftime(\"%Y-%m-%d\") # returns current date in YYYY-MM-DD format\n\n    if extra:\n        # Create log directory path\n        log_dir = os.path.join(\"runs\", timestamp, experiment_name, model_name, extra)\n    else:\n        log_dir = os.path.join(\"runs\", timestamp, experiment_name, model_name)\n        \n    print(f\"[INFO] Created SummaryWriter, saving to: {log_dir}...\")\n    return SummaryWriter(log_dir=log_dir)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T00:46:11.522462Z","iopub.execute_input":"2022-12-10T00:46:11.522844Z","iopub.status.idle":"2022-12-10T00:46:20.463671Z","shell.execute_reply.started":"2022-12-10T00:46:11.522814Z","shell.execute_reply":"2022-12-10T00:46:20.462610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"writer = create_writer(experiment_name='0.05upsampling', model_name='efficientnet_pretrained_unfrozen')","metadata":{"execution":{"iopub.status.busy":"2022-12-10T00:48:11.886313Z","iopub.execute_input":"2022-12-10T00:48:11.886734Z","iopub.status.idle":"2022-12-10T00:48:11.894599Z","shell.execute_reply.started":"2022-12-10T00:48:11.886696Z","shell.execute_reply":"2022-12-10T00:48:11.893544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = ['{}_loss', '{}_accuracy', '{}_roc', '{}_confusion', '{}_predictions', '{}_precision', '{}_recall', '{}_avpr', '{}_pr_curve']","metadata":{"execution":{"iopub.status.busy":"2022-12-10T02:25:29.766595Z","iopub.execute_input":"2022-12-10T02:25:29.766958Z","iopub.status.idle":"2022-12-10T02:25:29.771703Z","shell.execute_reply.started":"2022-12-10T02:25:29.766926Z","shell.execute_reply":"2022-12-10T02:25:29.770718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 25 # Number of epochs to run\nes_patience = 5  # Early Stopping patience - for how many epochs with no improvements to wait\nTTA = 3 # Test Time Augmentation rounds\n\noof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\npreds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)  # Predictions for test test\nresults = {i+1:{m.format(f):[] for m in metrics for f in ['train', 'val']} for i in range(FOLDS)}\n\nskf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):\n    print('=' * 20, 'Fold', fold, '=' * 20)  \n    \n    model_path = f'model_{fold}.pth'  # Path and filename to save model to\n    best_val = 0  # Best validation score within this fold\n    patience = es_patience  # Current patience counter\n    arch = EfficientNet.from_pretrained('efficientnet-b4')\n    model = Net(arch=arch, n_meta_features=len(meta_features))  # New model for each fold\n    model = model.to(device)\n    \n    \n    optim = torch.optim.Adam(model.parameters(), lr=0.001)\n    scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\n    criterion = nn.BCEWithLogitsLoss()\n    \n    train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                            train=True, \n                            train_transform=train_transform,\n                            meta_features=meta_features)\n    val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                            train=True, \n                            augment=0,\n                            meta_features=meta_features)\n    \n    train_loader = DataLoader(dataset=train, batch_size=16, shuffle=True, num_workers=2)\n    val_loader = DataLoader(dataset=val, batch_size=8, shuffle=False, num_workers=2)\n    test_loader = DataLoader(dataset=test, batch_size=8, shuffle=False, num_workers=2)\n    \n    for epoch in range(epochs):\n        start_time = time.time()\n        correct = 0\n        epoch_loss = 0\n        model.train()\n        \n        train_preds = torch.zeros((len(train.df), 1), dtype=torch.float32, device=device)\n        for j, (x, y) in enumerate(train_loader):\n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n            optim.zero_grad()\n            z = model(x)\n            loss = criterion(z, y.unsqueeze(1))\n            loss.backward()\n            optim.step()\n            pred = torch.round(torch.sigmoid(z))  # round off sigmoid to obtain predictions\n            try:\n                train_preds[j*train_loader.batch_size:j*train_loader.batch_size+x[0].shape[0]] = torch.sigmoid(z)\n            except Exception as e:\n                print(e)\n                print(f'Train preds {train_preds.shape}\\n Train loader {train_loader.batch_size}\\n J {j}\\n Pred:{torch.sigmoid(z).shape}')\n\n            correct += (pred.cpu() == y.cpu().unsqueeze(1)).sum().item()  # tracking number of correctly predicted samples\n            epoch_loss += loss.item()\n        train_acc = correct / len(train_idx)\n        train_roc = roc_auc_score(train.df['target'].values, train_preds.detach().cpu())\n        train_avpr = average_precision_score(train.df['target'].values, train_preds.detach().cpu())\n        \n        results[fold]['train_loss'].append(epoch_loss)\n        results[fold]['train_accuracy'].append(train_acc)\n        results[fold]['train_roc'].append(train_roc)\n        results[fold]['train_confusion'].append(confusion_matrix(train.df['target'].values, torch.round(train_preds.detach().cpu())))\n        results[fold]['train_avpr'].append(train_avpr)\n        try:\n            results[fold]['train_pr_curve'].append(precision_recall_curve(train.df['target'].values, train_preds.detach().cpu()))\n        except Exception as e:\n            print(f'PR train error {e}')\n            \n        \n        model.eval()  # switch model to the evaluation mode\n        val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n            # Predicting on validation set\n            val_epoch_loss=0\n            for j, (x_val, y_val) in enumerate(val_loader):\n                x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n                x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n                y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n                z_val = model(x_val)\n                val_loss = criterion(z_val, y_val.unsqueeze(1))\n                val_pred = torch.sigmoid(z_val)\n                val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n                val_epoch_loss += val_loss.item()\n            val_acc = accuracy_score(val.df['target'].values, torch.round(val_preds.cpu()))\n            val_roc = roc_auc_score(val.df['target'].values, val_preds.cpu())\n            val_avpr = average_precision_score(val.df['target'].values, val_preds.cpu())\n            \n            results[fold]['val_loss'].append(val_epoch_loss)\n            results[fold]['val_accuracy'].append(val_acc)\n            results[fold]['val_roc'].append(val_roc)\n            results[fold]['val_confusion'].append(confusion_matrix(val.df['target'].values, torch.round(val_preds.cpu())))\n            results[fold]['val_avpr'].append(val_avpr)\n            try:\n                results[fold]['val_pr_curve'].append(precision_recall_curve(val.df['target'].values, val_preds.detach().cpu()))\n            except Exception as e:\n                print(f'PR train error {e}')\n            \n            print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(\n            epoch + 1, \n            epoch_loss, \n            train_acc, \n            val_acc, \n            val_roc, \n            str(datetime.timedelta(seconds=time.time() - start_time))[:7]))\n            \n            \n            tn, fp, fn, tp = results[fold]['train_confusion'][-1].ravel()\n            val_tn, val_fp, val_fn, val_tp = results[fold]['val_confusion'][-1].ravel()\n            writer.add_scalars(main_tag=f'fold{fold}_loss', tag_scalar_dict={\"train_loss\": results[fold]['train_loss'][-1],\n                                            \"val_loss\": results[fold]['val_loss'][-1]},global_step=epoch)\n            writer.add_scalars(main_tag=f'fold{fold}_accuracy', tag_scalar_dict={\"train_accuracy\": results[fold]['train_accuracy'][-1],\n                                            \"val_accuracy\": results[fold]['val_accuracy'][-1]},global_step=epoch)\n            writer.add_scalars(main_tag=f'fold{fold}_roc', tag_scalar_dict={\"train_roc\": results[fold]['train_roc'][-1],\n                                            \"val_accuracy\": results[fold]['val_roc'][-1]},global_step=epoch)\n            \n            writer.add_scalars(main_tag=f'fold{fold}_tn', tag_scalar_dict={'train_tn':tn, 'val_tn':val_tn}, global_step=epoch)\n            writer.add_scalars(main_tag=f'fold{fold}_fp', tag_scalar_dict={'train_fp':fp, 'val_fp':val_fp}, global_step=epoch)\n            writer.add_scalars(main_tag=f'fold{fold}_fn', tag_scalar_dict={'train_fn':fn, 'val_fn':val_fn}, global_step=epoch)\n            writer.add_scalars(main_tag=f'fold{fold}_tp', tag_scalar_dict={'train_tp':tp, 'val_tp':val_tp}, global_step=epoch)\n            \n            writer.add_scalars(main_tag=f'fold{fold}_avpr', tag_scalar_dict={'train_avpr':results[fold]['train_avpr'][-1], 'val_avpr':results[fold]['val_avpr'][-1]})\n            try:\n                writer.add_graph(model, input_to_model=x_val)\n            except Exception as e:\n                print(e)\n                try:\n                    x_val[0] = torch.tensor(x_val[0], device='cpu', dtype=torch.float32)\n                    x_val[1] = torch.tensor(x_val[1], device='cpu', dtype=torch.float32)\n                    writer.add_graph(model, input_to_model=x_val)\n                except Exception as e:\n                    print(e)\n                    print('No graph, whatever')\n            try:\n                writer.add_pr_curve(f'train_fold{fold}_pr_curve',train.df['target'].values, train_preds, global_step=epoch)\n                writer.add_pr_curve(f'val_fold{fold}_pr_curve',val.df['target'].values, val_preds, global_step=epoch)\n            except Exception as e:\n                print(f'Error in add_pr_curve {e}')\n            try:\n                fig, ax =plt.subplots(figsize=(15,10))\n                PrecisionRecallDisplay.from_predictions(train.df['target'].values, train_preds.cpu(), ax=ax)\n                writer.add_figure(f'fold{fold}_pr_curve', fig, global_step=epoch)\n            except Exception as e:\n                print(f'Error in PrecisionRecallDisplay {e}')\n            try: \n                fig, ax =plt.subplots(figsize=(15,10))\n                PrecisionRecallDisplay.from_predictions(val.df['target'].values, val_preds.cpu(), ax=ax)\n                writer.add_figure(f'fold{fold}_pr_val_curve', fig, global_step=epoch)\n            except:\n                continue\n            \n            scheduler.step(val_avpr)\n                \n            if val_avpr >= best_val:\n                best_val = val_avpr\n                patience = es_patience  # Resetting patience since we have new best validation accuracy\n                torch.save(model, model_path)  # Saving current best model\n            else:\n                patience -= 1\n                if patience == 0:\n                    print('Early stopping. Best Val roc_auc: {:.3f}'.format(best_val))\n                    break\n                \n    model = torch.load(model_path)  # Loading best model of this fold\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n    with torch.no_grad():\n        # Predicting on validation set once again to obtain data for OOF\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n            z_val = model(x_val)\n            val_pred = torch.sigmoid(z_val)\n            val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n        oof[val_idx] = val_preds.cpu().numpy()\n        \n        # Predicting on test set\n        tta_preds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)\n        for _ in range(TTA):\n            for i, x_test in enumerate(test_loader):\n                x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)\n                x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)\n                z_test = model(x_test)\n                z_test = torch.sigmoid(z_test)\n                tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test\n        preds += tta_preds / TTA\n    \npreds /= skf.n_splits\nwriter.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open('results.json', 'w') as handle:\n    try:\n        handle.write(json.dumps(results))\n    except:\n        print('Oh well')","metadata":{"execution":{"iopub.status.busy":"2022-12-10T01:00:32.442822Z","iopub.status.idle":"2022-12-10T01:00:32.443591Z","shell.execute_reply.started":"2022-12-10T01:00:32.443331Z","shell.execute_reply":"2022-12-10T01:00:32.443355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r file.zip /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2022-12-10T01:00:32.445015Z","iopub.status.idle":"2022-12-10T01:00:32.445764Z","shell.execute_reply.started":"2022-12-10T01:00:32.445494Z","shell.execute_reply":"2022-12-10T01:00:32.445518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink('file.zip')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T01:00:32.447100Z","iopub.status.idle":"2022-12-10T01:00:32.447862Z","shell.execute_reply.started":"2022-12-10T01:00:32.447595Z","shell.execute_reply":"2022-12-10T01:00:32.447618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","metadata":{"execution":{"iopub.status.busy":"2022-12-08T13:32:18.067190Z","iopub.status.idle":"2022-12-08T13:32:18.072637Z","shell.execute_reply.started":"2022-12-08T13:32:18.072205Z","shell.execute_reply":"2022-12-08T13:32:18.072249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = torch.load('model_1.pth')","metadata":{"execution":{"iopub.status.busy":"2022-12-08T13:32:24.629227Z","iopub.execute_input":"2022-12-08T13:32:24.629655Z","iopub.status.idle":"2022-12-08T13:32:24.840219Z","shell.execute_reply.started":"2022-12-08T13:32:24.629617Z","shell.execute_reply":"2022-12-08T13:32:24.839163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.eval()","metadata":{"execution":{"iopub.status.busy":"2022-12-08T13:32:26.010203Z","iopub.execute_input":"2022-12-08T13:32:26.010596Z","iopub.status.idle":"2022-12-08T13:32:26.034155Z","shell.execute_reply.started":"2022-12-08T13:32:26.010561Z","shell.execute_reply":"2022-12-08T13:32:26.032779Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_loader = DataLoader(dataset=test, batch_size=8, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2022-12-08T13:32:30.541422Z","iopub.execute_input":"2022-12-08T13:32:30.541867Z","iopub.status.idle":"2022-12-08T13:32:30.548266Z","shell.execute_reply.started":"2022-12-08T13:32:30.541829Z","shell.execute_reply":"2022-12-08T13:32:30.546482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_preds_fold = {}\n# actual_values_fold = {}\n# for fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):\n#     print(len(val_idx))\n#     val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n#                             imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n#                             train=True, \n#                             transforms=test_transform,\n#                             meta_features=meta_features)\n#     val_loader = DataLoader(dataset=val, batch_size=8, shuffle=False, num_workers=2)\n#     val_preds= torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n#     actual_values= torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n\n#     with torch.no_grad():\n#         # Predicting on validation set once again to obtain data for OOF\n#         a=True\n#         for j, (x_val, y_val) in enumerate(val_loader):\n\n#             x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n#             x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n#             y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n#             z_val = model(x_val)\n#             val_pred = torch.sigmoid(z_val)\n# #             print(f'''Batch number: {j} \\n Start of pred: {j*val_loader.batch_size}, end of pred: {j*val_loader.batch_size + x_val[0].shape[0]} \n# #             \\n \n# #                   ''')\n#             if a:\n#                 a=False\n#             try:\n#                 actual_values[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = y_val.reshape(-1,1)\n#                 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n#             except Exception as e:\n#                 print(e)\n#         oof[val_idx] = val_preds.cpu().numpy()\n#         print(f'Finished fold {fold}')\n#     val_preds_fold[fold] = val_preds\n#     actual_values_fold[fold] = actual_values\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:18:37.259329Z","iopub.execute_input":"2022-12-08T14:18:37.259745Z","iopub.status.idle":"2022-12-08T14:23:07.510441Z","shell.execute_reply.started":"2022-12-08T14:18:37.259688Z","shell.execute_reply":"2022-12-08T14:23:07.508269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation","metadata":{}},{"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, precision_recall_fscore_support","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:40:30.267296Z","iopub.execute_input":"2022-12-08T14:40:30.267755Z","iopub.status.idle":"2022-12-08T14:40:30.273686Z","shell.execute_reply.started":"2022-12-08T14:40:30.267689Z","shell.execute_reply":"2022-12-08T14:40:30.272442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# av_cpu = [i.cpu() for i in actual_values_fold.values()]","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:28:51.764681Z","iopub.execute_input":"2022-12-08T14:28:51.765291Z","iopub.status.idle":"2022-12-08T14:28:51.773533Z","shell.execute_reply.started":"2022-12-08T14:28:51.765237Z","shell.execute_reply":"2022-12-08T14:28:51.772538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_cpu = [i.cpu() for i in val_preds_fold.values()]","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:29:12.958861Z","iopub.execute_input":"2022-12-08T14:29:12.959301Z","iopub.status.idle":"2022-12-08T14:29:12.965784Z","shell.execute_reply.started":"2022-12-08T14:29:12.959263Z","shell.execute_reply":"2022-12-08T14:29:12.964598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:59:04.069880Z","iopub.execute_input":"2022-12-08T14:59:04.070308Z","iopub.status.idle":"2022-12-08T14:59:04.091268Z","shell.execute_reply.started":"2022-12-08T14:59:04.070273Z","shell.execute_reply":"2022-12-08T14:59:04.089531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold = 3\n# threshold = 0.02\n# cm = confusion_matrix(av_cpu[fold], (pred_cpu[fold]>threshold).float())\n# disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n# print(precision_recall_fscore_support(av_cpu[fold], (pred_cpu[fold]>threshold).float()))\n# print(roc_auc_score(av_cpu[fold], (pred_cpu[fold]>threshold).float()))\n# disp.plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-08T15:13:15.117319Z","iopub.execute_input":"2022-12-08T15:13:15.117809Z","iopub.status.idle":"2022-12-08T15:13:15.427302Z","shell.execute_reply.started":"2022-12-08T15:13:15.117721Z","shell.execute_reply":"2022-12-08T15:13:15.426134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import roc_auc_score\n","metadata":{"execution":{"iopub.status.busy":"2022-12-08T15:11:24.953152Z","iopub.execute_input":"2022-12-08T15:11:24.954191Z","iopub.status.idle":"2022-12-08T15:11:24.972430Z","shell.execute_reply.started":"2022-12-08T15:11:24.954147Z","shell.execute_reply":"2022-12-08T15:11:24.971401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (pred_cpu[1]>0.01).float()","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:32:39.030682Z","iopub.execute_input":"2022-12-08T14:32:39.031671Z","iopub.status.idle":"2022-12-08T14:32:39.040903Z","shell.execute_reply.started":"2022-12-08T14:32:39.031633Z","shell.execute_reply":"2022-12-08T14:32:39.039128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:32:46.033609Z","iopub.execute_input":"2022-12-08T14:32:46.034107Z","iopub.status.idle":"2022-12-08T14:32:46.059961Z","shell.execute_reply.started":"2022-12-08T14:32:46.034059Z","shell.execute_reply":"2022-12-08T14:32:46.059025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_preds_fold[2]","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:14:41.759109Z","iopub.execute_input":"2022-12-08T14:14:41.759564Z","iopub.status.idle":"2022-12-08T14:14:41.770252Z","shell.execute_reply.started":"2022-12-08T14:14:41.759522Z","shell.execute_reply":"2022-12-08T14:14:41.769237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer Learning","metadata":{}},{"cell_type":"code","source":"# !pip install torchinfo\n# from torchinfo import summary\n# summary(arch, input_size=(16,3,256,256))\n# arch.classifier = nn.Sequential(nn.Dropout(p=0.3), nn.Linear(in_features=1408, out_feature))","metadata":{},"execution_count":null,"outputs":[]}]}