{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"# To define the behavior of the architecture\nAUGMENT = 0\nFROZEN = False\nCOMBINED = False\nEF = 'B4'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:38.983623Z","iopub.execute_input":"2022-12-23T11:53:38.983994Z","iopub.status.idle":"2022-12-23T11:53:38.989064Z","shell.execute_reply.started":"2022-12-23T11:53:38.983964Z","shell.execute_reply":"2022-12-23T11:53:38.988009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet_pytorch torchtoolbox","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:38.991645Z","iopub.execute_input":"2022-12-23T11:53:38.992008Z","iopub.status.idle":"2022-12-23T11:53:49.028139Z","shell.execute_reply.started":"2022-12-23T11:53:38.991972Z","shell.execute_reply":"2022-12-23T11:53:49.026915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torchtoolbox.transform as transforms\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os \nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:49.030237Z","iopub.execute_input":"2022-12-23T11:53:49.030654Z","iopub.status.idle":"2022-12-23T11:53:49.041578Z","shell.execute_reply.started":"2022-12-23T11:53:49.030612Z","shell.execute_reply":"2022-12-23T11:53:49.040335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 42\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:49.043362Z","iopub.execute_input":"2022-12-23T11:53:49.043731Z","iopub.status.idle":"2022-12-23T11:53:49.055837Z","shell.execute_reply.started":"2022-12-23T11:53:49.043698Z","shell.execute_reply":"2022-12-23T11:53:49.054814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:49.059276Z","iopub.execute_input":"2022-12-23T11:53:49.059918Z","iopub.status.idle":"2022-12-23T11:53:49.176008Z","shell.execute_reply.started":"2022-12-23T11:53:49.059883Z","shell.execute_reply":"2022-12-23T11:53:49.175012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.over_sampling import RandomOverSampler","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:49.177719Z","iopub.execute_input":"2022-12-23T11:53:49.178084Z","iopub.status.idle":"2022-12-23T11:53:49.857350Z","shell.execute_reply.started":"2022-12-23T11:53:49.178055Z","shell.execute_reply":"2022-12-23T11:53:49.856384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, meta_features = None, augment=0.05, log=False, train_transform=None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        # Don't augment if we're not training (validation or test) or if we actively disabled augment,\n        # the augment label lets us know whether it is one of the augmented images, \n        # which will mean it was duplicated by the RandomOverSampler\n        if train and augment!=0:\n            ros = RandomOverSampler(sampling_strategy=augment, random_state=SEED)\n            X, y = ros.fit_resample(df.drop(columns='target'), df[['target']])\n            df = X.join(y)\n        df['augment'] = df.duplicated()\n  \n            \n        self.df = df\n        self.imfolder = imfolder\n        if train_transform is None:\n            train_transform = transforms.Compose([\n                transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomVerticalFlip()\n            ])\n        test_transform = transforms.Compose([\n            transforms.ToTensor(),\n            # Normalize to imagenet means and stds\n            transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n        ])\n        self.transforms = {'Augment':train_transform, 'Normal':test_transform}\n        self.train = train\n        self.meta_features = meta_features\n        self.log=log\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n\n        if self.log:\n            print(im_path)\n        x = cv2.imread(im_path)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n        augment = self.df.iloc[index]['augment']\n        if self.train and augment:\n            x = self.transforms['Augment'](x)\n        x = self.transforms['Normal'](x)\n            \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n        \n    def __img_index__(self, image, disp:bool=False):\n        sub_df = self.df[self.df.image_name==image]\n        if disp:\n            print(sub_df)\n        return sub_df.index\n        \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass Net(nn.Module):\n    def __init__(self, arch, n_meta_features: int, freeze:bool=False, ef='B4', combine=False):\n        super(Net, self).__init__()\n        self.arch = arch\n        if freeze:\n            for mod in self.arch.modules():\n                mod.requires_grad_ = False\n\n            arch.classifier = torch.nn.Sequential(nn.Dropout(p=0.2, inplace=True),\n                                                nn.Linear(in_features=1280, \n                                                          out_features=500,\n                                                          bias=True))\n        if 'ResNet' in str(arch.__class__):\n            self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n        if 'EfficientNet' in str(arch.__class__) and ef='B4':\n            in_features=1792\n        else if ef='B6':\n            in_features=2304\n        self.arch._fc = nn.Linear(in_features=in_features, out_features=500, bias=True)\n\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        if combine:\n            self.ouput = nn.Sequential(nn.Linear(750, 300), nn.Linear(300,1))\n\n        else:\n            self.ouput = nn.Linear(750, 1)\n\n        \n        \n        \n    def forward(self, inputs):\n        \"\"\"\n        No sigmoid in forward because we are going to use BCEWithLogitsLoss\n        Which applies sigmoid for us when calculating a loss\n        \"\"\"\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:00:18.783842Z","iopub.execute_input":"2022-12-23T12:00:18.784460Z","iopub.status.idle":"2022-12-23T12:00:18.836095Z","shell.execute_reply.started":"2022-12-23T12:00:18.784416Z","shell.execute_reply":"2022-12-23T12:00:18.834197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arch = EfficientNet.from_pretrained('efficientnet-b6')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:49.883499Z","iopub.execute_input":"2022-12-23T11:53:49.883865Z","iopub.status.idle":"2022-12-23T11:53:53.256290Z","shell.execute_reply.started":"2022-12-23T11:53:49.883829Z","shell.execute_reply":"2022-12-23T11:53:53.255251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')\ntest_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:53.257834Z","iopub.execute_input":"2022-12-23T11:53:53.260525Z","iopub.status.idle":"2022-12-23T11:53:53.389560Z","shell.execute_reply.started":"2022-12-23T11:53:53.260496Z","shell.execute_reply":"2022-12-23T11:53:53.388585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = train_df.join(pd.get_dummies(train_df.diagnosis).iloc[:,:-1])\n# train_df['dum_male'] = pd.get_dummies(train_df.sex)['male']\n# train_df = train_df.drop(columns=['anatom_site_general_challenge', 'sex', 'benign_malignant', 'diagnosis'])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:53.390807Z","iopub.execute_input":"2022-12-23T11:53:53.392871Z","iopub.status.idle":"2022-12-23T11:53:53.396942Z","shell.execute_reply.started":"2022-12-23T11:53:53.392843Z","shell.execute_reply":"2022-12-23T11:53:53.395893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:53.398439Z","iopub.execute_input":"2022-12-23T11:53:53.399056Z","iopub.status.idle":"2022-12-23T11:53:53.447770Z","shell.execute_reply.started":"2022-12-23T11:53:53.399019Z","shell.execute_reply":"2022-12-23T11:53:53.446829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:53:53.449089Z","iopub.execute_input":"2022-12-23T11:53:53.449480Z","iopub.status.idle":"2022-12-23T11:53:53.455052Z","shell.execute_reply.started":"2022-12-23T11:53:53.449443Z","shell.execute_reply":"2022-12-23T11:53:53.454000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation ","metadata":{}},{"cell_type":"code","source":"skf = GroupKFold(n_splits=3)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:11.297022Z","iopub.execute_input":"2022-12-23T11:54:11.297389Z","iopub.status.idle":"2022-12-23T11:54:11.302259Z","shell.execute_reply.started":"2022-12-23T11:54:11.297357Z","shell.execute_reply":"2022-12-23T11:54:11.301104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tt=(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1)[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:11.530906Z","iopub.execute_input":"2022-12-23T11:54:11.531271Z","iopub.status.idle":"2022-12-23T11:54:11.537941Z","shell.execute_reply.started":"2022-12-23T11:54:11.531240Z","shell.execute_reply":"2022-12-23T11:54:11.536766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in tt:\n    train_idx=t[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:11.728636Z","iopub.execute_input":"2022-12-23T11:54:11.729278Z","iopub.status.idle":"2022-12-23T11:54:11.754972Z","shell.execute_reply.started":"2022-12-23T11:54:11.729244Z","shell.execute_reply":"2022-12-23T11:54:11.754039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(), \n    transforms.RandomApply([transforms.RandomSPNoise(p=0.3, prob=0.01), transforms.RandomGaussianNoise(p=0.3)], p=0.33),\n    transforms.RandomApply([transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0)], p=0.75)\n    \n])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:11.916727Z","iopub.execute_input":"2022-12-23T11:54:11.917078Z","iopub.status.idle":"2022-12-23T11:54:11.925069Z","shell.execute_reply.started":"2022-12-23T11:54:11.917049Z","shell.execute_reply":"2022-12-23T11:54:11.922375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n                        imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                        train=True, train_transform=train_transform,\n                        meta_features=meta_features, augment=0.05)\ntrain_loader = DataLoader(dataset=train, batch_size=16, shuffle=True, num_workers=2)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:12.110188Z","iopub.execute_input":"2022-12-23T11:54:12.110579Z","iopub.status.idle":"2022-12-23T11:54:12.210309Z","shell.execute_reply.started":"2022-12-23T11:54:12.110544Z","shell.execute_reply":"2022-12-23T11:54:12.209289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we observe the transformations being applied","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8,8))\nimg=cv2.imread(f'/kaggle/input/melanoma-external-malignant-256/train/train/{train.df[train.df.augment].iloc[0].image_name}.jpg')\nimg = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\nprint(train.df[train.df.augment].iloc[0].image_name)\nax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:12.855172Z","iopub.execute_input":"2022-12-23T11:54:12.855555Z","iopub.status.idle":"2022-12-23T11:54:13.232682Z","shell.execute_reply.started":"2022-12-23T11:54:12.855522Z","shell.execute_reply":"2022-12-23T11:54:13.231816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices = train.__img_index__(train.df[train.df.augment].iloc[0].image_name)\nfig, ax = plt.subplots(len(indices), figsize=(5,5*len(indices)))\nfor i, index in enumerate(indices):\n    image = train[index]\n    img = cv2.cvtColor(image[0][0].numpy().transpose(1, 2, 0), cv2.COLOR_RGB2BGR)\n    ax[i].imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:13.396082Z","iopub.execute_input":"2022-12-23T11:54:13.396655Z","iopub.status.idle":"2022-12-23T11:54:14.079224Z","shell.execute_reply.started":"2022-12-23T11:54:13.396618Z","shell.execute_reply":"2022-12-23T11:54:14.078325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/', \n                       train=False,\n                       train_transform=train_transform,  # For TTA\n                       meta_features=meta_features)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:14.081216Z","iopub.execute_input":"2022-12-23T11:54:14.082400Z","iopub.status.idle":"2022-12-23T11:54:14.095546Z","shell.execute_reply.started":"2022-12-23T11:54:14.082361Z","shell.execute_reply":"2022-12-23T11:54:14.094302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS=3","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:14.464020Z","iopub.execute_input":"2022-12-23T11:54:14.464372Z","iopub.status.idle":"2022-12-23T11:54:14.469089Z","shell.execute_reply.started":"2022-12-23T11:54:14.464342Z","shell.execute_reply":"2022-12-23T11:54:14.467828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = GroupKFold(n_splits=FOLDS)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:14.883014Z","iopub.execute_input":"2022-12-23T11:54:14.884729Z","iopub.status.idle":"2022-12-23T11:54:14.889691Z","shell.execute_reply.started":"2022-12-23T11:54:14.884681Z","shell.execute_reply":"2022-12-23T11:54:14.888613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter\nfrom sklearn.metrics import confusion_matrix, average_precision_score, precision_recall_curve, PrecisionRecallDisplay\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:15.173921Z","iopub.execute_input":"2022-12-23T11:54:15.174275Z","iopub.status.idle":"2022-12-23T11:54:15.699091Z","shell.execute_reply.started":"2022-12-23T11:54:15.174244Z","shell.execute_reply":"2022-12-23T11:54:15.697888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_writer(experiment_name: str, \n                  model_name: str, \n                  extra: str=None) -> torch.utils.tensorboard.writer.SummaryWriter():\n    \"\"\"Creates a torch.utils.tensorboard.writer.SummaryWriter() instance saving to a specific log_dir.\n\n    log_dir is a combination of runs/timestamp/experiment_name/model_name/extra.\n\n    Where timestamp is the current date in YYYY-MM-DD format.\n\n    Args:\n        experiment_name (str): Name of experiment.\n        model_name (str): Name of model.\n        extra (str, optional): Anything extra to add to the directory. Defaults to None.\n\n    Returns:\n        torch.utils.tensorboard.writer.SummaryWriter(): Instance of a writer saving to log_dir.\n\n    Example usage:\n        # Create a writer saving to \"runs/2022-06-04/data_10_percent/effnetb2/5_epochs/\"\n        writer = create_writer(experiment_name=\"data_10_percent\",\n                               model_name=\"effnetb2\",\n                               extra=\"5_epochs\")\n        # The above is the same as:\n        writer = SummaryWriter(log_dir=\"runs/2022-06-04/data_10_percent/effnetb2/5_epochs/\")\n    \"\"\"\n    from datetime import datetime\n    import os\n\n    # Get timestamp of current date (all experiments on certain day live in same folder)\n    timestamp = datetime.now().strftime(\"%Y-%m-%d\") # returns current date in YYYY-MM-DD format\n\n    if extra:\n        # Create log directory path\n        log_dir = os.path.join(\"runs\", timestamp, experiment_name, model_name, extra)\n    else:\n        log_dir = os.path.join(\"runs\", timestamp, experiment_name, model_name)\n        \n    print(f\"[INFO] Created SummaryWriter, saving to: {log_dir}...\")\n    return SummaryWriter(log_dir=log_dir)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:15.990573Z","iopub.execute_input":"2022-12-23T11:54:15.990922Z","iopub.status.idle":"2022-12-23T11:54:24.319935Z","shell.execute_reply.started":"2022-12-23T11:54:15.990892Z","shell.execute_reply":"2022-12-23T11:54:24.318938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"writer = create_writer(experiment_name=f'{AUGMENT}upsampling_stratified', model_name='efficientnet_pretrained_unfrozen')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.321735Z","iopub.execute_input":"2022-12-23T11:54:24.322096Z","iopub.status.idle":"2022-12-23T11:54:24.331635Z","shell.execute_reply.started":"2022-12-23T11:54:24.322061Z","shell.execute_reply":"2022-12-23T11:54:24.330229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = ['{}_loss', '{}_accuracy', '{}_roc', '{}_avpr', '{}_tn', '{}_tp', '{}_fn', '{}_fp', '{}_precision_curve', '{}_recall_curve', '{}_thresholds_curve']","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.333137Z","iopub.execute_input":"2022-12-23T11:54:24.333586Z","iopub.status.idle":"2022-12-23T11:54:24.342117Z","shell.execute_reply.started":"2022-12-23T11:54:24.333549Z","shell.execute_reply":"2022-12-23T11:54:24.341074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = {i+1:{m.format(f):[] for m in metrics for f in ['train', 'val']} for i in range(FOLDS)}\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.344794Z","iopub.execute_input":"2022-12-23T11:54:24.345316Z","iopub.status.idle":"2022-12-23T11:54:24.358393Z","shell.execute_reply.started":"2022-12-23T11:54:24.345283Z","shell.execute_reply":"2022-12-23T11:54:24.357440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.360959Z","iopub.execute_input":"2022-12-23T11:54:24.361609Z","iopub.status.idle":"2022-12-23T11:54:24.371508Z","shell.execute_reply.started":"2022-12-23T11:54:24.361572Z","shell.execute_reply":"2022-12-23T11:54:24.370489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NumpyEncoder(json.JSONEncoder):\n    \"\"\" Custom encoder for numpy data types \"\"\"\n    def default(self, obj):\n        if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,\n                            np.int16, np.int32, np.int64, np.uint8,\n                            np.uint16, np.uint32, np.uint64)):\n\n            return int(obj)\n\n        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):\n            return float(obj)\n        \n        elif isinstance(obj, (np.complex_, np.complex64, np.complex128)):\n            return {'real': obj.real, 'imag': obj.imag}\n        \n        elif isinstance(obj, (np.ndarray,)):\n            return obj.tolist()\n    \n        elif isinstance(obj, (np.bool_)):\n            return bool(obj)\n\n        elif isinstance(obj, (np.void)): \n            return None\n\n        return json.JSONEncoder.default(self, obj)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.372890Z","iopub.execute_input":"2022-12-23T11:54:24.373280Z","iopub.status.idle":"2022-12-23T11:54:24.384432Z","shell.execute_reply.started":"2022-12-23T11:54:24.373244Z","shell.execute_reply":"2022-12-23T11:54:24.383426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.385764Z","iopub.execute_input":"2022-12-23T11:54:24.386564Z","iopub.status.idle":"2022-12-23T11:54:24.396284Z","shell.execute_reply.started":"2022-12-23T11:54:24.386531Z","shell.execute_reply":"2022-12-23T11:54:24.395227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:54:24.397632Z","iopub.execute_input":"2022-12-23T11:54:24.398868Z","iopub.status.idle":"2022-12-23T11:54:24.409751Z","shell.execute_reply.started":"2022-12-23T11:54:24.398834Z","shell.execute_reply":"2022-12-23T11:54:24.408776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10 # Number of epochs to run\nes_patience = 50  # Early Stopping patience - for how many epochs with no improvements to wait\nTTA = 3 # Test Time Augmentation rounds\nfold=1\noof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\npreds = torch.zeros((len(train_df), 1), dtype=torch.float32, device=device)  # Predictions for test test\nresults = {i+1:{m.format(f):[] for m in metrics for f in ['train', 'val']} for i in range(FOLDS)}\n\ntrain_df, validation_df = train_test_split(train_df, test_size=0.2, train_size=0.8, stratify=train_df.target)\nvalidation_df, test_df = train_test_split(validation_df, test_size=0.5, train_size=0.5, stratify=validation_df.target)\n\nmodel_path = f'model.pth'  # Path and filename to save model to\nbest_val = 0  # Best validation score within this fold\npatience = es_patience  # Current patience counter\nif EF='B6':\n    arch = EfficientNet.from_pretrained('efficientnet-b6')\nelse:\n    arch = EfficientNet.from_pretrained('efficientnet-b4')\nmodel = Net(arch=arch, n_meta_features=len(meta_features), freeze=FROZEN, ef = EF, combined=COMBINED)  # New model for each fold\nmodel = model.to(device)\ntorch.save(model, model_path)\n\n\noptim = torch.optim.Adam(model.parameters(), lr=0.001)\nscheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\ncriterion = nn.BCEWithLogitsLoss()\n\ntrain = MelanomaDataset(df=train_df, \n                        imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                        train=True, \n                        train_transform=train_transform,\n                        meta_features=meta_features, augment=AUGMENT)\nval = MelanomaDataset(df=validation_df, \n                        imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                        train=True, \n                        augment=0,\n                        meta_features=meta_features)\ntest = MelanomaDataset(df=test_df, \n                        imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                        train=True, \n                        augment=0,\n                        meta_features=meta_features)\n\ntrain_loader = DataLoader(dataset=train, batch_size=16, shuffle=True, num_workers=2)\nval_loader = DataLoader(dataset=val, batch_size=8, shuffle=False, num_workers=2)\ntest_loader = DataLoader(dataset=test, batch_size=8, shuffle=False, num_workers=2)\n\nfor epoch in range(epochs):\n    start_time = time.time()\n    correct = 0\n    epoch_loss = 0\n    model.train()\n        \n    train_preds = torch.zeros((len(train.df), 1), dtype=torch.float32, device=device)\n    for j, (x, y) in enumerate(train_loader):\n        x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n        x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n        y = torch.tensor(y, device=device, dtype=torch.float32)\n        optim.zero_grad()\n        z = model(x)\n        loss = criterion(z, y.unsqueeze(1))\n        loss.backward()\n        optim.step()\n        pred = torch.round(torch.sigmoid(z))  # round off sigmoid to obtain predictions\n        try:\n            train_preds[j*train_loader.batch_size:j*train_loader.batch_size+x[0].shape[0]] = torch.sigmoid(z)\n        except Exception as e:\n            print(e)\n            print(f'Train preds {train_preds.shape}\\n Train loader {train_loader.batch_size}\\n J {j}\\n Pred:{torch.sigmoid(z).shape}')\n\n        correct += (pred.cpu() == y.cpu().unsqueeze(1)).sum().item()  # tracking number of correctly predicted samples\n        epoch_loss += loss.item()\n            \n    # Metrics computation\n    train_acc = correct / len(train.df)\n    train_roc = roc_auc_score(train.df['target'].values, train_preds.detach().cpu())\n    train_avpr = average_precision_score(train.df['target'].values, train_preds.detach().cpu())\n\n    results[fold]['train_loss'].append(epoch_loss)\n    results[fold]['train_accuracy'].append(train_acc)\n    results[fold]['train_roc'].append(train_roc)\n\n    cm = confusion_matrix(train.df['target'].values, torch.round(train_preds.detach().cpu()))\n    tn, fp, fn, tp = cm.ravel()\n    results[fold]['train_tn'].append(tn)\n    results[fold]['train_fp'].append(fp)\n    results[fold]['train_fn'].append(fn)\n    results[fold]['train_tp'].append(tp)\n    results[fold]['train_avpr'].append(train_avpr)\n    try:\n        pr_curve=precision_recall_curve(train.df['target'].values, train_preds.detach().cpu())\n        results[fold]['train_precision_curve'].append(list(pr_curve[0]))\n        results[fold]['train_recall_curve'].append(list(pr_curve[1]))\n        results[fold]['train_thresholds_curve'].append(list(pr_curve[2]))\n    except Exception as e:\n        print(f'PR train error {e}')\n            \n\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(validation_df), 1), dtype=torch.float32, device=device)\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        # Predicting on validation set\n        val_epoch_loss=0\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n            z_val = model(x_val)\n            val_loss = criterion(z_val, y_val.unsqueeze(1))\n            val_pred = torch.sigmoid(z_val)\n            val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n            val_epoch_loss += val_loss.item()\n        val_acc = accuracy_score(val.df['target'].values, torch.round(val_preds.cpu()))\n        val_roc = roc_auc_score(val.df['target'].values, val_preds.cpu())\n        val_avpr = average_precision_score(val.df['target'].values, val_preds.cpu())\n\n        results[fold]['val_loss'].append(val_epoch_loss)\n        results[fold]['val_accuracy'].append(val_acc)\n        results[fold]['val_roc'].append(val_roc)\n\n        cm = confusion_matrix(val.df['target'].values, torch.round(val_preds.cpu()))\n        val_tn, val_fp, val_fn, val_tp = cm.ravel()\n        results[fold]['val_tn'].append(val_tn)\n        results[fold]['val_fp'].append(val_fp)\n        results[fold]['val_fn'].append(val_fn)\n        results[fold]['val_tp'].append(val_tp)            \n        results[fold]['val_avpr'].append(val_avpr)\n        try:\n            pr_curve=precision_recall_curve(val.df['target'].values, val_preds.detach().cpu())\n            results[fold]['val_precision_curve'].append(list(pr_curve[0]))\n            results[fold]['val_recall_curve'].append(list(pr_curve[1]))\n            results[fold]['val_thresholds_curve'].append(list(pr_curve[2]))\n        except Exception as e:\n            print(f'PR val error {e}')\n\n\n        print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(\n        epoch + 1, \n        epoch_loss, \n        train_acc, \n        val_acc, \n        val_roc, \n        str(datetime.timedelta(seconds=time.time() - start_time))[:7]))\n        print(f'Train avpr: {train_avpr}, Validation avpr:{val_avpr} \\n')\n\n        with open('results.json', 'w') as handle:\n            try:\n                handle.write(json.dumps(results, cls=NumpyEncoder))\n            except Exception as e:\n                print(e)\n                print(results)\n                continue\n        writer.add_scalars(main_tag=f'fold{fold}_loss', tag_scalar_dict={\"train_loss\": results[fold]['train_loss'][-1],\n                                        \"val_loss\": results[fold]['val_loss'][-1]},global_step=epoch)\n        writer.add_scalars(main_tag=f'fold{fold}_accuracy', tag_scalar_dict={\"train_accuracy\": results[fold]['train_accuracy'][-1],\n                                        \"val_accuracy\": results[fold]['val_accuracy'][-1]},global_step=epoch)\n        writer.add_scalars(main_tag=f'fold{fold}_roc', tag_scalar_dict={\"train_roc\": results[fold]['train_roc'][-1],\n                                        \"val_roc\": results[fold]['val_roc'][-1]},global_step=epoch)\n\n        writer.add_scalars(main_tag=f'fold{fold}_tn', tag_scalar_dict={'train_tn':tn, 'val_tn':val_tn}, global_step=epoch)\n        writer.add_scalars(main_tag=f'fold{fold}_fp', tag_scalar_dict={'train_fp':fp, 'val_fp':val_fp}, global_step=epoch)\n        writer.add_scalars(main_tag=f'fold{fold}_fn', tag_scalar_dict={'train_fn':fn, 'val_fn':val_fn}, global_step=epoch)\n        writer.add_scalars(main_tag=f'fold{fold}_tp', tag_scalar_dict={'train_tp':tp, 'val_tp':val_tp}, global_step=epoch)\n\n        f2 = {'train_f2_score':(5*tp)/(5*tp+4*fn+fp) , 'val_f2_score':(5*val_tp)/(5*val_tp+4*val_fn+val_fp)}\n        writer.add_scalars(main_tag=f'fold{fold}_f2_score', tag_scalar_dict=f2, global_step=epoch)\n\n        writer.add_scalars(main_tag=f'fold{fold}_avpr', tag_scalar_dict={'train_avpr':results[fold]['train_avpr'][-1], 'val_avpr':results[fold]['val_avpr'][-1]},\n                          global_step=epoch)\n#             try:\n#                 writer.add_graph(model, input_to_model=x_val)\n#             except Exception as e:\n#                 print(e)\n#                 try:\n#                     x_val[0] = torch.tensor(x_val[0], device='cpu', dtype=torch.float32)\n#                     x_val[1] = torch.tensor(x_val[1], device='cpu', dtype=torch.float32)\n#                     writer.add_graph(model, input_to_model=x_val)\n#                 except Exception as e:\n#                     print(e)\n#                     print('No graph, whatever')\n#             try:\n#                 writer.add_pr_curve(f'train_fold{fold}_pr_curve',train.df['target'].values, train_preds, global_step=epoch)\n#                 writer.add_pr_curve(f'val_fold{fold}_pr_curve',val.df['target'].values, val_preds, global_step=epoch)\n#             except Exception as e:\n#                 print(f'Error in add_pr_curve {e}')\n        try:\n            fig, ax =plt.subplots(figsize=(15,10))\n            PrecisionRecallDisplay.from_predictions(train.df['target'].values, train_preds.cpu(), ax=ax)\n            writer.add_figure(f'fold{fold}_pr_curve', fig, global_step=epoch)\n        except Exception as e:\n            print(f'Error in PrecisionRecallDisplay {e}')\n        try: \n            fig, ax =plt.subplots(figsize=(15,10))\n            PrecisionRecallDisplay.from_predictions(val.df['target'].values, val_preds.cpu(), ax=ax)\n            writer.add_figure(f'fold{fold}_pr_val_curve', fig, global_step=epoch)\n        except:\n            continue\n\n        scheduler.step(val_roc)\n\n        if val_roc >= best_val or epoch==1:\n            print(f'Model saved for fold {fold} in {model_path}')\n#             os.system('ls')\n            best_val = val_roc\n            patience = es_patience  # Resetting patience since we have new best validation accuracy\n            torch.save(model, model_path)  # Saving current best model\n        else:\n            patience -= 1\n            if patience == 0:\n                print('Early stopping. Best Val roc_auc: {:.3f}'.format(best_val))\n                break\ntry:        \n    model = torch.load(model_path)  # Loading best model of this fold\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(validation_df), 1), dtype=torch.float32, device=device)\n    with torch.no_grad():\n        # Predicting on validation set once again to obtain data for OOF\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n            z_val = model(x_val)\n            val_pred = torch.sigmoid(z_val)\n            val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n        oof[validation_df] = val_preds.cpu().numpy()\n\n        # Predicting on test set\n        tta_preds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)\n        for _ in range(TTA):\n            for i, x_test in enumerate(test_loader):\n                x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)\n                x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)\n                z_test = model(x_test)\n                z_test = torch.sigmoid(z_test)\n                tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test\n        preds += tta_preds / TTA\nexcept Exception as e:\n    print(e)\n    print('Error on loading model')\nwriter.close()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:00:51.859594Z","iopub.execute_input":"2022-12-23T12:00:51.860222Z","iopub.status.idle":"2022-12-23T12:06:22.144780Z","shell.execute_reply.started":"2022-12-23T12:00:51.860179Z","shell.execute_reply":"2022-12-23T12:06:22.140551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"actual_results = {}\nfor fold in results.keys():\n    actual_results[fold] = {}\n    for metric in results[fold].keys():\n        actual_results[fold][metric] = results[fold][metric]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:04:06.525296Z","iopub.execute_input":"2022-12-16T15:04:06.525675Z","iopub.status.idle":"2022-12-16T15:04:06.532520Z","shell.execute_reply.started":"2022-12-16T15:04:06.525641Z","shell.execute_reply":"2022-12-16T15:04:06.531472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\ncls = ()\nwith open('results.json', 'w') as handle:\n    try:\n        handle.write(json.dumps(actual_results, cls=NumpyEncoder))\n    except Exception as e:\n        print(e)\n        print('Oh well')","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:04:08.438484Z","iopub.execute_input":"2022-12-16T15:04:08.438907Z","iopub.status.idle":"2022-12-16T15:04:08.780276Z","shell.execute_reply.started":"2022-12-16T15:04:08.438855Z","shell.execute_reply":"2022-12-16T15:04:08.779217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r runs.zip /kaggle/working/runs","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:04:11.898351Z","iopub.execute_input":"2022-12-16T15:04:11.898748Z","iopub.status.idle":"2022-12-16T15:04:13.197229Z","shell.execute_reply.started":"2022-12-16T15:04:11.898692Z","shell.execute_reply":"2022-12-16T15:04:13.195887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink('runs.zip')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-16T15:04:13.200292Z","iopub.execute_input":"2022-12-16T15:04:13.200717Z","iopub.status.idle":"2022-12-16T15:04:13.217422Z","shell.execute_reply.started":"2022-12-16T15:04:13.200660Z","shell.execute_reply":"2022-12-16T15:04:13.210808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","metadata":{"execution":{"iopub.status.busy":"2022-12-08T13:32:18.06719Z","iopub.status.idle":"2022-12-08T13:32:18.072637Z","shell.execute_reply.started":"2022-12-08T13:32:18.072205Z","shell.execute_reply":"2022-12-08T13:32:18.072249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = torch.load('/kaggle/input/models/model_2.pth')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:38:53.853006Z","iopub.execute_input":"2022-12-11T22:38:53.853410Z","iopub.status.idle":"2022-12-11T22:38:55.294836Z","shell.execute_reply.started":"2022-12-11T22:38:53.853371Z","shell.execute_reply":"2022-12-11T22:38:55.293731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.eval()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:38:55.300273Z","iopub.execute_input":"2022-12-11T22:38:55.302998Z","iopub.status.idle":"2022-12-11T22:38:55.330380Z","shell.execute_reply.started":"2022-12-11T22:38:55.302939Z","shell.execute_reply":"2022-12-11T22:38:55.329377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_loader = DataLoader(dataset=test, batch_size=8, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:38:55.335714Z","iopub.execute_input":"2022-12-11T22:38:55.336329Z","iopub.status.idle":"2022-12-11T22:38:55.343691Z","shell.execute_reply.started":"2022-12-11T22:38:55.336291Z","shell.execute_reply":"2022-12-11T22:38:55.342573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_preds_fold = {}\n# actual_values_fold = {}\n# for fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):\n#     print(len(val_idx))\n#     val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n#                             imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n#                             train=True, \n#                             augment=0,\n#                             meta_features=meta_features)\n#     val_loader = DataLoader(dataset=val, batch_size=8, shuffle=False, num_workers=2)\n#     val_preds= torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n#     actual_values= torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n\n#     with torch.no_grad():\n#         # Predicting on validation set once again to obtain data for OOF\n#         a=True\n#         for j, (x_val, y_val) in enumerate(val_loader):\n\n#             x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n#             x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n#             y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n#             z_val = model(x_val)\n#             val_pred = torch.sigmoid(z_val)\n# #             print(f'''Batch number: {j} \\n Start of pred: {j*val_loader.batch_size}, end of pred: {j*val_loader.batch_size + x_val[0].shape[0]} \n# #             \\n \n# #                   ''')\n#             if a:\n#                 a=False\n#             try:\n#                 actual_values[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = y_val.reshape(-1,1)\n#                 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n#             except Exception as e:\n#                 print(e)\n# #         oof[val_idx] = val_preds.cpu().numpy()\n#         print(f'Finished fold {fold}')\n#     val_preds_fold[fold] = val_preds\n#     actual_values_fold[fold] = actual_values\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:38:55.348745Z","iopub.execute_input":"2022-12-11T22:38:55.349144Z","iopub.status.idle":"2022-12-11T22:43:09.988766Z","shell.execute_reply.started":"2022-12-11T22:38:55.349058Z","shell.execute_reply":"2022-12-11T22:43:09.987583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation","metadata":{}},{"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, precision_recall_fscore_support, average_precision_score","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:43:09.991404Z","iopub.execute_input":"2022-12-11T22:43:09.992189Z","iopub.status.idle":"2022-12-11T22:43:09.998293Z","shell.execute_reply.started":"2022-12-11T22:43:09.992141Z","shell.execute_reply":"2022-12-11T22:43:09.997351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# av_cpu = [i.cpu() for i in actual_values_fold.values()]","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:43:09.999386Z","iopub.execute_input":"2022-12-11T22:43:10.000185Z","iopub.status.idle":"2022-12-11T22:43:10.017147Z","shell.execute_reply.started":"2022-12-11T22:43:10.000147Z","shell.execute_reply":"2022-12-11T22:43:10.016237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_cpu = [i.cpu() for i in val_preds_fold.values()]","metadata":{"execution":{"iopub.status.busy":"2022-12-11T22:43:10.019792Z","iopub.execute_input":"2022-12-11T22:43:10.021292Z","iopub.status.idle":"2022-12-11T22:43:10.028983Z","shell.execute_reply.started":"2022-12-11T22:43:10.021254Z","shell.execute_reply":"2022-12-11T22:43:10.028083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots(figsize=(6,3))\n# sns.ecdfplot(x=pred_cpu[1].numpy().reshape(len(pred_cpu[1])))\n# plt.savefig('kdeplot.jpg')\n# ax.set(xscale='log')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T23:33:28.530995Z","iopub.execute_input":"2022-12-11T23:33:28.531491Z","iopub.status.idle":"2022-12-11T23:33:29.222042Z","shell.execute_reply.started":"2022-12-11T23:33:28.531441Z","shell.execute_reply":"2022-12-11T23:33:29.220987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_cpu[1].numpy().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-11T23:27:33.015339Z","iopub.execute_input":"2022-12-11T23:27:33.015703Z","iopub.status.idle":"2022-12-11T23:27:33.026591Z","shell.execute_reply.started":"2022-12-11T23:27:33.015673Z","shell.execute_reply":"2022-12-11T23:27:33.025492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold = 0\n# threshold = 0.5\n# cm = confusion_matrix(av_cpu[fold], (pred_cpu[fold]>threshold).float())\n# disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n# print(precision_recall_fscore_support(av_cpu[fold], (pred_cpu[fold]>threshold).int()))\n# print(roc_auc_score(av_cpu[fold], (pred_cpu[fold]>threshold).float()))\n# print((cm[0][0]+cm[1][1])/cm.sum())\n# disp.plot()\n# plt.savefig(f'confusion_{threshold}.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T23:03:07.325893Z","iopub.execute_input":"2022-12-11T23:03:07.326258Z","iopub.status.idle":"2022-12-11T23:03:07.611966Z","shell.execute_reply.started":"2022-12-11T23:03:07.326226Z","shell.execute_reply":"2022-12-11T23:03:07.610944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import roc_auc_score\n","metadata":{"execution":{"iopub.status.busy":"2022-12-08T15:11:24.953152Z","iopub.execute_input":"2022-12-08T15:11:24.954191Z","iopub.status.idle":"2022-12-08T15:11:24.97243Z","shell.execute_reply.started":"2022-12-08T15:11:24.954147Z","shell.execute_reply":"2022-12-08T15:11:24.971401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (pred_cpu[1]>0.01).float()","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:32:39.030682Z","iopub.execute_input":"2022-12-08T14:32:39.031671Z","iopub.status.idle":"2022-12-08T14:32:39.040903Z","shell.execute_reply.started":"2022-12-08T14:32:39.031633Z","shell.execute_reply":"2022-12-08T14:32:39.039128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:32:46.033609Z","iopub.execute_input":"2022-12-08T14:32:46.034107Z","iopub.status.idle":"2022-12-08T14:32:46.059961Z","shell.execute_reply.started":"2022-12-08T14:32:46.034059Z","shell.execute_reply":"2022-12-08T14:32:46.059025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_preds_fold[2]","metadata":{"execution":{"iopub.status.busy":"2022-12-08T14:14:41.759109Z","iopub.execute_input":"2022-12-08T14:14:41.759564Z","iopub.status.idle":"2022-12-08T14:14:41.770252Z","shell.execute_reply.started":"2022-12-08T14:14:41.759522Z","shell.execute_reply":"2022-12-08T14:14:41.769237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer Learning","metadata":{}},{"cell_type":"code","source":"# !pip install torchinfo\n# from torchinfo import summary\n# summary(arch, input_size=(16,3,256,256))\n# arch.classifier = nn.Sequential(nn.Dropout(p=0.3), nn.Linear(in_features=1408, out_feature))","metadata":{},"execution_count":null,"outputs":[]}]}