{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1322494,"sourceType":"datasetVersion","datasetId":688574},{"sourceId":1339680,"sourceType":"datasetVersion","datasetId":756214},{"sourceId":6983586,"sourceType":"datasetVersion","datasetId":4013580},{"sourceId":6992158,"sourceType":"datasetVersion","datasetId":4018931},{"sourceId":7043425,"sourceType":"datasetVersion","datasetId":4052704}],"dockerImageVersionId":29981,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NOTEBOOK EDITED FROM https://www.kaggle.com/code/vishnus/a-simple-pytorch-starter-code-single-fold-93 \n# SIIM-ISIC Melanoma Classification\n","metadata":{}},{"cell_type":"code","source":"# !pip install efficientnet-pytorch","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-19T11:31:06.841173Z","iopub.execute_input":"2023-11-19T11:31:06.841518Z","iopub.status.idle":"2023-11-19T11:31:06.845087Z","shell.execute_reply.started":"2023-11-19T11:31:06.841489Z","shell.execute_reply":"2023-11-19T11:31:06.844232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport pandas as pd\nfrom torch.utils.data import Dataset,DataLoader\nfrom PIL import Image\nfrom torchvision import transforms as T\nimport torch.nn as nn\nimport torch\nimport torch.nn.functional as F\nfrom sklearn.model_selection import GroupKFold\nimport numpy as np\nfrom fastprogress.fastprogress import master_bar, progress_bar\nfrom sklearn.metrics import accuracy_score, roc_auc_score\n# from efficientnet_pytorch import EfficientNet\nfrom torchvision import models\nimport pdb\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensor\nimport matplotlib.pyplot as plt\n\nimport pickle ","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:45:52.465790Z","iopub.execute_input":"2023-11-26T22:45:52.466175Z","iopub.status.idle":"2023-11-26T22:45:52.474542Z","shell.execute_reply.started":"2023-11-26T22:45:52.466143Z","shell.execute_reply":"2023-11-26T22:45:52.473494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:45:52.885125Z","iopub.execute_input":"2023-11-26T22:45:52.885471Z","iopub.status.idle":"2023-11-26T22:45:52.889045Z","shell.execute_reply.started":"2023-11-26T22:45:52.885437Z","shell.execute_reply":"2023-11-26T22:45:52.888176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def list_files(path:Path):\n    return [o for o in path.iterdir()]","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:45:53.266361Z","iopub.execute_input":"2023-11-26T22:45:53.266734Z","iopub.status.idle":"2023-11-26T22:45:53.271800Z","shell.execute_reply.started":"2023-11-26T22:45:53.266701Z","shell.execute_reply":"2023-11-26T22:45:53.270704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"path = Path('../input/jpeg-melanoma-256x256/')\ndf_path = Path('../input/melanoma-256x256/')\nim_sz = 256\nbs = 64","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-26T22:46:15.569576Z","iopub.execute_input":"2023-11-26T22:46:15.569951Z","iopub.status.idle":"2023-11-26T22:46:15.574494Z","shell.execute_reply.started":"2023-11-26T22:46:15.569917Z","shell.execute_reply":"2023-11-26T22:46:15.573620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'concept_model%d' % 42","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:03.230730Z","iopub.execute_input":"2023-11-26T22:46:03.231172Z","iopub.status.idle":"2023-11-26T22:46:03.235551Z","shell.execute_reply.started":"2023-11-26T22:46:03.231129Z","shell.execute_reply":"2023-11-26T22:46:03.234496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fnames = list_files(path/'train')\ndf = pd.read_csv(df_path/'train.csv')\nprint(len(df))\ndf.head()\n\n\n# df.age_approx.unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:17.480612Z","iopub.execute_input":"2023-11-26T22:46:17.480971Z","iopub.status.idle":"2023-11-26T22:46:17.663255Z","shell.execute_reply.started":"2023-11-26T22:46:17.480941Z","shell.execute_reply":"2023-11-26T22:46:17.662320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pos = df[df[\"target\"]==1].sample(30000, replace=True)\ndf_neg = df#[df[\"target\"]==0]# .sample(580, replace=False)\ndf_aug =pd.concat([df_neg, df_pos])\n# df_neg = df[df['target']==0].sample()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:03.839291Z","iopub.execute_input":"2023-11-26T22:46:03.839634Z","iopub.status.idle":"2023-11-26T22:46:03.871473Z","shell.execute_reply.started":"2023-11-26T22:46:03.839589Z","shell.execute_reply":"2023-11-26T22:46:03.870826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_aug","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:03.913469Z","iopub.execute_input":"2023-11-26T22:46:03.913763Z","iopub.status.idle":"2023-11-26T22:46:03.917215Z","shell.execute_reply.started":"2023-11-26T22:46:03.913736Z","shell.execute_reply":"2023-11-26T22:46:03.916398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path/'train'","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:05.832351Z","iopub.execute_input":"2023-11-26T22:46:05.832739Z","iopub.status.idle":"2023-11-26T22:46:05.838213Z","shell.execute_reply.started":"2023-11-26T22:46:05.832706Z","shell.execute_reply":"2023-11-26T22:46:05.837464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset contains around 35K images out of which only 584 images are malignant. That makes only 1.8% of the total dataset.","metadata":{}},{"cell_type":"code","source":"df.target.value_counts(),df.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:22.249859Z","iopub.execute_input":"2023-11-26T22:46:22.250194Z","iopub.status.idle":"2023-11-26T22:46:22.258240Z","shell.execute_reply.started":"2023-11-26T22:46:22.250166Z","shell.execute_reply":"2023-11-26T22:46:22.257262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Samples with Melanoma\")\nimgs = df[df.target==1]['image_name'].values\n_, axs = plt.subplots(1, 5, figsize=(20, 8))\naxs = axs.flatten()\nfor f_name,ax in zip(imgs[:10],axs):\n    img = Image.open(path/f'train/{f_name}.jpg')\n    ax.imshow(img)\n    ax.axis('off')\nplt.show()\n\nprint(\"Samples without Melanoma\")\nimgs = df[df.target==0]['image_name'].values\n_, axs = plt.subplots(1, 5, figsize=(20, 8))\naxs = axs.flatten()\nfor f_name,ax in zip(imgs[:10],axs):\n    img = Image.open(path/f'train/{f_name}.jpg')\n    ax.imshow(img)\n    ax.axis('off')    \nplt.show()\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-26T22:46:06.230252Z","iopub.execute_input":"2023-11-26T22:46:06.230613Z","iopub.status.idle":"2023-11-26T22:46:07.263081Z","shell.execute_reply.started":"2023-11-26T22:46:06.230580Z","shell.execute_reply":"2023-11-26T22:46:07.262313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['diagnosis'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:08.596609Z","iopub.execute_input":"2023-11-26T22:46:08.596952Z","iopub.status.idle":"2023-11-26T22:46:08.606508Z","shell.execute_reply.started":"2023-11-26T22:46:08.596924Z","shell.execute_reply":"2023-11-26T22:46:08.605664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['sex_encoded'] = pd.factorize(df['sex'])[0]\ndf['anatomy_encoded'] = pd.factorize(df['anatom_site_general_challenge'])[0]\ndf['diagnosis_encoded'] = pd.factorize(df['diagnosis'])[0]","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:34.033917Z","iopub.execute_input":"2023-11-26T22:46:34.034315Z","iopub.status.idle":"2023-11-26T22:46:34.055017Z","shell.execute_reply.started":"2023-11-26T22:46:34.034279Z","shell.execute_reply":"2023-11-26T22:46:34.054115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# columns_plot = ['sex_encoded', 'anatomy_encoded', 'diagnosis_encoded', 'age_approx', 'target']\n# for col in columns_plot:\n#     plt.figure() \n#     df[[col]].boxplot()\n#     plt.title('Boxplot of ' + col)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:38.666271Z","iopub.execute_input":"2023-11-26T22:46:38.666618Z","iopub.status.idle":"2023-11-26T22:46:38.670652Z","shell.execute_reply.started":"2023-11-26T22:46:38.666588Z","shell.execute_reply":"2023-11-26T22:46:38.669504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axes = plt.subplots(nrows=2, ncols=3)\n# columns_plot = ['sex_encoded', 'anatomy_encoded', 'diagnosis_encoded', 'age_approx', 'target']\n\n# # Plot boxplots for each variable and label quartiles\n# for i, col in enumerate(columns_plot):\n#     ax = axes[i//3, i%3]\n#     ax.boxplot(df[col])\n#     quartiles = df[col].quantile([0.25, 0.5, 0.75])\n#     ax.text(0.05, quartiles[0.25], f'Q1: {quartiles[0.25]:.2f}', ha='center', va='bottom', color='blue')\n#     ax.text(0.05, quartiles[0.5], f'Q2: {quartiles[0.5]:.2f}', ha='center', va='bottom', color='green')\n#     ax.text(0.05, quartiles[0.75], f'Q3: {quartiles[0.75]:.2f}', ha='center', va='bottom', color='red')\n#     ax.set_title(col)\n\n# # plt.tight_layout()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:39.519832Z","iopub.execute_input":"2023-11-26T22:46:39.520179Z","iopub.status.idle":"2023-11-26T22:46:39.524468Z","shell.execute_reply.started":"2023-11-26T22:46:39.520148Z","shell.execute_reply":"2023-11-26T22:46:39.523425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.pairplot(lectures_nnan) # takes 20 minutes for 22 features\nimport matplotlib.pyplot as plt\ncolumns_plot = ['sex_encoded', 'anatomy_encoded', 'diagnosis_encoded', 'age_approx', 'target']\n\n# create a list of subplots\nplt.figure(figsize=(20, 10))\ngs = plt.GridSpec(2, 3)\naxs = [plt.subplot(gs[i]) for i in range(len(columns_plot))]\n\n# create boxplots for each column\nfor i, column in enumerate(columns_plot):\n    df[[column]].boxplot(ax=axs[i])\n#     axs[i].text(0.25, quartiles[0.25], f'Q1: {quartiles[0.25]:.2f}',ha='center', va='top', color='blue')\n#     axs[i].text(0.5, quartiles[0.5], f'Q2: {quartiles[0.5]:.2f}',ha='center', va='center',color='green')\n#     axs[i].text(0.15, quartiles[0.75], f'Q3: {quartiles[0.75]:.2f}',ha='center', va='bottom',color='red')\n    \n# customize the plot as desired\nplt.subplots_adjust(wspace=0.25, hspace=0.25)\nplt.suptitle('Box Plots per column feature')\nplt.legend()\n# show the resulting plot\nplt.show()\n# plt.savefig(\"boxplot-features\",dpi=500)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:39.931608Z","iopub.execute_input":"2023-11-26T22:46:39.931951Z","iopub.status.idle":"2023-11-26T22:46:40.709246Z","shell.execute_reply.started":"2023-11-26T22:46:39.931920Z","shell.execute_reply":"2023-11-26T22:46:40.708444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation\n\nWe use albumentations to perform augmentations. Since the dataset is small and we are not using any external dataset in this Kernel, an increased augmentation can be helpful. You can play with the augmentation argument to either increase or decrease the amount of data augmentation applied.","metadata":{}},{"cell_type":"code","source":"def get_augmentations(p=0.5):\n    imagenet_stats = {'mean':[0.485, 0.456, 0.406], 'std':[0.229, 0.224, 0.225]}\n    train_tfms = A.Compose([\n        A.Cutout(p=p),\n        A.RandomRotate90(p=p),\n        A.Flip(p=p),\n        A.OneOf([\n            A.RandomBrightnessContrast(brightness_limit=0.2,\n                                       contrast_limit=0.2,\n                                       ),\n            A.HueSaturationValue(\n                hue_shift_limit=20,\n                sat_shift_limit=50,\n                val_shift_limit=50)\n        ], p=p),\n        A.OneOf([\n            A.IAAAdditiveGaussianNoise(),\n            A.GaussNoise(),\n        ], p=p),\n        A.OneOf([\n            A.MotionBlur(p=0.2),\n            A.MedianBlur(blur_limit=3, p=0.1),\n            A.Blur(blur_limit=3, p=0.1),\n        ], p=p),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=p),\n        A.OneOf([\n            A.OpticalDistortion(p=0.3),\n            A.GridDistortion(p=0.1),\n            A.IAAPiecewiseAffine(p=0.3),\n        ], p=p), \n        ToTensor(normalize=imagenet_stats)\n        ])\n    \n    test_tfms = A.Compose([\n        ToTensor(normalize=imagenet_stats)\n        ])\n    return train_tfms, test_tfms","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:41.565693Z","iopub.execute_input":"2023-11-26T22:46:41.566021Z","iopub.status.idle":"2023-11-26T22:46:41.579121Z","shell.execute_reply.started":"2023-11-26T22:46:41.565993Z","shell.execute_reply":"2023-11-26T22:46:41.578062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train/Validation split\n- We use a simple 80/20 split based on the triple stratified K-Fold split. \n- We remove all the duplicate images.\n- TF record - id with values 12,13,14 are put into validation split and the rest into train split.","metadata":{}},{"cell_type":"code","source":"def get_train_val_split(df):\n    #Remove Duplicates\n    df = df[df.tfrecord != -1].reset_index(drop=True)\n    #We are splitting data based on triple stratified kernel provided here https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/165526\n    train_tf_records = list(range(len(df.tfrecord.unique())))[:12]\n    split_cond = df.tfrecord.apply(lambda x: x in train_tf_records)\n    train_df = df[split_cond].reset_index()\n    valid_df = df[~split_cond].reset_index()\n    return train_df,valid_df","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:42.668020Z","iopub.execute_input":"2023-11-26T22:46:42.668390Z","iopub.status.idle":"2023-11-26T22:46:42.675195Z","shell.execute_reply.started":"2023-11-26T22:46:42.668342Z","shell.execute_reply":"2023-11-26T22:46:42.674449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# class MelanomaDataset(Dataset):\n#     def __init__(self,df,im_path,transforms=None,is_test=False):\n#         self.df = df\n#         self.im_path = im_path\n#         self.transforms = transforms\n#         self.is_test = is_test\n        \n#     def __getitem__(self,idx):\n#         img_path = f\"{self.im_path}/{self.df.iloc[idx]['image_name']}.jpg\"\n#         img = Image.open(img_path)\n#         if self.transforms:\n#             img = self.transforms(**{\"image\": np.array(img)})[\"image\"]\n            \n#         if self.is_test:\n#             return img\n#         target = self.df.iloc[idx]['target']\n#         return img,torch.tensor([target],dtype=torch.float32)\n    \n#     def __len__(self):\n#         return self.df.shape[0]\n        ","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:44.066237Z","iopub.execute_input":"2023-11-26T22:46:44.066593Z","iopub.status.idle":"2023-11-26T22:46:44.070516Z","shell.execute_reply.started":"2023-11-26T22:46:44.066561Z","shell.execute_reply":"2023-11-26T22:46:44.069590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# adapted Dataset \n# transform = transforms.Compose([\n#             #transforms.Resize((resized_resol, resized_resol)),\n#             #transforms.RandomSizedCrop(resol),\n#             # transforms.ColorJitter(brightness=32/255, saturation=(0.5, 1.5)),\n#             # transforms.RandomResizedCrop(resol),\n#             transforms.RandomHorizontalFlip(),\n#             transforms.ToTensor(), #implicitly divides by 255\n#             transforms.Normalize(mean = [0.5, 0.5, 0.5], std = [2, 2, 2])\n#             #transforms.Normalize(mean = [ 0.485, 0.456, 0.406 ], std = [ 0.229, 0.224, 0.225 ]),\n#             ])\nclass MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None, test:bool=False):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        self.test = test\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n        # pdb.set_trace()\n        x = Image.open(im_path).convert('RGB') # cv2.imread(im_path)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n\n        if self.transforms:\n            x = self.transforms(**{\"image\": np.array(x)})[\"image\"]\n        \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, y, meta) # (x, meta), y\n        elif self.test:\n            return x\n#         else:\n#             # return (x, y)\n#             return (x, y, meta)\n        \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:44.424195Z","iopub.execute_input":"2023-11-26T22:46:44.424521Z","iopub.status.idle":"2023-11-26T22:46:44.437498Z","shell.execute_reply.started":"2023-11-26T22:46:44.424492Z","shell.execute_reply":"2023-11-26T22:46:44.436488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tfms, test_tfms = get_augmentations()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:44.864405Z","iopub.execute_input":"2023-11-26T22:46:44.864752Z","iopub.status.idle":"2023-11-26T22:46:44.869505Z","shell.execute_reply.started":"2023-11-26T22:46:44.864716Z","shell.execute_reply":"2023-11-26T22:46:44.868392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dummies(df : pd.DataFrame):\n    columns_one_hot = ['sex','anatom_site_general_challenge', 'diagnosis', 'age_approx']  # drop malignant_benign since this is y\n    temp_dummies = pd.get_dummies(df[columns_one_hot], columns=columns_one_hot)\n    # pdb.set_trace()\n    df = df.drop(columns_one_hot, axis = 1)\n    df = pd.concat([df, temp_dummies], axis = 1)\n    # for col in columns_one_hot:\n    #     # temp_dummies = pd.get_dummies(df[col])\n    #     # df = pd.concat([df, temp_dummies])\n    # pdb.set_trace()\n    # df.drop_duplicates(inplace=True, axis=1)\n    return df\ndef dummies_test(df : pd.DataFrame):\n    columns_one_hot = ['sex','anatom_site_general_challenge', 'age_approx']  # drop malignant_benign since this is y\n    temp_dummies = pd.get_dummies(df[columns_one_hot], columns=columns_one_hot)\n    # pdb.set_trace()\n    df = df.drop(columns_one_hot, axis = 1)\n    df = pd.concat([df, temp_dummies], axis = 1)\n    # for col in columns_one_hot:\n    #     # temp_dummies = pd.get_dummies(df[col])\n    #     # df = pd.concat([df, temp_dummies])\n    # pdb.set_trace()\n    # df.drop_duplicates(inplace=True, axis=1)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:45.292812Z","iopub.execute_input":"2023-11-26T22:46:45.293172Z","iopub.status.idle":"2023-11-26T22:46:45.302907Z","shell.execute_reply.started":"2023-11-26T22:46:45.293143Z","shell.execute_reply":"2023-11-26T22:46:45.301725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# my_path = Path('../input/melanoma-preprocess')\n# train_data = pd.read_csv(my_path/'one_hot_train_beta.csv')\n# train_data.drop(columns=['Unnamed: 0'], inplace=True)\n# train_data.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:45.883553Z","iopub.execute_input":"2023-11-26T22:46:45.883901Z","iopub.status.idle":"2023-11-26T22:46:45.887579Z","shell.execute_reply.started":"2023-11-26T22:46:45.883871Z","shell.execute_reply":"2023-11-26T22:46:45.886474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dummy = dummies(df)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:46.337173Z","iopub.execute_input":"2023-11-26T22:46:46.337551Z","iopub.status.idle":"2023-11-26T22:46:46.369087Z","shell.execute_reply.started":"2023-11-26T22:46:46.337517Z","shell.execute_reply":"2023-11-26T22:46:46.368180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dummy\ntrain_data=df_dummy\ntrain_data.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:46.976155Z","iopub.execute_input":"2023-11-26T22:46:46.976529Z","iopub.status.idle":"2023-11-26T22:46:46.983027Z","shell.execute_reply.started":"2023-11-26T22:46:46.976493Z","shell.execute_reply":"2023-11-26T22:46:46.982085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(feats))\n# feats","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:47.523520Z","iopub.execute_input":"2023-11-26T22:46:47.524011Z","iopub.status.idle":"2023-11-26T22:46:47.527476Z","shell.execute_reply.started":"2023-11-26T22:46:47.523974Z","shell.execute_reply":"2023-11-26T22:46:47.526416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# my_path = Path('../input/melanoma-preprocess')\n# train_data = pd.read_csv(my_path/'one_hot_train_beta.csv')\n# train_data.drop(columns=['Unnamed: 0'], inplace=True)\nirr_columns = ['sex_encoded', 'anatomy_encoded', 'diagnosis_encoded', 'patient_code', 'image_name', 'patient_id', 'benign_malignant', 'tfrecord', 'width', 'height', 'target']\nmeta_cols = ~train_data.columns.isin(irr_columns)\nfeats = train_data.columns[meta_cols]\ndataset = MelanomaDataset(df=train_data, imfolder=path/'train', meta_features=feats, train=True, transforms=train_tfms)\ndataset[0][2].shape","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:48.164479Z","iopub.execute_input":"2023-11-26T22:46:48.164802Z","iopub.status.idle":"2023-11-26T22:46:48.279468Z","shell.execute_reply.started":"2023-11-26T22:46:48.164774Z","shell.execute_reply":"2023-11-26T22:46:48.278674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model \n\nEfficientnets have proved themselves in the last year as a key to winning competitions. You can try different Efficientnet model by passing the right model name. Changing the number in the model name `efficientnet-b0` gives you different models. The larger the number the more complex/bigger and better the model is. ","metadata":{}},{"cell_type":"code","source":"class BaselineResnet(nn.Module):\n    def __init__(self,model_name='resnet18',num_classes=1, pretrained=True):\n        super().__init__()\n        self.model = models.resnet18(pretrained=pretrained) # aux_logits\n        self.model.fc = nn.Linear(self.model.fc.in_features, num_classes)\n        \n\n    def forward(self,x):\n        return torch.sigmoid(self.model(x))","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:49.363890Z","iopub.execute_input":"2023-11-26T22:46:49.364295Z","iopub.status.idle":"2023-11-26T22:46:49.371391Z","shell.execute_reply.started":"2023-11-26T22:46:49.364257Z","shell.execute_reply":"2023-11-26T22:46:49.370480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL IDEA AND CODE IS EXTRACTED FROM: https://arxiv.org/abs/2007.04612 and https://github.com/yewsiang/ConceptBottleneck\n","metadata":{}},{"cell_type":"code","source":"# temp = BaselineResnet()\n# temp(dataset[0][0][None, :])","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:50.410455Z","iopub.execute_input":"2023-11-26T22:46:50.410838Z","iopub.status.idle":"2023-11-26T22:46:50.414247Z","shell.execute_reply.started":"2023-11-26T22:46:50.410801Z","shell.execute_reply":"2023-11-26T22:46:50.413442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class End2EndModel(torch.nn.Module):\n    def __init__(self, pretrained=True, num_classes=len(feats), output_dim = 1, n_class_attr=2):\n        super(End2EndModel, self).__init__()\n        self.first_model = models.resnet18(pretrained=pretrained) # aux_logits\n        self.first_model.fc = nn.Linear(self.first_model.fc.in_features, num_classes)\n        self.sec_model = nn.Linear(num_classes, output_dim) # model2\n        \n    def forward_stage2(self, stage1_out):\n\n        stage2_inputs = stage1_out\n        # pdb.set_trace()\n        # stage2_inputs = torch.cat(stage2_inputs, dim=1)\n        all_out = [torch.sigmoid(self.sec_model(stage2_inputs))]\n        all_out.append(stage1_out)\n        return all_out\n    def forward(self, x):\n        outputs = self.first_model(x)\n        return self.forward_stage2(outputs)\n    \nclass AverageMeter(object):\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = torch.tensor(val).cpu()\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\ndef binary_accuracy(output, target):\n    \"\"\"\n    Computes the accuracy for multiple binary predictions\n    output and target are Torch tensors\n    \"\"\"\n    # pdb.set_trace()\n    # accuracy_score(output.cpu().detach().numpy().squeeze(), target.cpu().detach().numpy())\n    binary_preds = output.cpu().detach().numpy().squeeze() >= 0.5\n    target = target.cpu().detach().numpy()\n    #print(list(output.data.cpu().numpy()))\n    #print(list(pred.data[0].numpy()))\n    #print(list(target.data[0].numpy()))\n    #print(pred.size(), target.size())\n    acc = accuracy_score(binary_preds,target)\n    # acc = acc*100 / np.prod(np.array(target.size()))\n    return acc\n# def accuracy(output, target, topk=(1,)):\n#     \"\"\"\n#     Computes the precision@k for the specified values of k\n#     output and target are Torch tensors\n#     \"\"\"\n#     maxk = max(topk)\n#     batch_size = target.size(0)\n#     _, pred = output.topk(maxk, 1, True, True)\n#     pred = pred.t()\n#     temp = target.view(1, -1).expand_as(pred)\n#     temp = temp.cuda()\n#     correct = pred.eq(temp)\n\n#     res = []\n#     for k in topk:\n#         correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n#         res.append(correct_k.mul_(100.0 / batch_size))\n#     return res\ndef get_device():\n    return torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:50.818249Z","iopub.execute_input":"2023-11-26T22:46:50.818597Z","iopub.status.idle":"2023-11-26T22:46:50.836121Z","shell.execute_reply.started":"2023-11-26T22:46:50.818567Z","shell.execute_reply":"2023-11-26T22:46:50.835178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df,valid_df = get_train_val_split(train_data)\n# train_tfms,test_tfms = get_augmentations(p=0.5)\n# train_dl,valid_dl = get_data(train_df,valid_df,train_tfms,test_tfms,bs)\n# data_iter = iter(train_dl)  # Create an iterator for the data loader\n# first_batch = next(data_iter) ","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:51.501071Z","iopub.execute_input":"2023-11-26T22:46:51.501472Z","iopub.status.idle":"2023-11-26T22:46:51.505258Z","shell.execute_reply.started":"2023-11-26T22:46:51.501440Z","shell.execute_reply":"2023-11-26T22:46:51.504347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# out[0].cpu().detach.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:52.570351Z","iopub.execute_input":"2023-11-26T22:46:52.570717Z","iopub.status.idle":"2023-11-26T22:46:52.574339Z","shell.execute_reply.started":"2023-11-26T22:46:52.570684Z","shell.execute_reply":"2023-11-26T22:46:52.573478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model,val_rocs = fit(3, model,train_dl,valid_dl,opt)\n# torch.save(model.state_dict(),f'effb5.pth')","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:53.105331Z","iopub.execute_input":"2023-11-26T22:46:53.105670Z","iopub.status.idle":"2023-11-26T22:46:53.108907Z","shell.execute_reply.started":"2023-11-26T22:46:53.105640Z","shell.execute_reply":"2023-11-26T22:46:53.108120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions \n- Split data to train and validation split\n- Get model, choose different optimizer, freeze backbone, different learning rates/weight decay.\n- The training method by default uses cosine annealing for scheduling learning rate, you can experiment with.","metadata":{}},{"cell_type":"code","source":"# loss_meter = AverageMeter()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:54.070871Z","iopub.execute_input":"2023-11-26T22:46:54.071227Z","iopub.status.idle":"2023-11-26T22:46:54.075241Z","shell.execute_reply.started":"2023-11-26T22:46:54.071196Z","shell.execute_reply":"2023-11-26T22:46:54.074148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_device():\n    return torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n\ndef get_model_baseline(model_name='resnet18-alpha',lr=0.01,wd=0.00004,freeze_backbone=False,opt_fn=torch.optim.Adam,device='cuda'):\n    device = device if device else get_device()\n    model = BaselineResnet()\n    # scheduler = torch.optim.lr_scheduler.StepLR(opt_fn, step_size=20, gamma=0.1)\n    if freeze_backbone:\n        for parameter in model.backbone.parameters():\n            parameter.requires_grad = False\n    opt = opt_fn(model.parameters(),lr=lr,weight_decay=wd)\n    model = model.to(device)\n    return model, opt# , scheduler\n\ndef training_step_baseline(xb,yb,model,loss_fn,opt,device,scheduler, acc_meter, loss_meter):\n    # xb, yb, attrs_labels, model, loss_fn, opt, device,scheduler, acc_meter, loss_meter, attr_criterion=each_attr_criterion, attr_loss_weight=0.001\n    xb,yb = xb.to(device), yb.to(device)\n    out = model(xb)\n    #pdb.set_trace()\n    loss = loss_fn(out.squeeze(),yb.float())\n    acc = binary_accuracy(out, yb) #only care about class prediction accuracy\n    #pdb.set_trace()\n    acc_meter.update(acc, xb.size(0))\n    loss_meter.update(loss.item(), xb.shape[0])\n    opt.zero_grad()\n    loss.backward()\n    opt.step()\n    scheduler.step()\n    return loss_meter, acc_meter\n    \ndef validation_step_baseline(xb,yb,model,loss_fn,device, acc_meter, loss_meter):\n    xb,yb = xb.to(device), yb.to(device)\n    out = model(xb)\n    loss = loss_fn(out.squeeze(),yb.float())\n    acc = binary_accuracy(out, yb) #only care about class prediction accuracy\n    acc_meter.update(acc, xb.size(0))\n    loss_meter.update(loss.item(), xb.shape[0])\n    # pdb.set_trace()\n    return loss_meter, acc_meter, out\n\ndef get_data_baseline(train_df,valid_df,train_tfms,test_tfms,bs):\n    train_ds = MelanomaDataset(df=train_df, imfolder=path/'train', meta_features=feats, train=True, transforms=train_tfms)\n #MelanomaDataset(df=train_df,im_path=path/'train',transforms=train_tfms)\n    valid_ds = MelanomaDataset(df=valid_df, imfolder=path/'train', meta_features=feats, train=True, transforms=test_tfms) \n    #MelanomaDataset(df=valid_df,im_path=path/'train',transforms=test_tfms)\n    train_dl = DataLoader(dataset=train_ds,batch_size=bs,shuffle=True,num_workers=4)\n    valid_dl = DataLoader(dataset=valid_ds,batch_size=bs,shuffle=False,num_workers=4)\n    \n    return train_dl,valid_dl","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:54.429133Z","iopub.execute_input":"2023-11-26T22:46:54.429545Z","iopub.status.idle":"2023-11-26T22:46:54.450514Z","shell.execute_reply.started":"2023-11-26T22:46:54.429507Z","shell.execute_reply":"2023-11-26T22:46:54.449474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_iter = iter(train_dl)  # Create an iterator for the data loader\n# first_batch = next(data_iter) \n# # # each_attr_criterion\n# x,y, meta = first_batch\n# # ypred = basemdl(x.cuda())\n# acc_meter = AverageMeter()\n# loss_meter = AverageMeter()\n# # (xb,yb,model,loss_fn,device)\n# tempa, templ, outs = validation_step_baseline(x,y,basemdl,torch.nn.BCELoss(),'cuda', acc_meter, loss_meter)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:55.599256Z","iopub.execute_input":"2023-11-26T22:46:55.599620Z","iopub.status.idle":"2023-11-26T22:46:55.603261Z","shell.execute_reply.started":"2023-11-26T22:46:55.599586Z","shell.execute_reply":"2023-11-26T22:46:55.602472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_base(epochs,model,train_dl,valid_dl,opt,device=None,loss_fn=torch.nn.BCELoss()):\n    \n    device = device if device else get_device()\n    #scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, len(train_dl)*epochs)\n    scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=20, gamma=0.1)\n    train_losses = []\n    train_accuracies = []\n    val_losses = []\n    val_accuracies = []\n    val_rocs = [] \n    \n    #Creating progress bar\n    mb = master_bar(range(epochs))\n    mb.write(['epoch','train_loss', 'train_accuracy', 'valid_loss','val_roc', 'validation_accuracy'],table=True)\n\n    for epoch in mb:   \n        train_loss_meter = AverageMeter()\n        train_acc_meter = AverageMeter()\n        trn_loss,val_loss = 0.0,0.0\n        trn_acc, val_acc = 0.0, 0.0\n        val_preds = np.zeros((len(valid_dl.dataset),1))\n        val_targs = np.zeros((len(valid_dl.dataset),1))\n        \n        #Training\n        model.train()\n        \n        #For every batch \n        for xb,yb, _ in progress_bar(train_dl,parent=mb):\n            train_loss_meter, train_acc_meter = training_step_baseline(xb=xb,yb=yb,model=model,loss_fn=loss_fn,opt=opt,\n                                                                       device=device,scheduler=scheduler, \n                                                                       acc_meter=train_acc_meter, loss_meter=train_loss_meter)\n            trn_loss += train_loss_meter.avg\n            trn_acc += train_acc_meter.avg\n            trn_acc = trn_acc.item()\n            \n        trn_loss /= mb.child.total\n        trn_acc /= mb.child.total\n        val_loss_meter = AverageMeter()\n        val_acc_meter = AverageMeter()\n        #Validation\n        model.eval()\n        with torch.no_grad():\n            for i,(xb,yb, _) in enumerate(progress_bar(valid_dl,parent=mb)):\n                \n                val_loss_meter,val_acc_meter, out = validation_step_baseline(xb,yb,model,loss_fn,device, \n                                                                             acc_meter=val_acc_meter, loss_meter=val_loss_meter)\n                val_loss += val_loss_meter.avg\n                val_acc += val_acc_meter.avg\n                val_acc = val_acc.item()\n                bs = xb.shape[0]\n                \n                val_preds[i*bs:i*bs+bs] = out.cpu().detach().numpy()\n                val_targs[i*bs:i*bs+bs] = yb.cpu().detach().numpy()[:, None]\n        # pdb.set_trace()\n        val_loss /= mb.child.total\n        val_acc /= mb.child.total\n        val_roc = roc_auc_score(val_targs.reshape(-1),val_preds.reshape(-1))\n        val_rocs.append(val_roc)\n        #print(epoch, trn_loss, trn_acc, val_loss, val_roc, val_acc)\n        train_losses.append(trn_loss)\n        train_accuracies.append(trn_acc)\n        val_losses.append(val_loss)\n        val_accuracies.append(val_acc)\n        mb.write([epoch,f'{trn_loss:.6f}', f'{trn_acc:.6f}',f'{val_loss:.6f}',f'{val_roc:.6f}',f'{val_acc:.15f}'],table=True)\n    return model,val_rocs, train_losses, train_accuracies, val_losses, val_accuracies","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:56.837116Z","iopub.execute_input":"2023-11-26T22:46:56.837490Z","iopub.status.idle":"2023-11-26T22:46:56.863721Z","shell.execute_reply.started":"2023-11-26T22:46:56.837455Z","shell.execute_reply":"2023-11-26T22:46:56.862757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df,valid_df = get_train_val_split(train_data)\n# train_tfms,test_tfms = get_augmentations(p=0.5)\n# basetrain_dl,basevalid_dl = get_data_baseline(train_df,valid_df,train_tfms,test_tfms,bs)\n\ntrain_df,valid_df = get_train_val_split(train_data)\ntrain_tfms,test_tfms = get_augmentations(p=0.5)\nbasetrain_dl,basevalid_dl = get_data_baseline(train_df,valid_df,train_tfms,test_tfms,bs)\nbasemdl, baseopt = get_model_baseline(device='cpu')\nbasescheduler = torch.optim.lr_scheduler.StepLR(baseopt, step_size=2, gamma=0.1)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:57.421428Z","iopub.execute_input":"2023-11-26T22:46:57.421798Z","iopub.status.idle":"2023-11-26T22:46:58.674008Z","shell.execute_reply.started":"2023-11-26T22:46:57.421766Z","shell.execute_reply":"2023-11-26T22:46:58.673066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_base_rocs = []\n# all_base_train_losses = []\n# all_base_val_losses = []\n# all_base_train_accs = []\n# all_base_val_accs = []\n# for my_seed in [42, 56]:\n#     np.random.seed(my_seed)\n#     torch.manual_seed(my_seed)\n#     basemdl, baseopt = get_model_baseline()\n    \n#     basemodel,baseval_rocs, basetrain_losses, basetrain_accuracies, baseval_losses, baseval_accuracies = \\\n#     fit_base(5, basemdl,basetrain_dl,basevalid_dl,baseopt)\n    \n#     # save results\n#     all_base_rocs.append(baseval_rocs)\n#     all_base_train_losses.append(basetrain_losses)\n#     all_base_val_losses.append(baseval_losses)\n#     all_base_train_accs.append(basetrain_accuracies)\n#     all_base_val_accs.append(baseval_accuracies)\n#     torch.save(basemodel.state_dict(),'base-resnet-alpha_%d.pth' % my_seed)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:59.146162Z","iopub.execute_input":"2023-11-26T22:46:59.146508Z","iopub.status.idle":"2023-11-26T22:46:59.150920Z","shell.execute_reply.started":"2023-11-26T22:46:59.146479Z","shell.execute_reply":"2023-11-26T22:46:59.149718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink('base-resnet-alpha_42.pth')\n# # FileLink('base-resnet-alpha_56.pth')","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:59.493536Z","iopub.execute_input":"2023-11-26T22:46:59.493853Z","iopub.status.idle":"2023-11-26T22:46:59.497417Z","shell.execute_reply.started":"2023-11-26T22:46:59.493824Z","shell.execute_reply":"2023-11-26T22:46:59.496346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"each_attr_criterion = [torch.nn.BCEWithLogitsLoss() for _ in range(len(feats))]\n\ndef get_model_conceptb(model_name='joint-concept-alpha',lr=0.01,wd=0.00004,freeze_backbone=False,opt_fn=torch.optim.Adam,device='cuda'):\n    device = device if device else get_device()\n    model = End2EndModel()\n    # scheduler = torch.optim.lr_scheduler.StepLR(opt_fn, step_size=20, gamma=0.1)\n    if freeze_backbone:\n        for parameter in model.backbone.parameters():\n            parameter.requires_grad = False\n    opt = opt_fn(model.parameters(),lr=lr,weight_decay=wd)\n    model = model.to(device)\n    return model, opt# , scheduler\n# run_epoch(model, optimizer, loader, loss_meter, acc_meter, criterion, attr_criterion, args, is_training):\n    \ndef training_step_conceptb(xb, yb, attrs_labels, model, loss_fn, opt, device,scheduler, acc_meter, loss_meter, attr_criterion=each_attr_criterion, \n                  attr_loss_weight=0.001):\n    \n    xb, yb, attrs_labels = torch.tensor(xb).to(device), \\\n                           torch.tensor(yb).to(device), \\\n                           torch.tensor(attrs_labels).float().to(device)\n    model = model.to(device)\n    outputs = model(xb) # in this case it would be a list where out[0] was the concepts, and out[1] are the labels\n    losses = []\n    out_start = 0\n    # pdb.set_trace()\n    loss_main = loss_fn(outputs[0].squeeze(), yb.float())\n    losses.append(loss_main)\n    out_start = 1\n    # pdb.set_trace()\n    for i in range(len(attr_criterion)):\n        # pdb.set_trace()\n        losses.append(attr_loss_weight * (1.0 * attr_criterion[i](outputs[out_start][:, i], attrs_labels[:, i])))# \\\n    # pdb.set_trace()                         \n    acc = binary_accuracy(outputs[0], yb) #only care about class prediction accuracy\n    acc_meter.update(acc, xb.size(0))\n    total_loss = losses[0] + sum(losses[1:])\n    total_loss = total_loss / (1 + attr_loss_weight * len(attr_criterion))\n    loss_meter.update(total_loss.item(), xb.shape[0])\n    \n    # loss = loss_fn(out,yb)\n    opt.zero_grad()\n    total_loss.backward()\n    opt.step()\n    scheduler.step()\n    return loss_meter, acc_meter\n    \ndef validation_step_conceptb(xb, yb, attrs_labels, model, loss_fn, opt, device,scheduler, acc_meter, loss_meter, attr_criterion=each_attr_criterion, \\\n                             attr_loss_weight=0.001):\n    \n    xb, yb, attrs_labels = torch.tensor(xb).to(device), \\\n                           torch.tensor(yb).to(device), \\\n                           torch.tensor(attrs_labels).float().to(device)\n    model = model.to(device)\n    outputs = model(xb) # in this case it would be a list where out[0] was the labels, and out[1] are the concepts\n    losses = []\n    out_start = 0\n    # pdb.set_trace()\n    loss_main = loss_fn(outputs[0].squeeze(), yb.float())\n    losses.append(loss_main)\n    out_start = 1\n    # pdb.set_trace()\n    for i in range(len(attr_criterion)):\n        # pdb.set_trace()\n        losses.append(attr_loss_weight * (1.0 * attr_criterion[i](outputs[out_start][:, i], attrs_labels[:, i])))# \\\n    # pdb.set_trace()                         \n    acc = binary_accuracy(outputs[0], yb) #only care about class prediction accuracy\n    acc_meter.update(acc, xb.size(0))\n    total_loss = losses[0] + sum(losses[1:])\n    total_loss = total_loss / (1 + attr_loss_weight * len(attr_criterion))\n    loss_meter.update(total_loss.item(), xb.shape[0])\n    #pdb.set_trace()\n#     # loss = loss_fn(out,yb)\n#     opt.zero_grad()\n#     total_loss.backward()\n#     opt.step()\n#     scheduler.step()\n    return loss_meter, acc_meter, outputs\n\ndef get_data(train_df,valid_df,train_tfms,test_tfms, meta_features=feats, bs=64):\n#     my_path = Path('../input/melanoma-preprocess')  \n#     train_data = pd.read_csv(my_path/'one_hot_train_beta.csv')\n#     train_data.drop(columns=['Unnamed: 0'], inplace=True)\n#     irr_columns = ['image_name', 'patient_id', 'benign_malignant', 'tfrecord', 'width', 'height', 'target']\n#     meta_cols = ~train_data.columns.isin(irr_columns)\n#     feats = train_data.columns[meta_cols]\n#     dataset = MelanomaDataset(df=train_data, imfolder=path/'train', meta_features=feats, train=True, transforms=transform)\n\n    train_ds = MelanomaDataset(df=train_df, imfolder=path/'train', meta_features=feats, train=True, transforms=train_tfms)\n #MelanomaDataset(df=train_df,im_path=path/'train',transforms=train_tfms)\n    valid_ds = MelanomaDataset(df=valid_df, imfolder=path/'train', meta_features=feats, train=True, transforms=test_tfms) \n    #MelanomaDataset(df=valid_df,im_path=path/'train',transforms=test_tfms)\n    train_dl = DataLoader(dataset=train_ds,batch_size=bs,shuffle=True,num_workers=4)\n    valid_dl = DataLoader(dataset=valid_ds,batch_size=bs,shuffle=False,num_workers=4)\n    return train_dl,valid_dl\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:46:59.842324Z","iopub.execute_input":"2023-11-26T22:46:59.842704Z","iopub.status.idle":"2023-11-26T22:46:59.879801Z","shell.execute_reply.started":"2023-11-26T22:46:59.842669Z","shell.execute_reply":"2023-11-26T22:46:59.878920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_iter = iter(train_dl)  # Create an iterator for the data loader\n# first_batch = next(data_iter) \n# # # each_attr_criterion\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:00.685179Z","iopub.execute_input":"2023-11-26T22:47:00.685559Z","iopub.status.idle":"2023-11-26T22:47:00.689419Z","shell.execute_reply.started":"2023-11-26T22:47:00.685522Z","shell.execute_reply":"2023-11-26T22:47:00.688288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # train_ts, test_ts = get_augmentations(p=0.5)\n# # tdf, vdf = get_train_val_split(train_data)\n# # load, valload, ts, vs = get_data(tdf, vdf, train_ts, test_ts )\n# train_df,valid_df = get_train_val_split(train_data)\n# train_tfms,test_tfms = get_augmentations(p=0.5)\n# train_dl,valid_dl = get_data(train_df,valid_df,train_tfms,test_tfms,bs)\n# model, opt = get_model_conceptb()\n# scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=20, gamma=0.1)\n# x,y, meta = first_batch\n# # ypred = basemdl(x.cuda())\n# acc_meter = AverageMeter()\n# loss_meter = AverageMeter()\n# # (xb,yb,model,loss_fn,device)\n# tempa, templ, outs = validation_step_conceptb(x,y, meta , model, torch.nn.BCELoss(), opt, 'cuda',scheduler, acc_meter, loss_meter, attr_criterion=each_attr_criterion, \\\n#                              attr_loss_weight=0.001)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:01.008253Z","iopub.execute_input":"2023-11-26T22:47:01.008584Z","iopub.status.idle":"2023-11-26T22:47:01.012402Z","shell.execute_reply.started":"2023-11-26T22:47:01.008554Z","shell.execute_reply":"2023-11-26T22:47:01.011471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_iter = iter(ts)  # Create an iterator for the data loader\n# first_batch = next(data_iter) \n# x, y, feats = first_batch\n# feats.shape\n# tempa.avg","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:01.531115Z","iopub.execute_input":"2023-11-26T22:47:01.531479Z","iopub.status.idle":"2023-11-26T22:47:01.535895Z","shell.execute_reply.started":"2023-11-26T22:47:01.531447Z","shell.execute_reply":"2023-11-26T22:47:01.534731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_step_conceptb(xb, yb, attrs_labels, model, loss_fn, opt, device,scheduler, acc_meter, loss_meter, attr_criterion=each_attr_criterion, attr_loss_weight=0.001)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:01.878170Z","iopub.execute_input":"2023-11-26T22:47:01.878647Z","iopub.status.idle":"2023-11-26T22:47:01.884171Z","shell.execute_reply.started":"2023-11-26T22:47:01.878605Z","shell.execute_reply":"2023-11-26T22:47:01.883049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(epochs,model,train_dl,valid_dl,opt,device=None,loss_fn=torch.nn.BCELoss()):\n    \n    device = device if device else get_device()\n    #scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, len(train_dl)*epochs)\n    scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=20, gamma=0.1)\n    train_losses = []\n    train_accuracies = []\n    val_losses = []\n    val_accuracies = []\n    val_rocs = [] \n    \n    #Creating progress bar\n    mb = master_bar(range(epochs))\n    mb.write(['epoch','train_loss', 'train_accuracy', 'valid_loss','val_roc', 'validation_accuracy'],table=True)\n\n    for epoch in mb:   \n        train_loss_meter = AverageMeter()\n        train_acc_meter = AverageMeter()\n        trn_loss,val_loss = 0.0,0.0\n        trn_acc, val_acc = 0.0, 0.0\n        val_preds = np.zeros((len(valid_dl.dataset),1))\n        val_targs = np.zeros((len(valid_dl.dataset),1))\n        \n        #Training\n        model.train()\n        \n        #For every batch \n        for xb,yb, attrs_labels in progress_bar(train_dl,parent=mb):\n            train_loss_meter, train_acc_meter = training_step_conceptb(xb=xb, yb=yb, attrs_labels=attrs_labels, model=model, loss_fn=loss_fn, \n                                                                       opt=opt, device=device,scheduler=scheduler, acc_meter=train_acc_meter, \n                                                                       loss_meter=train_loss_meter,\n                                                                       attr_criterion=each_attr_criterion, attr_loss_weight=0.001)\n            trn_loss += train_loss_meter.avg\n            trn_acc += train_acc_meter.avg\n            trn_acc = trn_acc.item()\n            \n        trn_loss /= mb.child.total\n        trn_acc /= mb.child.total\n        val_loss_meter = AverageMeter()\n        val_acc_meter = AverageMeter()\n        #Validation\n        model.eval()\n        with torch.no_grad():\n            for i,(xb,yb, attrs_labels) in enumerate(progress_bar(valid_dl,parent=mb)):\n                \n                val_loss_meter,val_acc_meter, out = validation_step_conceptb(xb, yb, attrs_labels, model, loss_fn, opt, device, \n                                                                             scheduler, acc_meter=val_acc_meter, \n                                                                             loss_meter=val_loss_meter, attr_criterion=each_attr_criterion, attr_loss_weight=0.001)\n                val_loss += val_loss_meter.avg\n                val_acc += val_acc_meter.avg\n                val_acc = val_acc.item()\n                bs = xb.shape[0]\n                \n                val_preds[i*bs:i*bs+bs] = out[0].cpu().detach().numpy()\n                val_targs[i*bs:i*bs+bs] = yb.cpu().detach().numpy()[:, None]\n        # pdb.set_trace()\n        val_loss /= mb.child.total\n        val_acc /= mb.child.total\n        val_roc = roc_auc_score(val_targs.reshape(-1),val_preds.reshape(-1))\n        val_rocs.append(val_roc)\n        #print(epoch, trn_loss, trn_acc, val_loss, val_roc, val_acc)\n        train_losses.append(trn_loss)\n        train_accuracies.append(trn_acc)\n        val_losses.append(val_loss)\n        val_accuracies.append(val_acc)\n        mb.write([epoch,f'{trn_loss:.6f}', f'{trn_acc:.6f}',f'{val_loss:.6f}',f'{val_roc:.6f}',f'{val_acc:.6f}'],table=True)\n    return model,val_rocs, train_losses, train_accuracies, val_losses, val_accuracies","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:02.596089Z","iopub.execute_input":"2023-11-26T22:47:02.596465Z","iopub.status.idle":"2023-11-26T22:47:02.623750Z","shell.execute_reply.started":"2023-11-26T22:47:02.596433Z","shell.execute_reply":"2023-11-26T22:47:02.622791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv(df_path/'train.csv')\ntrain_df,valid_df = get_train_val_split(train_data)\ntrain_tfms,test_tfms = get_augmentations(p=0.5)\ntrain_dl,valid_dl = get_data(train_df,valid_df,train_tfms,test_tfms,bs)\nmodel, opt = get_model_conceptb(device='cpu')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-26T22:47:03.925128Z","iopub.execute_input":"2023-11-26T22:47:03.925527Z","iopub.status.idle":"2023-11-26T22:47:04.270683Z","shell.execute_reply.started":"2023-11-26T22:47:03.925492Z","shell.execute_reply":"2023-11-26T22:47:04.269918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I trained the model on my local machine. You can uncomment below code to start training here.","metadata":{}},{"cell_type":"code","source":"train_model_base = False\nif train_model_base:\n    all_base_rocs = []\n    all_base_train_losses = []\n    all_base_val_losses = []\n    all_base_train_accs = []\n    all_base_val_accs = []\n    for my_seed in [42, 56]:\n        np.random.seed(my_seed)\n        torch.manual_seed(my_seed)\n        model, opt = get_model_conceptb()\n\n        model,val_rocs, train_losses, train_accuracies, val_losses, val_accuracies = \\\n        fit(5, model,train_dl,valid_dl,opt)\n\n\n        # save results\n        all_base_rocs.append(val_rocs)\n        all_base_train_losses.append(train_losses)\n        all_base_val_losses.append(val_losses)\n        all_base_train_accs.append(train_accuracies)\n        all_base_val_accs.append(val_accuracies)\n        torch.save(model.state_dict(),'base_resnet_beta_%d.pth' % my_seed)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:05.003003Z","iopub.execute_input":"2023-11-26T22:47:05.003346Z","iopub.status.idle":"2023-11-26T22:47:05.007887Z","shell.execute_reply.started":"2023-11-26T22:47:05.003315Z","shell.execute_reply":"2023-11-26T22:47:05.006717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model = False\nif train_model:\n    all_base_rocs = []\n    all_base_train_losses = []\n    all_base_val_losses = []\n    all_base_train_accs = []\n    all_base_val_accs = []\n    my_seed=42\n    np.random.seed(my_seed)\n    torch.manual_seed(my_seed)\n    model, opt = get_model_conceptb(device='cuda')\n\n    model,val_rocs, train_losses, train_accuracies, val_losses, val_accuracies = \\\n    fit(50, model,train_dl,valid_dl,opt)\n\n\n    # save results\n    all_base_rocs.append(val_rocs)\n    all_base_train_losses.append(train_losses)\n    all_base_val_losses.append(val_losses)\n    all_base_train_accs.append(train_accuracies)\n    all_base_val_accs.append(val_accuracies)\n    torch.save(model.state_dict(),'concept_bottleneck_gamma_%d_50.pth' % my_seed)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:05.380883Z","iopub.execute_input":"2023-11-26T22:47:05.381230Z","iopub.status.idle":"2023-11-26T22:47:10.852001Z","shell.execute_reply.started":"2023-11-26T22:47:05.381188Z","shell.execute_reply":"2023-11-26T22:47:10.849556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink('concept_bottleneck_gamma_42_50.pth')","metadata":{"execution":{"iopub.status.busy":"2023-11-24T18:11:46.820159Z","iopub.status.idle":"2023-11-24T18:11:46.820771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate test predictions\n\n- By default we use the same data augmentation techniques that we applied during training. \n- Tweak the TTA parameter in `get_preds()` to increase the number of times TTA is applied.\n- If you do not want TTA, change `transforms` to `test_transforms`.\n","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:16.101805Z","iopub.execute_input":"2023-11-26T22:47:16.102218Z","iopub.status.idle":"2023-11-26T22:47:16.107276Z","shell.execute_reply.started":"2023-11-26T22:47:16.102169Z","shell.execute_reply":"2023-11-26T22:47:16.106088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tfms","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:16.641099Z","iopub.execute_input":"2023-11-26T22:47:16.641529Z","iopub.status.idle":"2023-11-26T22:47:16.646624Z","shell.execute_reply.started":"2023-11-26T22:47:16.641488Z","shell.execute_reply":"2023-11-26T22:47:16.645897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_tfms = A.Compose([\n#     A.RandomRotate90(p=p),\n#         A.Flip(p=p),\n#         A.OneOf([\n#             A.RandomBrightnessContrast(brightness_limit=0.2,\n#                                        contrast_limit=0.2,\n#                                        ),\n#             A.HueSaturationValue(\n#                 hue_shift_limit=20,\n#                 sat_shift_limit=50,\n#                 val_shift_limit=50)\n#         ], p=p),\n#         A.OneOf([\n#             A.IAAAdditiveGaussianNoise(),\n#             A.GaussNoise(),\n#         ], p=p),\n#     ToTensor(normalize=imagenet_stats)\n#     ])\n# os.getcwd()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:17.213275Z","iopub.execute_input":"2023-11-26T22:47:17.213859Z","iopub.status.idle":"2023-11-26T22:47:17.219448Z","shell.execute_reply.started":"2023-11-26T22:47:17.213816Z","shell.execute_reply":"2023-11-26T22:47:17.218416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concept_bottleneck_gamma_42","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:17.813610Z","iopub.execute_input":"2023-11-26T22:47:17.813952Z","iopub.status.idle":"2023-11-26T22:47:17.818101Z","shell.execute_reply.started":"2023-11-26T22:47:17.813923Z","shell.execute_reply":"2023-11-26T22:47:17.817115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(path/'test.csv')\n# (self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None)\ntrained_conceptmodel, opt = get_model_conceptb(device='cpu')\n# trained_conceptmodel.load_state_dict(torch.load(f'/kaggle/input/model-checkpoints-5-epochs/concept_bottleneck_beta_42.pth',map_location='cpu'))\n# trained_conceptmodel.load_state_dict(torch.load(f'concept_bottleneck_gamma_42.pth',map_location='cuda'))","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:18.232497Z","iopub.execute_input":"2023-11-26T22:47:18.232831Z","iopub.status.idle":"2023-11-26T22:47:18.573669Z","shell.execute_reply.started":"2023-11-26T22:47:18.232803Z","shell.execute_reply":"2023-11-26T22:47:18.572801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.read_csv(path/'test.csv')\n# # (self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None)\n# trained_basemodel, opt = get_model_baseline(device='cpu')\n# trained_basemodel.load_state_dict(torch.load(f'/kaggle/input/model-checkpoints-5-epochs/base-resnet-alpha_42.pth',map_location='cpu'))","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:23.260760Z","iopub.execute_input":"2023-11-26T22:47:23.261099Z","iopub.status.idle":"2023-11-26T22:47:23.265171Z","shell.execute_reply.started":"2023-11-26T22:47:23.261069Z","shell.execute_reply":"2023-11-26T22:47:23.263912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = dummies_test(test_df)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:23.591520Z","iopub.execute_input":"2023-11-26T22:47:23.591853Z","iopub.status.idle":"2023-11-26T22:47:23.608332Z","shell.execute_reply.started":"2023-11-26T22:47:23.591824Z","shell.execute_reply":"2023-11-26T22:47:23.607671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# my_path_test = Path('../input/melanoma-preprocess-test')\n# test_data = pd.read_csv(my_path_test/'one_hot_test_data.csv')\n# train_data.drop(columns=['Unnamed: 0'], inplace=True)\nirr_columns_test = ['image_name', 'patient_id', 'benign_malignant', 'tfrecord', 'width', 'height', 'target']\nmeta_cols_test = ~test_data.columns.isin(irr_columns_test)\nfeats_test = test_data.columns[meta_cols_test]\ntest_ds = MelanomaDataset(df=test_data, imfolder=path/'test', meta_features=feats_test, train=False, transforms=test_tfms, test=True)\ntest_ds[0]","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:24.181046Z","iopub.execute_input":"2023-11-26T22:47:24.181407Z","iopub.status.idle":"2023-11-26T22:47:24.230594Z","shell.execute_reply.started":"2023-11-26T22:47:24.181362Z","shell.execute_reply":"2023-11-26T22:47:24.229751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.read_csv(path/'test.csv')\n\n# # model, opt = get_model_conceptb()\n# # model.load_state_dict(torch.load(f'../input/melanomaefficientnetb5/effb5.pth',map_location=device))\n\n# #Testing with lighter augmentation\n# test_ds = MelanomaDataset(df=test_df,imfolder=path/'test', train=False, transforms=test_tfms,meta_features=None)\ntrain_dl,valid_dl = get_data(train_df,valid_df,train_tfms,test_tfms,bs)\ntest_dl = DataLoader(dataset=test_ds,batch_size=bs,shuffle=False,num_workers=4)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-26T22:47:27.655137Z","iopub.execute_input":"2023-11-26T22:47:27.655500Z","iopub.status.idle":"2023-11-26T22:47:27.661037Z","shell.execute_reply.started":"2023-11-26T22:47:27.655465Z","shell.execute_reply":"2023-11-26T22:47:27.659860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = MelanomaDataset(df=train_df, imfolder=path/'train', meta_features=feats, train=True, transforms=train_tfms)\nvalid_ds = MelanomaDataset(df=valid_df, imfolder=path/'train', meta_features=feats, train=True, transforms=test_tfms) \n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T22:47:28.018752Z","iopub.execute_input":"2023-11-26T22:47:28.019096Z","iopub.status.idle":"2023-11-26T22:47:28.024497Z","shell.execute_reply.started":"2023-11-26T22:47:28.019063Z","shell.execute_reply":"2023-11-26T22:47:28.023676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_data_iter = iter(test_dl)  # Create an iterator for the data loader\n# first_test_batch = next(test_data_iter) \n# first_test_batch.shape\ndef get_preds_base_f1(model, data_dl,device=None,tta=3):\n    if device is None:\n        device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    preds = np.zeros(len(data_dl))\n    labels = np.zeros(len(data_dl))\n    test_preds = []\n    ground_truths = []\n    with torch.no_grad():\n        for xb, yb, _ in data_dl:\n            xb = xb.to(device)\n            out = model(xb)\n            # out = torch.sigmoid(out)\n            test_preds.extend(out.cpu().numpy())\n            ground_truths.extend(yb.cpu().numpy())\n        # preds += np.array(test_preds).reshape(-1)\n        # labels += np.array(ground_truths).reshape(-1)\n    return test_preds, ground_truths\n\n#Changing tta to 25 from 10\ntrained_basemodel, opt = get_model_baseline(device='cuda')\ntrained_basemodel.load_state_dict(torch.load(f'/kaggle/input/model-checkpoints-5-epochs/base-resnet-alpha_42.pth',map_location='cpu'))\npreds_base_train_f1, train_labels = get_preds_base_f1(trained_basemodel, train_dl,tta=1) \npreds_base_valid_f1, valid_labels = get_preds_base_f1(trained_basemodel, valid_dl,tta=1) \n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T23:08:46.955998Z","iopub.execute_input":"2023-11-26T23:08:46.956343Z","iopub.status.idle":"2023-11-26T23:14:22.716367Z","shell.execute_reply.started":"2023-11-26T23:08:46.956312Z","shell.execute_reply":"2023-11-26T23:14:22.715325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_conceptmodel(train_ds[0][0][None, :].cuda())","metadata":{"execution":{"iopub.status.busy":"2023-11-26T23:38:58.540249Z","iopub.execute_input":"2023-11-26T23:38:58.540639Z","iopub.status.idle":"2023-11-26T23:38:58.678313Z","shell.execute_reply.started":"2023-11-26T23:38:58.540601Z","shell.execute_reply":"2023-11-26T23:38:58.677550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# f1 = multiclass_f1_score(predicted_labels, true_labels, num_classes=2, average='macro')\nactual = np.repeat([1, 0], repeats=[160, 240])\n\n# Define array of predicted classes\npred = np.repeat([1, 0, 1, 0], repeats=[120, 40, 70, 170])\n\n# Calculate F1 score\nf1 = f1_score(actual, pred)\nf1","metadata":{"execution":{"iopub.status.busy":"2023-11-26T23:40:25.096730Z","iopub.execute_input":"2023-11-26T23:40:25.097069Z","iopub.status.idle":"2023-11-26T23:40:25.107548Z","shell.execute_reply.started":"2023-11-26T23:40:25.097039Z","shell.execute_reply":"2023-11-26T23:40:25.106473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TP = sum((np.array(preds_base_valid_f1) >= 0.5).reshape(-1) == np.array(valid_labels))\nFP = sum((np.array(preds_base_valid_f1) >= 0.5).reshape(-1) != np.array(valid_labels))\nTN = sum((np.array(preds_base_valid_f1) < 0.5).reshape(-1) == np.array(valid_labels))\nFN = sum((np.array(preds_base_valid_f1) < 0.5).reshape(-1) != np.array(valid_labels))\nprecision = TP/(TP + FP)\nrecall = TP / (TP + FN)\nf1 = 2 * (precision * recall) / (precision + recall)\nprint(f1)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T23:53:58.224898Z","iopub.execute_input":"2023-11-26T23:53:58.225250Z","iopub.status.idle":"2023-11-26T23:53:58.349336Z","shell.execute_reply.started":"2023-11-26T23:53:58.225215Z","shell.execute_reply":"2023-11-26T23:53:58.348341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TP = sum((np.array(preds_base_valid_f1) >= 0.5).reshape(-1) == np.array(valid_labels))\n# FP = sum((np.array(preds_base_valid_f1) >= 0.5).reshape(-1) != np.array(valid_labels))\n# TN = sum((np.array(preds_base_valid_f1) < 0.5).reshape(-1) == np.array(valid_labels))\n# FN = sum((np.array(preds_base_valid_f1) < 0.5).reshape(-1) != np.array(valid_labels))\n# precision = TP/(TP + FP)\n# recall = TP / (TP + FN)\n# f1 = 2 * (precision * recall) / (precision + recall)\n# print(f1)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T00:02:29.150712Z","iopub.execute_input":"2023-11-27T00:02:29.151102Z","iopub.status.idle":"2023-11-27T00:02:29.154608Z","shell.execute_reply.started":"2023-11-27T00:02:29.151062Z","shell.execute_reply":"2023-11-27T00:02:29.153651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = (np.array(preds_base_train_f1) >= 0.5).reshape(-1).astype(int).tolist()\nprint(f1_score(train_labels, temp, average='binary' ))\nprint(f1_score((np.array(preds_base_valid_f1) >= 0.5).reshape(-1), np.array(valid_labels)))","metadata":{"execution":{"iopub.status.busy":"2023-11-27T00:02:29.892800Z","iopub.execute_input":"2023-11-27T00:02:29.893132Z","iopub.status.idle":"2023-11-27T00:02:29.983646Z","shell.execute_reply.started":"2023-11-27T00:02:29.893103Z","shell.execute_reply":"2023-11-27T00:02:29.982686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_data_iter = iter(test_dl)  # Create an iterator for the data loader\n# first_test_batch = next(test_data_iter) \n# first_test_batch.shape\ndef get_preds_concept_f1(model, data_dl,device=None,tta=3):\n    if device is None:\n        device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    preds = np.zeros(len(data_dl))\n    labels = np.zeros(len(data_dl))\n    ground_truths=[]\n    for tta_id in range(tta):\n        test_preds = []\n        with torch.no_grad():\n            for xb, yb, _ in data_dl:\n                xb = xb.to(device)\n                out = model(xb)\n                # out = torch.sigmoid(out)\n                test_preds.extend(out[0].cpu().numpy())\n                ground_truths.extend(yb.cpu().numpy())\n            # preds += np.array(test_preds).reshape(-1)\n            # preds += np.array(test_preds).reshape(-1)\n            # labels += np.array(ground_truths).reshape(-1)\n            \n        print(f'TTA {tta_id}')\n    preds /= tta\n    return test_preds, ground_truths\n\ntrained_conceptmodel, opt = get_model_conceptb(device='cuda')\ntrained_conceptmodel.load_state_dict(torch.load(f'/kaggle/input/model-checkpoints-5-epochs/concept_bottleneck_beta_42.pth',map_location='cpu'))\npreds_concept_train_f1, concept_train_labels = get_preds_concept_f1(trained_conceptmodel, train_dl,tta=1) \npreds_concept_valid_f1, concept_valid_labels = get_preds_concept_f1(trained_conceptmodel, valid_dl,tta=1) \n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-26T23:14:22.718158Z","iopub.execute_input":"2023-11-26T23:14:22.718485Z","iopub.status.idle":"2023-11-26T23:19:54.423469Z","shell.execute_reply.started":"2023-11-26T23:14:22.718453Z","shell.execute_reply":"2023-11-26T23:19:54.422257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.array(preds_concept_train_f1) < 0.5","metadata":{"execution":{"iopub.status.busy":"2023-11-26T23:46:24.546589Z","iopub.execute_input":"2023-11-26T23:46:24.546985Z","iopub.status.idle":"2023-11-26T23:46:24.567049Z","shell.execute_reply.started":"2023-11-26T23:46:24.546949Z","shell.execute_reply":"2023-11-26T23:46:24.566318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f1_score(np.array(preds_concept_train_f1) < 0.5, np.array(concept_train_labels)))\nprint(f1_score(np.array(preds_concept_valid_f1) < 0.5, np.array(concept_valid_labels)))","metadata":{"execution":{"iopub.status.busy":"2023-11-27T00:02:43.704588Z","iopub.execute_input":"2023-11-27T00:02:43.704926Z","iopub.status.idle":"2023-11-27T00:02:43.741929Z","shell.execute_reply.started":"2023-11-27T00:02:43.704896Z","shell.execute_reply":"2023-11-27T00:02:43.740952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_preds_base(model,device=None,tta=3):\n    if device is None:\n        device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    preds = np.zeros(len(test_ds))\n    for tta_id in range(tta):\n        test_preds = []\n        with torch.no_grad():\n            for xb in test_dl:\n                xb = xb.to(device)\n                out = model(xb)\n                # out = torch.sigmoid(out)\n                test_preds.extend(out.cpu().numpy())\n            preds += np.array(test_preds).reshape(-1)\n        print(f'TTA {tta_id}')\n    preds /= tta\n    return preds\n\n#Changing tta to 25 from 10\npreds_base = get_preds_base(trained_basemodel,tta=1) \n","metadata":{"execution":{"iopub.status.busy":"2023-11-24T13:27:47.788645Z","iopub.execute_input":"2023-11-24T13:27:47.789341Z","iopub.status.idle":"2023-11-24T13:40:59.131452Z","shell.execute_reply.started":"2023-11-24T13:27:47.789149Z","shell.execute_reply":"2023-11-24T13:40:59.130044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_preds(model,device=None,tta=3):\n    if device is None:\n        device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    preds = np.zeros(len(test_ds))\n    for tta_id in range(tta):\n        test_preds = []\n        with torch.no_grad():\n            for xb in test_dl:\n                xb = xb.to(device)\n                out = model(xb)\n                # out = torch.sigmoid(out)\n                test_preds.extend(out[0].cpu().numpy())\n            preds += np.array(test_preds).reshape(-1)\n        print(f'TTA {tta_id}')\n    preds /= tta\n    return preds\n\n#Changing tta to 25 from 10\npreds = get_preds(model,tta=1)  ","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-24T18:12:49.939040Z","iopub.execute_input":"2023-11-24T18:12:49.939644Z","iopub.status.idle":"2023-11-24T18:26:19.611700Z","shell.execute_reply.started":"2023-11-24T18:12:49.939605Z","shell.execute_reply":"2023-11-24T18:26:19.610339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = pd.read_csv(path/'sample_submission.csv')\nsubm.target = preds\nsubm.to_csv('submission_bottleneck_50_epochs.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-24T18:26:20.324790Z","iopub.execute_input":"2023-11-24T18:26:20.325151Z","iopub.status.idle":"2023-11-24T18:26:20.398998Z","shell.execute_reply.started":"2023-11-24T18:26:20.325117Z","shell.execute_reply":"2023-11-24T18:26:20.397612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_iter = iter(test_dl)\nfirst_batch = next(test_iter)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:09:21.270199Z","iopub.execute_input":"2023-11-19T16:09:21.270588Z","iopub.status.idle":"2023-11-19T16:09:22.127661Z","shell.execute_reply.started":"2023-11-19T16:09:21.270551Z","shell.execute_reply":"2023-11-19T16:09:22.125904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_outputs = trained_conceptmodel(first_batch.to('cuda'))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:10:13.056816Z","iopub.execute_input":"2023-11-19T16:10:13.057153Z","iopub.status.idle":"2023-11-19T16:10:13.085545Z","shell.execute_reply.started":"2023-11-19T16:10:13.057125Z","shell.execute_reply":"2023-11-19T16:10:13.084658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-24T14:10:10.701585Z","iopub.execute_input":"2023-11-24T14:10:10.701987Z","iopub.status.idle":"2023-11-24T14:10:10.709033Z","shell.execute_reply.started":"2023-11-24T14:10:10.701950Z","shell.execute_reply":"2023-11-24T14:10:10.707970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# INTERPRETABILITY FROM THE TAIL OF THE RESNET","metadata":{}},{"cell_type":"code","source":"feat_dict = {\n    'sex': list(range(0, 2)),\n    'anatomical_site': list(range(2, 8)),\n    'diagnosis': list(range(8, 17)),\n    'age': list(range(17, 35))\n}\n\n# def inspection(test_output):\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:19:29.489274Z","iopub.execute_input":"2023-11-19T16:19:29.489657Z","iopub.status.idle":"2023-11-19T16:19:29.495137Z","shell.execute_reply.started":"2023-11-19T16:19:29.489623Z","shell.execute_reply":"2023-11-19T16:19:29.494227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(1):\ni=42\nprint('new sample')\nfor k, v in feat_dict.items():\n    print(k)\n    test_outputs[1][0][feat_dict[k]]\n    single_test = test_outputs[1][i][feat_dict[k]]\n    print(single_test)\n    print(max(single_test))\n    print(feats[feat_dict[k]][torch.argmax(single_test).cpu().detach().numpy()])\n    print()\n    # print(argmax(single_test))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:47:13.909531Z","iopub.execute_input":"2023-11-19T16:47:13.909860Z","iopub.status.idle":"2023-11-19T16:47:13.931351Z","shell.execute_reply.started":"2023-11-19T16:47:13.909832Z","shell.execute_reply":"2023-11-19T16:47:13.930404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Samples without Melanoma\")\nidx = subm['image_name'][42]\nimg = Image.open(path/f'test/{idx}.jpg')\nplt.imshow(img)\nplt.axis('off')    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:46:56.809147Z","iopub.execute_input":"2023-11-19T16:46:56.809511Z","iopub.status.idle":"2023-11-19T16:46:56.907053Z","shell.execute_reply.started":"2023-11-19T16:46:56.809481Z","shell.execute_reply":"2023-11-19T16:46:56.906195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # feats.columns\n# indices_columns = [i for i, col in enumerate(feats) if col.startswith('age_approx')]\n# indices_columns","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:31:09.521060Z","iopub.execute_input":"2023-11-19T16:31:09.521406Z","iopub.status.idle":"2023-11-19T16:31:09.525123Z","shell.execute_reply.started":"2023-11-19T16:31:09.521376Z","shell.execute_reply":"2023-11-19T16:31:09.524062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.sec_model.state_dict()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trained_conceptmodel.sec_model.state_dict()\n# weights_test = trained_conceptmodel.sec_model.state_dict()['weight'].squeeze()\n# bias_test = trained_conceptmodel.sec_model.state_dict()['bias'].item()\n\n# model.sec_model.state_dict()\nweights_test = model.sec_model.state_dict()['weight'].squeeze()\nbias_test = model.sec_model.state_dict()['bias'].item()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:40:16.167047Z","iopub.execute_input":"2023-11-19T16:40:16.167395Z","iopub.status.idle":"2023-11-19T16:40:16.172951Z","shell.execute_reply.started":"2023-11-19T16:40:16.167366Z","shell.execute_reply":"2023-11-19T16:40:16.172070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"functional_form = ' + '.join([f'{weights_test[i]:.2f} * {feats[i]}' for i in range(len(weights_test))]) + f' + {bias_test:2f}'\nfunctional_form","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:42:24.679873Z","iopub.execute_input":"2023-11-19T16:42:24.680216Z","iopub.status.idle":"2023-11-19T16:42:24.689323Z","shell.execute_reply.started":"2023-11-19T16:42:24.680186Z","shell.execute_reply":"2023-11-19T16:42:24.688268Z"},"trusted":true},"execution_count":null,"outputs":[]}]}