{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1225697,"sourceType":"datasetVersion","datasetId":701123},{"sourceId":1253590,"sourceType":"datasetVersion","datasetId":720563},{"sourceId":1322494,"sourceType":"datasetVersion","datasetId":688574},{"sourceId":1339680,"sourceType":"datasetVersion","datasetId":756214}],"dockerImageVersionId":29928,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Versions:\n* v9: ColorJitter transformation added **[0.896]**\n* v10: Changed the dataset to [this one](https://www.kaggle.com/shonenkov/melanoma-merged-external-data-512x512-jpeg) with external data. **[0.894]**\n* v11: Switched to [another dataset](https://www.kaggle.com/nroman/melanoma-external-malignant-256/) which I've created by myself. Also switched from StratifiedKFold to GroupKFold **[0.916]**\n* v12: Switched to efficientnet-b1 **[0.919]**\n* v13: Using meta featues: sex and age **[0.918]**\n* v14: anatom_site_general_challenge meta feature added as one-hot encoded matrix **[0.923]**\n* v16: Fixed OOF - now it contains only data from original training dataset, without extarnal data. Also switched back to StratifiedKFold. Added DrawHair augmentation. **[0.909]**\n* v18: Too many things were changed at the same time. All experiments should have only one small change each, so it would be easy to understand how changes affect the result. Said that I rolled back everything, keeping only OOF fix, to make sure it work.\n* v19: Added 'Hair' augmentation. OOF rework posponed untill the best time, since there is some bug in my code for it. **[0.925]**\n* v20: Advanced Hair Augmentation technique used. Read more about it here: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/159176 **[0.923]**\n* v21: Microscope augmentation added instead of Cutout. Read more here: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/159476 **[0.914]**\n* v22: Changed the dataset to [this one](https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256) by Chris Deotte. More info [here](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/165526) **[0.900]**\n* v23: All the same as v22 but effnet-b0 instead of b1 and more epochs per fold. **[0.895]**\n* v24: effnet-b01 and more epochs. **[0.9092]**\n* v25: Fixed a mistake in a way of filling preds. See [this comment](https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet/comments?scriptVersionId=39125585#913846). **[0.9016]**\n* v26: Fix for another mistake. This time with a way of averaging TTA. See [this comment](https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet/comments#955916) **[0.915]**\n* v27: Back to [my dataset](https://www.kaggle.com/nroman/melanoma-external-malignant-256/)","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:11.837607Z","iopub.execute_input":"2025-10-11T17:22:11.837977Z","iopub.status.idle":"2025-10-11T17:22:11.851057Z","shell.execute_reply.started":"2025-10-11T17:22:11.837942Z","shell.execute_reply":"2025-10-11T17:22:11.850122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Import Fix\n\n**Fixed**: Changed `import torchtoolbox.transform as transforms` to `import torchvision.transforms as transforms` due to compatibility issues. The `torchtoolbox.transform` module has internal dependencies that conflict with newer torchvision versions.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install efficientnet_pytorch torchtoolbox","metadata":{"_kg_hide-output":true,"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:11.85297Z","iopub.execute_input":"2025-10-11T17:22:11.85332Z","iopub.status.idle":"2025-10-11T17:22:17.082245Z","shell.execute_reply.started":"2025-10-11T17:22:11.853279Z","shell.execute_reply":"2025-10-11T17:22:17.081476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(47)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.084606Z","iopub.execute_input":"2025-10-11T17:22:17.084858Z","iopub.status.idle":"2025-10-11T17:22:17.090745Z","shell.execute_reply.started":"2025-10-11T17:22:17.084832Z","shell.execute_reply":"2025-10-11T17:22:17.090005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\n\nclass MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n        x = cv2.imread(im_path)\n        x = Image.fromarray(cv2.cvtColor(x, cv2.COLOR_BGR2RGB))  # Convert from cv2 (BGR numpy) to PIL (RGB)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n        \n        if self.transforms:\n            x = self.transforms(x)\n            \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass Net(nn.Module):\n    def __init__(self, arch, n_meta_features: int):\n        super(Net, self).__init__()\n        self.arch = arch\n        if 'ResNet' in str(arch.__class__):\n            self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n        if 'EfficientNet' in str(arch.__class__):\n            self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.ouput = nn.Linear(500 + 250, 1)\n        \n    def forward(self, inputs):\n        \"\"\"\n        No sigmoid in forward because we are going to use BCEWithLogitsLoss\n        Which applies sigmoid for us when calculating a loss\n        \"\"\"\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.093006Z","iopub.execute_input":"2025-10-11T17:22:17.093351Z","iopub.status.idle":"2025-10-11T17:22:17.108719Z","shell.execute_reply.started":"2025-10-11T17:22:17.093318Z","shell.execute_reply":"2025-10-11T17:22:17.107923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Fix #1: Convert cv2 images to PIL format\n\n**Error:** `TypeError: Unexpected type <class 'numpy.ndarray'>` in torchvision transforms\n\n**Root Cause:** The MelanomaDataset class uses cv2.imread() which returns numpy arrays (BGR format), but torchvision transforms expect PIL Images.\n\n**Solution:** Convert BGR numpy array (from cv2) to RGB PIL Image before applying transforms in the __getitem__ method. Add: `x = Image.fromarray(cv2.cvtColor(x, cv2.COLOR_BGR2RGB))` after imread.","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.109821Z","iopub.execute_input":"2025-10-11T17:22:17.110058Z","iopub.status.idle":"2025-10-11T17:22:17.124682Z","shell.execute_reply.started":"2025-10-11T17:22:17.110033Z","shell.execute_reply":"2025-10-11T17:22:17.123864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\n\nclass AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n    \n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        # Convert PIL to numpy\n        img = np.array(img)\n        \n        n_hairs = random.randint(0, self.hairs)\n        \n        if not n_hairs:\n            return Image.fromarray(img)\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n            \n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n            \n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n            \n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n            \n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n                \n        return Image.fromarray(img)\n    \n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.125863Z","iopub.execute_input":"2025-10-11T17:22:17.126065Z","iopub.status.idle":"2025-10-11T17:22:17.138568Z","shell.execute_reply.started":"2025-10-11T17:22:17.126046Z","shell.execute_reply":"2025-10-11T17:22:17.137943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DrawHair:\n    \"\"\"\n    Draw a random number of pseudo hairs\n\n    Args:\n        hairs (int): maximum number of hairs to draw\n        width (tuple): possible width of the hair in pixels\n    \"\"\"\n\n    def __init__(self, hairs:int = 4, width:tuple = (1, 2)):\n        self.hairs = hairs\n        self.width = width\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        if not self.hairs:\n            return img\n        \n        width, height, _ = img.shape\n        \n        for _ in range(random.randint(0, self.hairs)):\n            # The origin point of the line will always be at the top half of the image\n            origin = (random.randint(0, width), random.randint(0, height // 2))\n            # The end of the line \n            end = (random.randint(0, width), random.randint(0, height))\n            color = (0, 0, 0)  # color of the hair. Black.\n            cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.139569Z","iopub.execute_input":"2025-10-11T17:22:17.139809Z","iopub.status.idle":"2025-10-11T17:22:17.151903Z","shell.execute_reply.started":"2025-10-11T17:22:17.139787Z","shell.execute_reply":"2025-10-11T17:22:17.151211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\n\nclass Microscope:\n    \"\"\"\n    Cutting out the edges around the center circle of the image\n    Imitating a picture, taken through the microscope\n    Args:\n        p (float): probability of applying an augmentation\n    \"\"\"\n    def __init__(self, p: float = 0.5):\n        self.p = p\n    \n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to apply transformation to.\n        Returns:\n            PIL Image: Image with transformation.\n        \"\"\"\n        # Convert PIL to numpy\n        img = np.array(img)\n        \n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8),  # image placeholder\n                        (img.shape[0]//2, img.shape[1]//2),  # center point of circle\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15),  # radius\n                        (0, 0, 0),  # color\n                        -1)\n            mask = circle - 255\n            img = np.multiply(img, mask)\n        \n        return Image.fromarray(img)\n    \n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.152727Z","iopub.execute_input":"2025-10-11T17:22:17.152962Z","iopub.status.idle":"2025-10-11T17:22:17.163912Z","shell.execute_reply.started":"2025-10-11T17:22:17.15294Z","shell.execute_reply":"2025-10-11T17:22:17.163048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs'),\n    transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    Microscope(p=0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.164937Z","iopub.execute_input":"2025-10-11T17:22:17.165165Z","iopub.status.idle":"2025-10-11T17:22:17.17747Z","shell.execute_reply.started":"2025-10-11T17:22:17.165144Z","shell.execute_reply":"2025-10-11T17:22:17.176774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"arch = EfficientNet.from_pretrained('efficientnet-b1')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.178597Z","iopub.execute_input":"2025-10-11T17:22:17.178944Z","iopub.status.idle":"2025-10-11T17:22:17.376709Z","shell.execute_reply.started":"2025-10-11T17:22:17.178909Z","shell.execute_reply":"2025-10-11T17:22:17.375716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')\ntest_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.37912Z","iopub.execute_input":"2025-10-11T17:22:17.379378Z","iopub.status.idle":"2025-10-11T17:22:17.456375Z","shell.execute_reply.started":"2025-10-11T17:22:17.379352Z","shell.execute_reply":"2025-10-11T17:22:17.455629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.459062Z","iopub.execute_input":"2025-10-11T17:22:17.459334Z","iopub.status.idle":"2025-10-11T17:22:17.493415Z","shell.execute_reply.started":"2025-10-11T17:22:17.459308Z","shell.execute_reply":"2025-10-11T17:22:17.492776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.494588Z","iopub.execute_input":"2025-10-11T17:22:17.494844Z","iopub.status.idle":"2025-10-11T17:22:17.499287Z","shell.execute_reply.started":"2025-10-11T17:22:17.49482Z","shell.execute_reply":"2025-10-11T17:22:17.498409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/', \n                       train=False,\n                       transforms=train_transform,  # For TTA\n                       meta_features=meta_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.500293Z","iopub.execute_input":"2025-10-11T17:22:17.500617Z","iopub.status.idle":"2025-10-11T17:22:17.51151Z","shell.execute_reply.started":"2025-10-11T17:22:17.500584Z","shell.execute_reply":"2025-10-11T17:22:17.510773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"skf = GroupKFold(n_splits=5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:22:17.512463Z","iopub.execute_input":"2025-10-11T17:22:17.51273Z","iopub.status.idle":"2025-10-11T17:22:17.522317Z","shell.execute_reply.started":"2025-10-11T17:22:17.512688Z","shell.execute_reply":"2025-10-11T17:22:17.521575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epochs = 12  # Number of epochs to run\nes_patience = 3  # Early Stopping patience - for how many epochs with no improvements to wait\nTTA = 3 # Test Time Augmentation rounds\n\noof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\npreds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)  # Predictions for test test\n\nskf = KFold(n_splits=5, shuffle=True, random_state=47)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):\n    print('=' * 20, 'Fold', fold, '=' * 20)  \n    \n    model_path = f'model_{fold}.pth'  # Path and filename to save model to\n    best_val = 0  # Best validation score within this fold\n    patience = es_patience  # Current patience counter\n    arch = EfficientNet.from_pretrained('efficientnet-b1')\n    model = Net(arch=arch, n_meta_features=len(meta_features))  # New model for each fold\n    model = model.to(device)\n    \n    optim = torch.optim.Adam(model.parameters(), lr=0.001)\n    scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\n    criterion = nn.BCEWithLogitsLoss()\n    \n    train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                            train=True, \n                            transforms=train_transform,\n                            meta_features=meta_features)\n    val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                            train=True, \n                            transforms=test_transform,\n                            meta_features=meta_features)\n    \n    train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)\n    val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)\n    test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)\n    \n    for epoch in range(epochs):\n        start_time = time.time()\n        correct = 0\n        epoch_loss = 0\n        model.train()\n        \n        for x, y in train_loader:\n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n            optim.zero_grad()\n            z = model(x)\n            loss = criterion(z, y.unsqueeze(1))\n            loss.backward()\n            optim.step()\n            pred = torch.round(torch.sigmoid(z))  # round off sigmoid to obtain predictions\n            correct += (pred.cpu() == y.cpu().unsqueeze(1)).sum().item()  # tracking number of correctly predicted samples\n            epoch_loss += loss.item()\n        train_acc = correct / len(train_idx)\n        \n        model.eval()  # switch model to the evaluation mode\n        val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n            # Predicting on validation set\n            for j, (x_val, y_val) in enumerate(val_loader):\n                x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n                x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n                y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n                z_val = model(x_val)\n                val_pred = torch.sigmoid(z_val)\n                val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n            val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))\n            val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())\n            \n            print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(\n            epoch + 1, \n            epoch_loss, \n            train_acc, \n            val_acc, \n            val_roc, \n            str(datetime.timedelta(seconds=time.time() - start_time))[:7]))\n            \n            scheduler.step(val_roc)\n                \n            if val_roc >= best_val:\n                best_val = val_roc\n                patience = es_patience  # Resetting patience since we have new best validation accuracy\n                torch.save(model, model_path)  # Saving current best model\n            else:\n                patience -= 1\n                if patience == 0:\n                    print('Early stopping. Best Val roc_auc: {:.3f}'.format(best_val))\n                    break\n                \n    model = torch.load(model_path)  # Loading best model of this fold\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n    with torch.no_grad():\n        # Predicting on validation set once again to obtain data for OOF\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n            z_val = model(x_val)\n            val_pred = torch.sigmoid(z_val)\n            val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred\n        oof[val_idx] = val_preds.cpu().numpy()\n        \n        # Predicting on test set\n        tta_preds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)\n        for _ in range(TTA):\n            for i, x_test in enumerate(test_loader):\n                x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)\n                x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)\n                z_test = model(x_test)\n                z_test = torch.sigmoid(z_test)\n                tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test\n        preds += tta_preds / TTA\n    \npreds /= skf.n_splits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:34:08.749093Z","iopub.execute_input":"2025-10-11T17:34:08.749397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.kdeplot(pd.Series(preds.cpu().numpy().reshape(-1,)));","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:33:55.288049Z","iopub.status.idle":"2025-10-11T17:33:55.28862Z","shell.execute_reply":"2025-10-11T17:33:55.288323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Saving OOF predictions so stacking would be easier\npd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:33:55.289829Z","iopub.status.idle":"2025-10-11T17:33:55.290376Z","shell.execute_reply":"2025-10-11T17:33:55.290095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nsub['target'] = preds.cpu().numpy().reshape(-1,)\nsub.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T17:33:55.291386Z","iopub.status.idle":"2025-10-11T17:33:55.291962Z","shell.execute_reply":"2025-10-11T17:33:55.291657Z"}},"outputs":[],"execution_count":null}]}