{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1236170,"sourceType":"datasetVersion","datasetId":708434}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n# Set random seed for reproducibility\nimport torch\nseed = 42\ntorch.manual_seed(seed)\nnp.random.seed(seed)\n\nimport os\ninput_directory = '/kaggle/input'\n\nfor dirpath, dirnames, filenames in os.walk(input_directory):\n    for dirname in dirnames:\n        print(dirname)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-31T18:30:54.410084Z","iopub.execute_input":"2024-01-31T18:30:54.410445Z","iopub.status.idle":"2024-01-31T18:33:35.010858Z","shell.execute_reply.started":"2024-01-31T18:30:54.410417Z","shell.execute_reply":"2024-01-31T18:33:35.009776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom albumentations import Compose, Flip, CropAndPad, Transpose\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:33:35.013029Z","iopub.execute_input":"2024-01-31T18:33:35.013883Z","iopub.status.idle":"2024-01-31T18:33:36.902833Z","shell.execute_reply.started":"2024-01-31T18:33:35.013846Z","shell.execute_reply":"2024-01-31T18:33:36.901831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nwandb.init(project='Skin Melanoma Detection Project', save_code=True,)","metadata":{"execution":{"iopub.status.busy":"2024-01-31T17:57:37.920633Z","iopub.execute_input":"2024-01-31T17:57:37.921532Z","iopub.status.idle":"2024-01-31T17:59:11.963292Z","shell.execute_reply.started":"2024-01-31T17:57:37.921487Z","shell.execute_reply":"2024-01-31T17:59:11.962399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:40:53.51945Z","iopub.execute_input":"2024-01-31T18:40:53.519804Z","iopub.status.idle":"2024-01-31T18:40:53.650552Z","shell.execute_reply.started":"2024-01-31T18:40:53.519775Z","shell.execute_reply":"2024-01-31T18:40:53.649583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train['target'].value_counts())\ntrain['target'].hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:41:02.382893Z","iopub.execute_input":"2024-01-31T18:41:02.383766Z","iopub.status.idle":"2024-01-31T18:41:02.598531Z","shell.execute_reply.started":"2024-01-31T18:41:02.383732Z","shell.execute_reply":"2024-01-31T18:41:02.597655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"mode:{train['age_approx'].mode()}\")\ntrain['age_approx'].hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:41:04.076548Z","iopub.execute_input":"2024-01-31T18:41:04.076888Z","iopub.status.idle":"2024-01-31T18:41:04.227937Z","shell.execute_reply.started":"2024-01-31T18:41:04.076864Z","shell.execute_reply":"2024-01-31T18:41:04.227042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['age_approx'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:41:05.584176Z","iopub.execute_input":"2024-01-31T18:41:05.585037Z","iopub.status.idle":"2024-01-31T18:41:05.594143Z","shell.execute_reply.started":"2024-01-31T18:41:05.585006Z","shell.execute_reply":"2024-01-31T18:41:05.593192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['age_approx'].hist(bins=len(train['age_approx'].unique()))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:41:06.763976Z","iopub.execute_input":"2024-01-31T18:41:06.764875Z","iopub.status.idle":"2024-01-31T18:41:06.934398Z","shell.execute_reply.started":"2024-01-31T18:41:06.764838Z","shell.execute_reply":"2024-01-31T18:41:06.93352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#imgs = os.listdir('/kaggle/input/melanoma224/jpeg224/train')\n\n# Get image paths\nimgs = [f'/kaggle/input/siim-isic-melanoma-classification/jpeg/train/{img}.jpg' for img in train['image_name']]\nimgs[:5]","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:41:08.162991Z","iopub.execute_input":"2024-01-31T18:41:08.163366Z","iopub.status.idle":"2024-01-31T18:41:08.182178Z","shell.execute_reply.started":"2024-01-31T18:41:08.163336Z","shell.execute_reply":"2024-01-31T18:41:08.18134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import cv2\n\nclass Dataset(torch.utils.data.Dataset):\n    def __init__(self, images, meta, labels=None, train=True, cache=False, trans=None):\n        self.cache = cache\n        self.images = [self.load_img(img) for img in images] if cache else images\n        meta = oh_encoder.transform(meta).toarray()\n        self.meta = torch.tensor(meta).float()\n        self.train = train\n        self.trans = trans\n        if train: self.labels = [torch.tensor([label]).float() for label in labels]\n\n    def __len__(self):\n        return len(self.images)\n    def load_img(self, img):\n        full_path = os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/train', img + '.jpg')\n        return cv2.imread(full_path)\n   \n\n    def __getitem__(self, ix):\n        img = self.images[ix] if self.cache else self.load_img(self.images[ix])\n        if self.trans:\n            img = self.trans(image=img)['image']\n        if self.train:\n            return torch.from_numpy(img), self.meta[ix], self.labels[ix][0]  # Extract the value from the list\n        return torch.from_numpy(img), self.meta[ix]\n","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:41:16.137101Z","iopub.execute_input":"2024-01-31T18:41:16.137534Z","iopub.status.idle":"2024-01-31T18:41:16.151672Z","shell.execute_reply.started":"2024-01-31T18:41:16.137498Z","shell.execute_reply":"2024-01-31T18:41:16.150532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''import random\n\nr, c = 3, 5\nfig = plt.figure(figsize=(2*c, 2*r))\nfor _r in range(r):\n    for _c in range(c):\n        plt.subplot(r, c, _r*c + _c + 1)\n        ix = random.randint(0, len(dataset['train']) - 1)\n        img, meta, label = dataset['train'][ix]\n        plt.imshow(img)\n        plt.title(label)\n        plt.axis('off')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:42:15.628814Z","iopub.execute_input":"2024-01-31T18:42:15.629525Z","iopub.status.idle":"2024-01-31T18:42:15.63543Z","shell.execute_reply.started":"2024-01-31T18:42:15.629494Z","shell.execute_reply":"2024-01-31T18:42:15.634235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model building\nclass NN(torch.nn.Module):\n    def __init__(self, inputs=29):\n        super().__init__()\n        \n        # Encoder for images\n        resnet = torchvision.models.resnet101(weights=True)\n        for param in resnet.parameters():\n            param.requires_grad = False\n        self.encoder = torch.nn.Sequential(*list(resnet.children())[:-1])\n        \n        # Encoder for metadata\n        self.meta_encoder = torch.nn.Sequential(\n            torch.nn.Linear(inputs, 100),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Linear(100, 3),\n            torch.nn.ReLU(inplace=True))\n        \n        # Head of the model\n        self.head = torch.nn.Linear(2048 + 3, 1)\n\n    def forward(self, x, y):\n        B, H, W, C = x.shape\n        x = (x / 255.).float().view(B, C, H, W)\n        x = self.encoder(x)\n        x = x.view(B, -1)\n        y = self.meta_encoder(y)\n        z = torch.cat([x, y], -1)\n        z = self.head(z)\n\n        return z","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:43:27.370034Z","iopub.execute_input":"2024-01-31T18:43:27.370408Z","iopub.status.idle":"2024-01-31T18:43:27.379832Z","shell.execute_reply.started":"2024-01-31T18:43:27.370378Z","shell.execute_reply":"2024-01-31T18:43:27.378785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nBATCH_SIZE = 64\n\nmodel = NN()\n\noutput = model(torch.randn(BATCH_SIZE, 224, 224, 3), torch.randn(BATCH_SIZE, 29))\noutput.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:44:05.980369Z","iopub.execute_input":"2024-01-31T18:44:05.98117Z","iopub.status.idle":"2024-01-31T18:44:25.275087Z","shell.execute_reply.started":"2024-01-31T18:44:05.981136Z","shell.execute_reply":"2024-01-31T18:44:25.27413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train set and validation set","metadata":{}},{"cell_type":"code","source":"import torchvision.transforms as transforms\n\n# Data Transformation\ninput_size = 224\n\ndata_transforms = {\n    'train': transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.Resize(input_size),\n        transforms.CenterCrop(input_size),\n        # Add extra transformations for data augmentation\n        transforms.RandomApply([\n            transforms.RandomChoice([\n                transforms.RandomAffine(degrees=20),\n                transforms.RandomAffine(degrees=0, scale=(0.1, 0.15)),\n                transforms.RandomAffine(degrees=0, translate=(0.2, 0.2)),\n                # transforms.RandomAffine(degrees=0, shear=0.15),\n                transforms.RandomHorizontalFlip(p=1.0)\n            ])\n        ]),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5], [0.225])\n    ]),\n    'val': transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.Resize(input_size),\n        transforms.CenterCrop(input_size),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5], [0.225])\n    ]),\n    'test': transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.Resize(input_size),\n        transforms.CenterCrop(input_size),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5], [0.225])\n    ])\n}\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:50:45.305177Z","iopub.execute_input":"2024-01-31T18:50:45.305846Z","iopub.status.idle":"2024-01-31T18:50:45.314834Z","shell.execute_reply.started":"2024-01-31T18:50:45.305813Z","shell.execute_reply":"2024-01-31T18:50:45.313834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\n# Assuming you have a variable 'num_folds' defined earlier\nnum_folds = 5\n\n# Initialize StratifiedKFold\nskf = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=42)\n\n# Iterate through each fold\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train, train['target'])):\n    print(f\"Fold {fold + 1}/{num_folds}\")\n    \n    # Create train and validation sets for the current fold\n    train_fold, val_fold = train.iloc[train_idx], train.iloc[val_idx]\n    \n    # Display target distribution in the current fold\n    print(f\"Train target distribution in fold {fold + 1}:\\n{train_fold['target'].value_counts()}\")\n    print(f\"Validation target distribution in fold {fold + 1}:\\n{val_fold['target'].value_counts()}\")\n    \n    # Train set and validation set\n    dataset = {\n        'train': Dataset(train_fold['image_name'], train_fold['target'], cache=True, trans=data_transforms['train']),\n        'val': Dataset(val_fold['image_name'], val_fold['target'], cache=True, trans=data_transforms['val']),\n    }\n\n    # Create data loaders\n    BATCH_SIZE = 64\n    dataloader = {\n        'train': torch.utils.data.DataLoader(dataset['train'], batch_size=BATCH_SIZE, shuffle=True),\n        'val': torch.utils.data.DataLoader(dataset['val'], batch_size=BATCH_SIZE),\n    }\n","metadata":{"execution":{"iopub.status.busy":"2024-01-31T18:51:55.303508Z","iopub.execute_input":"2024-01-31T18:51:55.304226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"imgs, meta, labels = next(iter(dataloader['train']))\n\nimgs.shape, meta.shape, labels.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# AUC error metric","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef AUC(outputs, labels):\n    outputs = torch.sigmoid(outputs)\n    outputs = outputs.detach().cpu().numpy()\n    labels = labels.detach().cpu().numpy()\n    auc = roc_auc_score(labels, outputs)\n    return auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"model = NN().to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.01)\ncriterion = torch.nn.BCEWithLogitsLoss()  # this loss applies sigmoid itself, and it's better mathematically\n\nepochs = 12\nvalidation_steps = 15\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# WandB Config\nwandb.config.epochs = epochs\nwandb.config.validation_steps = validation_steps\n\nmb = master_bar(range(1, epochs + 1))\nbest_auc = 0\n\nfor epoch in mb:\n    train_loss = []\n    model.train()\n    for batch in progress_bar(dataloader['train'], parent=mb):\n        imgs, meta, labels = batch\n        imgs, meta, labels = imgs.to(device), meta.to(device), labels.to(device)\n        outputs = model(imgs, meta)\n        optimizer.zero_grad()\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss.append(loss.item())\n        mb.child.comment = f'loss: {np.mean(train_loss):.5f}'\n\n    # Log training loss to wandb\n    wandb.log({\"train_loss\": np.mean(train_loss)})\n\n    val_loss = []\n    model.eval()\n    validation_step = 0\n    val_outputs = torch.tensor([])\n    val_targets = torch.tensor([])\n    with torch.no_grad():\n        for batch in progress_bar(dataloader['val'], parent=mb):\n            imgs, meta, labels = batch\n            imgs, meta, labels = imgs.to(device), meta.to(device), labels.to(device)\n            outputs = model(imgs, meta)\n            loss = criterion(outputs, labels)\n            val_loss.append(loss.item())\n            mb.child.comment = f'val_loss: {np.mean(val_loss):.5f}'\n            val_outputs = torch.cat([val_outputs, outputs.cpu()])\n            val_targets = torch.cat([val_targets, labels.cpu()])\n            validation_step += 1\n            if validation_step > validation_steps:\n                break\n\n    auc = AUC(val_outputs, val_targets)\n\n    # Log validation metrics to wandb\n    wandb.log({\"val_loss\": np.mean(val_loss), \"auc\": auc})\n\n    if auc > best_auc:\n        best_auc = auc\n        # Save the model state_dict and log to wandb\n        torch.save(model.state_dict(), 'model.pth')\n        wandb.save('model.pth')\n\n    # Log metrics to wandb\n    wandb.log({\"epoch\": epoch, \"best_auc\": best_auc})\n\n    mb.write(f'epoch: {epoch} | train_loss: {np.mean(train_loss):.5f} | val_loss: {np.mean(val_loss):.5f} | auc: {auc:.5f}')\n\n# Finish wandb run\nwandb.finish()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/melanoma224/test.csv')\ntest['path'] = [f'/kaggle/input/melanoma224/jpeg224/test/{img}.jpg' for img in test['image_name']]\n\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = Dataset(test['path'], test[cols], train=False)\ntest_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=200)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs, meta = next(iter(test_dataloader))\nimgs.shape, meta.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the model for Testing","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n# Initialize wandb\nwandb.init(project='Skin Melanoma Detection Project', name=\"testingModel_run\")\n\n# Instantiate your model\nmodel = NN()\n\n# Load the state dictionary\nmodel_state_dict = torch.load('/kaggle/working/model.pth')\n\n# Load the state dictionary into the model\nmodel.load_state_dict(model_state_dict)\n\n# Move the model to the desired device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Set the model to evaluation mode\nmodel.eval()\n\n# Log model to wandb\nwandb.watch(model)\n\n# Initialize a list to store predictions\npreds_list = []\n\n# No need to calculate gradients during testing\nwith torch.no_grad():\n    for imgs, meta in tqdm(test_dataloader, desc=\"Testing\"):\n        imgs, meta = imgs.to(device), meta.to(device)\n        \n        # Forward pass and sigmoid activation\n        outputs = model(imgs, meta)\n        outputs = torch.sigmoid(outputs)\n        \n        # Append predictions to the list\n        preds_list.append(outputs.view(-1).cpu().numpy())\n\n# Concatenate predictions and convert to a tensor\npreds = torch.tensor(np.concatenate(preds_list)).to(device)\n\n# Replace \"metric_name\" and \"metric_value\" with actual metric names and values\n#wandb.log({\"metric_name\": metric_value})\n\n# Log predictions to wandb\nwandb.log({\"predictions\": wandb.Histogram(preds.cpu().numpy())})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert the preds to Dataframe","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({'image_name': test['image_name'].values, 'target': preds.cpu().numpy()})\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}