{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1243687,"sourceType":"datasetVersion","datasetId":690737},{"sourceId":7513364,"sourceType":"datasetVersion","datasetId":4376202}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports ","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom sklearn.model_selection import StratifiedKFold\nfrom torchvision.models import resnet50, ResNet50_Weights\nimport wandb\nimport time\nfrom sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-14T09:54:39.291535Z","iopub.execute_input":"2024-02-14T09:54:39.292516Z","iopub.status.idle":"2024-02-14T09:54:46.953916Z","shell.execute_reply.started":"2024-02-14T09:54:39.292483Z","shell.execute_reply":"2024-02-14T09:54:46.952845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Params Configuration","metadata":{}},{"cell_type":"code","source":"\n# Define global parameters\nPARAMS = {\n    'width': 256,\n    'height': 256,\n    'B': 0.5,\n    'split': 0.8,\n    'epochs': 2,\n    'learning_rate': 0.0001,\n    'batch_size': 32,\n    'folds': 3,  # Number of folds for stratified k-fold\n    'optimizer': 'adam',  # 'adam', 'rmsprop', 'sgd', etc.\n    'loss_function': 'BCE', # 'cross_entropy','BCE'\n    'momentum': 0,  # Add momentum for SGD optimizer\n    'model_architecture': 'resnet50',  # Change to any other model\n    'IMG_PATHS': ['/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/',\n                  '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/'],\n}\nwandb.init(project=\"SIIM ISIC RESNET50\", save_code=True, config=PARAMS)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:54:46.956001Z","iopub.execute_input":"2024-02-14T09:54:46.956308Z","iopub.status.idle":"2024-02-14T09:58:37.166543Z","shell.execute_reply.started":"2024-02-14T09:54:46.956281Z","shell.execute_reply":"2024-02-14T09:58:37.165389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the Data from Dataset","metadata":{}},{"cell_type":"code","source":"# Load data\n#train_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntrain_dir = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\ntest_dir = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\ntrain_df = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/marking.csv')\ntest_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:37.167876Z","iopub.execute_input":"2024-02-14T09:58:37.168299Z","iopub.status.idle":"2024-02-14T09:58:37.905218Z","shell.execute_reply.started":"2024-02-14T09:58:37.168257Z","shell.execute_reply":"2024-02-14T09:58:37.904320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:37.906704Z","iopub.execute_input":"2024-02-14T09:58:37.907076Z","iopub.status.idle":"2024-02-14T09:58:38.462566Z","shell.execute_reply.started":"2024-02-14T09:58:37.907047Z","shell.execute_reply":"2024-02-14T09:58:38.461574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Data Frame with Image id and Target","metadata":{}},{"cell_type":"code","source":"labels=[]\ndata=[]\nfor i in range(train_df.shape[0]):\n    data.append(train_dir + train_df['image_id'].iloc[i]+'.jpg')\n    labels.append(train_df['target'].iloc[i])\ndf=pd.DataFrame(data)\ndf.columns=['images']\ndf['target']=labels\ndf","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:38.465740Z","iopub.execute_input":"2024-02-14T09:58:38.466110Z","iopub.status.idle":"2024-02-14T09:58:41.239777Z","shell.execute_reply.started":"2024-02-14T09:58:38.466078Z","shell.execute_reply":"2024-02-14T09:58:41.238968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:41.241085Z","iopub.execute_input":"2024-02-14T09:58:41.241418Z","iopub.status.idle":"2024-02-14T09:58:42.046485Z","shell.execute_reply.started":"2024-02-14T09:58:41.241387Z","shell.execute_reply":"2024-02-14T09:58:42.045467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test DataFrame ","metadata":{}},{"cell_type":"code","source":"test_data=[]\nfor i in range(test_df.shape[0]):\n    test_data.append(test_dir + test_df['image_name'].iloc[i]+'.jpg')\ndf_test=pd.DataFrame(test_data)\ndf_test.columns=['images']\ndf_test","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:42.047963Z","iopub.execute_input":"2024-02-14T09:58:42.048330Z","iopub.status.idle":"2024-02-14T09:58:42.790414Z","shell.execute_reply.started":"2024-02-14T09:58:42.048296Z","shell.execute_reply":"2024-02-14T09:58:42.789431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# Define preprocessing transforms\npreprocess_train = transforms.Compose([\n    transforms.Resize((PARAMS['height'], PARAMS['width'])),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    #transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), #Resnet50 Normalization\n])\n\npreprocess_val = transforms.Compose([\n    transforms.Resize((PARAMS['height'], PARAMS['width'])),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:42.791613Z","iopub.execute_input":"2024-02-14T09:58:42.791978Z","iopub.status.idle":"2024-02-14T09:58:43.237280Z","shell.execute_reply.started":"2024-02-14T09:58:42.791931Z","shell.execute_reply":"2024-02-14T09:58:43.236475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataSet Definition","metadata":{}},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, data_paths, labels, transform=None, mode='train'):\n        self.data = data_paths\n        self.labels = labels\n        self.transform = transform\n        self.mode = mode\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = self.data[idx]\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = Image.fromarray(img)        \n\n        if self.transform is not None:\n            img = self.transform(img)\n\n        if self.mode == 'test':\n            return img\n        else:\n            labels = torch.tensor(self.labels[idx])\n            \n            return img, labels","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:43.238384Z","iopub.execute_input":"2024-02-14T09:58:43.238719Z","iopub.status.idle":"2024-02-14T09:58:43.759390Z","shell.execute_reply.started":"2024-02-14T09:58:43.238677Z","shell.execute_reply":"2024-02-14T09:58:43.758168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Definition","metadata":{}},{"cell_type":"code","source":"class ResNet50(nn.Module):\n    def __init__(self):\n        super(ResNet50, self).__init__()\n        self.model = models.resnet50(pretrained=True)  # Load pre-trained ResNet50\n        #for param in self.model.parameters():\n            #param.requires_grad = False  # Freeze the parameters of the ResNet50 model\n        num_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(num_features, 1)  # Change the output to 1 for binary classification\n\n    def forward(self, x):\n        return self.model(x)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:43.760944Z","iopub.execute_input":"2024-02-14T09:58:43.761386Z","iopub.status.idle":"2024-02-14T09:58:44.312848Z","shell.execute_reply.started":"2024-02-14T09:58:43.761348Z","shell.execute_reply":"2024-02-14T09:58:44.311966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimizer and Criterion Definition","metadata":{}},{"cell_type":"code","source":"def get_optimizer_and_criterion(PARAMS, model):\n    # Update the loss function based on the PARAMS\n    if PARAMS['loss_function'] == 'cross_entropy':\n        criterion = nn.CrossEntropyLoss()\n    elif PARAMS['loss_function'] == 'BCE':\n        criterion = nn.BCEWithLogitsLoss()\n    # Add more conditions for other loss functions as needed\n    else:\n        raise ValueError(f\"Unsupported loss function: {PARAMS['loss_function']}\")\n\n    # Update the optimizer based on the PARAMS\n    if PARAMS['optimizer'] == 'adam':\n        optimizer = optim.Adam(model.parameters(), lr=PARAMS['learning_rate'])\n    elif PARAMS['optimizer'] == 'rmsprop':\n        optimizer = optim.RMSprop(model.parameters(), lr=PARAMS['learning_rate'])\n    elif PARAMS['optimizer'] == 'sgd':\n        optimizer = optim.SGD(model.parameters(), lr=PARAMS['learning_rate'], momentum=PARAMS['momentum'])\n    else:\n        raise ValueError(f\"Unsupported optimizer: {PARAMS['optimizer']}\")\n\n    return optimizer, criterion\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:44.314305Z","iopub.execute_input":"2024-02-14T09:58:44.314999Z","iopub.status.idle":"2024-02-14T09:58:44.805060Z","shell.execute_reply.started":"2024-02-14T09:58:44.314941Z","shell.execute_reply":"2024-02-14T09:58:44.804120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Loop Definition","metadata":{}},{"cell_type":"code","source":"def train_model(train_loader, val_loader, model, optimizer, criterion, device, num_epochs):\n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        correct_train = 0\n        total_train = 0\n        start_time = time.time() \n        print(f'Epoch [{epoch + 1}/{num_epochs}] Training...')\n        \n        for inputs, labels in train_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels.float().unsqueeze(1))\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n\n            # Calculate accuracy\n            predicted = (torch.sigmoid(outputs) >= 0.5).flatten().cpu().numpy()\n            total_train += labels.size(0)\n            correct_train += (predicted == labels.cpu().numpy()).sum().item()\n\n        # Print training loss and accuracy at the end of each epoch\n        end_time = time.time()  # Record the end time for the epoch\n        epoch_time = end_time - start_time  # Calculate the time taken for the epoch\n        average_loss = running_loss / len(train_loader)\n        train_accuracy = correct_train / total_train  \n        # Log training accuracy and loss to WandB\n        wandb.log({\"train_loss\": average_loss, \"train_accuracy\": train_accuracy})\n        print(f'Training Loss: {average_loss}, Training Accuracy: {train_accuracy}, Time: {epoch_time} seconds')\n\n        \n        # Evaluate the model on the validation set\n        model.eval()\n        correct_val = 0\n        total_val = 0\n        val_running_loss = 0.0    \n        start_val_time = time.time()\n        y_true = []\n        y_pred = []\n        with torch.no_grad():\n            for inputs, labels in val_loader:\n                inputs, labels = inputs.to(device), labels.to(device)\n                outputs = model(inputs)\n                val_loss = criterion(outputs, labels.float().unsqueeze(1))\n                val_running_loss += val_loss.item()\n\n                # Calculate validation accuracy\n                predicted = (torch.sigmoid(outputs) >= 0.5).flatten().cpu().numpy()\n                y_true.extend(labels.cpu().numpy())\n                y_pred.extend(predicted)\n                total_val += labels.size(0)\n                correct_val += (predicted == labels.cpu().numpy()).sum().item()\n\n        # Print validation loss and accuracy\n        validation_loss = val_running_loss / len(val_loader)\n        validation_accuracy = correct_val / total_val\n        conf_matrix = confusion_matrix(y_true, y_pred)\n        precision = precision_score(y_true, y_pred)\n        recall = recall_score(y_true, y_pred)\n        f1 = f1_score(y_true, y_pred)\n        end_val_time = time.time()  # Record the end time for validation\n        val_epoch_time = end_val_time - start_val_time\n        # Log Validation accuracy and loss to WandB\n        wandb.log({\"validation_loss\": validation_loss, \"validation_accuracy\": validation_accuracy})\n        print(f'Validation Loss: {validation_loss}, Validation Accuracy: {validation_accuracy}, Validation Time: {val_epoch_time} seconds')\n        # Assuming y_true contains the true labels and y_pred contains the predicted labels\n        print(f'Confusion Matrix: {conf_matrix},Precision: {precision},Recall: {recall},F1 Score: {f1}')\n        wandb.log({\"confusion_matrix\": wandb.plot.confusion_matrix(probs=None, y_true=y_true, preds=y_pred, class_names=[\"class_0\", \"class_1\"]),\n                    \"precision\": precision,\"recall\": recall,\"f1_score\": f1})","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:44.808867Z","iopub.execute_input":"2024-02-14T09:58:44.809201Z","iopub.status.idle":"2024-02-14T09:58:45.278822Z","shell.execute_reply.started":"2024-02-14T09:58:44.809166Z","shell.execute_reply":"2024-02-14T09:58:45.277864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Loop Definition","metadata":{}},{"cell_type":"code","source":"def test_model(test_loader, model, device):\n    model.eval()\n    predictions = []\n    image_names = []\n\n    image_names = test_df['image_name'].tolist()\n    test_start_time = time.time()\n    with torch.no_grad():\n        for i, data in enumerate(test_loader):\n            data = data.to(device)\n            outputs = model(data)\n\n            # Generate predictions for submission using sigmoid\n            probabilities = (torch.sigmoid(outputs) >= 0.5).cpu().numpy()\n            for prob in probabilities:\n                predictions.append(int(prob.item()))\n                \n    test_end_time = time.time()\n    test_time = test_end_time - test_start_time\n    # Save predictions to CSV file\n    submission_df = pd.DataFrame({'image_name': image_names, 'target': predictions})\n    #submission_df = pd.DataFrame({'id': range(1, len(predictions) + 1), 'target': predictions})\n    submission_df.to_csv('submission.csv', index=False)\n    print(f'Test Evaluation and Submission CSV is generated in : {test_time} seconds')","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:45.280286Z","iopub.execute_input":"2024-02-14T09:58:45.280883Z","iopub.status.idle":"2024-02-14T09:58:45.767321Z","shell.execute_reply.started":"2024-02-14T09:58:45.280845Z","shell.execute_reply":"2024-02-14T09:58:45.766331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initialization of Model,Optimizer and device","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n    \n# Initialize model and optimizer\nmodel = ResNet50().to(device)\noptimizer,criterion = get_optimizer_and_criterion(PARAMS,model)  \nprint(model)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:45.770618Z","iopub.execute_input":"2024-02-14T09:58:45.770904Z","iopub.status.idle":"2024-02-14T09:58:48.014929Z","shell.execute_reply.started":"2024-02-14T09:58:45.770878Z","shell.execute_reply":"2024-02-14T09:58:48.014004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Implementation of stratified k-fold cross-validation","metadata":{}},{"cell_type":"code","source":"# Implement stratified k-fold cross-validation\nskf = StratifiedKFold(n_splits=PARAMS['folds'], shuffle=True, random_state=42)\nfor fold, (train_index, val_index) in enumerate(skf.split(train_df['image_id'], train_df['target'])):\n    print(f'Fold {fold + 1}')    \n    # --- Read in Data ---\n    train_data = df.iloc[train_index].reset_index(drop=True)\n    valid_data = df.iloc[val_index].reset_index(drop=True)\n    \n    # Create datasets and data loaders\n    train_dataset=ImageDataset(data_paths=train_data['images'].values,labels=train_data['target'].values,transform=preprocess_train)\n    val_dataset=ImageDataset(data_paths=valid_data['images'].values,labels=train_data['target'].values,transform=preprocess_val)\n        \n    train_loader = DataLoader(train_dataset, batch_size=PARAMS['batch_size'], shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=PARAMS['batch_size'], shuffle=False)       \n    \n    train_model(train_loader, val_loader, model, optimizer, criterion, device, PARAMS['epochs'])      \n    \nprint('Training completed.')","metadata":{"execution":{"iopub.status.busy":"2024-02-14T09:58:48.016177Z","iopub.execute_input":"2024-02-14T09:58:48.016527Z","iopub.status.idle":"2024-02-14T12:03:05.986381Z","shell.execute_reply.started":"2024-02-14T09:58:48.016495Z","shell.execute_reply":"2024-02-14T12:03:05.984450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Definition Call","metadata":{}},{"cell_type":"code","source":"test_dataset=ImageDataset(data_paths=df_test['images'].values,labels=None,transform=preprocess_val,mode ='test')\n    \ntest_loader = DataLoader(test_dataset, batch_size=PARAMS['batch_size'], shuffle=False)\n\n# Call the test_model function\ntest_model(test_loader,model,device)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T12:03:05.987632Z","iopub.execute_input":"2024-02-14T12:03:05.988021Z"},"trusted":true},"execution_count":null,"outputs":[]}]}