{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1225697,"sourceType":"datasetVersion","datasetId":701123},{"sourceId":1331863,"sourceType":"datasetVersion","datasetId":712479}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torchvision.models import resnet50\n\ndef set_seed(seed=42):\n   \"\"\"Sets a seed for reproducibility across libraries.\n\n   Args:\n       seed (int, optional): The seed value to use. Defaults to 42.\n   \"\"\"\n   np.random.seed(seed)\n   random.seed(seed)\n   torch.manual_seed(seed)\n   torch.cuda.manual_seed_all(seed)\n   torch.backends.cudnn.deterministic = True\n   torch.backends.cudnn.benchmark = False\n   os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Device available: {device}')\n\n# Constants\nOUTPUT_SIZE = 1\nTRAIN_PATH = '../input/melanoma-external-malignant-256/train/train/'\nTEST_PATH = '../input/melanoma-external-malignant-256/test/test/'\n\n# Data preparation\nmy_train = pd.read_csv('../input/siim-melanoma-prep-data/train_clean.csv')\nmy_train.drop(['path_dicom', 'path_jpeg', 'diagnosis'], axis=1, inplace=True)\n\nroman_train = pd.read_csv('../input/../input/melanoma-external-malignant-256/train_concat.csv')\nroman_train['patient_id'] = roman_train['patient_id'].fillna(0)\n\ncategorical_cols = ['sex', 'anatom_site_general_challenge']\nroman_train[categorical_cols] = roman_train[categorical_cols].astype(str)\nlabel_encoder = LabelEncoder()\nroman_train[categorical_cols] = roman_train[categorical_cols].apply(lambda col: label_encoder.fit_transform(col))\n\nroman_train.columns = my_train.columns\ncommon_images = my_train['dcm_name'].unique()\nnew_data = roman_train[~roman_train['dcm_name'].isin(common_images)]\ntrain_df = pd.concat([my_train, new_data], axis=0, ignore_index=True)\n\ntest_df = pd.read_csv('../input/siim-melanoma-prep-data/test_clean.csv')\nexisting_cols = ['path_dicom', 'path_jpeg', 'diagnosis']\nto_drop = [col for col in existing_cols if col in test_df.columns]\ntest_df.drop(to_drop, axis=1, inplace=True)\n\ntrain_df['path_jpg'] = train_df['dcm_name'].apply(lambda name: os.path.join(TRAIN_PATH, f\"{name}.jpg\"))\ntest_df['path_jpg'] = test_df['dcm_name'].apply(lambda name: os.path.join(TEST_PATH, f\"{name}.jpg\"))\n\ntrain_df['age'] = train_df['age'].fillna(-1)\nscaler = StandardScaler()\ntrain_df[['sex', 'age', 'anatomy']] = scaler.fit_transform(train_df[['sex', 'age', 'anatomy']])\ntest_df[['sex', 'age', 'anatomy']] = scaler.transform(test_df[['sex', 'age', 'anatomy']])\n\nprint(f\"Length of Train: {len(train_df)}\\nLength of Test: {len(test_df)}\")\n\nclass MelanomaDataset(Dataset):\n   def __init__(self, df, transform=None, is_train=True, is_valid=False, is_test=False):\n       self.df = df\n       self.transform = transform\n       self.is_train = is_train\n       self.is_valid = is_valid\n       self.is_test = is_test\n\n       if is_train or is_test:\n           self.transform = transforms.Compose([\n               transforms.RandomResizedCrop(224),\n               transforms.RandomHorizontalFlip(),\n               transforms.RandomVerticalFlip(),\n               transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1),\n               transforms.RandomRotation(degrees=90),\n               transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n               transforms.ToTensor()\n           ])\n       else:\n           self.transform = transforms.Compose([\n               transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n               transforms.ToTensor()\n           ])\n\n   def __len__(self):\n       return len(self.df)\n\n   def __getitem__(self, idx):\n       image_path = self.df.iloc[idx]['path_jpg']\n       image = cv2.imread(image_path)\n       image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n       if self.transform:\n           image = self.transform(image)\n\n       tabular_data = self.df.iloc[idx][['sex', 'age', 'anatomy']].values.astype(np.float32)\n\n       if self.is_train or self.is_valid:\n           target = self.df.iloc[idx]['target']\n           return (image, tabular_data), target\n       else:\n           return (image, tabular_data)\n\n# Create datasets and data loaders\ntrain_transform = transforms.Compose([\n   transforms.RandomResizedCrop(224),\n   transforms.RandomHorizontalFlip(),\n   transforms.RandomVerticalFlip(),\n   transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1),\n   transforms.RandomRotation(degrees=90),\n   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n   transforms.ToTensor()\n])\n\nvalid_transform = transforms.Compose([\n   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n   transforms.ToTensor()\n])\n\ntest_transform = transforms.Compose([\n   transforms.RandomResizedCrop(224),\n   transforms.RandomHorizontalFlip(),\n   transforms.RandomVerticalFlip(),\n   transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.1),\n   transforms.RandomRotation(degrees=90),\n   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n   transforms.ToTensor()\n])\n\ntrain_dataset = MelanomaDataset(train_df, transform=train_transform, is_train=True)\nvalid_dataset = MelanomaDataset(train_df, transform=valid_transform, is_valid=True)\ntest_dataset = MelanomaDataset(test_df, transform=test_transform, is_test=True)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4)\nvalid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2024-05-16T16:29:09.087126Z","iopub.execute_input":"2024-05-16T16:29:09.087564Z","iopub.status.idle":"2024-05-16T16:29:09.486895Z","shell.execute_reply.started":"2024-05-16T16:29:09.087534Z","shell.execute_reply":"2024-05-16T16:29:09.485931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNet50Network(nn.Module):\n    def __init__(self, output_size, no_columns):\n        super().__init__()\n        self.no_columns, self.output_size = no_columns, output_size\n\n        # Define Feature part (IMAGE)\n        self.features = resnet50(pretrained=True)  # 1000 neurons out\n        num_ftrs = self.features.fc.in_features\n        self.features.fc = nn.Identity()  # Remove the last fully-connected layer\n\n        # (CSV data)\n        self.csv_branch = nn.Sequential(\n            nn.Linear(self.no_columns, 500),\n            nn.BatchNorm1d(500),\n            nn.ReLU(),\n            nn.Dropout(p=0.2)\n        )\n\n        # Define Classification part\n        self.classification = nn.Sequential(\n            nn.Linear(num_ftrs + 500, 512),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(512, output_size)\n        )\n\n    def forward(self, image, csv_data, prints=False):\n        if prints:\n            print('Input Image shape:', image.shape, '\\n' +\n                  'Input csv_data shape:', csv_data.shape)\n\n        # Image CNN\n        image_features = self.features(image)\n        if prints:\n            print('Features Image shape:', image_features.shape)\n\n        # CSV FNN\n        csv_features = self.csv_branch(csv_data)\n        if prints:\n            print('CSV Data:', csv_features.shape)\n\n        # Concatenate layers from image with layers from csv_data\n        concat_features = torch.cat((image_features, csv_features), dim=1)\n\n        # CLASSIF\n        out = self.classification(concat_features)\n        if prints:\n            print('Out shape:', out.shape)\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-05-16T16:29:13.992673Z","iopub.execute_input":"2024-05-16T16:29:13.993056Z","iopub.status.idle":"2024-05-16T16:29:14.003469Z","shell.execute_reply.started":"2024-05-16T16:29:13.993025Z","shell.execute_reply":"2024-05-16T16:29:14.002512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n!pip install efficientnet-pytorch\nfrom efficientnet_pytorch import EfficientNet\n\nclass EfficientNetwork(nn.Module):\n    def __init__(self, output_size, no_columns, b4=False, b2=False):\n        super().__init__()\n        self.b4, self.b2, self.no_columns = b4, b2, no_columns\n        \n        # Define Feature part (IMAGE)\n        if b4:\n            self.features = EfficientNet.from_pretrained('efficientnet-b4')\n            feature_dim = 1792\n        elif b2:\n            self.features = EfficientNet.from_pretrained('efficientnet-b2')\n            feature_dim = 1408\n        else:\n            self.features = EfficientNet.from_pretrained('efficientnet-b7')\n            feature_dim = 2560\n        \n        # (CSV)\n        self.csv_branch = nn.Sequential(\n            nn.Linear(self.no_columns, 250),\n            nn.BatchNorm1d(250),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(250, 250),\n            nn.BatchNorm1d(250),\n            nn.ReLU(),\n            nn.Dropout(p=0.2)\n        )\n        \n        # Define Classification part\n        self.classification = nn.Sequential(\n            nn.Linear(feature_dim + 250, 512),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(512, output_size)\n        )\n        \n    def forward(self, image, csv_data, prints=False):    \n        if prints:\n            print('Input Image shape:', image.shape, '\\n' +\n                  'Input csv_data shape:', csv_data.shape)\n        \n        # IMAGE CNN\n        image_features = self.features.extract_features(image)\n        if prints:\n            print('Features Image shape:', image_features.shape)\n        \n        if self.b4:\n            image_features = F.adaptive_avg_pool2d(image_features, 1).reshape(-1, 1792)\n        elif self.b2:\n            image_features = F.adaptive_avg_pool2d(image_features, 1).reshape(-1, 1408)\n        else:\n            image_features = F.adaptive_avg_pool2d(image_features, 1).reshape(-1, 2560)\n        if prints:\n            print('Image Reshaped shape:', image_features.shape)\n        \n        # CSV FNN\n        csv_features = self.csv_branch(csv_data)\n        if prints:\n            print('CSV Data:', csv_features.shape)\n        \n        # Concatenate\n        concat_features = torch.cat((image_features, csv_features), dim=1)\n        \n        # CLASSIF\n        out = self.classification(concat_features)\n        if prints:\n            print('Out shape:', out.shape)\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2024-05-16T16:31:25.126615Z","iopub.execute_input":"2024-05-16T16:31:25.127004Z","iopub.status.idle":"2024-05-16T16:31:40.830573Z","shell.execute_reply.started":"2024-05-16T16:31:25.126975Z","shell.execute_reply":"2024-05-16T16:31:40.829496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport time\nimport datetime\nimport gc\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import GroupKFold\n\ndef train_folds(preds_submission, model, train_df, test_df, device, version='v1'):\n    train_len = len(train_df)\n    test_len = len(test_df)\n    oof = np.zeros(shape=(train_len, 1))\n\n    # ----- STATICS -----\n    k = 6  # number of folds in Group K Fold\n    epochs = 15\n    patience = 3\n    TTA = 3\n    num_workers = 8\n    learning_rate = 0.0005\n    weight_decay = 0.0\n    lr_patience = 1  # 1 model not improving until lr is decreasing\n    lr_factor = 0.4  # by how much the lr is decreasing\n    batch_size1 = 32\n    batch_size2 = 16\n    vertical_flip = 0.5\n    horizontal_flip = 0.5\n    # -------------------\n\n    # Create Object\n    group_fold = GroupKFold(n_splits=k)\n\n    # Generate indices to split data into training and test set.\n    folds = group_fold.split(X=np.zeros(train_len),\n                             y=train_df['target'],\n                             groups=train_df['ID'].tolist())\n\n    # Creates a .txt file that will contain the logs\n    log_file = f\"logs_{version}.txt\"\n    with open(log_file, \"w+\") as f:\n        f.write(\"\")  # Clear the file\n\n    for fold, (train_index, valid_index) in enumerate(folds):\n        # Append to .txt\n        with open(log_file, 'a+') as f:\n            print('-'*10, 'Fold:', fold+1, '-'*10, file=f)\n        print('-'*10, 'Fold:', fold+1, '-'*10)\n\n        # --- Create Instances ---\n        best_roc = None\n        patience_f = patience\n\n        model = model.to(device)\n        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=lr_patience, verbose=True, factor=lr_factor)\n        criterion = nn.BCEWithLogitsLoss()\n\n        # --- Read in Data ---\n        train_data = train_df.iloc[train_index].reset_index(drop=True)\n        valid_data = train_df.iloc[valid_index].reset_index(drop=True)\n\n        # Create Data instances\n        train_dataset = MelanomaDataset(train_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, is_train=True)\n        valid_dataset = MelanomaDataset(valid_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, is_valid=True)\n        test_dataset = MelanomaDataset(test_df, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, is_test=True)\n\n        # Dataloaders\n        train_loader = DataLoader(train_dataset, batch_size=batch_size1, shuffle=True, num_workers=num_workers)\n        valid_loader = DataLoader(valid_dataset, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n        test_loader = DataLoader(test_dataset, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n\n        # === EPOCHS ===\n        for epoch in range(epochs):\n            start_time = time.time()\n            correct = 0\n            train_losses = 0\n\n            # === TRAIN ===\n            model.train()\n\n            for (images, csv_data), labels in train_loader:\n                images = images.to(device, dtype=torch.float32)\n                csv_data = csv_data.to(device, dtype=torch.float32)\n                labels = labels.to(device, dtype=torch.float32)\n\n                # Clear gradients\n                optimizer.zero_grad()\n\n                # Log Probabilities & Backpropagation\n                out = model(images, csv_data)\n                loss = criterion(out, labels.unsqueeze(1))\n                loss.backward()\n                optimizer.step()\n\n                # --- Save information after this batch ---\n                train_losses += loss.item()\n                train_preds = torch.round(torch.sigmoid(out))  # 0 and 1\n                correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n\n            # Compute Train Accuracy\n            train_acc = correct / len(train_index)\n\n            # === EVAL ===\n            model.eval()\n\n            valid_preds = torch.zeros(size=(len(valid_index), 1), device=device, dtype=torch.float32)\n\n            with torch.no_grad():\n                for k, ((images, csv_data), labels) in enumerate(valid_loader):\n                    images = images.to(device, dtype=torch.float32)\n                    csv_data = csv_data.to(device, dtype=torch.float32)\n                    labels = labels.to(device, dtype=torch.float32)\n\n                    out = model(images, csv_data)\n                    pred = torch.sigmoid(out)\n                    valid_preds[k*images.shape[0]: k*images.shape[0] + images.shape[0]] = pred\n\n                # Compute accuracy\n                valid_acc = accuracy_score(valid_data['target'].values, torch.round(valid_preds.cpu()))\n                # Compute ROC\n                valid_roc = roc_auc_score(valid_data['target'].values, valid_preds.cpu())\n\n                # Compute time on Train + Eval\n                duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n                # PRINT INFO\n                log_msg = f\"{duration} | Epoch: {epoch+1}/{epochs} | Loss: {train_losses:.4} | Train Acc: {train_acc:.3} | Valid Acc: {valid_acc:.3} | ROC: {valid_roc:.3}\"\n                with open(log_file, 'a+') as f:\n                    print(log_msg, file=f)\n                print(log_msg)\n\n                # Update scheduler (for learning_rate)\n                scheduler.step(valid_roc)\n\n                # Update best_roc\n                if best_roc is None or valid_roc > best_roc:\n                    best_roc = valid_roc\n                    patience_f = patience\n                    torch.save(model.state_dict(), f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                else:\n                    patience_f -= 1\n                    if patience_f == 0:\n                        with open(log_file, 'a+') as f:\n                            print(f'Early stopping (no improvement since {patience} models) | Best ROC: {best_roc}', file=f)\n                        print(f'Early stopping (no improvement since {patience} models) | Best ROC: {best_roc}')\n                        break\n\n        # === INFERENCE ===\n        best_model_path = '../working/' + [file for file in os.listdir('../working') if str(round(best_roc, 3)) in file and 'Fold' + str(fold + 1) in file][0]\n        model = EfficientNetwork(output_size=output_size, no_columns=no_columns, b4=False, b2=True).to(device)\n        model.load_state_dict(torch.load(best_model_path))\n        model.eval()\n\n        with torch.no_grad():\n            # --- EVAL ---\n            valid_preds = torch.zeros(size=(len(valid_index), 1), device=device, dtype=torch.float32)\n\n            for k, ((images, csv_data), _) in enumerate(valid_loader):\n                images = images.to(device, dtype=torch.float32)\n                csv_data = csv_data.to(device, dtype=torch.float32)\n\n                out = model(images, csv_data)\n                pred = torch.sigmoid(out)\n                valid_preds[k*images.shape[0]: k*images.shape[0] + images.shape[0]] = pred\n\n            # Save info to OOF\n            oof[valid_index] = valid_preds.cpu().numpy()\n\n            # --- TEST ---\n            # Now (Finally) prediction for our TEST data\n            preds_submission = torch.zeros(size=(test_len, 1), device=device, dtype=torch.float32)\n            for i in range(TTA):\n                for k, (images, csv_data) in enumerate(test_loader):\n                    images = images.to(device, dtype=torch.float32)\n                    csv_data = csv_data.to(device, dtype=torch.float32)\n\n                    out = model(images, csv_data)\n                    out = torch.sigmoid(out)\n\n                    preds_submission[k*images.shape[0]: k*images.shape[0] + images.shape[0]] += out\n\n            # Divide Predictions by TTA (to average the results during TTA)\n            preds_submission /= TTA\n\n        # === CLEANING ===\n        del train_dataset, valid_dataset, train_loader, valid_loader, images, labels\n        gc.collect()\n\n    return oof, preds_submission.cpu().numpy()","metadata":{"execution":{"iopub.status.busy":"2024-05-16T16:35:19.717810Z","iopub.execute_input":"2024-05-16T16:35:19.718201Z","iopub.status.idle":"2024-05-16T16:35:19.753866Z","shell.execute_reply.started":"2024-05-16T16:35:19.718172Z","shell.execute_reply":"2024-05-16T16:35:19.752890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ----- STATICS -----\noutput_size = 1\nno_columns = 3\nversion = 'v7'\n# -------------------\n\n# --- EffNet B2 ---\nmodel = EfficientNetwork(output_size=output_size, no_columns=no_columns, b4=False, b2=True).to(device)\n\n# # ===== Uncomment and Train =====\n# oof, preds_submission = train_folds(preds_submission=torch.zeros(size=(test_len, 1), device=device, dtype=torch.float32), model=model, train_df=train_df, test_df=test_df, device=device, version=version)\n\n# # Save OOF values\n# save_oof = pd.DataFrame(data=oof, columns=['oof'])\n# save_oof.to_csv(f'oof_{version}.csv', index=False)\n\n# Print the logs during training\nlog_file = '../input/siim-melanoma-prep-data/logs_v7.txt'\nwith open(log_file, \"r\") as f:\n    contents = f.read()\n    print(contents)","metadata":{"execution":{"iopub.status.busy":"2024-05-16T16:37:25.694528Z","iopub.execute_input":"2024-05-16T16:37:25.694933Z","iopub.status.idle":"2024-05-16T16:37:26.615547Z","shell.execute_reply.started":"2024-05-16T16:37:25.694903Z","shell.execute_reply":"2024-05-16T16:37:26.614489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import OOF (pretrained)\noof = pd.read_csv('../input/siim-melanoma-prep-data/oof_v7.csv')\n\n# ROC on full Training data\nprint('OOF ROC: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","metadata":{"execution":{"iopub.status.busy":"2024-05-16T16:43:52.727441Z","iopub.execute_input":"2024-05-16T16:43:52.727879Z","iopub.status.idle":"2024-05-16T16:43:52.780672Z","shell.execute_reply.started":"2024-05-16T16:43:52.727846Z","shell.execute_reply":"2024-05-16T16:43:52.779769Z"},"trusted":true},"execution_count":null,"outputs":[]}]}