{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# SIIM-COVID19: Classification Only PyTorch Starter\n\nThis notebook is just a classification only starter. \n\nI will be making an Object Detection notebook soon!\n\n### Please leave an upvote if you are forking it or you found it helpful : )","metadata":{}},{"cell_type":"markdown","source":"Credits to [this](https://www.kaggle.com/yujiariyasu/catch-up-on-positive-samples-plot-submission-csv) for some of the functions I am using here.","metadata":{}},{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y;","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-22T13:29:40.646934Z","iopub.execute_input":"2021-05-22T13:29:40.647337Z","iopub.status.idle":"2021-05-22T13:30:40.57379Z","shell.execute_reply.started":"2021-05-22T13:29:40.647231Z","shell.execute_reply":"2021-05-22T13:30:40.572896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/timmeffnetv2\")\n\nimport platform\nimport numpy as np\nimport pandas as pd\nimport os\nfrom tqdm.notebook import tqdm\nimport cv2\nimport pydicom\nimport gdcm\nimport glob\nimport gc\nfrom math import ceil\nimport matplotlib.pyplot as plt\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport timm\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-22T13:30:40.577231Z","iopub.execute_input":"2021-05-22T13:30:40.577516Z","iopub.status.idle":"2021-05-22T13:30:44.367824Z","shell.execute_reply.started":"2021-05-22T13:30:40.577483Z","shell.execute_reply":"2021-05-22T13:30:44.366907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_study = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:30:44.370781Z","iopub.execute_input":"2021-05-22T13:30:44.371128Z","iopub.status.idle":"2021-05-22T13:30:44.423463Z","shell.execute_reply.started":"2021-05-22T13:30:44.371085Z","shell.execute_reply":"2021-05-22T13:30:44.422747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"../input/siim-covid19-detection/train/\"\ntrain_study['StudyInstanceUID'] = train_study['id'].apply(lambda x: x.replace('_study', ''))\ntrain = train_image.merge(train_study, on='StudyInstanceUID')\n\n# Make a path folder\npaths = []\nfor instance_id in tqdm(train['StudyInstanceUID']):\n    paths.append(glob.glob(os.path.join(TRAIN_DIR, instance_id +\"/*/*\"))[0])\n\ntrain['path'] = paths\n\ntrain = train.drop(['id_x', 'id_y'], axis=1)\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:30:44.424881Z","iopub.execute_input":"2021-05-22T13:30:44.42514Z","iopub.status.idle":"2021-05-22T13:31:04.826137Z","shell.execute_reply.started":"2021-05-22T13:30:44.425116Z","shell.execute_reply":"2021-05-22T13:31:04.825187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    train_pcent = 0.85\n    model_name = 'tf_efficientnet_b4'\n    image_size = (400, 400)\n    TRAIN_BS = 32\n    VALID_BS = 16\n    num_workers = 8\n    NB_EPOCHS = 3\n    scaler = GradScaler()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:31:04.828747Z","iopub.execute_input":"2021-05-22T13:31:04.828999Z","iopub.status.idle":"2021-05-22T13:31:04.854835Z","shell.execute_reply.started":"2021-05-22T13:31:04.828974Z","shell.execute_reply":"2021-05-22T13:31:04.854083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:31:04.857012Z","iopub.execute_input":"2021-05-22T13:31:04.857345Z","iopub.status.idle":"2021-05-22T13:31:04.976122Z","shell.execute_reply.started":"2021-05-22T13:31:04.857311Z","shell.execute_reply":"2021-05-22T13:31:04.975294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EfficientNetModel(nn.Module):\n    \"\"\"\n    Model Class for EfficientNet Model\n    \"\"\"\n    def __init__(self, num_classes=4, model_name=Config.model_name, pretrained=True):\n        super(EfficientNetModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained, in_chans=3)\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass NFNetModel(nn.Module):\n    \"\"\"\n    Model Class for EfficientNet Model\n    \"\"\"\n    def __init__(self, num_classes=4, model_name=Config.model_name, pretrained=True):\n        super(NFNetModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained, in_chans=3)\n        self.model.head.fc = nn.Linear(self.model.head.fc.in_features, num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:31:04.977546Z","iopub.execute_input":"2021-05-22T13:31:04.977948Z","iopub.status.idle":"2021-05-22T13:31:04.988715Z","shell.execute_reply.started":"2021-05-22T13:31:04.97791Z","shell.execute_reply":"2021-05-22T13:31:04.987821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, train_dataloader, valid_dataloader, model, optimizer, loss_fn, val_loss_fn, agc=False, device=\"cuda:0\"):\n        \"\"\"\n        Constructor for Trainer class\n        \"\"\"\n        self.train = train_dataloader\n        self.valid = valid_dataloader\n        self.optim = optim\n        self.loss_fn = loss_fn\n        self.val_loss_fn = val_loss_fn\n        self.device = device\n        self.agc = agc\n    \n    def train_one_cycle(self):\n        \"\"\"\n        Runs one epoch of training, backpropagation and optimization\n        \"\"\"\n        model.train()\n        train_prog_bar = tqdm(self.train, total=len(self.train))\n\n        all_train_labels = []\n        all_train_preds = []\n        \n        running_loss = 0\n        \n        for xtrain, ytrain in train_prog_bar:\n            xtrain = xtrain.to(self.device).float()\n            ytrain = ytrain.to(self.device).float()\n            xtrain = xtrain.permute(0, 3, 1, 2)\n            \n            with autocast():\n                # Get predictions\n                z = model(xtrain)\n\n                # Training\n                train_loss = self.loss_fn(z, ytrain)\n                scaler.scale(train_loss).backward()\n                \n                if self.agc:\n                    adaptive_clip_grad(model.parameters(), clip_factor=0.01, eps=1e-3, norm_type=2.0)\n                \n                scaler.step(self.optim)\n                scaler.update()\n                self.optim.zero_grad(set_to_none=True)\n\n                # For averaging and reporting later\n                running_loss += train_loss\n\n                # Convert the predictions and corresponding labels to right form\n                train_predictions = torch.argmax(z, 1).detach().cpu().numpy()\n                train_labels = ytrain.detach().cpu().numpy()\n\n                # Append current predictions and current labels to a list\n                all_train_labels += [train_predictions]\n                all_train_preds += [train_labels]\n\n            # Show the current loss to the progress bar\n            train_pbar_desc = f'loss: {train_loss.item():.4f}'\n            train_prog_bar.set_description(desc=train_pbar_desc)\n        \n        # Now average the running loss over all batches and return\n        train_running_loss = running_loss / len(self.train)\n        print(f\"Final Training Loss: {train_running_loss:.4f}\")\n        \n        # Free up memory\n        del all_train_labels, all_train_preds, train_predictions, train_labels, xtrain, ytrain, z\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n        return train_running_loss\n\n    def valid_one_cycle(self):\n        \"\"\"\n        Runs one epoch of prediction\n        \"\"\"        \n        model.eval()\n        \n        valid_prog_bar = tqdm(self.valid, total=len(self.valid))\n        \n        with torch.no_grad():\n            all_valid_labels = []\n            all_valid_preds = []\n            \n            running_loss = 0\n            \n            for xval, yval in valid_prog_bar:\n                xval = xval.to(self.device).float()\n                yval = yval.to(self.device).float()\n                xval = xval.permute(0, 3, 1, 2)\n                \n                val_z = model(xval)\n                \n                val_loss = self.val_loss_fn(val_z, yval)\n                \n                running_loss += val_loss.item()\n                \n                val_pred = torch.argmax(val_z, 1).detach().cpu().numpy()\n                val_label = yval.detach().cpu().numpy()\n                \n                all_valid_labels += [val_label]\n                all_valid_preds += [val_pred]\n            \n                # Show the current loss\n                valid_pbar_desc = f\"loss: {val_loss.item():.4f}\"\n                valid_prog_bar.set_description(desc=valid_pbar_desc)\n            \n            # Get the final loss\n            final_loss_val = running_loss / len(self.valid)\n            \n            # Get Validation Accuracy\n            all_valid_labels = np.concatenate(all_valid_labels)\n            all_valid_preds = np.concatenate(all_valid_preds)\n            \n            print(f\"Final Validation Loss: {final_loss_val:.4f}\")\n            \n            # Free up memory\n            del all_valid_labels, all_valid_preds, val_label, val_pred, xval, yval, val_z\n            gc.collect()\n            torch.cuda.empty_cache()\n            \n        return (final_loss_val, model)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:31:04.990946Z","iopub.execute_input":"2021-05-22T13:31:04.991394Z","iopub.status.idle":"2021-05-22T13:31:05.00987Z","shell.execute_reply.started":"2021-05-22T13:31:04.991358Z","shell.execute_reply":"2021-05-22T13:31:05.008939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SIIMData(Dataset):\n    def __init__(self, df, is_train=True, augments=None, img_size=Config.image_size):\n        super().__init__()\n        self.df = df.sample(frac=1).reset_index(drop=True)\n        self.is_train = is_train\n        self.augments = augments\n        self.img_size = img_size\n        \n    def __getitem__(self, idx):\n        image_id = self.df['StudyInstanceUID'].values[idx]\n        \n        image_path = self.df['path'].values[idx]\n        image = dicom2array(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        image = cv2.resize(image, Config.image_size)\n        \n        # Augments must be albumentations\n        if self.augments:\n            image = self.augments(image=image)['image']\n        else:\n            image = torch.tensor(image)\n        \n        if self.is_train:\n            label = self.df[self.df['StudyInstanceUID'] == image_id].values.tolist()[0][3:7]\n            return image, torch.tensor(label)\n        \n        return image\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:31:05.011426Z","iopub.execute_input":"2021-05-22T13:31:05.011823Z","iopub.status.idle":"2021-05-22T13:31:05.022647Z","shell.execute_reply.started":"2021-05-22T13:31:05.011787Z","shell.execute_reply":"2021-05-22T13:31:05.021851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Code\nif __name__ == '__main__':\n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n        DEVICE = torch.device('cuda:0')\n    else:\n        print(\"\\n[INFO] GPU not found. Using CPU: {}\\n\".format(platform.processor()))\n        DEVICE = torch.device('cpu')\n    \n    nb_training_samples = int(Config.train_pcent * train.path.shape[0])\n    train_data = train[:nb_training_samples]\n    valid_data = train[nb_training_samples:]\n\n    print(f\"[INFO] Training on {train_data.shape[0]} samples ({int(Config.train_pcent*100)}%) and validation on {valid_data.shape[0]} ({ceil(abs(1 - Config.train_pcent) * 100)}%) samples\")\n    \n    # Make Training and Validation Datasets\n    training_set = SIIMData(\n        df=train_data\n    )\n\n    validation_set = SIIMData(\n        df=valid_data\n    )\n\n    train_loader = DataLoader(\n        training_set,\n        batch_size=Config.TRAIN_BS,\n        shuffle=True,\n        num_workers=8,\n        pin_memory=True\n    )\n\n    valid_loader = DataLoader(\n        validation_set,\n        batch_size=Config.VALID_BS,\n        shuffle=False,\n        num_workers=8\n    )\n    \n    if \"efficient\" in Config.model_name or \"eff\" in Config.model_name:        \n        model = EfficientNetModel().to(DEVICE)\n    \n    elif \"nfnet\" in Config.model_name:\n        model = NFNetModel().to(DEVICE)\n    \n    else:\n        raise RuntimeError(\"Must specify a valid model type to train.\")\n    \n    print(f\"Training Model: {Config.model_name}\")\n    \n    optim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\n    loss_fn_train = nn.BCEWithLogitsLoss()\n    loss_fn_val = nn.BCEWithLogitsLoss()\n\n    trainer = Trainer(\n        train_dataloader=train_loader,\n        valid_dataloader=valid_loader,\n        model=model,\n        optimizer=optim,\n        loss_fn=loss_fn_train,\n        val_loss_fn=loss_fn_val,\n        agc=False,\n        device=DEVICE,\n    )\n\n    train_losses_eff = []\n    valid_losses_eff = []\n\n    scaler = GradScaler()\n\n    for epoch in range(Config.NB_EPOCHS):\n        print(f\"{'-'*20} EPOCH: {epoch+1}/{Config.NB_EPOCHS} {'-'*20}\")\n\n        # Run one training epoch\n        current_train_loss = trainer.train_one_cycle()\n        train_losses_eff.append(current_train_loss)\n\n        # Run one validation epoch\n        current_val_loss, op_model = trainer.valid_one_cycle()\n        valid_losses_eff.append(current_val_loss)\n\n        # Empty CUDA cache\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T13:31:05.023928Z","iopub.execute_input":"2021-05-22T13:31:05.0243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}