{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Tez: a simple pytorch trainer\n\nInference Kernel: https://www.kaggle.com/abhishek/ranzcr-tez-inference-efficientnet5\n\nPlease note: a few things have been changed in this training kernel to make it hard for you to reproduce the inference score ;)"},{"metadata":{"trusted":true},"cell_type":"code","source":"tez_path = '../input/tez-lib/'\neffnet_path = '../input/efficientnet-pytorch/'\nimport sys\nsys.path.append(tez_path)\nsys.path.append(effnet_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import argparse\nimport os\n\nimport albumentations\nimport pandas as pd\nimport tez\nimport torch\nimport torch.nn as nn\nfrom efficientnet_pytorch import EfficientNet\nfrom sklearn import metrics, model_selection, preprocessing\nfrom tez.callbacks import EarlyStopping\nfrom tez.datasets import ImageDataset\nfrom torch.nn import functional as F","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"INPUT_PATH = \"../input/ranzcr-clip-catheter-line-classification/\"\nIMAGE_PATH = \"../input/ranzcr-clip-catheter-line-classification/train/\"\nFOLDS_PATH = \"../input/ranzcr-folds/\"\nTRAIN_BATCH_SIZE = 32\nVALID_BATCH_SIZE = 32\nEPOCHS = 200\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RanzcrModel(tez.Model):\n    def __init__(self):\n        super().__init__()\n\n        self.effnet = EfficientNet.from_pretrained(\"efficientnet-b5\")\n\n        self.effnet._conv_stem.in_channels = 1\n        weight = self.effnet._conv_stem.weight.mean(1, keepdim=True)\n        self.effnet._conv_stem.weight = torch.nn.Parameter(weight)\n\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(2048, 11)\n        self.step_scheduler_after = \"epoch\"\n        self.step_scheduler_metric = \"valid_auc\"\n\n    def monitor_metrics(self, outputs, targets):\n        outputs = outputs.cpu().detach().numpy()\n        targets = targets.cpu().detach().numpy()\n        # NOTE: this is not correct :)\n        auc = 0\n        for j in range(outputs.shape[1]):\n            try:\n                auc += metrics.roc_auc_score(\n                    targets[:, j], outputs[:, j]\n                )\n            except ValueError:\n                auc += 0.5\n        return {\"auc\": auc / 11}\n\n    def fetch_optimizer(self):\n        opt = torch.optim.Adam(self.parameters(), lr=1e-3)\n        return opt\n\n    def fetch_scheduler(self):\n        rlr = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            self.optimizer,\n            verbose=True,\n            factor=0.7,\n            mode=\"max\",\n            patience=2,\n            threshold=0.01,\n        )\n        return rlr\n\n    def forward(self, image, targets=None):\n        batch_size, _, _, _ = image.shape\n\n        x = self.effnet.extract_features(image)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        outputs = self.out(self.dropout(x))\n        if targets is not None:\n            loss = nn.BCEWithLogitsLoss()(\n                outputs, targets.type_as(outputs)\n            )\n            metrics = self.monitor_metrics(outputs, targets)\n            return outputs, loss, metrics\n        return outputs, None, {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_aug = albumentations.Compose(\n    [\n        albumentations.RandomResizedCrop(IMAGE_SIZE, IMAGE_SIZE),\n        albumentations.Normalize(\n            mean=[0.485],\n            std=[0.229],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)\n\nvalid_aug = albumentations.Compose(\n    [\n        albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE, p=1.0),\n        albumentations.Normalize(\n            mean=[0.485],\n            std=[0.229],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)\n\n\n# change folds here\ncurrent_fold = 0\ndfx = pd.read_csv(os.path.join(FOLDS_PATH, \"train_folds.csv\"))\ntargets = [\n    \"ETT - Abnormal\",\n    \"ETT - Borderline\",\n    \"ETT - Normal\",\n    \"NGT - Abnormal\",\n    \"NGT - Borderline\",\n    \"NGT - Incompletely Imaged\",\n    \"NGT - Normal\",\n    \"CVC - Abnormal\",\n    \"CVC - Borderline\",\n    \"CVC - Normal\",\n    \"Swan Ganz Catheter Present\",\n]\ndf_train = dfx[dfx.kfold != current_fold].reset_index(drop=True)\ndf_valid = dfx[dfx.kfold == current_fold].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_paths = [\n    os.path.join(IMAGE_PATH, x + \".jpg\") for x in df_train.StudyInstanceUID.values\n]\nvalid_image_paths = [\n    os.path.join(IMAGE_PATH, x + \".jpg\") for x in df_valid.StudyInstanceUID.values\n]\n\ntrain_targets = df_train[targets].values\nvalid_targets = df_valid[targets].values\n\ntrain_dataset = ImageDataset(\n    image_paths=train_image_paths,\n    targets=train_targets,\n    augmentations=train_aug,\n    grayscale=True,\n)\n\nvalid_dataset = ImageDataset(\n    image_paths=valid_image_paths,\n    targets=valid_targets,\n    augmentations=valid_aug,\n    grayscale=True,\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = RanzcrModel()\n\nes = EarlyStopping(\n    monitor=\"valid_auc\",\n    model_path=f\"effnet5_fold_{current_fold}.bin\",\n    patience=5,\n    mode=\"max\",\n)\n\nmodel.fit(\n    train_dataset,\n    valid_dataset=valid_dataset,\n    train_bs=TRAIN_BATCH_SIZE,\n    valid_bs=VALID_BATCH_SIZE,\n    device=\"cuda\",\n    epochs=EPOCHS,\n    callbacks=[es],\n    fp16=True,\n)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}