{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install -q -U pip\n!pip install -q -U seaborn\n!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport ast\nimport random\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nfrom matplotlib_venn import venn2, venn3\nimport seaborn as sns\nimport sys\nfrom datetime import datetime\nfrom tqdm.notebook import tqdm\nfrom pprint import pprint\nimport cv2, glob, time, random, os, ast, glob\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\n# https://nvlabs.github.io/iccv2019-mixed-precision-tutorial/files/dusan_stosic_intro_to_mixed_precision_training.pdf\n# https://analyticsindiamag.com/pytorch-mixed-precision-training/\n# https://pytorch.org/docs/stable/notes/amp_examples.html\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts, CosineAnnealingLR\nfrom torch.optim import Adam, AdamW, SGD\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n# sys.setrecursionlimit(10**6)\n!git clone https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer.git\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('./Ranger-Deep-Learning-Optimizer/ranger')\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\nTRAIN_PATH = '../input/ranzcr-clip-catheter-line-classification/train'\nfrom ranger import Ranger  # this is from ranger.py\nfrom ranger913A import RangerVA  # this is from ranger913A.py\nfrom rangerqh import RangerQH  # this is from rangerqh.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_DIR = \"../input/ranzcr-clip-catheter-line-classification/\"\nprint(os.listdir(BASE_DIR))\ndf_train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"), index_col=0)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**StudyInstanceUID** - unique ID for each image <br>\n**ETT** - Abnormal - endotracheal tube placement abnormal<br>\n**ETT** - Borderline - endotracheal tube placement borderline abnormal<br>\n**ETT** - Normal - endotracheal tube placement normal<br>\n**NGT** - Abnormal - nasogastric tube placement abnormal<br>\n**NGT** - Borderline - nasogastric tube placement borderline abnormal<br>\n**NGT** - Incompletely Imaged - nasogastric tube placement inconclusive due to imaging<br>\n**NGT** - Normal - nasogastric tube placement borderline normal<br>\n**CVC** - Abnormal - central venous catheter placement abnormal<br>\n**CVC** - Borderline - central venous catheter placement borderline abnormal<br>\n**CVC** - Normal - central venous catheter placement normal<br>\nSwan Ganz Catheter Present<br>\n**PatientID** - unique ID for each patient in the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\ndf_tmp = df_train.iloc[:, :-1].sum()\nsns.barplot(x=df_tmp.values, y=df_tmp.index)\nplt.xticks(fontsize=14)\nplt.yticks(fontsize=14)\nplt.xlabel(\"Number of images\", fontsize=15)\nplt.title(\"Distribution of labels\", fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of unique patients: \", df_train[\"PatientID\"].unique().shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 6))\ndf_tmp = df_train[\"PatientID\"].value_counts()\nsns.countplot(x=df_tmp.values)\nplt.xticks(fontsize=12, rotation=90)\nplt.yticks(fontsize=14)\nplt.xlabel(\"Number of observations\", fontsize=15)\nplt.ylabel(\"Number of patients\", fontsize=15)\nplt.title(\"Distribution of observations by PatientID\", fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Annotations**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_annot = pd.read_csv(os.path.join(BASE_DIR, \"train_annotations.csv\"))\ndf_annot.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image_with_annotations(row_ind):\n    row = df_annot.iloc[row_ind]\n    image_path = os.path.join(BASE_DIR, \"train\", row[\"StudyInstanceUID\"] + \".jpg\")\n    label = row[\"label\"]\n    data = np.array(ast.literal_eval(row[\"data\"]))\n    \n    plt.figure(figsize=(10, 5))\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    plt.subplot(1, 2, 1)\n    plt.imshow(image)\n    plt.subplot(1, 2, 2)\n    plt.imshow(image)\n    plt.scatter(data[:, 0], data[:, 1])\n    \n    plt.suptitle(label, fontsize=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_with_annotations(8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef visualize_annotations(file_id):\n    plt.figure(figsize=(8, 8))\n    \n    image = cv2.imread(os.path.join(BASE_DIR, \"train\", file_id + \".jpg\"))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    plt.imshow(image)\n    \n    df_patient = df_annot.loc[df_annot[\"StudyInstanceUID\"] == file_id]\n    \n    if df_patient.shape[0]:        \n        labels = df_patient[\"label\"].values.tolist()\n        lines = df_patient[\"data\"].apply(ast.literal_eval).values.tolist()\n\n        for line, label in zip(lines, labels):         \n            line = np.asarray(line)\n            plt.scatter(line[:, 0], line[:, 1], s=40, label=label)\n        \n        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0, prop={'size': 20})\n        \n    plt.tick_params(axis=\"x\", labelsize=15)\n    plt.tick_params(axis=\"y\", labelsize=15)\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_ids = [\n    \"1.2.826.0.1.3680043.8.498.83331936392921199432218327504041001669\",\n    \"1.2.826.0.1.3680043.8.498.11693509889426445054876979814173446281\",\n    \"1.2.826.0.1.3680043.8.498.15159015355212130418020059688126994534\",\n    \"1.2.826.0.1.3680043.8.498.92067938763801985117661596637576203997\",\n]\n\nfor image_id in image_ids:\n    visualize_annotations(image_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **ETT Abnormal**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_batch(image_ids):\n    plt.figure(figsize=(16, 10))\n    \n    for ind, image_id in enumerate(image_ids):\n        plt.subplot(2, 3, ind + 1)\n        image = cv2.imread(os.path.join(BASE_DIR, \"train\", f\"{image_id}.jpg\"))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n        plt.axis(\"off\")\n    \n    plt.show()\n\n    \ndef plot_statistics(df, col):\n    plt.figure(figsize=(16, 2))\n    sns.countplot(y=df[col])\n    \n    plt.xticks(fontsize=12)\n    plt.yticks(fontsize=12)\n    plt.xlabel(\"Number of observations\", fontsize=15)\n    plt.ylabel(col, fontsize=15)\n    plt.title(f\"Distribution of {col}\", fontsize=16);\n    \n    plt.show()\n    \ndef process_class(col_name):\n    plot_statistics(df_train, col_name)\n    tmp_df = df_train[df_train[col_name] == 1]\n    visualize_batch(random.sample(tmp_df.index.tolist(), 6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"process_class(\"ETT - Abnormal\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_annotations(\"1.2.826.0.1.3680043.8.498.93345761486297843389996628528592497280\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"process_class(\"ETT - Borderline\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"process_class(\"ETT - Normal\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Abnormal**"},{"metadata":{"trusted":true},"cell_type":"code","source":"process_class(\"NGT - Abnormal\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Borderline**"},{"metadata":{"trusted":true},"cell_type":"code","source":"process_class(\"NGT - Borderline\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Incomplete Images**"},{"metadata":{"trusted":true},"cell_type":"code","source":"process_class(\"NGT - Incompletely Imaged\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Venn Diagrams**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_venn2(col_1, col_2):\n    plt.figure(figsize=(6, 6))\n    \n    area_10 = df_train[col_1].sum()\n    area_01 = df_train[col_2].sum()\n    area_11 = df_train[(df_train[col_1] == 1) & (df_train[col_2] == 1)].shape[0]\n\n    venn2(\n        subsets=(area_10, area_01, area_11), \n        set_labels=(col_1, col_2),\n        alpha=0.5,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_venn2(\"ETT - Abnormal\", \"NGT - Abnormal\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_venn2(\"ETT - Abnormal\", \"CVC - Abnormal\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_venn3(col_1, col_2, col_3):\n    plt.figure(figsize=(6, 6))\n    \n    area_100 = df_train[col_1].sum()\n    area_010 = df_train[col_2].sum()\n    area_110 = df_train[(df_train[col_1] == 1) & (df_train[col_2] == 1)].shape[0]\n    area_001 = df_train[col_3].sum()\n    area_101 = df_train[(df_train[col_1] == 1) & (df_train[col_3] == 1)].shape[0]\n    area_011 = df_train[(df_train[col_2] == 1) & (df_train[col_3] == 1)].shape[0]\n    area_111 = df_train[(df_train[col_1] == 1) & (df_train[col_2] == 1) & (df_train[col_3] == 1)].shape[0]\n\n#     print(area_100, area_010, area_110, area_001, area_101, area_011, area_111)\n\n    venn3(\n        subsets=(area_100, area_010, area_110, area_001, area_101, area_011, area_111), \n        set_labels=(col_1, col_2, col_3), \n        alpha=0.5\n    );","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_venn3(\n    \"ETT - Abnormal\",\n    \"NGT - Abnormal\",\n    \"CVC - Abnormal\",\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_venn3(\n    \"ETT - Normal\",\n    \"NGT - Normal\",\n    \"CVC - Normal\",\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_venn3(\n    \"ETT - Borderline\",\n    \"NGT - Borderline\",\n    \"CVC - Borderline\",\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv(os.path.join(BASE_DIR, \"sample_submission.csv\"), index_col=0)\ndf_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.to_csv(\"submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 8 # 8 for bigger architectures\nVAL_BATCH_SIZE = 16\nEPOCHS = 4 # train upto 10 epochs\nIMG_SIZE = 684 # 384 for bigger architectures\nif BATCH_SIZE == 8:\n    ITER_FREQ = 400\nelse:\n    ITER_FREQ = 200\nNUM_WORKERS = 8\nMEAN = [0.485, 0.456, 0.406]\nSTD = [0.229, 0.224, 0.225]\nSEED = 416\nN_FOLDS = 4\nTR_FOLDS = [0,1,2,3,4]\nSTART_FOLD = 0\n\nMODEL_PATH = None\nMODEL_ARCH = 'tf_efficientnet_b5_ns' # tf_efficientnet_b4_ns, tf_efficientnet_b5_ns, resnext50_32x4d\nITERS_TO_ACCUMULATE = 1\n\nLR = 5e-4\nMIN_LR = 1e-6 # SAM, CosineAnnealingWarmRestarts\nWEIGHT_DECAY = 1e-6\nMOMENTUM = 0.9\nT_0 = EPOCHS # SAM, CosineAnnealingWarmRestarts\nMAX_NORM = 1000\nT_MAX = 5 # CosineAnnealingLR\n\nBASE_OPTIMIZER = SGD #for SAM, Ranger\nOPTIMIZER = 'Ranger' # Ranger, Adam, AdamP, SGD, SAM\n\nSCHEDULER = 'CosineAnnealingLR' # ReduceLROnPlateau, CosineAnnealingLR, CosineAnnealingWarmRestarts, OneCycleLR\nSCHEDULER_UPDATE = 'epoch' # batch\n\nCRITERION = 'BCE' # CrossEntropyLoss, TaylorSmoothedLoss, LabelSmoothedLoss\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):    \n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.val = 0\n        self.sum = 0\n        self.avg = 0\n        self.count = 0\n        \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val*n\n        self.count += n\n        self.avg = self.sum / self.count\n\ndef seed_torch(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch(SEED)\n\ndef macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(target_cols)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n#     print(np.round(aucs, 4))\n    return np.mean(aucs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR = '../input/ranzcr-clip-catheter-line-classification/train/'\ntrain_df = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\nfolds = pd.read_csv('../input/ranzcr-folds/train_folds.csv')\ntrain_annotations = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train_annotations.csv')\n\ntarget_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                 'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                 'Swan Ganz Catheter Present']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLOR_MAP = {'ETT - Abnormal': (255, 0, 0),\n             'ETT - Borderline': (0, 255, 0),\n             'ETT - Normal': (0, 0, 255),\n             'NGT - Abnormal': (255, 255, 0),\n             'NGT - Borderline': (255, 0, 255),\n             'NGT - Incompletely Imaged': (0, 255, 255),\n             'NGT - Normal': (128, 0, 0),\n             'CVC - Abnormal': (0, 128, 0),\n             'CVC - Borderline': (0, 0, 128),\n             'CVC - Normal': (128, 128, 0),\n             'Swan Ganz Catheter Present': (128, 0, 128),\n            }\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RanzcrDataset(Dataset):\n    def __init__(self, df, df_annotations, annot_size=50, transform=None):\n        self.df = df\n        self.df_annotations = df_annotations\n        self.annot_size = annot_size\n        self.image_id = df['StudyInstanceUID'].values\n        self.labels = df[target_cols].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.image_id[idx]\n        file_path = f'{TRAIN_DIR}{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        query_string = f\"StudyInstanceUID == '{file_name}'\"\n        df = self.df_annotations.query(query_string)\n        for i, row in df.iterrows():\n            label = row[\"label\"]\n            data = np.array(ast.literal_eval(row[\"data\"]))\n            for d in data:\n                image[d[1]-self.annot_size//2:d[1]+self.annot_size//2,\n                      d[0]-self.annot_size//2:d[0]+self.annot_size//2,\n                      :] = COLOR_MAP[label]\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label\ndef get_transform(*, train=True):\n    \n    if train:\n        return A.Compose([\n            A.RandomResizedCrop(IMG_SIZE, IMG_SIZE, scale=(0.85, 1.0)),\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.2, brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2)),\n            A.HueSaturationValue(p=0.2, hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            A.ShiftScaleRotate(p=0.2, shift_limit=0.0625, scale_limit=0.2, rotate_limit=20),\n            A.CoarseDropout(p=0.2),\n            A.Cutout(p=0.2, max_h_size=16, max_w_size=16, fill_value=(0., 0., 0.), num_holes=16),\n            A.Normalize(mean=MEAN, std=STD),\n            ToTensorV2(),\n        ])\n    else:\n        return A.Compose([\n#             A.CenterCrop(IMG_SIZE, IMG_SIZE),\n            A.Resize(IMG_SIZE, IMG_SIZE),\n            A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNet200D(nn.Module):\n    def __init__(self, model_arch, n_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, n_classes)\n        if pretrained:\n            pretrained_path = '../input/startingpointschestx/resnet200d_320_chestx.pth'\n            checkpoint = torch.load(pretrained_path)['model']\n            for key in list(checkpoint.keys()):\n                if 'model.' in key:\n                    checkpoint[key.replace('model.', '')] = checkpoint[key]\n                    del checkpoint[key]\n            self.model.load_state_dict(checkpoint) \n            print(f'load {model_arch} pretrained model')\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, n_classes)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return features, pooled_features, output\n\nclass SeResnet152D(nn.Module): \n    def __init__(self, model_arch, n_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, n_classes)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n            \nclass CustomEffNet(nn.Module):\n    def __init__(self, model_arch, n_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=False)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_classes)\n        if pretrained:\n            pretrained_path = '../input/startingpointschestx/tf_efficientnet_b5_ns_chestx.pth'\n            checkpoint = torch.load(pretrained_path)['model']\n            for key in list(checkpoint.keys()):\n                if 'model.' in key:\n                    checkpoint[key.replace('model.', '')] = checkpoint[key]\n                    del checkpoint[key]\n            self.model.load_state_dict(checkpoint) \n            print(f'load {model_arch} pretrained model')\n        n_features = self.model.classifier.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.classifier = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.classifier = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.classifier(pooled_features)\n        return features, pooled_features, output \n    \nclass CustomResNext(nn.Module):\n    def __init__(self, model_arch, n_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, n_classes)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GetCriterion(criterion_name, criterion=None):\n#     if criterion_name == 'BiTemperedLoss':\n#         criterion = BiTemperedLogistic()\n#     elif criterion_name == 'SymmetricCrossEntropyLoss':\n#         criterion = SymmetricCrossEntropy()\n    if criterion_name == 'CrossEntropyLoss':\n        criterion = nn.CrossEntropyLoss()\n    elif criterion_name == 'CustomLoss':\n        criterion = CustomLoss(WEIGHTS)\n    elif criterion_name == 'BCE':\n        criterion = nn.BCEWithLogitsLoss()\n    return criterion\n    \n    \ndef GetScheduler(scheduler_name, optimizer, batches=None):\n    #['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts', 'OneCycleLR', 'GradualWarmupSchedulerV2']\n    if scheduler_name == 'OneCycleLR':\n        return torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr = 1e-2,epochs = CFG.EPOCHS,\n                                                   steps_per_epoch = batches+1,pct_start = 0.1)\n    if scheduler_name == 'CosineAnnealingWarmRestarts':\n        return torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = T_0, T_mult=1,\n                                                                    eta_min=MIN_LR, last_epoch=-1)\n    elif scheduler_name == 'CosineAnnealingLR':\n        return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=T_MAX, eta_min=0, last_epoch=-1)\n    elif scheduler_name == 'ReduceLROnPlateau':\n        return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1, threshold=0.0001,\n                                                          cooldown=0, min_lr=MIN_LR)\n#     elif scheduler_name == 'GradualWarmupSchedulerV2':\n#         return GradualWarmupSchedulerV2(optimizer=optimizer)\n    \ndef GetOptimizer(optimizer_name,parameters):\n    #['Adam','Ranger']\n    if optimizer_name == 'Adam':\n#         if CFG.scheduler_name == 'GradualWarmupSchedulerV2':\n#             return torch.optim.Adam(parameters, lr=CFG.LR_START, weight_decay=CFG.weight_decay, amsgrad=False)\n#         else:\n        return torch.optim.Adam(parameters, lr=LR, weight_decay=WEIGHT_DECAY, amsgrad=False)\n    elif optimizer_name == 'AdamW':\n#         if CFG.scheduler_name == 'GradualWarmupSchedulerV2':\n#             return torch.optim.AdamW(parameters, lr=CFG.LR_START, weight_decay=CFG.weight_decay, amsgrad=False)\n#         else:\n        return torch.optim.Adam(parameters, lr=LR, weight_decay=WEIGHT_DECAY, amsgrad=False)\n    elif optimizer_name == 'AdamP':\n#         if CFG.scheduler_name == 'GradualWarmupSchedulerV2':\n#             return AdamP(parameters, lr=CFG.LR_START, weight_decay=CFG.weight_decay)\n#         else:\n        return AdamP(parameters, lr=LR, weight_decay=WEIGHT_DECAY)\n    elif optimizer_name == 'Ranger':\n        return Ranger(parameters, lr = LR, alpha = 0.5, k = 6, N_sma_threshhold = 5, \n                      betas = (0.95,0.999), weight_decay=WEIGHT_DECAY)\n    elif optimizer_name == 'SAM':\n        return SAM(parameters, BASE_OPTIMIZER, lr=0.1, momentum=0.9,weight_decay=0.0005)\n    \n    elif optimizer_name == 'AdamP':\n        return AdamP(parameters, lr=LR, weight_decay=WEIGHT_DECAY)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(model, dataloader, device, epoch, optimizer, criterion, scheduler):\n    \n    data_time = AverageMeter()\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    accuracies = AverageMeter()\n    model.train()\n    scaler = GradScaler()\n    start_time = time.time()\n    loader = tqdm(dataloader, total=len(dataloader))\n    for step, (images, labels) in enumerate(loader):\n        \n        images = images.to(device).float()\n        labels = labels.to(device)\n        data_time.update(time.time() - start_time)\n\n        with autocast():\n\n            _, _, output = model(images)\n            loss = criterion(output, labels)\n            losses.update(loss.item(), BATCH_SIZE)\n            scaler.scale(loss).backward()\n            grad_norm = nn.utils.clip_grad_norm_(model.parameters(), max_norm = MAX_NORM)\n            if (step+1) % ITERS_TO_ACCUMULATE == 0:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n        \n        if scheduler is not None and SCHEDULER_UPDATE == 'batch':\n            scheduler.step()\n\n        batch_time.update(time.time() - start_time)\n        start_time = time.time()\n        \n        if step % ITER_FREQ == 0:\n            \n            print('Epoch: [{0}][{1}/{2}]\\t'\n                  'Batch Time {batch_time.val:.3f}s ({batch_time.avg:.3f}s)\\t'\n                  'Data Time {data_time.val:.3f}s ({data_time.avg:.3f}s)\\t'\n                  'Loss: {loss.val:.4f} ({loss.avg:.4f})'.format((epoch+1),\n                                                                    step, len(dataloader),\n                                                                    batch_time=batch_time,\n                                                                    data_time=data_time,\n                                                                    loss=losses))\n                                                                             #accuracy=accuracies))\n        # To check the loss real-time while iterating over data.   'Accuracy {accuracy.val:.4f} ({accuracy.avg:.4f})'\n        loader.set_description(f'Training Epoch {epoch+1}/{EPOCHS}')\n        loader.set_postfix(loss=losses.avg) #accuracy=accuracies.avg)\n#         del images, labels\n    if scheduler is not None and SCHEDULER_UPDATE == 'epoch':\n        scheduler.step()\n        \n    return losses.avg#, accuracies.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def valid_fn(epoch, model, criterion, val_loader, device, scheduler):\n    \n    model.eval()\n    losses = AverageMeter()\n    accuracies = AverageMeter()\n    PREDS = []\n    TARGETS = []\n    loader = tqdm(val_loader, total=len(val_loader))\n    with torch.no_grad():  # without torch.no_grad() will make the CUDA run OOM.\n        for step, (images, labels) in enumerate(loader):\n\n            images = images.to(device)\n            labels = labels.to(device)\n\n            _, _, output = model(images)\n            loss = criterion(output, labels)\n            losses.update(loss.item(), BATCH_SIZE)\n            PREDS += [output.sigmoid()]\n            TARGETS += [labels.detach().cpu()]\n            loader.set_description(f'Validating Epoch {epoch+1}/{EPOCHS}')\n            loader.set_postfix(loss=losses.avg)#, accuracy=accuracies.avg)\n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    roc_auc = macro_multilabel_auc(TARGETS, PREDS)\n    if scheduler is not None:\n        scheduler.step()\n        \n    return losses.avg, roc_auc# accuracies.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def engine(device, folds, fold, model_path=None):\n    \n    trn_idx = folds[folds['kfold'] != fold].index\n    val_idx = folds[folds['kfold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n\n    train_folds = train_folds[train_folds['StudyInstanceUID'].isin(train_annotations['StudyInstanceUID'].unique())].reset_index(drop=True)\n    valid_folds = valid_folds[valid_folds['StudyInstanceUID'].isin(train_annotations['StudyInstanceUID'].unique())].reset_index(drop=True)\n\n    train_data = RanzcrDataset(train_folds, train_annotations, transform=get_transform())\n    val_data = RanzcrDataset(valid_folds, train_annotations, transform=get_transform(train=False))        \n    \n    train_loader = DataLoader(train_data,\n                              batch_size=BATCH_SIZE, \n                              shuffle=True, \n                              num_workers=NUM_WORKERS,\n                              pin_memory=True, # enables faster data transfer to CUDA-enabled GPUs.\n                              drop_last=True)\n    val_loader = DataLoader(val_data,\n                            batch_size=VAL_BATCH_SIZE,\n                            num_workers=NUM_WORKERS,\n                            shuffle=False, \n                            pin_memory=True,\n                            drop_last=False)\n\n    if model_path is not None:\n        model = torch.load(model_path)\n        START_EPOCH = int(model_path.split('_')[-1])\n    else:\n        model = CustomEffNet(MODEL_ARCH, 11, True)\n        START_EPOCH = 0\n    model.to(device)\n    \n    params = filter(lambda p: p.requires_grad, model.parameters())    \n    optimizer = GetOptimizer(OPTIMIZER, params)\n\n    criterion = GetCriterion(CRITERION).to(device)    \n    val_criterion = GetCriterion(CRITERION).to(device)\n\n    scheduler = GetScheduler(SCHEDULER, optimizer)\n    \n    loss = []\n    accuracy = []\n    for epoch in range(START_EPOCH, EPOCHS):\n        \n        epoch_start = time.time()        \n        avg_loss = train_fn(model, train_loader, device, epoch, optimizer, criterion, scheduler)\n\n        torch.cuda.empty_cache()\n        avg_val_loss, roc_auc_score = valid_fn(epoch, model, val_criterion, val_loader, device, scheduler)\n        epoch_end = time.time() - epoch_start\n        \n        print(f'Validation accuracy after epoch {epoch+1}: {roc_auc_score:.4f}')\n        loss.append(avg_loss)\n#         accuracy.append(avg_accuracy)\n        \n        content = f'Fold {fold} Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f} roc_auc_score: {roc_auc_score:.4f} time: {epoch_end:.0f}s'\n        with open(f'stage1_{MODEL_ARCH}_{IMG_SIZE}.txt', 'a') as appender:\n            appender.write(content + '\\n')                                         # avg_train_accuracy: {avg_accuracy:.4f}\n        \n        # Save the model to use it for inference.\n        torch.save(model.state_dict(), f'stage1_{MODEL_ARCH}_fold_{fold}_epoch_{(epoch+1)}.pth')\n#         torch.save(model, f'stage1_{MODEL_ARCH}_fold_{fold}_epoch_{(epoch+1)}')\n        torch.cuda.empty_cache()\n    \n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    \n    if MODEL_PATH is not None:\n        START_FOLD = int(MODEL_PATH.split('_')[-3])\n    \n    for fold in range(START_FOLD, N_FOLDS):\n        if fold == 3:\n            break\n        print(f'===== Fold {fold} Starting =====')\n        fold_start = time.time()\n        logs = engine(DEVICE, folds, fold, MODEL_PATH)\n        print(f'Time taken in fold {fold}: {time.time()-fold_start}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}