{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Reference to load DICOM Images: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\nand https://www.kaggle.com/tanlikesmath/siim-covid-19-detection-a-simple-eda","metadata":{"gradient":{"editing":false}}},{"cell_type":"markdown","source":"# Classification Training Script\nThis script is written to be modified and facilitate different types of experiments.","metadata":{"gradient":{"editing":false}}},{"cell_type":"code","source":"!pip install timm","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:26.418578Z","iopub.execute_input":"2021-07-22T12:56:26.419079Z","iopub.status.idle":"2021-07-22T12:56:33.328374Z","shell.execute_reply.started":"2021-07-22T12:56:26.419026Z","shell.execute_reply":"2021-07-22T12:56:33.327042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\nimport time\nimport random\n\nimport numpy as np  # linear algebra!\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\nimport PIL\n\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import roc_auc_score\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm import tqdm\nfrom tqdm.contrib.concurrent import process_map\n\nimport torch\nimport torchvision\nfrom torch.utils.data.dataset import Dataset\nimport torch.cuda.amp as amp\n\nimport timm\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nDATA_DIR = \"/kaggle/input/siim-covid19-detection\"\nRESIZE_DIR = \"/kaggle/working/\"\n\nSIZE = (384, 384)\nFOLDS = 5\nNUM_CLASSES = 4\nBATCHSIZE = 48\nSEED = 420\nMODEL_TYPE = \"4_CLASS\"\nMODEL_NAME = \"vit_small_r26_s32_384\"","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:33.332742Z","iopub.execute_input":"2021-07-22T12:56:33.333354Z","iopub.status.idle":"2021-07-22T12:56:33.350003Z","shell.execute_reply.started":"2021-07-22T12:56:33.333311Z","shell.execute_reply":"2021-07-22T12:56:33.349084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir \"/kaggle/working/train_{SIZE[0]}x{SIZE[1]}\"\n!tar -xzf \"/kaggle/input/jpeg-and-archive-{SIZE[0]}/train_{SIZE[0]}x{SIZE[1]}.tar.gz\" -C \"/kaggle/working/train_{SIZE[0]}x{SIZE[1]}\" .","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:33.352044Z","iopub.execute_input":"2021-07-22T12:56:33.35786Z","iopub.status.idle":"2021-07-22T12:56:35.834498Z","shell.execute_reply.started":"2021-07-22T12:56:33.357818Z","shell.execute_reply":"2021-07-22T12:56:35.833306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make results reproducible\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n\n\nseed_everything(SEED)","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:35.838343Z","iopub.execute_input":"2021-07-22T12:56:35.838675Z","iopub.status.idle":"2021-07-22T12:56:35.846836Z","shell.execute_reply.started":"2021-07-22T12:56:35.838642Z","shell.execute_reply":"2021-07-22T12:56:35.845984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean column names in CSV's\nWe clean up a few column names in the `train_image_level.csv` and the `train_study_level.csv`, to merge these two csvs into one.\nWe also rename the columns to simplified names to use later on.","metadata":{"gradient":{"editing":false}}},{"cell_type":"code","source":"train_images_df = pd.read_csv(os.path.join(DATA_DIR, \"train_image_level.csv\"))\ntrain_study_df = pd.read_csv(os.path.join(DATA_DIR, \"train_study_level.csv\"))\ntrain_images_df[\"StudyInstanceUID\"] = train_images_df[\"StudyInstanceUID\"] + \"_study\"\n\ntrain_study_df.columns = train_study_df.columns.map(lambda x: x.split(\" \")[0])\n\ntrain_study_df.rename(columns={\"id\": \"study_id\"}, inplace=True)\ntrain_images_df.rename(columns={\"StudyInstanceUID\": \"study_id\"}, inplace=True)","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:35.848253Z","iopub.execute_input":"2021-07-22T12:56:35.848771Z","iopub.status.idle":"2021-07-22T12:56:35.891947Z","shell.execute_reply.started":"2021-07-22T12:56:35.848734Z","shell.execute_reply":"2021-07-22T12:56:35.8911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label Map Creation\nAt prediction time, we will need these labels in string form. These maps can also be modified to train different types of models, like Binary classification.","metadata":{"gradient":{"editing":false}}},{"cell_type":"code","source":"if NUM_CLASSES == 4:\n    NAME_TO_LABEL_MAP = {\"Negative\": 0, \"Typical\": 1, \"Indeterminate\": 2, \"Atypical\": 3}\n\n\ndef get_str_label(row):\n    for k in NAME_TO_LABEL_MAP:\n        if row[k]:\n            return k\n    return None\n\n\ndef get_int_label(row):\n    for k in NAME_TO_LABEL_MAP:\n        if row[k]:\n            return NAME_TO_LABEL_MAP[k]\n    return None","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:35.894912Z","iopub.execute_input":"2021-07-22T12:56:35.895177Z","iopub.status.idle":"2021-07-22T12:56:35.902171Z","shell.execute_reply.started":"2021-07-22T12:56:35.895145Z","shell.execute_reply":"2021-07-22T12:56:35.901333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Group K Fold\nWe create a `fold` column to be used while we run cross validation.\nWe choose the `study_id` column created above to split into the number of groups defined by the `FOLDS` variable.","metadata":{}},{"cell_type":"code","source":"train_study_df[\"int_label\"] = train_study_df.apply(get_int_label, axis=1)\ntrain_study_df[\"str_label\"] = train_study_df.apply(get_str_label, axis=1)\n\ngkf = GroupKFold(n_splits=FOLDS)\ntrain_study_df[\"fold\"] = -1\nfor fold, (train_idx, val_idx) in enumerate(\n    gkf.split(train_study_df, groups=train_study_df[\"study_id\"].tolist())\n):\n    train_study_df.loc[val_idx, \"fold\"] = fold","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:35.903876Z","iopub.execute_input":"2021-07-22T12:56:35.904114Z","iopub.status.idle":"2021-07-22T12:56:36.137146Z","shell.execute_reply.started":"2021-07-22T12:56:35.904091Z","shell.execute_reply":"2021-07-22T12:56:36.136239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plotting Group Counts\nWe see from the plots below that each group is fairly balanced. The `Typical` category is the most common followed by `Negative`.","metadata":{}},{"cell_type":"code","source":"sns.catplot(x=\"str_label\", col=\"fold\", data=train_study_df, kind=\"count\")","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:36.14119Z","iopub.execute_input":"2021-07-22T12:56:36.141461Z","iopub.status.idle":"2021-07-22T12:56:36.918134Z","shell.execute_reply.started":"2021-07-22T12:56:36.141435Z","shell.execute_reply":"2021-07-22T12:56:36.917344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean Study level data\n1. As per the recommendations made [here](https://www.kaggle.com/c/siim-covid19-detection/discussion/246597) for studies with more than one image it appears that there only one which has bounding boxes. \n\n2. As per the recommendation made [here](https://www.kaggle.com/c/siim-covid19-detection/discussion/240250#1351079), the label for only the one with bounding boxes is retained since the other images were not looked at by the annotators.\n\n3. For studies that have more than one image but no bounding boxes associated with them, it is unclear as to which image was looked at therefore all images are retained in those studies.","metadata":{}},{"cell_type":"code","source":"train_samples_df = pd.merge(\n    train_images_df, train_study_df, on=\"study_id\", how=\"inner\"\n).reset_index(drop=True)\n\nbox_and_images_counts_df = (\n    train_samples_df.groupby(\"study_id\")[[\"id\", \"boxes\"]]\n    .count()\n    .sort_values(ascending=False, by=\"id\")\n    .reset_index()\n)\nbox_and_images_counts_df.rename(\n    columns={\"id\": \"id_count\", \"boxes\": \"boxes_count\"}, inplace=True\n)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:36.920241Z","iopub.execute_input":"2021-07-22T12:56:36.920722Z","iopub.status.idle":"2021-07-22T12:56:36.951135Z","shell.execute_reply.started":"2021-07-22T12:56:36.920684Z","shell.execute_reply":"2021-07-22T12:56:36.95034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=box_and_images_counts_df, x=\"id_count\")","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:36.952359Z","iopub.execute_input":"2021-07-22T12:56:36.95275Z","iopub.status.idle":"2021-07-22T12:56:37.109522Z","shell.execute_reply.started":"2021-07-22T12:56:36.952714Z","shell.execute_reply":"2021-07-22T12:56:37.10855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df = pd.merge(\n    train_samples_df, box_and_images_counts_df, how=\"inner\", on=\"study_id\"\n)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.111032Z","iopub.execute_input":"2021-07-22T12:56:37.111403Z","iopub.status.idle":"2021-07-22T12:56:37.125833Z","shell.execute_reply.started":"2021-07-22T12:56:37.111364Z","shell.execute_reply":"2021-07-22T12:56:37.124974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df.sort_values([\"id_count\", \"boxes_count\"], ascending=False, inplace=True)\ntrain_samples_df.head(5)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.127409Z","iopub.execute_input":"2021-07-22T12:56:37.127779Z","iopub.status.idle":"2021-07-22T12:56:37.149314Z","shell.execute_reply.started":"2021-07-22T12:56:37.127742Z","shell.execute_reply":"2021-07-22T12:56:37.148545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dicom helpers","metadata":{}},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=PIL.Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = PIL.Image.fromarray(array)\n\n    if keep_ratio:\n        im.thumbnail((size[0], size[1]), resample)\n    else:\n        im = im.resize((size[0], size[1]), resample)\n\n    return im\n\n\ndef dicom2array(path, size, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    data = resize(data, size)\n    return data","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.150623Z","iopub.execute_input":"2021-07-22T12:56:37.150971Z","iopub.status.idle":"2021-07-22T12:56:37.160109Z","shell.execute_reply.started":"2021-07-22T12:56:37.150936Z","shell.execute_reply":"2021-07-22T12:56:37.159079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Caching Images\n\nIt is imperative that the simplistic logic below, executes without errors. This improves training speeds very significantly. It merely convers the DICOM into JPG and caches it. Even fully expanded a 512x512 image dataset is only a few 100MB.","metadata":{}},{"cell_type":"code","source":"%%time\nTRAIN_DATA_PATH = None\n\nif os.path.exists(os.path.join(RESIZE_DIR, 'train_{}x{}'.format(SIZE[0], SIZE[1]))):\n    TRAIN_DATA_PATH = os.path.join(RESIZE_DIR, 'train_{}x{}'.format(SIZE[0], SIZE[1]))\n    print(\"{} Exists\".format(TRAIN_DATA_PATH))\nelse:\n    TRAIN_DATA_PATH = os.path.join(RESIZE_DIR, 'train_{}x{}'.format(SIZE[0], SIZE[1]))\n    print(\"Creating Training dir at {}\".format(TRAIN_DATA_PATH))\n    os.makedirs(TRAIN_DATA_PATH)\n    filenames = glob.glob(os.path.join(DATA_DIR, \"train/*/*/*.dcm\"))\n\n    def persist_image(path):\n        im = dicom2array(path, SIZE)\n        fname = os.path.basename(os.path.splitext(path)[-2])\n        jpg_fname = os.path.join(TRAIN_DATA_PATH, \"{}.jpg\".format(fname))\n        im.save(jpg_fname)\n        return jpg_fname\n\n    process_map(persist_image, filenames, max_workers=8, chunksize=1)\n","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.161747Z","iopub.execute_input":"2021-07-22T12:56:37.162132Z","iopub.status.idle":"2021-07-22T12:56:37.174843Z","shell.execute_reply.started":"2021-07-22T12:56:37.162076Z","shell.execute_reply":"2021-07-22T12:56:37.17374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jpg_counts = !ls -l {TRAIN_DATA_PATH} | wc -l\n# assert int(jpg_counts[0]) - 2 == train_images_df.shape[0]","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.176515Z","iopub.execute_input":"2021-07-22T12:56:37.177017Z","iopub.status.idle":"2021-07-22T12:56:37.280885Z","shell.execute_reply.started":"2021-07-22T12:56:37.17698Z","shell.execute_reply":"2021-07-22T12:56:37.279607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jpg_counts","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.284589Z","iopub.execute_input":"2021-07-22T12:56:37.284925Z","iopub.status.idle":"2021-07-22T12:56:37.294592Z","shell.execute_reply.started":"2021-07-22T12:56:37.284894Z","shell.execute_reply":"2021-07-22T12:56:37.293586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cleaning Study Level data\nAs per the recommendations in the beginning of the section, in order to have clean data for studies that have multiple images we only retain the one with the bounding box. This is used later in the Dataset loader created below.","metadata":{}},{"cell_type":"code","source":"def keep_row(row):\n    # keep as negative sample for study with 0 bboxes with opacity\n    # or non-null bounding box\n    if row[\"boxes_count\"] and not pd.isna(row[\"boxes\"]):\n        return True\n    if row[\"boxes_count\"] == 0:\n        return True\n    else:\n        return False","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.297694Z","iopub.execute_input":"2021-07-22T12:56:37.29794Z","iopub.status.idle":"2021-07-22T12:56:37.305043Z","shell.execute_reply.started":"2021-07-22T12:56:37.297917Z","shell.execute_reply":"2021-07-22T12:56:37.304073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_img_path(row):\n    study_id = row[\"study_id\"][:-6]\n    img_id = row[\"id\"][:-6]\n    paths = glob.glob(os.path.join(TRAIN_DATA_PATH, \"{}.jpg\".format(img_id)))\n    for path in paths:\n        if img_id in path:\n            return path\n    return None","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.308253Z","iopub.execute_input":"2021-07-22T12:56:37.3086Z","iopub.status.idle":"2021-07-22T12:56:37.316171Z","shell.execute_reply.started":"2021-07-22T12:56:37.308574Z","shell.execute_reply":"2021-07-22T12:56:37.315323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df[\"path\"] = train_samples_df.apply(get_img_path, axis=1)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.319483Z","iopub.execute_input":"2021-07-22T12:56:37.319798Z","iopub.status.idle":"2021-07-22T12:56:37.500678Z","shell.execute_reply.started":"2021-07-22T12:56:37.31977Z","shell.execute_reply":"2021-07-22T12:56:37.499746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df[\"keep_row\"] = train_samples_df.apply(keep_row, axis=1)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.502564Z","iopub.execute_input":"2021-07-22T12:56:37.502934Z","iopub.status.idle":"2021-07-22T12:56:37.611943Z","shell.execute_reply.started":"2021-07-22T12:56:37.502898Z","shell.execute_reply":"2021-07-22T12:56:37.611128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df = train_samples_df[train_samples_df[\"keep_row\"]]\ntrain_samples_df.head(5)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.613305Z","iopub.execute_input":"2021-07-22T12:56:37.613642Z","iopub.status.idle":"2021-07-22T12:56:37.636208Z","shell.execute_reply.started":"2021-07-22T12:56:37.61359Z","shell.execute_reply":"2021-07-22T12:56:37.635494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df[\"path\"][0]","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.638527Z","iopub.execute_input":"2021-07-22T12:56:37.638881Z","iopub.status.idle":"2021-07-22T12:56:37.646236Z","shell.execute_reply.started":"2021-07-22T12:56:37.638848Z","shell.execute_reply":"2021-07-22T12:56:37.645159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df.sort_values([\"study_id\"], inplace=True)\ntrain_samples_df.set_index(\"study_id\", inplace=True)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.651095Z","iopub.execute_input":"2021-07-22T12:56:37.651337Z","iopub.status.idle":"2021-07-22T12:56:37.665709Z","shell.execute_reply.started":"2021-07-22T12:56:37.651313Z","shell.execute_reply":"2021-07-22T12:56:37.664929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_df[\n    (train_samples_df[\"id_count\"] > 1) & (train_samples_df[\"boxes_count\"] == 0)\n].head(5)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.667963Z","iopub.execute_input":"2021-07-22T12:56:37.668319Z","iopub.status.idle":"2021-07-22T12:56:37.686664Z","shell.execute_reply.started":"2021-07-22T12:56:37.668282Z","shell.execute_reply":"2021-07-22T12:56:37.685588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dev = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\ndev","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.688346Z","iopub.execute_input":"2021-07-22T12:56:37.689023Z","iopub.status.idle":"2021-07-22T12:56:37.69749Z","shell.execute_reply.started":"2021-07-22T12:56:37.688986Z","shell.execute_reply":"2021-07-22T12:56:37.696407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Class\nFor the others, we retain all images and randomly choose which image to train on. At prediction time, we will average predictions made, but this is not yet implemented instead we randomly produce a prediction by choosing an image within the study.","metadata":{"gradient":{"editing":false}}},{"cell_type":"code","source":"class XRayDatasetFromDF(Dataset):\n    def __init__(self, df, train=True, augment=True, normalize=False, size=(384, 384)):\n        self.df = df\n        self.name_to_label_map = {\n            \"Negative\": 0,\n            \"Typical\": 1,\n            \"Indeterminate\": 2,\n            \"Atypical\": 3,\n        }\n        self.study_ids = df.index.sort_values()\n        self.path_suffix = (\n            os.path.join(DATA_DIR, \"train\") if train else os.path.join(DATA_DIR, \"test\")\n        )\n        self._augment = augment\n        self._normalize = normalize\n        self._size = size\n        self._transform_list = [\n            # A.Resize(size[0], size[1], p=1)\n        ]\n\n        if self._augment:\n            self._transform_list.extend(\n                [\n                    A.VerticalFlip(p=0.5),\n                    A.HorizontalFlip(p=0.5),\n                    A.ShiftScaleRotate(\n                        scale_limit=0.20,\n                        rotate_limit=10,\n                        shift_limit=0.1,\n                        p=0.5,\n                        border_mode=cv2.BORDER_CONSTANT,\n                        value=0,\n                    ),\n                    A.RandomBrightnessContrast(p=0.5),\n                    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n                    ToTensorV2(),\n                ]\n            )\n        elif self._normalize and not self._augment:  # test mode\n            self._transform_list.extend(\n                [\n                    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n                    ToTensorV2(),\n                ]\n            )\n        self._transforms = A.Compose(self._transform_list)\n\n    def __len__(self):\n        return len(self.study_ids)\n\n    def assign_label(self, row):\n        for k in self.name_to_label_map:\n            if row[k]:\n                return self.name_to_label_map[k]\n\n    def __getitem__(self, idx):\n        study_id = self.study_ids[idx]\n        study_imgs = self.df.loc[study_id]\n        path = None\n        label = None\n\n        if len(study_imgs.shape) == 1:\n            path = study_imgs[\"path\"]\n            label = study_imgs[\"int_label\"]\n        else:\n            row = study_imgs.sample(1).loc[study_id]\n            path = row[\"path\"]\n            label = row[\"int_label\"]\n\n        # ideally, we'd clean up the df,\n        # but may be we use it to produce predictions as well.\n        dicom_arr = (\n            cv2.imread(path)\n            if path.endswith(\".jpg\")\n            else dicom2array(path, size=self._size)\n        )\n        img = cv2.cvtColor(dicom_arr, cv2.COLOR_BGR2RGB)\n        img = self._transforms(image=img)[\"image\"]\n\n        return img, label","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.699324Z","iopub.execute_input":"2021-07-22T12:56:37.699861Z","iopub.status.idle":"2021-07-22T12:56:37.717424Z","shell.execute_reply.started":"2021-07-22T12:56:37.699688Z","shell.execute_reply":"2021-07-22T12:56:37.716625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter:\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.718902Z","iopub.execute_input":"2021-07-22T12:56:37.719304Z","iopub.status.idle":"2021-07-22T12:56:37.7274Z","shell.execute_reply.started":"2021-07-22T12:56:37.719268Z","shell.execute_reply":"2021-07-22T12:56:37.726507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(\n    epoch,\n    step,\n    dataloader,\n    batchsize,\n    model,\n    optimizer,\n    loss_fn,\n    log_every=10,\n    scaler=None,\n):\n\n    steps = len(dataloader)\n    batchsize = batchsize\n    dataiter = iter(dataloader)\n\n    time_now = time.time()\n\n    loss_avg = AverageMeter()\n    acc_avg = AverageMeter()\n\n    model.train()\n    loader = tqdm(range(steps))\n\n    enable_autocast = not (scaler == None)\n\n    for i in loader:\n        optimizer.zero_grad()\n        data, targets = next(dataiter)\n        data = data.to(dev)\n        targets = targets.to(dev)\n\n        with amp.autocast(enabled=enable_autocast):\n            output = model(data)\n            loss = loss_fn(output, targets)\n\n        if enable_autocast:\n            scaled_loss = scaler.scale(loss)\n            scaled_loss.backward()\n            loss_avg.update(scaled_loss.item(), batchsize)\n        else:\n            loss_avg.update(loss.item(), batchsize)\n            loss.backward()\n\n        if scaler:\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            optimizer.step()\n\n        time_spent = time.time() - time_now\n        time_now = time.time()\n        preds = output.argmax(axis=1)\n        acc = (preds == targets).sum().item() / batchsize\n\n        acc_avg.update(acc, batchsize)\n        #         if step % log_every == 0:\n        #             print(\"{}, Epoch : {}, Step : {}, Training Loss : {:.5f}, Run Time : {:.5g}\"\n        #                   .format(time.strftime(\"%Y-%m-%d %H:%M:%S\"), epoch, step, loss.item(), time_spent))\n        loader.set_description(\n            \"Training Epoch : {}, Time Spent {:.5g}, Step {}\".format(\n                epoch, time_spent, step\n            )\n        )\n        loader.set_postfix(loss=loss_avg.avg, acc=acc_avg.avg)\n\n        step += 1\n\n    return acc_avg.avg, loss_avg.avg, step","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.72894Z","iopub.execute_input":"2021-07-22T12:56:37.729574Z","iopub.status.idle":"2021-07-22T12:56:37.741741Z","shell.execute_reply.started":"2021-07-22T12:56:37.729539Z","shell.execute_reply":"2021-07-22T12:56:37.740743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apk(actual, predicted, k=10):\n    if len(predicted) > k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\n\ndef mean_average_precision(actual, predicted, k=10):\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.743362Z","iopub.execute_input":"2021-07-22T12:56:37.743775Z","iopub.status.idle":"2021-07-22T12:56:37.75329Z","shell.execute_reply.started":"2021-07-22T12:56:37.743713Z","shell.execute_reply":"2021-07-22T12:56:37.752276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_epoch(epoch, step, dataloader, batchsize, model, loss_fn):\n    num_val = len(dataloader)\n    probs_list = []\n    targets_list = []\n    dataiter = iter(dataloader)\n    model.eval()\n\n    loss_avg = AverageMeter()\n    acc_avg = AverageMeter()\n\n    with torch.no_grad():\n        for i in tqdm(range(num_val)):\n            data, targets = next(dataiter)\n            data = data.to(dev)\n            targets = targets.to(dev)\n\n            outputs = model(data)\n            loss = loss_fn(outputs, targets)\n\n            probs = outputs.softmax(axis=1)\n\n            loss_avg.update(loss.item(), batchsize)\n\n            probs_list.append(probs)\n            targets_list.append(targets)\n\n        probs = torch.cat(probs_list).cpu().numpy()\n        targets = torch.cat(targets_list).cpu().numpy()\n\n    print(classification_report(targets, np.argmax(probs, axis=1)))\n    print(confusion_matrix(targets, np.argmax(probs, axis=1)))\n    auc = roc_auc_score(targets, probs, average=\"macro\", multi_class=\"ovo\")\n\n    topk = (-probs).argsort(axis=1)[:, :2]\n    mapk = mean_average_precision(targets[:, np.newaxis].tolist(), topk.tolist(), k=2)\n    print(\"MAP@2 Score at Epoch {} and Step {}: {}\".format(epoch, step, mapk))\n    print(\"AUC Score at Epoch {} and Step {}: {}\".format(epoch, step, auc))\n    return mapk, auc, loss_avg.avg, probs, targets","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.754771Z","iopub.execute_input":"2021-07-22T12:56:37.755282Z","iopub.status.idle":"2021-07-22T12:56:37.768051Z","shell.execute_reply.started":"2021-07-22T12:56:37.755243Z","shell.execute_reply":"2021-07-22T12:56:37.767106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(\n    model,\n    loss_fn,\n    epochs,\n    batchsize,\n    optimizer,\n    scheduler,\n    save_path,\n    train_dl,\n    validation_dl,\n    use_mp=True,\n):\n    if use_mp:\n        scaler = amp.GradScaler()\n    else:\n        scaler = None\n    if not os.path.exists(save_path):\n        os.mkdir(save_path)\n\n    train_vs_val = []\n\n    loss_valid_best = float(\"inf\")\n    mapk_valid_best = -float(\"inf\")\n    auc_valid_best = -float(\"inf\")\n    step = 0\n\n    for epoch in range(epochs):\n\n        train_acc_avg, train_loss_avg, step = train_epoch(\n            epoch, step, train_dl, batchsize, model, optimizer, loss_fn, scaler=scaler\n        )\n\n        time_now = time.time()\n        val_mapk, val_auc, val_loss_avg, probs, targets = valid_epoch(\n            epoch, step, validation_dl, batchsize * 2, model, loss_fn\n        )\n        time_spent = time.time() - time_now\n        train_vs_val.extend(\n            [\n                (epoch, val_loss_avg, \"Validation Loss\"),\n                (epoch, val_mapk, \"Validation MAP@2\"),\n                (epoch, val_auc, \"Validation AUC\"),\n                (epoch, train_loss_avg, \"Training Loss\"),\n                (epoch, train_acc_avg, \"Training Accuracy\"),\n            ]\n        )\n\n        if scheduler:\n            scheduler.step(val_loss_avg)\n            print(\n                \"Setting Learning Rate to: {:.6f}\".format(\n                    optimizer.param_groups[-1][\"lr\"]\n                )\n            )\n\n        if mapk_valid_best < val_mapk:\n            print(\"Found Model with best Map@2 {} at epoch {}\".format(val_mapk, epoch))\n            torch.save(\n                {\n                    \"epoch\": epoch,\n                    \"map_at_2\": val_mapk,\n                    \"auc\": val_auc,\n                    \"probs\": probs,\n                    \"targets\": targets,\n                    \"state_dict\": model.module.state_dict(),\n                },\n                os.path.join(save_path, \"best_map_at_2.pth\"),\n            )\n            mapk_valid_best = val_mapk\n        if auc_valid_best < val_auc:\n            print(\"Found Model with best AUC {} at epoch {}\".format(val_auc, epoch))\n            torch.save(\n                {\n                    \"epoch\": epoch,\n                    \"map_at_2\": val_mapk,\n                    \"auc\": val_auc,\n                    \"probs\": probs,\n                    \"targets\": targets,\n                    \"state_dict\": model.module.state_dict(),\n                },\n                os.path.join(save_path, \"best_auc.pth\"),\n            )\n            auc_valid_best = val_auc\n\n        print(\n            \"{}, Epoch : {}, Step : {}, Validation Loss : {:.5f}, Run Time : {:.5g}\".format(\n                time.strftime(\"%Y-%m-%d %H:%M:%S\"),\n                epoch,\n                step,\n                val_loss_avg,\n                time_spent,\n            )\n        )\n\n    return train_vs_val","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.769485Z","iopub.execute_input":"2021-07-22T12:56:37.769872Z","iopub.status.idle":"2021-07-22T12:56:37.785849Z","shell.execute_reply.started":"2021-07-22T12:56:37.769835Z","shell.execute_reply":"2021-07-22T12:56:37.785003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_lr(\n    model,\n    optimizer,\n    training_dl,\n    batch_size,\n    loss_fn,\n    init_value=1e-8,\n    final_value=10,\n    beta=0.98,\n    update_every=10,\n    num_epochs=3,\n    use_mp=True,\n):\n    if use_mp:\n        scaler = amp.GradScaler()\n    else:\n        scaler = None\n    enable_autocast = not (scaler == None)\n\n    num_samples = num_epochs * len(training_dl) - 1\n    print(num_samples)\n    multiplier = (final_value / init_value) ** (1 / num_samples)\n    lr = init_value\n\n    for group in optimizer.param_groups:\n        group[\"lr\"] = lr\n\n    avg_loss = 0\n    best_loss = 0\n    batch_num = 0\n\n    losses = []\n    log_lrs = []\n    smoothed_losses = []\n    for epoch in range(num_epochs):\n        for x_pos in training_dl:\n            optimizer.zero_grad()\n            batch_num += 1\n            pos_img, pos_target = x_pos\n\n            idx_rand = torch.randperm(batch_size, requires_grad=False).to(dev)\n            imgs = pos_img[idx_rand]\n            target = pos_target[idx_rand]\n            imgs = imgs.to(dev)\n            target = target.to(dev)\n\n            if enable_autocast:\n                scaled_loss = scaler.scale(loss)\n                scaled_loss.backward()\n                loss_avg.update(scaled_loss.item(), batchsize)\n            else:\n                loss_avg.update(loss.item(), batchsize)\n                loss.backward()\n\n            if scaler:\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                optimizer.step()\n            # Smoothed Loss Computation\n            avg_loss = beta * avg_loss + (1 - beta) * loss.item()\n            smoothed_loss = avg_loss / (1 - beta ** batch_num)\n            # Stop if the loss is exploding\n            if batch_num > 1 and smoothed_loss > 2 * best_loss:\n                print(\"boom\")\n                return log_lrs, losses, smoothed_losses\n            # Record the best loss\n            if smoothed_loss < best_loss or batch_num == 1:\n                best_loss = smoothed_loss\n\n            # Store values\n\n            losses.append(loss.detach().cpu())\n            log_lrs.append(np.log10(lr))\n            smoothed_losses.append(smoothed_loss)\n\n            with torch.no_grad():\n                lr *= multiplier\n                for group in optimizer.param_groups:\n                    group[\"lr\"] = lr\n            if epoch * len(training_dl) // batch_size + batch_num % update_every == 0:\n                print(\"Setting LR to be: {:.5g}\".format(lr))\n    return log_lrs, losses, smoothed_losses","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.787362Z","iopub.execute_input":"2021-07-22T12:56:37.787724Z","iopub.status.idle":"2021-07-22T12:56:37.803709Z","shell.execute_reply.started":"2021-07-22T12:56:37.787688Z","shell.execute_reply":"2021-07-22T12:56:37.802824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = timm.create_model(model_name=\"tf_efficientnetv2_s\", pretrained=True, in_chans=3)\n# model.classifier = torch.nn.Linear(in_features=model.classifier.in_features, out_features=NUM_CLASSES)\n\n# model = timm.create_model(\n#     model_name=\"vit_small_r26_s32_384\", pretrained=True, in_chans=3\n# )\n# model.head = torch.nn.Linear(\n#     in_features=model.head.in_features, out_features=NUM_CLASSES\n# )\n\n# model = torch.nn.DataParallel(model)\n# model = model.to(dev)\n# optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=1e-2)\n# # optimizer = torch.optim.SGD(model.parameters(), lr=3e-4, momentum=0.9, weight_decay=1e-3)\n# loss_fn = torch.nn.CrossEntropyLoss().to(dev)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.805152Z","iopub.execute_input":"2021-07-22T12:56:37.805554Z","iopub.status.idle":"2021-07-22T12:56:37.812534Z","shell.execute_reply.started":"2021-07-22T12:56:37.80552Z","shell.execute_reply":"2021-07-22T12:56:37.811725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_ds = XRayDatasetFromDF(train_samples_df, augment=True)\n# training_dl = torch.utils.data.DataLoader(dataset=training_ds,\n#                                           batch_size=BATCHSIZE,\n#                                           pin_memory=True,\n#                                           num_workers=8,\n#                                           drop_last=True,\n#                                           shuffle=True,\n#                                           prefetch_factor=8)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.81383Z","iopub.execute_input":"2021-07-22T12:56:37.81418Z","iopub.status.idle":"2021-07-22T12:56:37.820295Z","shell.execute_reply.started":"2021-07-22T12:56:37.814145Z","shell.execute_reply":"2021-07-22T12:56:37.819319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# log_lrs, losses, smoothed_losses = find_lr(model, optimizer, training_dl, BATCHSIZE, loss_fn)","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.821754Z","iopub.execute_input":"2021-07-22T12:56:37.82214Z","iopub.status.idle":"2021-07-22T12:56:37.828737Z","shell.execute_reply.started":"2021-07-22T12:56:37.822103Z","shell.execute_reply":"2021-07-22T12:56:37.827922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# i = 0\n# j = 6\n\n# plt.plot(log_lrs[i:-j], smoothed_losses[i:-j], c=\"r\")\n# plt.plot(log_lrs[i:-j], losses[i:-j], c=\"b\")","metadata":{"gradient":{},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.830003Z","iopub.execute_input":"2021-07-22T12:56:37.830474Z","iopub.status.idle":"2021-07-22T12:56:37.837321Z","shell.execute_reply.started":"2021-07-22T12:56:37.830436Z","shell.execute_reply":"2021-07-22T12:56:37.836474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(FOLDS):\n    print(\"Training Fold {}\".format(fold))\n    model = timm.create_model(model_name=MODEL_NAME, pretrained=True, in_chans=3)\n    model.head = torch.nn.Linear(\n        in_features=model.head.in_features, out_features=NUM_CLASSES\n    )\n\n    model = torch.nn.DataParallel(model)\n    model = model.to(dev)\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-2)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer, mode=\"min\", factor=0.1, patience=5, verbose=False, min_lr=1e-7\n    )\n\n    loss_fn = torch.nn.CrossEntropyLoss().to(dev)\n\n    training_ds = XRayDatasetFromDF(\n        train_samples_df[train_samples_df[\"fold\"] != fold], augment=True\n    )\n    validation_ds = XRayDatasetFromDF(\n        train_samples_df[train_samples_df[\"fold\"] == fold],\n        augment=False,\n        normalize=True,\n    )\n    print(\"{} train len {} val len\".format(len(training_ds), len(validation_ds)))\n\n    training_dl = torch.utils.data.DataLoader(\n        dataset=training_ds,\n        batch_size=BATCHSIZE,\n        pin_memory=True,\n        num_workers=8,\n        drop_last=True,\n        shuffle=True,\n        prefetch_factor=6,\n    )\n    validation_dl = torch.utils.data.DataLoader(\n        dataset=validation_ds,\n        batch_size=BATCHSIZE * 2,\n        pin_memory=True,\n        num_workers=8,\n        drop_last=False,\n        prefetch_factor=8,\n    )\n\n    print(\n        \"{} training data loader size {} validation dataloader size\".format(\n            len(training_dl), len(validation_dl)\n        )\n    )\n\n    train_vs_val = train_model(\n        model=model,\n        loss_fn=loss_fn,\n        epochs=20,\n        optimizer=optimizer,\n        scheduler=scheduler,\n        batchsize=BATCHSIZE,\n        save_path=\"{}-{}\".format(MODEL_NAME, fold),\n        train_dl=training_dl,\n        validation_dl=validation_dl,\n    )\n\n    fold_report = pd.DataFrame.from_records(\n        data=train_vs_val, columns=[\"Epoch\", \"Loss\", \"Type\"]\n    )\n    sns.lineplot(data=fold_report, x=\"Epoch\", y=\"Loss\", hue=\"Type\")","metadata":{"gradient":{"editing":false},"execution":{"iopub.status.busy":"2021-07-22T12:56:37.838767Z","iopub.execute_input":"2021-07-22T12:56:37.839209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}