{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":26680,"databundleVersionId":2283525,"sourceType":"competition"},{"sourceId":2338117,"sourceType":"datasetVersion","datasetId":1411570},{"sourceId":63354367,"sourceType":"kernelVersion"}],"dockerImageVersionId":30097,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, math\nimport psutil, random \n\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n\nimport cv2; print(cv2.__version__)\nimport tensorflow as tf; print(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T21:16:15.750191Z","iopub.execute_input":"2021-07-16T21:16:15.750646Z","iopub.status.idle":"2021-07-16T21:16:20.918537Z","shell.execute_reply.started":"2021-07-16T21:16:15.750552Z","shell.execute_reply":"2021-07-16T21:16:20.91716Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MIXED_PRECISION = True\nXLA_ACCELERATE  = False\n\nGPUS = tf.config.experimental.list_physical_devices('GPU')\nif GPUS:\n    try:\n        for GPU in GPUS:\n            tf.config.experimental.set_memory_growth(GPU, True)\n            logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n            print(len(GPUS), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\") \n    except RuntimeError as  RE:\n        print(RE)\n\nif MIXED_PRECISION:\n    policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    tf.keras.mixed_precision.experimental.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')\n    \nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T21:16:20.920259Z","iopub.execute_input":"2021-07-16T21:16:20.920597Z","iopub.status.idle":"2021-07-16T21:16:23.017705Z","shell.execute_reply.started":"2021-07-16T21:16:20.920561Z","shell.execute_reply":"2021-07-16T21:16:23.015611Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocess","metadata":{}},{"cell_type":"code","source":"study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv'); print(study_df.shape)\nstudy_df['StudyInstanceUID'] = study_df['id'].apply(lambda x: x.replace('_study', ''))\ndel study_df['id']\n\ndef hot_to_sparse(row):\n    return(row.index[row.apply(lambda x: x==1)][0])\nstudy_df['diagnosis'] = study_df.apply(lambda row:hot_to_sparse(row), axis=1)\ncls = {\n    'Typical Appearance':1,                    \n    'Negative for Pneumonia':2,                \n    'Indeterminate Appearance':3,                     \n    'Atypical Appearance':4,    \n}\nstudy_df['sparse_gt'] = study_df.diagnosis.map(cls) \n\nimage_df = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv'); print(image_df.shape)\ntrain = image_df.merge(study_df, on='StudyInstanceUID')\ntrain['id'] = train['id'].apply(lambda x: x.replace('_image', ''))\ndisplay(train.head()); print(train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T21:16:23.019808Z","iopub.execute_input":"2021-07-16T21:16:23.020176Z","iopub.status.idle":"2021-07-16T21:16:24.071756Z","shell.execute_reply.started":"2021-07-16T21:16:23.020134Z","shell.execute_reply":"2021-07-16T21:16:24.070784Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class2 = pd.read_csv('../input/siim-cov19-csv-2class/train.csv')\n\nclass2['id'] = class2['id'].apply(lambda x: x.replace('_image', ''))\nclass2['sparse_gt'] = train['sparse_gt']\ntrain = class2.copy()\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-07-16T21:18:04.590007Z","iopub.execute_input":"2021-07-16T21:18:04.590411Z","iopub.status.idle":"2021-07-16T21:18:04.633905Z","shell.execute_reply.started":"2021-07-16T21:18:04.590375Z","shell.execute_reply":"2021-07-16T21:18:04.632824Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ROI Segment: Cropped Bounding Box","metadata":{}},{"cell_type":"code","source":"def vis(path1, path2, n_images, is_random=True, figsize=(16, 16)):\n    '''\n    https://gist.github.com/innat/00de7561033ba373745d425c6da7bf8c\n    '''\n    image_names = os.listdir(path1)\n    masks_names = os.listdir(path2)\n    \n    for i in range(n_images):\n        if is_random:\n            image_name = random.choice(masks_names)\n            masks_name = image_name\n        else:\n            image_name = masks_names[i]\n            masks_name = masks_names[i]\n            \n        img = cv2.resize(cv2.imread(os.path.join(path1, image_name)), (512, 512))\n        msk = cv2.resize(cv2.imread(os.path.join(path2, masks_name)), (512, 512))\n        \n        plt.figure(figsize=(20,20))\n        plt.subplot(121); plt.imshow(img);\n        plt.subplot(122); plt.imshow(msk);\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:49.763147Z","iopub.execute_input":"2021-07-10T00:28:49.763512Z","iopub.status.idle":"2021-07-10T00:28:49.773017Z","shell.execute_reply.started":"2021-07-10T00:28:49.763475Z","shell.execute_reply":"2021-07-10T00:28:49.772116Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_path = '../input/covid19-detection-890pxpng-study'\nTRAIN_IMG_PATH =  os.path.join(base_path, 'train/')\nTRAIN_MSK_PATH = os.path.join(base_path, 'ROI Mask/')\n\nvis(TRAIN_IMG_PATH, TRAIN_MSK_PATH, 5, is_random=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:49.774422Z","iopub.execute_input":"2021-07-10T00:28:49.774822Z","iopub.status.idle":"2021-07-10T00:28:52.491028Z","shell.execute_reply.started":"2021-07-10T00:28:49.774759Z","shell.execute_reply":"2021-07-10T00:28:52.490081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=3, shuffle=True, random_state=101)\nfor index, (train_index, val_index) in enumerate(skf.split(X=train.index, y=train.sparse_gt)):\n    train.loc[val_index, 'fold'] = index\n    \nprint(train.groupby(['fold', train.sparse_gt]).size())","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:52.492433Z","iopub.execute_input":"2021-07-10T00:28:52.492763Z","iopub.status.idle":"2021-07-10T00:28:53.192067Z","shell.execute_reply.started":"2021-07-10T00:28:52.49273Z","shell.execute_reply":"2021-07-10T00:28:53.191201Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Generator","metadata":{}},{"cell_type":"code","source":"import albumentations\nfrom albumentations import *\n\n# For Validation \ndef albu_transforms_train(data_resize): \n    return albumentations.Compose([\n           albumentations.Resize(data_resize, data_resize),\n           albumentations.RandomResizedCrop(data_resize, data_resize, scale=(0.9, 1), p=1), \n           albumentations.HorizontalFlip(p=0.5),\n           albumentations.ShiftScaleRotate(p=0.5),\n           #albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n           albumentations.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n           albumentations.CLAHE(clip_limit=(1,4), p=0.5),\n     \n          \n       \n          IAAPiecewiseAffine(p=0.2),\n          IAASharpen(p=0.2),\n          #albumentations.Cutout(max_h_size=int(data_resize * 0.1), max_w_size=int(data_resize * 0.1), num_holes=5, p=0.5),\n          #albumentations.Normalize(),\n        ])\n\n\n# For Validation \ndef albu_transforms_valid(data_resize): \n    return albumentations.Compose([\n        albumentations.Resize(data_resize, data_resize)\n        #A.ToFloat(), # no need if use keras.applicaiton.EfficientNets Bx\n        ], p=1.)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:53.193355Z","iopub.execute_input":"2021-07-10T00:28:53.19385Z","iopub.status.idle":"2021-07-10T00:28:54.080126Z","shell.execute_reply.started":"2021-07-10T00:28:53.193807Z","shell.execute_reply":"2021-07-10T00:28:54.079345Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Covid19Generator(tf.keras.utils.Sequence):\n    def __init__(self, img_path, msk_path, data, batch_size, random_state, \n                 idim, mdim, shuffle=True, transform=None, is_train=False):\n        self.idim = idim\n        self.mdim = mdim  \n        self.data = data\n        self.shuffle  = shuffle\n        self.random_state = random_state\n        \n        self.img_path = img_path\n        self.msk_path = msk_path\n        self.is_train = is_train\n        \n        self.augment  = transform\n        self.batch_size = batch_size\n        \n        self.list_idx = data.index.values\n        self.label = self.data[['none']] if self.is_train else np.nan\n        self.on_epoch_end()\n        \n    def __len__(self):\n        return int(np.floor(len(self.list_idx) / self.batch_size))\n    \n    def __getitem__(self, index):\n        batch_idx = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n        idx = [self.list_idx[k] for k in batch_idx]\n        \n        Data = np.zeros((self.batch_size,) + self.idim + (3,), dtype=\"float32\")\n        Mask = np.zeros((self.batch_size,) + self.mdim + (1,), dtype=\"float32\")\n        Target = np.zeros((self.batch_size, 1), dtype = np.float32)\n\n        for i, k in enumerate(idx):\n            # load the image file using cv2\n            image = cv2.imread(self.img_path + self.data['id'][k] + '.png')[:, :, [2, 1, 0]]\n            mask = cv2.imread(self.msk_path + self.data['id'][k] + '.png', 0)\n            \n            try:\n                mask = cv2.resize(mask, self.mdim)[:, :, np.newaxis]\n            except:\n                mask = np.zeros_like(cv2.resize(image[:,:,:1], self.mdim))[:, :, np.newaxis]\n          \n            res = self.augment(image=image)\n            image = res['image']\n            \n            # mask normalization must\n            mask = mask.astype(np.float32)/255.0 \n\n            # assign \n            if self.is_train:\n                Data[i,] = image\n                Mask[i,] = mask\n                Target[i,] = self.label.iloc[k,].values #.values\n            else:\n                Data[i,] =  image \n        \n        inps = {'input': Data}\n        outs = {'clss': Target, 'segg': Mask}\n        return inps, outs\n    \n    def on_epoch_end(self):\n        self.indices = np.arange(len(self.list_idx))\n        if self.shuffle:\n            np.random.seed(self.random_state)\n            np.random.shuffle(self.indices)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:54.082614Z","iopub.execute_input":"2021-07-10T00:28:54.082997Z","iopub.status.idle":"2021-07-10T00:28:54.098695Z","shell.execute_reply.started":"2021-07-10T00:28:54.082963Z","shell.execute_reply":"2021-07-10T00:28:54.097631Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt \nfrom pylab import rcParams\n\n# helper function to plot sample \ndef plot_imgs(dataset_show, row, col):\n    rcParams['figure.figsize'] = 20,10\n    for i in range(row):\n        f, ax = plt.subplots(1,col)\n        for p in range(col):\n            idx = np.random.randint(0, len(dataset_show))\n            img, label = dataset_show[idx]\n            ax[p].grid(False)\n            ax[p].imshow(label['segg'][0], cmap='gray')\n            ax[p].set_title(label['clss'][0])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:54.100316Z","iopub.execute_input":"2021-07-10T00:28:54.100679Z","iopub.status.idle":"2021-07-10T00:28:54.115524Z","shell.execute_reply.started":"2021-07-10T00:28:54.100629Z","shell.execute_reply":"2021-07-10T00:28:54.114738Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fold = 0\nimg_size = 324\nmsk_sizze = 11\nbatch_size = 8\n\ndef count_data_items(length, b_max):\n    batch_size = sorted([int(length/n) for n in range(1, length+1) \\\n                         if length % n == 0 and length/n <= b_max], reverse=True)[0]  \n    steps  = length / batch_size \n    return batch_size, steps\n\ndef fold_generator(fold):\n    # for way one - data generator\n    train_labels = train[train.fold != fold].reset_index(drop=True)\n    val_labels = train[train.fold == fold].reset_index(drop=True)\n\n    train_generator = Covid19Generator(TRAIN_IMG_PATH, TRAIN_MSK_PATH,\n                              train_labels, \n                              batch_size, 1234, (img_size, img_size), (msk_sizze, msk_sizze),\n                              shuffle = True, is_train = True,\n                              transform = albu_transforms_train(img_size))\n    \n    valid_batch, valid_step = count_data_items(len(val_labels), batch_size)\n\n    val_generator = Covid19Generator(TRAIN_IMG_PATH, TRAIN_MSK_PATH,\n                              val_labels, \n                              valid_batch, 1234, (img_size, img_size), (msk_sizze, msk_sizze),\n                              shuffle = False, is_train = True,\n                              transform = albu_transforms_valid(img_size))\n\n    return train_generator, val_generator, train_labels, val_labels, valid_step\n\n\n# first fold \ntrain_gen, val_gen, train_len, val_len, val_step = fold_generator(fold)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:54.11683Z","iopub.execute_input":"2021-07-10T00:28:54.117199Z","iopub.status.idle":"2021-07-10T00:28:54.144505Z","shell.execute_reply.started":"2021-07-10T00:28:54.117147Z","shell.execute_reply":"2021-07-10T00:28:54.143789Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_imgs(train_gen, 5, 4) # plotting only 16x mask","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:28:54.145758Z","iopub.execute_input":"2021-07-10T00:28:54.146101Z","iopub.status.idle":"2021-07-10T00:29:08.544154Z","shell.execute_reply.started":"2021-07-10T00:28:54.146066Z","shell.execute_reply":"2021-07-10T00:29:08.543329Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras import Model \nfrom tensorflow.keras import Sequential \nfrom tensorflow.keras import Input \nfrom tensorflow.keras import layers \nfrom tensorflow.keras import applications \n\nclass CovidNet(Model):\n    def __init__(self):\n        super(CovidNet, self).__init__()\n        self.base = applications.EfficientNetB7(input_shape=(324, 324, 3),\n                                                  include_top=False,\n                                                  weights='imagenet')\n        # desired model \n        self.base = Model(\n                [self.base.inputs], \n                [self.base.get_layer('top_activation').output, self.base.output]\n            )\n        \n        # tail / head for the classifier \n        self.tail = Sequential(\n            [\n                layers.GlobalAveragePooling2D(),\n                layers.Dropout(0.2),\n                #layers.BatchNormalization(),\n                layers.Dense(1,activation='sigmoid'),\n                #layers.Sigmoid()\n            ]\n        )\n        \n        # tail / head for the mask \n        self.msk = Sequential(\n            [\n                layers.Conv2D(filters=324, kernel_size=(1, 1), strides=(1, 1), padding=\"same\"),\n                layers.ReLU(),\n                layers.BatchNormalization(),\n                layers.Conv2D(filters=1, kernel_size=(1,1), padding=\"same\")\n            ]\n        )\n\n    # feed-forwarding  \n    def call(self, inputs, training=None, **kwargs):\n        segg, clss = self.base(inputs['input'])\n\n        return {\n            'clss': self.tail(clss), \n            'segg': self.msk(segg)\n        }\n    \n\ntf.keras.backend.clear_session()\nmodel = CovidNet()\nmodel.build(input_shape={'input': (None, 324, 324, 3)})\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:29:47.887167Z","iopub.execute_input":"2021-07-10T00:29:47.887527Z","iopub.status.idle":"2021-07-10T00:29:55.897194Z","shell.execute_reply.started":"2021-07-10T00:29:47.887497Z","shell.execute_reply":"2021-07-10T00:29:55.896368Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules\nfrom tensorflow.keras.optimizers.schedules import LearningRateSchedule, ExponentialDecay\n\nclass WarmupLearningRateSchedule(LearningRateSchedule):\n    \"\"\"Provides a variety of learning rate decay schedules with warm up.\"\"\"\n\n    def __init__(self,\n               initial_lr,\n               steps_per_epoch=None,\n               lr_decay_type='exponential',\n               decay_factor=0.97,\n               decay_epochs=2.4,\n               total_steps=None,\n               warmup_epochs=5,\n               minimal_lr=0):\n        super(WarmupLearningRateSchedule, self).__init__()\n        self.initial_lr = initial_lr\n        self.steps_per_epoch = steps_per_epoch\n        self.lr_decay_type = lr_decay_type\n        self.decay_factor = decay_factor\n        self.decay_epochs = decay_epochs\n        self.total_steps = total_steps\n        self.warmup_epochs = warmup_epochs\n        self.minimal_lr = minimal_lr\n\n    def __call__(self, step):\n        if self.lr_decay_type == 'exponential':\n            assert self.steps_per_epoch is not None\n            decay_steps = self.steps_per_epoch * self.decay_epochs\n            lr = ExponentialDecay(self.initial_lr, decay_steps, \n                                  self.decay_factor, staircase=True)(step)\n        elif self.lr_decay_type == 'cosine':\n            assert self.total_steps is not None\n            lr = 0.5 * self.initial_lr * (\n              1 + tf.cos(np.pi * tf.cast(step, tf.float32) / self.total_steps))\n            \n        elif self.lr_decay_type == 'linear':\n            assert self.total_steps is not None\n            lr = (1.0 - tf.cast(step, tf.float32) / self.total_steps) * self.initial_lr\n        elif self.lr_decay_type == 'constant':\n            lr = self.initial_lr\n        else:\n            assert False, 'Unknown lr_decay_type : %s' % self.lr_decay_type\n\n        if self.minimal_lr:\n            lr = tf.math.maximum(lr, self.minimal_lr)\n\n        if self.warmup_epochs:\n            warmup_steps = int(self.warmup_epochs * self.steps_per_epoch)\n            warmup_lr = (\n              self.initial_lr * tf.cast(step, tf.float32) /\n              tf.cast(warmup_steps, tf.float32))\n            lr = tf.cond(step < warmup_steps, lambda: warmup_lr, lambda: lr)\n\n        return lr\n\n    def get_config(self):\n        return {\n            'initial_lr': self.initial_lr,\n            'steps_per_epoch': self.steps_per_epoch,\n            'lr_decay_type': self.lr_decay_type,\n            'decay_factor': self.decay_factor,\n            'decay_epochs': self.decay_epochs,\n            'total_steps': self.total_steps,\n            'warmup_epochs': self.warmup_epochs,\n            'minimal_lr': self.minimal_lr,\n        }","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:29:17.088561Z","iopub.status.idle":"2021-07-10T00:29:17.08894Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"steps_per_epoch  = np.ceil(float(len(train_len)) / batch_size) \nvalidation_steps = val_step \nepochs = 20\n\nlr_sched = 'cosine'\nlr_base = 0.003\nlr_min=1e-6\nlr_decay_epoch = 2.4\nlr_warmup_epoch = 5\nlr_decay_factor = 0.97\n\nscaled_lr = lr_base * (batch_size / 256.0)\nscaled_lr_min = lr_min * (batch_size / 256.0)\ntotal_steps = steps_per_epoch * epochs\n\nlearning_rate = WarmupLearningRateSchedule(\n    scaled_lr,\n    steps_per_epoch=steps_per_epoch,\n    decay_epochs=lr_decay_epoch,\n    warmup_epochs=lr_warmup_epoch,\n    decay_factor=lr_decay_factor,\n    lr_decay_type=lr_sched,\n    total_steps=total_steps,\n    minimal_lr=scaled_lr_min)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:29:17.089888Z","iopub.status.idle":"2021-07-10T00:29:17.090533Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:29:17.091647Z","iopub.status.idle":"2021-07-10T00:29:17.092238Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras import losses \nfrom tensorflow.keras import metrics\nfrom tensorflow.keras import optimizers\n\n# bind all\nmodel.compile(\n    loss = {\n        'clss': losses.CategoricalCrossentropy(\n            label_smoothing=0, from_logits=False),\n        'segg': losses.BinaryCrossentropy(from_logits=True)\n    },\n    \n    metrics = {\n        'clss': [\n            metrics.AUC(curve='ROC', multi_label=True),\n            metrics.SpecificityAtSensitivity(0.60, name='@sensitivity')\n        ]\n    },\n    \n    optimizer = optimizers.Adam()\n)\n\n# list of call backs \nfrom tensorflow.keras import callbacks\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n        f'model_{fold}.h5', save_best_only=True, monitor='val_loss', mode='min')\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')\n\n# fitter \nmodel.fit(train_gen, \n          steps_per_epoch=steps_per_epoch,\n          validation_data=val_gen, \n          validation_steps=validation_steps,\n          callbacks=[checkpoint, lr_reducer], \n          workers=psutil.cpu_count(), verbose=2,\n          epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T00:29:17.093356Z","iopub.status.idle":"2021-07-10T00:29:17.093896Z"},"trusted":true},"outputs":[],"execution_count":null}]}