{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Densenet** \n* max_lr = 0.0631\n* momentum = 0.9\n<br/>\n\n**CNBM**\n* max_lr = 0.0316\n* momentum = 0.99\n<br/>\n\n**seresnet**\n* max_lr = 0.02\n* momentum = 0.95\n"},{"metadata":{"_uuid":"f0d95b709e9c79a2fd7ab085be1b90e4044d1688"},"cell_type":"markdown","source":"### One Cycle Policy with Keras\nHighly inspired by following paper:\n- [A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay](http://arxiv.org/abs/1803.09820) (Leslie N. Smith)\n\nI have implemented the One Cycle Policy algorithm developed by Leslie N. Smith into the Keras Callback class. Leslie Smith suggests in this paper a slight modification of cyclical learning rate policy for super convergence using one cycle that is smaller than the total number of iterations/epochs and allow the learning rate todecrease several orders of magnitude less than the initial learning rate for the remaining miterations. In his experiments this policy allows the accuracy to plateau before the training ends. This approach (among others) helped me to improve my score."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"from sklearn.utils import shuffle\nimport pandas as pd\nimport os\n# Save train labels to dataframe\ndf = pd.read_csv(\"../input/train_labels.csv\")\n\n# Save test labels to dataframe\n\n\ndf = shuffle(df)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# For demonstration only\n#df = df[:10000]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"ef9a6aee05dec5ec4ba97fd5e8d2724ff4f743ae","scrolled":true,"trusted":true},"cell_type":"code","source":"# Split data set  to train and validation sets\nfrom sklearn.model_selection import train_test_split\n\n# Use stratify= df['label'] to get balance ratio 1/1 in train and validation sets\ndf_train, df_val = train_test_split(df, test_size=0.1, stratify= df['label'])\n\n# Check balancing\nprint(\"Train data: \" + str(len(df_train[df_train[\"label\"] == 1]) + len(df_train[df_train[\"label\"] == 0])))\nprint(\"True positive in train data: \" +  str(len(df_train[df_train[\"label\"] == 1])))\nprint(\"True negative in train data: \" +  str(len(df_train[df_train[\"label\"] == 0])))\nprint(\"Valid data: \" + str(len(df_val[df_val[\"label\"] == 1]) + len(df_val[df_val[\"label\"] == 0])))\nprint(\"True positive in validation data: \" +  str(len(df_val[df_val[\"label\"] == 1])))\nprint(\"True negative in validation data: \" +  str(len(df_val[df_val[\"label\"] == 0])))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"89f0b8d6439c876464b9e44062852a9eb747426f","trusted":true},"cell_type":"code","source":"# Train List\ntrain_list = df_train['id'].tolist()\ntrain_list = ['../input/train/'+ name + \".tif\" for name in train_list]\n\n# Validation List\nval_list = df_val['id'].tolist()\nval_list = ['../input/train/'+ name + \".tif\" for name in val_list]\n\n\n# Names library\nid_label_map = {k:v for k,v in zip(df.id.values, df.label.values)}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"5d5639a7f9ebeb8fa044f5743bb6237d73f7fe09","trusted":true},"cell_type":"code","source":"# Functions for generators\ndef get_id_from_path(file_path):\n    return file_path.split(os.path.sep)[-1].replace('.tif', '')\n\ndef chunker(seq, size):\n    return (seq[pos:pos + size] for pos in range(0, len(seq), size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install albumentations\nimport albumentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_list = df['id'].tolist()\n# train_list = ['../input/train/'+ name + \".tif\" for name in train_list]\n# id_label_map = {k:v for k,v in zip(df.id.values, df.label.values)}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"94bfd61b2dfcd0f4616e47ede87ab9ddb1a2f83e","trusted":true},"cell_type":"code","source":"# Import Pretrained Models\nimport keras\nfrom keras.applications.densenet import DenseNet201, preprocess_input\nfrom keras.layers import Dense, Input, Dropout, MaxPooling2D, Concatenate, GlobalAveragePooling2D, GlobalMaxPooling2D, Flatten, Concatenate\nfrom keras.models import Model\nimport pandas as pd\nfrom random import shuffle\nimport numpy as np\nimport cv2\nimport glob\nimport gc\nimport os\nimport tensorflow as tf\nfrom keras.regularizers import l2\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Dense, Dropout, Flatten, Activation, Input, BatchNormalization, Add, GlobalAveragePooling2D,AveragePooling2D,GlobalMaxPooling2D,concatenate\nfrom keras.layers import Lambda, Reshape, DepthwiseConv2D, ZeroPadding2D, Add, MaxPooling2D,Activation, Flatten, Conv2D, Dense, Input, Dropout, Concatenate, GlobalMaxPooling2D, GlobalAveragePooling2D, BatchNormalization\n\nfrom keras.models import Sequential\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint,TensorBoard,TerminateOnNaN\nfrom keras.optimizers import Adam,RMSprop\nfrom keras.models import Model,load_model\nfrom keras.applications import NASNetMobile,MobileNetV2,densenet,resnet50,xception\n\nfrom keras_applications.resnext import ResNeXt50\nfrom albumentations import Resize,Compose, RandomRotate90, Transpose, Flip, OneOf, CLAHE, IAASharpen, IAAEmboss, RandomBrightnessContrast, JpegCompression, Blur, GaussNoise, HueSaturationValue, ShiftScaleRotate, Normalize\n\n\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nfrom skimage import data, exposure\nimport itertools\nimport shutil\nimport matplotlib.pyplot as plt\nos.environ['TF_CPP_MIN_LOG_LEVEL']='2'\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nfrom keras import backend as K\nclass LRFinder(Callback):\n    def __init__(self,\n                 num_samples,\n                 batch_size,\n                 minimum_lr=1e-5,\n                 maximum_lr=10.,\n                 lr_scale='exp',\n                 validation_data=None,\n                 validation_sample_rate=5,\n                 stopping_criterion_factor=4.,\n                 loss_smoothing_beta=0.98,\n                 save_dir=None,\n                 verbose=True):\n        \n        super(LRFinder, self).__init__()\n\n        if lr_scale not in ['exp', 'linear']:\n            raise ValueError(\"`lr_scale` must be one of ['exp', 'linear']\")\n\n        if validation_data is not None:\n            self.validation_data = validation_data\n            self.use_validation_set = True\n\n            if validation_sample_rate > 0 or validation_sample_rate < 0:\n                self.validation_sample_rate = validation_sample_rate\n            else:\n                raise ValueError(\"`validation_sample_rate` must be a positive or negative integer other than o\")\n        else:\n            self.use_validation_set = False\n            self.validation_sample_rate = 0\n\n        self.num_samples = num_samples\n        self.batch_size = batch_size\n        self.initial_lr = minimum_lr\n        self.final_lr = maximum_lr\n        self.lr_scale = lr_scale\n        self.stopping_criterion_factor = stopping_criterion_factor\n        self.loss_smoothing_beta = loss_smoothing_beta\n        self.save_dir = save_dir\n        self.verbose = verbose\n\n        self.num_batches_ = num_samples // batch_size\n        self.current_lr_ = minimum_lr\n\n        if lr_scale == 'exp':\n            self.lr_multiplier_ = (maximum_lr / float(minimum_lr)) ** (\n                1. / float(self.num_batches_))\n        else:\n            extra_batch = int((num_samples % batch_size) != 0)\n            self.lr_multiplier_ = np.linspace(\n                minimum_lr, maximum_lr, num=self.num_batches_ + extra_batch)\n\n        # If negative, use entire validation set\n        if self.validation_sample_rate < 0:\n            self.validation_sample_rate = self.validation_data[0].shape[0] // batch_size\n\n        self.current_batch_ = 0\n        self.current_epoch_ = 0\n        self.best_loss_ = 1e6\n        self.running_loss_ = 0.\n\n        self.history = {}\n\n    def on_train_begin(self, logs=None):\n\n        self.current_epoch_ = 1\n        K.set_value(self.model.optimizer.lr, self.initial_lr)\n\n        warnings.simplefilter(\"ignore\")\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.current_batch_ = 0\n\n        if self.current_epoch_ > 1:\n            warnings.warn(\n                \"\\n\\nLearning rate finder should be used only with a single epoch. \"\n                \"Hereafter, the callback will not measure the losses.\\n\\n\")\n\n    def on_batch_begin(self, batch, logs=None):\n        self.current_batch_ += 1\n\n    def on_batch_end(self, batch, logs=None):\n        if self.current_epoch_ > 1:\n            return\n\n        if self.use_validation_set:\n            X, Y = self.validation_data[0], self.validation_data[1]\n\n            # use 5 random batches from test set for fast approximate of loss\n            num_samples = self.batch_size * self.validation_sample_rate\n\n            if num_samples > X.shape[0]:\n                num_samples = X.shape[0]\n\n            idx = np.random.choice(X.shape[0], num_samples, replace=False)\n            x = X[idx]\n            y = Y[idx]\n\n            values = self.model.evaluate(x, y, batch_size=self.batch_size, verbose=False)\n            loss = values[0]\n        else:\n            loss = logs['loss']\n\n        # smooth the loss value and bias correct\n        running_loss = self.loss_smoothing_beta * loss + (\n            1. - self.loss_smoothing_beta) * loss\n        running_loss = running_loss / (\n            1. - self.loss_smoothing_beta**self.current_batch_)\n\n        # stop logging if loss is too large\n        if self.current_batch_ > 1 and self.stopping_criterion_factor is not None and (\n                running_loss >\n                self.stopping_criterion_factor * self.best_loss_):\n\n            if self.verbose:\n                print(\" - LRFinder: Skipping iteration since loss is %d times as large as best loss (%0.4f)\"\n                      % (self.stopping_criterion_factor, self.best_loss_))\n            return\n\n        if running_loss < self.best_loss_ or self.current_batch_ == 1:\n            self.best_loss_ = running_loss\n\n        current_lr = K.get_value(self.model.optimizer.lr)\n\n        self.history.setdefault('running_loss_', []).append(running_loss)\n        if self.lr_scale == 'exp':\n            self.history.setdefault('log_lrs', []).append(np.log10(current_lr))\n        else:\n            self.history.setdefault('log_lrs', []).append(current_lr)\n\n        # compute the lr for the next batch and update the optimizer lr\n        if self.lr_scale == 'exp':\n            current_lr *= self.lr_multiplier_\n        else:\n            current_lr = self.lr_multiplier_[self.current_batch_ - 1]\n\n        K.set_value(self.model.optimizer.lr, current_lr)\n\n        # save the other metrics as well\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n\n        if self.verbose:\n            if self.use_validation_set:\n                print(\" - LRFinder: val_loss: %1.4f - lr = %1.8f \" %\n                      (values[0], current_lr))\n            else:\n                print(\" - LRFinder: lr = %1.8f \" % current_lr)\n\n    def on_epoch_end(self, epoch, logs=None):\n        if self.save_dir is not None and self.current_epoch_ <= 1:\n            if not os.path.exists(self.save_dir):\n                os.makedirs(self.save_dir)\n\n            losses_path = os.path.join(self.save_dir, 'losses.npy')\n            lrs_path = os.path.join(self.save_dir, 'lrs.npy')\n\n            np.save(losses_path, self.losses)\n            np.save(lrs_path, self.lrs)\n\n            if self.verbose:\n                print(\"\\tLR Finder : Saved the losses and learning rate values in path : {%s}\"\n                      % (self.save_dir))\n\n        self.current_epoch_ += 1\n\n        warnings.simplefilter(\"default\")\n\n    def plot_schedule(self, clip_beginning=None, clip_endding=None):\n        \"\"\"\n        Plots the schedule from the callback itself.\n        # Arguments:\n            clip_beginning: Integer or None. If positive integer, it will\n                remove the specified portion of the loss graph to remove the large\n                loss values in the beginning of the graph.\n            clip_endding: Integer or None. If negative integer, it will\n                remove the specified portion of the ending of the loss graph to\n                remove the sharp increase in the loss values at high learning rates.\n        \"\"\"\n        try:\n            import matplotlib.pyplot as plt\n            plt.style.use('seaborn-white')\n        except ImportError:\n            print(\n                \"Matplotlib not found. Please use `pip install matplotlib` first.\"\n            )\n            return\n\n        if clip_beginning is not None and clip_beginning < 0:\n            clip_beginning = -clip_beginning\n\n        if clip_endding is not None and clip_endding > 0:\n            clip_endding = -clip_endding\n\n        losses = self.losses\n        lrs = self.lrs\n\n        if clip_beginning:\n            losses = losses[clip_beginning:]\n            lrs = lrs[clip_beginning:]\n\n        if clip_endding:\n            losses = losses[:clip_endding]\n            lrs = lrs[:clip_endding]\n\n        plt.plot(lrs, losses)\n        plt.title('Learning rate vs Loss')\n        plt.xlabel('learning rate')\n        plt.ylabel('loss')\n        plt.show()\n\n    @classmethod\n    def restore_schedule_from_dir(cls,\n                                  directory,\n                                  clip_beginning=None,\n                                  clip_endding=None):\n        \"\"\"\n        Loads the training history from the saved numpy files in the given directory.\n        # Arguments:\n            directory: String. Path to the directory where the serialized numpy\n                arrays of the loss and learning rates are saved.\n            clip_beginning: Integer or None. If positive integer, it will\n                remove the specified portion of the loss graph to remove the large\n                loss values in the beginning of the graph.\n            clip_endding: Integer or None. If negative integer, it will\n                remove the specified portion of the ending of the loss graph to\n                remove the sharp increase in the loss values at high learning rates.\n        Returns:\n            tuple of (losses, learning rates)\n        \"\"\"\n        if clip_beginning is not None and clip_beginning < 0:\n            clip_beginning = -clip_beginning\n\n        if clip_endding is not None and clip_endding > 0:\n            clip_endding = -clip_endding\n\n        losses_path = os.path.join(directory, 'losses.npy')\n        lrs_path = os.path.join(directory, 'lrs.npy')\n\n        if not os.path.exists(losses_path) or not os.path.exists(lrs_path):\n            print(\"%s and %s could not be found at directory : {%s}\" %\n                  (losses_path, lrs_path, directory))\n\n            losses = None\n            lrs = None\n\n        else:\n            losses = np.load(losses_path)\n            lrs = np.load(lrs_path)\n\n            if clip_beginning:\n                losses = losses[clip_beginning:]\n                lrs = lrs[clip_beginning:]\n\n            if clip_endding:\n                losses = losses[:clip_endding]\n                lrs = lrs[:clip_endding]\n\n        return losses, lrs\n\n    @classmethod\n    def plot_schedule_from_file(cls,\n                                directory,\n                                clip_beginning=None,\n                                clip_endding=None):\n        \"\"\"\n        Plots the schedule from the saved numpy arrays of the loss and learning\n        rate values in the specified directory.\n        # Arguments:\n            directory: String. Path to the directory where the serialized numpy\n                arrays of the loss and learning rates are saved.\n            clip_beginning: Integer or None. If positive integer, it will\n                remove the specified portion of the loss graph to remove the large\n                loss values in the beginning of the graph.\n            clip_endding: Integer or None. If negative integer, it will\n                remove the specified portion of the ending of the loss graph to\n                remove the sharp increase in the loss values at high learning rates.\n        \"\"\"\n        try:\n            import matplotlib.pyplot as plt\n            plt.style.use('seaborn-white')\n        except ImportError:\n            print(\"Matplotlib not found. Please use `pip install matplotlib` first.\")\n            return\n\n        losses, lrs = cls.restore_schedule_from_dir(\n            directory,\n            clip_beginning=clip_beginning,\n            clip_endding=clip_endding)\n\n        if losses is None or lrs is None:\n            return\n        else:\n            plt.plot(lrs, losses)\n            plt.title('Learning rate vs Loss')\n            plt.xlabel('learning rate')\n            plt.ylabel('loss')\n            plt.show()\n\n    @property\n    def lrs(self):\n        return np.array(self.history['log_lrs'])\n\n    @property\n    def losses(self):\n        return np.array(self.history['running_loss_'])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"0a58666bdbd7cb0aa548868a07d34eed44b076d1","trusted":true},"cell_type":"code","source":"def do_train_augmentations():\n    return Compose([\n        #Resize(196,196),\n        RandomRotate90(p=0.5),\n        Transpose(p=0.5),\n        Flip(p=0.5),\n        OneOf([CLAHE(clip_limit=2),\n              IAASharpen(),\n              IAAEmboss(),\n              RandomBrightnessContrast(),\n              JpegCompression(),\n              Blur(),\n              GaussNoise()],\n              p=0.5),\n        HueSaturationValue(p=0.5),\n        ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),\n        Normalize(p=1)])\n\n\ndef do_inference_aug():\n    return Compose([\n       # Resize(196,196),\n        RandomRotate90(p=0.5),\n        Transpose(p=0.5),\n        Flip(p=0.5),Normalize(p=1)])\n\n\ndef data_gen(list_files,id_label_map,batch_size,aug_func):\n    aug = aug_func()\n    while True:\n        shuffle(list_files)\n        for block in chunker(list_files,batch_size):\n            x = [aug(image = cv2.imread(addr))['image'] for addr in block]\n            y = [id_label_map[get_id_from_path(addr)] for addr in block]\n            yield np.array(x),np.array(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import BatchNormalization\nfrom keras.layers import Conv2D\nfrom keras.layers import Dense\nfrom keras.layers import GlobalAveragePooling2D\nfrom keras.layers import GlobalMaxPooling2D\nfrom keras.layers import Input\nfrom keras.layers import MaxPool2D\nfrom keras.layers import ReLU\nfrom keras.layers import add\nfrom keras.models import Model\nfrom keras.utils import get_source_inputs\n\nfrom keras import backend as K\nfrom keras_applications.imagenet_utils import _obtain_input_shape\n\n\nfrom keras.layers import Conv2D, AveragePooling2D, UpSampling2D\nfrom keras.layers import add\n\n\ndef initial_octconv(ip, filters, kernel_size=(3, 3), strides=(1, 1),\n                    alpha=0.5, padding='same', dilation=None, bias=False):\n\n    if dilation is None:\n        dilation = (1, 1)\n\n    high_low_filters = int(alpha * filters)\n    high_high_filters = filters - high_low_filters\n\n    if strides[0] > 1:\n        ip = AveragePooling2D()(ip)\n\n    # High path\n    x_high = Conv2D(high_high_filters, kernel_size, padding=padding,\n                    dilation_rate=dilation, use_bias=bias,\n                    kernel_initializer='he_normal')(ip)\n\n    # Low path\n    x_high_low = AveragePooling2D()(ip)\n    x_low = Conv2D(high_low_filters, kernel_size, padding=padding,\n                   dilation_rate=dilation, use_bias=bias,\n                   kernel_initializer='he_normal')(x_high_low)\n\n    return x_high, x_low\n\n\ndef final_octconv(ip_high, ip_low, filters, kernel_size=(3, 3), strides=(1, 1),\n                  padding='same', dilation=None, bias=False):\n\n    if dilation is None:\n        dilation = (1, 1)\n\n    if strides[0] > 1:\n        avg_pool = AveragePooling2D()\n\n        ip_high = avg_pool(ip_high)\n        ip_low = avg_pool(ip_low)\n\n    # High path\n    x_high_high = Conv2D(filters, kernel_size, padding=padding,\n                         dilation_rate=dilation, use_bias=bias,\n                         kernel_initializer='he_normal')(ip_high)\n\n    # Low path\n    x_low_high = Conv2D(filters, kernel_size, padding=padding,\n                        dilation_rate=dilation, use_bias=bias,\n                        kernel_initializer='he_normal')(ip_low)\n\n    x_low_high = UpSampling2D(interpolation='nearest')(x_low_high)\n\n    # Merge paths\n    x = add([x_high_high, x_low_high])\n\n    return x\n\n\ndef octconv_block(ip_high, ip_low, filters, kernel_size=(3, 3), strides=(1, 1),\n                  alpha=0.5, padding='same', dilation=None, bias=False):\n\n    if dilation is None:\n        dilation = (1, 1)\n\n    low_low_filters = high_low_filters = int(alpha * filters)\n    high_high_filters = low_high_filters = filters - low_low_filters\n\n    avg_pool = AveragePooling2D()\n\n    if strides[0] > 1:\n        ip_high = avg_pool(ip_high)\n        ip_low = avg_pool(ip_low)\n\n    # High path\n    x_high_high = Conv2D(high_high_filters, kernel_size, padding=padding,\n                         dilation_rate=dilation, use_bias=bias,\n                         kernel_initializer='he_normal')(ip_high)\n\n    x_low_high = Conv2D(low_high_filters, kernel_size, padding=padding,\n                        dilation_rate=dilation, use_bias=bias,\n                        kernel_initializer='he_normal')(ip_low)\n    x_low_high = UpSampling2D(interpolation='nearest')(x_low_high)\n\n    # Low path\n    x_low_low = Conv2D(low_low_filters, kernel_size, padding=padding,\n                       dilation_rate=dilation, use_bias=bias,\n                       kernel_initializer='he_normal')(ip_low)\n\n    x_high_low = avg_pool(ip_high)\n    x_high_low = Conv2D(high_low_filters, kernel_size, padding=padding,\n                        dilation_rate=dilation, use_bias=bias,\n                        kernel_initializer='he_normal')(x_high_low)\n\n    # Merge paths\n    x_high = add([x_high_high, x_low_high])\n    x_low = add([x_low_low, x_high_low])\n\n    return x_high, x_low\n\n\ndef _conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1),\n                padding='same', bias=False):\n    x = Conv2D(filters, kernel_size, strides=strides, padding=padding, use_bias=bias,\n               kernel_initializer='he_normal')(ip)\n\n    return x\n\n\ndef _conv_bn_relu(ip, filters, kernel_size=(3, 3), strides=(1, 1),\n                  padding='same', bias=False, activation=True):\n\n    channel_axis = 1 if K.image_data_format() == 'channels_first' else -1\n\n    x = _conv_block(ip, filters, kernel_size, strides, padding, bias)\n    x = BatchNormalization(axis=channel_axis)(x)\n    if activation:\n        x = ReLU()(x)\n\n    return x\n\n\ndef _initial_oct_conv_bn_relu(ip, filters, kernel_size=(3, 3), strides=(1, 1),\n                              alpha=0.5, padding='same', dilation=None, bias=False,\n                              activation=True):\n\n    channel_axis = 1 if K.image_data_format() == 'channels_first' else -1\n\n    x_high, x_low = initial_octconv(ip, filters, kernel_size, strides, alpha,\n                                    padding, dilation, bias)\n\n    relu = ReLU()\n    x_high = BatchNormalization(axis=channel_axis)(x_high)\n    if activation:\n        x_high = relu(x_high)\n\n    x_low = BatchNormalization(axis=channel_axis)(x_low)\n    if activation:\n        x_low = relu(x_low)\n\n    return x_high, x_low\n\n\ndef _final_oct_conv_bn_relu(ip_high, ip_low, filters, kernel_size=(3, 3), strides=(1, 1),\n                            padding='same', dilation=None, bias=False, activation=True):\n\n    channel_axis = 1 if K.image_data_format() == 'channels_first' else -1\n\n    x = final_octconv(ip_high, ip_low, filters, kernel_size, strides,\n                      padding, dilation, bias)\n\n    x = BatchNormalization(axis=channel_axis)(x)\n    if activation:\n        x = ReLU()(x)\n\n    return x\n\n\ndef _oct_conv_bn_relu(ip_high, ip_low, filters, kernel_size=(3, 3), strides=(1, 1),\n                      alpha=0.5, padding='same', dilation=None, bias=False, activation=True):\n\n    channel_axis = 1 if K.image_data_format() == 'channels_first' else -1\n\n    x_high, x_low = octconv_block(ip_high, ip_low, filters, kernel_size, strides, alpha,\n                                  padding, dilation, bias)\n\n    relu = ReLU()\n    x_high = BatchNormalization(axis=channel_axis)(x_high)\n    if activation:\n        x_high = relu(x_high)\n\n    x_low = BatchNormalization(axis=channel_axis)(x_low)\n    if activation:\n        x_low = relu(x_low)\n\n    return x_high, x_low\n\n\ndef _octresnet_bottleneck_block(ip, filters, alpha=0.5, strides=(1, 1),\n                                downsample_shortcut=False, first_block=False,\n                                expansion=4):\n\n    if first_block:\n        x_high_res, x_low_res = _initial_oct_conv_bn_relu(ip, filters, kernel_size=(1, 1),\n                                                          alpha=alpha)\n\n        x_high, x_low = _oct_conv_bn_relu(x_high_res, x_low_res, filters, kernel_size=(3, 3),\n                                          strides=strides, alpha=alpha)\n\n    else:\n        x_high_res, x_low_res = ip\n        x_high, x_low = _oct_conv_bn_relu(x_high_res, x_low_res, filters, kernel_size=(1, 1),\n                                          alpha=alpha)\n\n        x_high, x_low = _oct_conv_bn_relu(x_high, x_low, filters, kernel_size=(3, 3),\n                                          strides=strides, alpha=alpha)\n\n    final_out_filters = int(filters * expansion)\n    x_high, x_low = _oct_conv_bn_relu(x_high, x_low, filters=final_out_filters,\n                                      kernel_size=(1, 1), alpha=alpha, activation=False)\n\n    if downsample_shortcut:\n        x_high_res, x_low_res = _oct_conv_bn_relu(x_high_res, x_low_res,\n                                                  final_out_filters, kernel_size=(1, 1),\n                                                  strides=strides, activation=False)\n\n    x_high = add([x_high, x_high_res])\n    x_low = add([x_low, x_low_res])\n\n    x_high = ReLU()(x_high)\n    x_low = ReLU()(x_low)\n\n    return x_high, x_low\n\n\ndef _octresnet_final_bottleneck_block(ip, filters, alpha=0.5, strides=(1, 1),\n                                      downsample_shortcut=False,\n                                      expansion=4):\n\n    x_high_res, x_low_res = ip\n\n    x_high, x_low = _oct_conv_bn_relu(x_high_res, x_low_res, filters, kernel_size=(1, 1),\n                                      alpha=alpha)\n\n    x_high, x_low = _oct_conv_bn_relu(x_high, x_low, filters, kernel_size=(3, 3),\n                                      strides=strides, alpha=alpha)\n\n    final_filters = int(filters * expansion)\n    x_high = _final_oct_conv_bn_relu(x_high, x_low, final_filters, kernel_size=(1, 1),\n                                     activation=False)\n\n    if downsample_shortcut:\n        x_high_res = _final_oct_conv_bn_relu(x_high_res, x_low_res, final_filters, kernel_size=(1, 1),\n                                             strides=strides, activation=False)\n\n    x = add([x_high, x_high_res])\n    x = ReLU()(x)\n\n    return x\n\n\ndef _bottleneck_original(ip, filters, strides=(1, 1), downsample_shortcut=False,\n                         expansion=4):\n\n    final_filters = int(filters * expansion)\n\n    shortcut = ip\n\n    x = _conv_bn_relu(ip, filters, kernel_size=(1, 1))\n    x = _conv_bn_relu(x, filters, kernel_size=(3, 3), strides=strides)\n    x = _conv_bn_relu(x, final_filters, kernel_size=(1, 1), activation=False)\n\n    if downsample_shortcut:\n        shortcut = _conv_block(shortcut, final_filters, kernel_size=(1, 1),\n                               strides=strides)\n\n    x = add([x, shortcut])\n    x = ReLU()(x)\n\n    return x\n\n\ndef OctaveResNet(block,\n                 layers,\n                 include_top=True,\n                 weights=None,\n                 input_tensor=None,\n                 input_shape=None,\n                 pooling=None,\n                 classes=1000,\n                 alpha=0.5,\n                 expansion=1,\n                 initial_filters=64,\n                 initial_strides=False,\n                 **kwargs):\n\n    if not (weights in {'imagenet', None} or os.path.exists(weights)):\n        raise ValueError('The `weights` argument should be either '\n                         '`None` (random initialization), `imagenet` '\n                         '(pre-training on ImageNet), '\n                         'or the path to the weights file to be loaded.')\n\n    if weights == 'imagenet' and include_top and classes != 1000:\n        raise ValueError('If using `weights` as `\"imagenet\"` with `include_top`'\n                         ' as true, `classes` should be 1000')\n\n    assert alpha >= 0. and alpha <= 1., \"`alpha` must be between 0 and 1\"\n\n    assert type(layers) in [list, tuple], \"`layers` must be a list/tuple of integers\"\n\n    # Determine proper input shape\n    input_shape = _obtain_input_shape(input_shape,\n                                      default_size=224,\n                                      min_size=32,\n                                      data_format=K.image_data_format(),\n                                      require_flatten=include_top,\n                                      weights=weights)\n\n    if input_tensor is None:\n        img_input = Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n\n    if initial_strides:\n        initial_strides = (2, 2)\n\n    else:\n        initial_strides = (1, 1)\n\n    x = _conv_bn_relu(img_input, filters=64, kernel_size=(7, 7), strides=initial_strides)\n\n    if initial_strides:\n        x = MaxPool2D((3, 3), strides=(2, 2), padding='same')(x)\n\n    num_filters = initial_filters\n    num_blocks = len(layers)\n\n    for i in range(num_blocks - 1):\n        for j in range(layers[i]):\n            if j == 0:\n                strides = (2, 2)\n                downsample_shortcut = True\n\n            else:\n                strides = (1, 1)\n                downsample_shortcut = False\n\n            # first block has no downsample, no shortcut\n            if i == 0 and j == 0:\n                first_block = True\n                strides = (1, 1)\n                downsample_shortcut = True\n\n            else:\n                first_block = False\n\n            x = block(x, num_filters, alpha, strides, downsample_shortcut, first_block, expansion)\n\n        # double number of filters per block\n        num_filters *= 2\n\n    # final block\n    for j in range(layers[-1]):\n        if j == 0:\n            strides = (2, 2)\n            x = _octresnet_final_bottleneck_block(x, num_filters, alpha, strides,\n                                                  downsample_shortcut=True, expansion=expansion)\n\n        else:\n            strides = (1, 1)\n            x = _bottleneck_original(x, num_filters, strides, expansion=expansion)\n\n    if include_top:\n        x = GlobalAveragePooling2D(name='avg_pool')(x)\n        x = Dense(classes, activation='softmax', name='fc')(x)\n    else:\n        if pooling == 'avg':\n            x = GlobalAveragePooling2D(name='avg_pool')(x)\n        elif pooling == 'max':\n            x = GlobalMaxPooling2D(name='max_pool')(x)\n\n    # Ensure that the model takes into account\n    # any potential predecessors of `input_tensor`.\n    if input_tensor is not None:\n        inputs = get_source_inputs(input_tensor)\n    else:\n        inputs = img_input\n\n    model = Model(inputs, x, name='OctaveResNet')\n\n    return model\n\n\ndef OctaveResNet50(include_top=True,\n                   weights=None,\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   alpha=0.5,\n                   expansion=4,\n                   initial_filters=64,\n                   initial_strides=True,\n                   **kwargs):\n\n    return OctaveResNet(_octresnet_bottleneck_block,\n                        [3, 4, 6, 3],\n                        include_top,\n                        weights,\n                        input_tensor,\n                        input_shape,\n                        pooling,\n                        classes,\n                        alpha,\n                        expansion,\n                        initial_filters,\n                        initial_strides,\n                        **kwargs)\n\n\ndef OctaveResNet101(include_top=True,\n                    weights=None,\n                    input_tensor=None,\n                    input_shape=None,\n                    pooling=None,\n                    classes=1000,\n                    alpha=0.5,\n                    expansion=4,\n                    initial_filters=64,\n                    initial_strides=True,\n                    **kwargs):\n\n    return OctaveResNet(_octresnet_bottleneck_block,\n                        [3, 4, 23, 3],\n                        include_top,\n                        weights,\n                        input_tensor,\n                        input_shape,\n                        pooling,\n                        classes,\n                        alpha,\n                        expansion,\n                        initial_filters,\n                        initial_strides,\n                        **kwargs)\n\n\ndef OctaveResNet152(include_top=True,\n                    weights=None,\n                    input_tensor=None,\n                    input_shape=None,\n                    pooling=None,\n                    classes=1000,\n                    alpha=0.5,\n                    expansion=4,\n                    initial_filters=64,\n                    initial_strides=True,\n                    **kwargs):\n\n    return OctaveResNet(_octresnet_bottleneck_block,\n                        [3, 8, 36, 3],\n                        include_top,\n                        weights,\n                        input_tensor,\n                        input_shape,\n                        pooling,\n                        classes,\n                        alpha,\n                        expansion,\n                        initial_filters,\n                        initial_strides,\n                        **kwargs)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def densenet_model(input_shape,batch_size = 1024):\n    base_model = OctaveResNet50(input_shape=input_shape, include_top=False,\n                           alpha=0.5, expansion=4,\n                           initial_filters=64,\n                           initial_strides=False)\n    x = base_model.output\n\n    out1 = GlobalMaxPooling2D()(x)\n    out2 = GlobalAveragePooling2D()(x)\n    #out3 = Flatten()(x)\n    out = concatenate([out1,out2])\n    out = BatchNormalization(epsilon = 1e-5)(out)\n    out = Dropout(0.4)(out)\n    fc = Dense(512,activation = 'relu')(out)\n    fc = BatchNormalization(epsilon = 1e-5)(fc)\n    fc = Dropout(0.3)(fc)\n    fc = Dense(256,activation = 'relu')(fc)\n    fc = BatchNormalization(epsilon = 1e-5)(fc)\n    fc = Dropout(0.3)(fc)\n    X = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', bias_initializer='zeros')(fc)\n    model =  Model(inputs=base_model.input, outputs=X)\n    #model.compile(optimizer=tf.keras.optimizers.Adam(lr = 0.0001), loss=tf.keras.losses.binary_crossentropy, metrics=['acc'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res_model = densenet_model((96,96,3))\nprint(res_model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* #### Here is the implementation of One Cycle Policy in the Keras Callback Class"},{"metadata":{"_uuid":"6f0cdeea4e204cbe8c4518de1e9502b317e6f97a","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport warnings\n\nfrom keras.callbacks import Callback\nfrom keras import backend as K\n\n\n# Code is ported from https://github.com/fastai/fastai\nclass OneCycleLR(Callback):\n    def __init__(self,\n                 max_lr,\n                 end_percentage=0.1,\n                 scale_percentage=None,\n                 maximum_momentum=0.95,\n                 minimum_momentum=0.85,\n                 verbose=True):\n        \"\"\" This callback implements a cyclical learning rate policy (CLR).\n        This is a special case of Cyclic Learning Rates, where we have only 1 cycle.\n        After the completion of 1 cycle, the learning rate will decrease rapidly to\n        100th its initial lowest value.\n        # Arguments:\n            max_lr: Float. Initial learning rate. This also sets the\n                starting learning rate (which will be 10x smaller than\n                this), and will increase to this value during the first cycle.\n            end_percentage: Float. The percentage of all the epochs of training\n                that will be dedicated to sharply decreasing the learning\n                rate after the completion of 1 cycle. Must be between 0 and 1.\n            scale_percentage: Float or None. If float, must be between 0 and 1.\n                If None, it will compute the scale_percentage automatically\n                based on the `end_percentage`.\n            maximum_momentum: Optional. Sets the maximum momentum (initial)\n                value, which gradually drops to its lowest value in half-cycle,\n                then gradually increases again to stay constant at this max value.\n                Can only be used with SGD Optimizer.\n            minimum_momentum: Optional. Sets the minimum momentum at the end of\n                the half-cycle. Can only be used with SGD Optimizer.\n            verbose: Bool. Whether to print the current learning rate after every\n                epoch.\n        # Reference\n            - [A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, weight_decay, and weight decay](https://arxiv.org/abs/1803.09820)\n            - [Super-Convergence: Very Fast Training of Residual Networks Using Large Learning Rates](https://arxiv.org/abs/1708.07120)\n        \"\"\"\n        super(OneCycleLR, self).__init__()\n\n        if end_percentage < 0. or end_percentage > 1.:\n            raise ValueError(\"`end_percentage` must be between 0 and 1\")\n\n        if scale_percentage is not None and (scale_percentage < 0. or scale_percentage > 1.):\n            raise ValueError(\"`scale_percentage` must be between 0 and 1\")\n\n        self.initial_lr = max_lr\n        self.end_percentage = end_percentage\n        self.scale = float(scale_percentage) if scale_percentage is not None else float(end_percentage)\n        self.max_momentum = maximum_momentum\n        self.min_momentum = minimum_momentum\n        self.verbose = verbose\n\n        if self.max_momentum is not None and self.min_momentum is not None:\n            self._update_momentum = True\n        else:\n            self._update_momentum = False\n\n        self.clr_iterations = 0.\n        self.history = {}\n\n        self.epochs = None\n        self.batch_size = None\n        self.samples = None\n        self.steps = None\n        self.num_iterations = None\n        self.mid_cycle_id = None\n\n    def _reset(self):\n        \"\"\"\n        Reset the callback.\n        \"\"\"\n        self.clr_iterations = 0.\n        self.history = {}\n\n    def compute_lr(self):\n        \"\"\"\n        Compute the learning rate based on which phase of the cycle it is in.\n        - If in the first half of training, the learning rate gradually increases.\n        - If in the second half of training, the learning rate gradually decreases.\n        - If in the final `end_percentage` portion of training, the learning rate\n            is quickly reduced to near 100th of the original min learning rate.\n        # Returns:\n            the new learning rate\n        \"\"\"\n        if self.clr_iterations > 2 * self.mid_cycle_id:\n            current_percentage = (self.clr_iterations - 2 * self.mid_cycle_id)\n            current_percentage /= float((self.num_iterations - 2 * self.mid_cycle_id))\n            new_lr = self.initial_lr * (1. + (current_percentage *\n                                              (1. - 100.) / 100.)) * self.scale\n\n        elif self.clr_iterations > self.mid_cycle_id:\n            current_percentage = 1. - (\n                self.clr_iterations - self.mid_cycle_id) / self.mid_cycle_id\n            new_lr = self.initial_lr * (1. + current_percentage *\n                                        (self.scale * 100 - 1.)) * self.scale\n\n        else:\n            current_percentage = self.clr_iterations / self.mid_cycle_id\n            new_lr = self.initial_lr * (1. + current_percentage *\n                                        (self.scale * 100 - 1.)) * self.scale\n\n        if self.clr_iterations == self.num_iterations:\n            self.clr_iterations = 0\n\n        return new_lr\n\n    def compute_momentum(self):\n        \"\"\"\n         Compute the momentum based on which phase of the cycle it is in.\n        - If in the first half of training, the momentum gradually decreases.\n        - If in the second half of training, the momentum gradually increases.\n        - If in the final `end_percentage` portion of training, the momentum value\n            is kept constant at the maximum initial value.\n        # Returns:\n            the new momentum value\n        \"\"\"\n        if self.clr_iterations > 2 * self.mid_cycle_id:\n            new_momentum = self.max_momentum\n\n        elif self.clr_iterations > self.mid_cycle_id:\n            current_percentage = 1. - ((self.clr_iterations - self.mid_cycle_id) / float(\n                                        self.mid_cycle_id))\n            new_momentum = self.max_momentum - current_percentage * (\n                self.max_momentum - self.min_momentum)\n\n        else:\n            current_percentage = self.clr_iterations / float(self.mid_cycle_id)\n            new_momentum = self.max_momentum - current_percentage * (\n                self.max_momentum - self.min_momentum)\n\n        return new_momentum\n\n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        self.epochs = self.params['epochs']\n        self.batch_size = 192\n        self.samples = len(train_list)\n        self.steps = self.params['steps']\n\n        if self.steps is not None:\n            self.num_iterations = self.epochs * self.steps\n        else:\n            if (self.samples % self.batch_size) == 0:\n                remainder = 0\n            else:\n                remainder = 1\n            self.num_iterations = (self.epochs + remainder) * self.samples // self.batch_size\n\n        self.mid_cycle_id = int(self.num_iterations * ((1. - self.end_percentage)) / float(2))\n\n        self._reset()\n        K.set_value(self.model.optimizer.lr, self.compute_lr())\n\n        if self._update_momentum:\n            if not hasattr(self.model.optimizer, 'momentum'):\n                raise ValueError(\"Momentum can be updated only on SGD optimizer !\")\n\n            new_momentum = self.compute_momentum()\n            K.set_value(self.model.optimizer.momentum, new_momentum)\n\n    def on_batch_end(self, epoch, logs=None):\n        logs = logs or {}\n\n        self.clr_iterations += 1\n        new_lr = self.compute_lr()\n\n        self.history.setdefault('lr', []).append(\n            K.get_value(self.model.optimizer.lr))\n        K.set_value(self.model.optimizer.lr, new_lr)\n\n        if self._update_momentum:\n            if not hasattr(self.model.optimizer, 'momentum'):\n                raise ValueError(\"Momentum can be updated only on SGD optimizer !\")\n\n            new_momentum = self.compute_momentum()\n\n            self.history.setdefault('momentum', []).append(\n                K.get_value(self.model.optimizer.momentum))\n            K.set_value(self.model.optimizer.momentum, new_momentum)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n\n    def on_epoch_end(self, epoch, logs=None):\n        if self.verbose:\n            if self._update_momentum:\n                print(\" - lr: %0.5f - momentum: %0.2f \" %\n                      (self.history['lr'][-1], self.history['momentum'][-1]))\n\n            else:\n                print(\" - lr: %0.5f \" % (self.history['lr'][-1]))\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0c409e4875937c8e4eacb532e507f70dedede9f","trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"# Define Ony Cycle Policy parameters and train model\n########################################################################################\nfrom keras.optimizers import Adam, SGD\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n#from clr import OneCycleLR\n# CLR parameters\n\nbatch_size = 192\nepochs = 38\n# lr_callback = LRFinder(len(train_list), batch_size,\n#                        1e-5, 1.,\n#                        # validation_data=(X_val, Y_val),\n#                        lr_scale='exp', save_dir='weights/')\nlr_manager = OneCycleLR(max_lr=0.02, end_percentage=0.1, scale_percentage=None,\n                        maximum_momentum=0.9,minimum_momentum=0.8)\n\nres_model.compile(loss='binary_crossentropy', optimizer=SGD(0.002, momentum=0.9, nesterov=True), metrics=['accuracy'])\n    \ncallbacks = [lr_manager,\n           ModelCheckpoint(filepath='octresnet_one_cycle_model.h5', monitor='val_loss',mode='min',verbose=1,save_best_only=True)]\n\nhistory = res_model.fit_generator(data_gen(train_list, id_label_map, batch_size,do_train_augmentations),\n                              validation_data=data_gen(val_list, id_label_map, batch_size,do_inference_aug),\n                              epochs = epochs,\n                              steps_per_epoch = (len(train_list) // batch_size) + 1,\n                              validation_steps = (len(val_list) // batch_size) + 1,\n                              callbacks=callbacks,\n                              verbose = 1)\nplt.plot(history.history['loss'], label='train')\nplt.plot(history.history['val_loss'], label='valid')\nplt.title(\"model loss\")\nplt.ylabel(\"loss\")\nplt.xlabel(\"epoch\")\nplt.legend([\"train\", \"valid\"], loc=\"upper left\")\nplt.savefig('loss_performance.png')\nplt.clf()\nplt.plot(history.history['acc'], label='train')\nplt.plot(history.history['val_acc'], label='valid')\nplt.title(\"model acc\")\nplt.ylabel(\"acc\")\nplt.xlabel(\"epoch\")\nplt.legend([\"train\", \"valid\"], loc=\"upper left\")\nplt.savefig('acc_performance.png')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def do_inference_aug():\n    return Compose([\n       # Resize(196,196),\n        RandomRotate90(p=0.5),\n        Transpose(p=0.5),\n        Flip(p=0.5),Normalize(p=1)])\n\n\ndef data_gen(list_files,batch_size,aug_func):\n    aug = aug_func()\n    while True:\n        #shuffle(list_files)\n        for block in chunker(list_files,batch_size):\n            x = [aug(image = cv2.imread(addr))['image'] for addr in block]\n            y = [id_label_map[get_id_from_path(addr)] for addr in block]\n            yield np.array(x),np.array(y)\n\n\npreds = res_model.predict_generator(data_gen(val_list,1,do_inference_aug),steps = len(val_list))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds = np.array(preds)\ny_preds[preds >= 0.5] = 1\ny_preds[preds < 0.5] = 0\ntrue = df_val['label'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score,confusion_matrix,classification_report\nroc_auc_score(true,preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn.metrics as metrics\n# calculate the fpr and tpr for all thresholds of the classification\n\nfpr, tpr, threshold = metrics.roc_curve(true, preds)\nroc_auc = metrics.auc(fpr, tpr)\n\n# method I: plt\nimport matplotlib.pyplot as plt\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, 'g', label = 'AUC = %0.2f' % roc_auc)\nplt.legend(loc = 'lower right')\nplt.plot([0, 1], [0, 1],'r--')\nplt.xlim([0, 1])\nplt.ylim([0, 1])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show()\nplt.savefig('octresnet_auc_roc.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(true,y_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    plt.savefig('octresnet_cm.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm,['no_tumor_tissue', 'has_tumor_tissue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"report = classification_report(true,y_preds,target_names=['no_tumor_tissue', 'has_tumor_tissue'])\nprint(report)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lr_callback.plot_schedule(clip_beginning=200, clip_endding=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"# # Define Ony Cycle Policy parameters and train model\n# ########################################################################################\n# import gc\n# from keras.optimizers import Adam, SGD\n# from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n# import keras.backend as K\n# # CLR parameters\n\n# batch_size = 256\n# epochs = 1\n# for momentum in [0.9,0.95,0.99]:\n#     #K.clear_session()\n#     lr_finder = LRFinder(len(train_list), batch_size, minimum_lr=.0001, maximum_lr=.001,\n#                          lr_scale='linear',\n#                          #validation_data=data_gen(val_list, id_label_map, batch_size, do_inference_aug),  # use the validation data for losses\n#                          #validation_sample_rate=5,\n#                          save_dir='weights/momentum/momentum-%s' % str(momentum), verbose=True)\n#     res_model = densenet_model((96,96,3))\n#     res_model.compile(loss='binary_crossentropy', optimizer=SGD(0.0001, momentum=momentum, nesterov=True), metrics=['accuracy'])\n\n#     # clr =  CyclicLR(base_lr=base_lr,\n#     #                 max_lr=max_lr,\n#     #                 step_size=step_size,\n#     #                 max_m=max_m,\n#     #                 base_m=base_m,\n#     #                 cyclical_momentum=cyclical_momentum)\n\n#     callbacks = [lr_finder]\n#                 #ModelCheckpoint(filepath='best_model.h5', monitor='val_loss',mode='min',verbose=1,save_best_only=True)]\n\n#     history = res_model.fit_generator(data_gen(train_list, id_label_map, batch_size,do_train_augmentations),\n#                                   #validation_data=data_gen(val_list, id_label_map, batch_size, do_inference_aug),\n#                                   epochs = epochs,\n#                                   steps_per_epoch = (len(train_list) // batch_size) + 1,\n#                                  #validation_steps = (len(val_list) // batch_size) + 1,\n#                                   callbacks=callbacks,\n#                                   verbose = 1)\n#     del history\n#     del res_model\n#     gc.collect()\n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}