{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* v1 input size 224=>288 | lr 1e-3 1e-4 1e-4 => 1e-3 1e-4 2e-5\n* v2 input size 288=>300 | new resize function | new mix up a 1.0=>0.4 | augmentation affine rotate (-180,180) = (-10,10)\n* v3 input size 300=>224\n* v4 batch size\n* v5 use image edge instead of image\n* v6 back to image\n* v7 use categorical_crossentropy instead of binary_crossentropy\n* v8 fix bug in data preprocessing\n* v9 add clr\n* v10 add hsv\n* v11 new focal loss | remove hsv"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy\nfrom keras.applications.densenet import preprocess_input\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score, fbeta_score\nfrom keras.utils import Sequence\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nbeta_f2=2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_classes = 1103\n#batch_size = 100\nimg_size = 224\nnb_epochs = 25","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imet_path = '../input/imet-2019-fgvc6/'\n#imet_path = 'F:/ChromeDownload/IMET/'\n\nmodel_weights_path = '../input/densenet121weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5'\n#model_weights_path = 'F:/Edge Downloads/keras_model_weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5'\n#model_weights_path = 'F:/Edge Downloads/keras_model_weights/densenet169_weights_tf_dim_ordering_tf_kernels_notop.h5'\n#model_weights_path = 'F:/Edge Downloads/keras_model_weights/densenet201_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\ntrain_df_path = imet_path + 'train.csv'\nlabel_df_path = imet_path + 'labels.csv'\nsub_df_path = imet_path + 'sample_submission.csv'\ntrain_img_path = imet_path + 'train/'\ntest_img_path = imet_path + 'test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from keras.legacy import interfaces\nfrom keras.optimizers import Optimizer\nfrom keras import backend as K\n\nclass AdamAccumulate_v1(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')\n\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, dtype='int64')\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n\n        self.updates = [K.update(self.iterations, self.iterations + 1)]\n\n        flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)\n        flag = K.cast(flag, K.floatx())\n\n        self.updates.append(K.update(self.effective_iterations,\n                                     self.effective_iterations + K.cast(flag, 'int64')))\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.effective_iterations,\n                                                      K.dtype(self.decay))))\n\n        t = K.cast(self.effective_iterations, K.floatx()) + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n                     (1. - K.pow(self.beta_1, t)))\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):\n\n            gg_t = (1 - flag) * (gg + g)\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * (gg + flag * g) / K.cast(self.accum_iters, K.floatx())\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(\n                (gg + flag * g) / K.cast(self.accum_iters, K.floatx()))\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - flag * lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                p_t = p - flag * lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append((m, flag * m_t + (1 - flag) * m))\n            self.updates.append((v, flag * v_t + (1 - flag) * v))\n            self.updates.append((gg, gg_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n\nclass AdamAccumulate(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):\n        if accum_iters < 1:\n            raise ValueError('accum_iters must be >= 1')\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n        self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * completed_updates))\n\n        t = completed_updates + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n        # self.iterations incremented after processing a batch\n        # batch:              1 2 3 4 5 6 7 8 9\n        # self.iterations:    0 1 2 3 4 5 6 7 8\n        # update_switch = 1:        x       x    (if accum_iters=4)\n        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n        update_switch = K.cast(update_switch, K.floatx())\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n            sum_grad = tg + g\n            avg_grad = sum_grad / self.accum_iters_float\n\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n            else:\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load dataset info\npath_to_train = imet_path+'train/'\ndata = pd.read_csv(imet_path+'train.csv')\n\ntrain_dataset_info = []\nfor name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')):\n    train_dataset_info.append({\n        'path':os.path.join(path_to_train, name),\n        'labels':np.array([int(label) for label in labels])})\ntrain_dataset_info = np.array(train_dataset_info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gamma = 2.0\nepsilon = K.epsilon()\ndef old_focal_loss(y_true, y_pred):\n    pt = y_pred * y_true + (1-y_pred) * (1-y_true)\n    pt = K.clip(pt, epsilon, 1-epsilon)\n    CE = -K.log(pt)\n    FL = K.pow(1-pt, gamma) * CE\n    loss = K.sum(FL, axis=1)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/imet-2019-fgvc6/train.csv')\ntrain_labels = np.zeros((train_df.shape[0], 1103))\n\nfor row_index, row in enumerate(train_df['attribute_ids']):\n    for label in row.split():\n        train_labels[row_index, int(label)] = 1\nlabels_sum = np.sum(train_labels,axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n\ntrain_index = None\nvalid_index = None\n\nclass data_generator(Sequence):\n\n    def mix_up(x, y):\n        x = np.array(x, np.float32)\n        lam = np.random.beta(0.4, 0.4)\n        ori_index = np.arange(int(len(x)))\n        index_array = np.arange(int(len(x)))\n        np.random.shuffle(index_array)        \n        \n        mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n        mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n        \n        return mixed_x, mixed_y\n    \n    def create_train(dataset_info, batch_size, shape, augument=True, mix=False):\n        assert shape[2] == 3\n        while True:\n            dataset_info = shuffle(dataset_info)\n            for start in range(0, len(dataset_info), batch_size):\n                end = min(start + batch_size, len(dataset_info))\n                batch_images = []\n                X_train_batch = dataset_info[start:end]\n                batch_labels = np.zeros((len(X_train_batch), nb_classes))\n                for i in range(len(X_train_batch)):\n                    image = data_generator.load_image(\n                        X_train_batch[i]['path'], shape)   \n                    if augument:\n                        image = data_generator.augment(image)\n                    #batch_images.append(preprocess_input(image))\n                    batch_images.append((image)/255.0)\n                    batch_labels[i][X_train_batch[i]['labels']] = 1\n                if(mix):\n                    batch_images, batch_labels = data_generator.mix_up(batch_images, batch_labels)    \n                yield np.array(batch_images, np.float32), batch_labels\n                \n    def create_valid(valid_dataset_info, batch_size, shape, augument=False):\n        global valid_index\n        assert shape[2] == 3\n        while True:\n            #valid_dataset_info = dataset_info[valid_index]\n            for start in range(0, len(valid_dataset_info), batch_size):\n                end = min(start + batch_size, len(valid_dataset_info))\n                batch_images = []\n                X_train_batch = valid_dataset_info[start:end]\n                batch_labels = np.zeros((len(X_train_batch), nb_classes))\n                for i in range(len(X_train_batch)):\n                    image = data_generator.load_image(\n                        X_train_batch[i]['path'], shape)   \n                    if augument:\n                        image = data_generator.augment(image)\n                    #batch_images.append(preprocess_input(image))\n                    batch_images.append((image)/255.0)\n                    batch_labels[i][X_train_batch[i]['labels']] = 1\n                yield np.array(batch_images, np.float32), batch_labels\n\n    def load_image(path, shape):\n        #img_pil = Image.open(path+'.png')\n        #img_cv = np.array(img_pil.filter(PIL.ImageFilter.FIND_EDGES))\n        img_cv = cv2.imread(path+'.png')\n        img_cv = cv2.cvtColor(img_cv,cv2.COLOR_BGR2RGB)\n        if img_cv.shape[1]==300 and img_cv.shape[0]>=450:\n            over_high = img_cv.shape[0]-300\n            start = np.random.randint(over_high//3,over_high,1)[0]\n            img_cv = img_cv[start:start+300,:,:]\n        elif img_cv.shape[0]==300 and img_cv.shape[1]>450:\n            over_width = img_cv.shape[1]-300\n            start = np.random.randint(over_width//3,over_width,1)[0]\n            img_cv = img_cv[:,start:start+300,:]\n        img_cv = cv2.resize(img_cv,(img_size,img_size))\n        return img_cv\n\n    def augment(image):\n        augment_img = iaa.Sequential([\n            iaa.SomeOf((0,4),[\n                #iaa.Crop(percent=(0, 0.1)),\n                iaa.ContrastNormalization((0.5, 1.5)),\n                iaa.Multiply((0.9, 1.1), per_channel=0.2),\n                iaa.Fliplr(0.5),\n                iaa.GaussianBlur(sigma=(0, 0.6)),\n                #iaa.Affine(\n                #        scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n                #        translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n                #        rotate=(-10, 10),\n                #    )\n            ])], random_order=True)\n\n        image_aug = augment_img.augment_image(image)\n        return image_aug","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,Concatenate)\nfrom keras import applications\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# reference link: https://gist.github.com/drscotthawley/d1818aabce8d1bf082a6fb37137473ae\nfrom keras.callbacks import Callback\n\ndef get_1cycle_schedule(lr_max=1e-3, n_data_points=8000, epochs=200, batch_size=40, verbose=0):          \n    \"\"\"\n    Creates a look-up table of learning rates for 1cycle schedule with cosine annealing\n    See @sgugger's & @jeremyhoward's code in fastai library: https://github.com/fastai/fastai/blob/master/fastai/train.py\n    Wrote this to use with my Keras and (non-fastai-)PyTorch codes.\n    Note that in Keras, the LearningRateScheduler callback (https://keras.io/callbacks/#learningratescheduler) only operates once per epoch, not per batch\n      So see below for Keras callback\n\n    Keyword arguments:\n    lr_max            chosen by user after lr_finder\n    n_data_points     data points per epoch (e.g. size of training set)\n    epochs            number of epochs\n    batch_size        batch size\n    Output:  \n    lrs               look-up table of LR's, with length equal to total # of iterations\n    Then you can use this in your PyTorch code by counting iteration number and setting\n          optimizer.param_groups[0]['lr'] = lrs[iter_count]\n    \"\"\"\n    if verbose > 0:\n        print(\"Setting up 1Cycle LR schedule...\")\n    pct_start, div_factor = 0.3, 25.        # @sgugger's parameters in fastai code\n    lr_start = lr_max/div_factor\n    lr_end = lr_start/1e4\n    n_iter = (n_data_points * epochs // batch_size) + 1    # number of iterations\n    a1 = int(n_iter * pct_start)\n    a2 = n_iter - a1\n\n    # make look-up table\n    lrs_first = np.linspace(lr_start, lr_max, a1)            # linear growth\n    lrs_second = (lr_max-lr_end)*(1+np.cos(np.linspace(0,np.pi,a2)))/2 + lr_end  # cosine annealing\n    lrs = np.concatenate((lrs_first, lrs_second))\n    return lrs\n\n\nclass OneCycleScheduler(Callback):\n    \"\"\"My modification of Keras' Learning rate scheduler to do 1Cycle learning\n       which increments per BATCH, not per epoch\n    Keyword arguments\n        **kwargs:  keyword arguments to pass to get_1cycle_schedule()\n        Also, verbose: int. 0: quiet, 1: update messages.\n\n    Sample usage (from my train.py):\n        lrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=X_train.shape[0],\n        epochs=epochs, batch_size=batch_size, verbose=1)\n    \"\"\"\n    def __init__(self, **kwargs):\n        super(OneCycleScheduler, self).__init__()\n        self.verbose = kwargs.get('verbose', 0)\n        self.lrs = get_1cycle_schedule(**kwargs)\n        self.iteration = 0\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = self.lrs[self.iteration]\n        K.set_value(self.model.optimizer.lr, lr)         # here's where the assignment takes place\n        if self.verbose > 0:\n            print('\\nIteration %06d: OneCycleScheduler setting learning '\n                  'rate to %s.' % (self.iteration, lr))\n        self.iteration += 1\n\n    def on_epoch_end(self, epoch, logs=None):  # this is unchanged from Keras LearningRateScheduler\n        logs = logs or {}\n        logs['lr'] = K.get_value(self.model.optimizer.lr)\n        self.iteration = 0\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nimport keras.backend as K\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n            ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n    def __init__(self, base_lr=1e-5, max_lr=1e-2, step_size=1000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n        self._reset()\n        \n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.        \n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)        \n\n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n       \n    def on_batch_end(self, epoch, logs=None):        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)        \n        K.set_value(self.model.optimizer.lr, self.clr())        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.regularizers import l2\n\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    #base_model = ResNet50V2(include_top=False,weights=None,input_tensor=input_tensor)\n    #base_model.load_weights('../input/keras-pretrain-model-weights/resnet50v2_weights_tf_dim_ordering_tf_kernels_notop.h5')\n#     x = Conv2D(32, kernel_size=(1,1), activation='relu')(base_model.output)\n#     x = Flatten()(x)\n    model = applications.DenseNet121(weights=None,include_top=False,input_tensor=input_tensor)\n    model.load_weights(model_weights_path)\n    x0 = model.output\n    x1 = GlobalAveragePooling2D()(x0)\n    x2 = GlobalMaxPooling2D()(x0)\n    x = Concatenate()([x1,x2])\n    \n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu', kernel_regularizer=l2(5e-4))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    \n    #x = GlobalAveragePooling2D()(base_model.output)\n    #x = Dropout(0.5)(x)\n    #x = Dense(1024, activation='relu', kernel_regularizer=l2(5e-4))(x)\n    #x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation='sigmoid', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create callbacks list\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\n                             \nfrom sklearn.model_selection import train_test_split\n\nepochs = 17; warmup_batch_size = 128;mixup_batch_size = 64\nbatch_size = 64\ncheckpoint = ModelCheckpoint('denseNet121_focal.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, \n                                   verbose=1, mode='auto', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=7)\n\ncsv_logger = CSVLogger(filename='training_log.csv',\n                       separator=',',\n                       append=True)\n\n\n# split data into train, valid\nindexes = np.arange(train_dataset_info.shape[0])\ntrain_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)\n\n# create train and valid datagens\ntrain_generator = data_generator.create_train(\n    train_dataset_info[train_indexes], batch_size, (img_size,img_size,3), augument=True, mix=False)\ntrain_mixup = data_generator.create_train(\n    train_dataset_info[train_indexes], mixup_batch_size, (img_size,img_size,3), augument=False, mix=True)\ntrain_generator_warmup = data_generator.create_train(\n    train_dataset_info[train_indexes], warmup_batch_size, (img_size,img_size,3), augument=False)\nvalidation_generator = data_generator.create_valid(\n    train_dataset_info[valid_indexes], batch_size, (img_size,img_size,3), augument=False)\n\nlrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=len(train_indexes),\n        epochs=1, batch_size=warmup_batch_size, verbose=0)\n# callbacks_list = [checkpoint, csv_logger, lrsched]\n# callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model(\n    input_shape=(img_size,img_size,3), \n    n_out=nb_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# warm up model\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-6,0):\n    model.layers[i].trainable = True\n\nmodel.compile(\n    loss='binary_crossentropy',\n    #loss='categorical_crossentropy',\n    #loss = [focal_loss(labels_sum)],\n    optimizer=Adam(1e-3))\n    # optimizer=AdamAccumulate(lr=1e-3, accum_iters=2))\n\nmodel.fit_generator(\n    train_generator_warmup,\n    steps_per_epoch=np.ceil(float(len(train_indexes)) / float(warmup_batch_size)),\n    epochs=2,\n    max_queue_size=16, \n    workers=2, \n    use_multiprocessing=True,\n    verbose=1,\n    class_weight = 'auto'\n    #callbacks = lrsched\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# train all layers\nfor layer in model.layers:\n    layer.trainable = True\n\n# callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]\nmodel.compile(\n            loss='binary_crossentropy',\n            #loss='categorical_crossentropy',\n            #loss = [focal_loss(labels_sum)],\n             #loss=focal_loss,\n            #optimizer=Adam(lr=1e-4))\n            optimizer=AdamAccumulate(lr=1e-4, accum_iters=4))\n\nmodel.fit_generator(\n    train_mixup,\n    steps_per_epoch=np.ceil(float(len(train_indexes)) / float(mixup_batch_size)),\n    # validation_data=validation_generator,\n    # validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)),\n    epochs=7,\n    verbose=1,\n    class_weight = 'auto',\n    max_queue_size=16, \n    workers=WORKERS, \n    use_multiprocessing=True)\n    # callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nclr = CyclicLR(base_lr=1e-5,max_lr=6e-5,mode='triangular2', step_size=np.ceil(len(train_indexes)/batch_size))\n\nmodel.compile(\n    loss='binary_crossentropy',\n    #loss='categorical_crossentropy',\n    #loss = [focal_loss(labels_sum)],\n     #loss=focal_loss,\n    optimizer=Adam(lr=3e-5))\n    # optimizer=AdamAccumulate(lr=1e-4, accum_iters=4))\n\n#callbacks_list = [checkpoint, csv_logger, clr]\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat]\nmodel.fit_generator(\n    train_generator,\n    steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)),\n    validation_data=validation_generator,\n    validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)),\n    epochs=epochs,\n    verbose=1,\n    class_weight = 'auto',\n    max_queue_size=16, workers=WORKERS, use_multiprocessing=True,\n    callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../working/'))\nmodel.save('../input/denseNet121.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')\n#model = keras.models.load_model('../input/denseNet121.h5',custom_objects={'focal_loss':focal_loss})\npredicted = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''Search for the best threshold regarding the validation set'''\n\nBATCH = 512\nfullValGen = data_generator.create_valid(\n    train_dataset_info[valid_indexes], BATCH, (img_size,img_size,3))\n\nn_val = round(train_dataset_info.shape[0]*0.15)//BATCH\nprint(n_val)\n\nlastFullValPred = np.empty((0, nb_classes))\nlastFullValLabels = np.empty((0, nb_classes))\nfor i in tqdm(range(n_val+1)): \n    im, lbl = next(fullValGen)\n    scores = model.predict(im)\n    lastFullValPred = np.append(lastFullValPred, scores, axis=0)\n    lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)\nprint(lastFullValPred.shape, lastFullValLabels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def my_f2(y_true, y_pred):\n    assert y_true.shape[0] == y_pred.shape[0]\n\n    tp = np.sum((y_true == 1) & (y_pred == 1), axis=1)\n    tn = np.sum((y_true == 0) & (y_pred == 0), axis=1)\n    fp = np.sum((y_true == 0) & (y_pred == 1), axis=1)\n    fn = np.sum((y_true == 1) & (y_pred == 0), axis=1)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f2 = (1+beta_f2**2)*p*r / (p*beta_f2**2 + r + 1e-15)\n\n    return np.mean(f2)\n\ndef find_best_fixed_threshold(preds, targs, do_plot=True):\n    score = []\n    thrs = np.arange(0, 0.5, 0.01)\n    for thr in tqdm(thrs):\n        score.append(my_f2(targs, (preds > thr).astype(int) ))\n    score = np.array(score)\n    pm = score.argmax()\n    best_thr, best_score = thrs[pm], score[pm].item()\n    print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')\n    if do_plot:\n        plt.plot(thrs, score)\n        plt.vlines(x=best_thr, ymin=score.min(), ymax=score.max())\n        plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);\n        plt.show()\n    return best_thr, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, name in tqdm(enumerate(submit['id'])):\n    path = os.path.join('../input/imet-2019-fgvc6/test/', name)\n    image = data_generator.load_image(path, (img_size,img_size,3))\n    #score_predict = model.predict(preprocess_input(image[np.newaxis]))\n    score_predict = model.predict((image[np.newaxis])/255.0)\n    # print(score_predict)\n    label_predict = np.arange(nb_classes)[score_predict[0]>=best_thr]\n    # print(label_predict)\n    str_predict_label = ' '.join(str(l) for l in label_predict)\n    predicted.append(str_predict_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['attribute_ids'] = predicted\nsubmit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}