{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# --- random seed -------------\nSEED = 123456\nimport os\nimport random as rn\nimport numpy as np\nfrom tensorflow import set_random_seed\n\nos.environ['PYTHONHASHSEED']=str(SEED)\nnp.random.seed(SEED)\nset_random_seed(SEED)\nrn.seed(SEED)\nSAVED_MODEL_NAME = 'model.h5'\n\n# import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nimport pandas as pd\n\ndf_train = pd.read_csv(\"../input/aptos2019-blindness-detection/train.csv\") #.sample(200)\n\ndf_test = pd.read_csv(\"../input/aptos2019-blindness-detection/test.csv\") #.sample(200)\ndf_submit = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\") #.sample(200)\n# diagnosis_encoded = pd.get_dummies(df_train.diagnosis)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! mkdir '../input/train_images/'\n! mkdir '../input/test_images/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport cv2\nfrom matplotlib import pyplot as plt\n# -------------------------------------------\n#def crop_image1(img,tol=7):\n#    # img is image data\n#    # tol  is tolerance\n#        \n#    mask = img>tol\n#    return img[np.ix_(mask.any(1),mask.any(0))]\n# -------------------------------------------\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n    \n# -------------------------------------------\ndef circle_crop(img, sigmaX=10):   \n    image = cv2.imread(img)\n    #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    #image = crop_image_from_gray(image)\n    image = cv2.resize(image, (299, 299))\n    #image = cv2.addWeighted (image,4, cv2.GaussianBlur(image, (0,0) ,sigmaX), -4, 128)    \n    return image\n# --- train images preprocess ----------------------------------------\nmy_id = 0\nfor index, row in df_train.iterrows():\n    my_id = my_id + 1\n    if (my_id%100)==0: print('train ... '+str(my_id))\n    my_pic_name = row.id_code\n    im_path = \"../input/aptos2019-blindness-detection/train_images/\"+my_pic_name+\".png\"    \n    image = circle_crop(im_path)\n    cv2.imwrite(\"../input/train_images/\" +my_pic_name+\".png\", image)\n# --- test images preprocess ----------------------------------------\nmy_id = 0\nfor index, row in df_test.iterrows():\n    my_id = my_id + 1\n    if (my_id%100)==0: print('test ... '+str(my_id))\n    my_pic_name = row.id_code\n    im_path = \"../input/aptos2019-blindness-detection/test_images/\"+my_pic_name+\".png\"    \n    image = circle_crop(im_path,sigmaX=30)\n    cv2.imwrite(\"../input/test_images/\" +my_pic_name+\".png\", image)\n# -------------------------------------------\n#tlog('Images preprosess Total time:'+str((timeit.default_timer()-start_time)/3600)+' hours.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras import initializers\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import backend as K\nfrom keras.activations import elu\nfrom keras.optimizers import Adam\nfrom keras.models import Sequential\nfrom keras.engine import Layer, InputSpec\nfrom keras.utils.generic_utils import get_custom_objects\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import cohen_kappa_score\n\nclass RAdam(keras.optimizers.Optimizer):\n    \"\"\"RAdam optimizer.\n    # Arguments\n        lr: float >= 0. Learning rate.\n        beta_1: float, 0 < beta < 1. Generally close to 1.\n        beta_2: float, 0 < beta < 1. Generally close to 1.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n        decay: float >= 0. Learning rate decay over each update.\n        weight_decay: float >= 0. Weight decay for each param.\n        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n            algorithm from the paper \"On the Convergence of Adam and\n            Beyond\".\n        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n        min_lr: float >= 0. Minimum learning rate after warmup.\n    # References\n        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n    \"\"\"\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n        super(RAdam, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n            self.total_steps = K.variable(total_steps, name='total_steps')\n            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n            self.min_lr = K.variable(lr, name='min_lr')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.initial_weight_decay = weight_decay\n        self.initial_total_steps = total_steps\n        self.amsgrad = amsgrad\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n\n        if self.initial_total_steps > 0:\n            warmup_steps = self.total_steps * self.warmup_proportion\n            decay_steps = self.total_steps - warmup_steps\n            lr = K.switch(\n                t <= warmup_steps,\n                lr * (t / warmup_steps),\n                lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),\n            )\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n        else:\n            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        beta_1_t = K.pow(self.beta_1, t)\n        beta_2_t = K.pow(self.beta_2, t)\n\n        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n\n        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n            m_corr_t = m_t / (1.0 - beta_1_t)\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n\n            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n                         (sma_t - 2.0) / (sma_inf - 2.0) *\n                         sma_inf / sma_t)\n\n            p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n\n            if self.initial_weight_decay > 0:\n                p_t += self.weight_decay * p\n\n            p_t = p - lr * p_t\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {\n            'lr': float(K.get_value(self.lr)),\n            'beta_1': float(K.get_value(self.beta_1)),\n            'beta_2': float(K.get_value(self.beta_2)),\n            'decay': float(K.get_value(self.decay)),\n            'weight_decay': float(K.get_value(self.weight_decay)),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n            'total_steps': float(K.get_value(self.total_steps)),\n            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n            'min_lr': float(K.get_value(self.min_lr)),\n        }\n        base_config = super(RAdam, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nCopyright 2017 TensorFlow Authors and Kent Sommer\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n   http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n'''\nimport numpy as np\n\n# Sys\nimport warnings\n# Keras Core\nfrom keras.layers.convolutional import MaxPooling2D, Convolution2D, AveragePooling2D\nfrom keras.layers import Input, Dropout, Dense, Flatten, Activation\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.merge import concatenate\nfrom keras import regularizers\nfrom keras import initializers\nfrom keras.models import Model\n# Backend\nfrom keras import backend as K\n# Utils\nfrom keras.utils.layer_utils import convert_all_kernels_in_model\nfrom keras.utils.data_utils import get_file\n\n\n#########################################################################################\n# Implements the Inception Network v4 (http://arxiv.org/pdf/1602.07261v1.pdf) in Keras. #\n#########################################################################################\n\nWEIGHTS_PATH = 'https://github.com/kentsommer/keras-inceptionV4/releases/download/2.1/inception-v4_weights_tf_dim_ordering_tf_kernels.h5'\nWEIGHTS_PATH_NO_TOP = 'https://github.com/kentsommer/keras-inceptionV4/releases/download/2.1/inception-v4_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\n\ndef preprocess_input(x):\n    x = np.divide(x, 255.0)\n    x = np.subtract(x, 0.5)\n    x = np.multiply(x, 2.0)\n    return x\n\n\ndef conv2d_bn(x, nb_filter, num_row, num_col,\n              padding='same', strides=(1, 1), use_bias=False):\n    \"\"\"\n    Utility function to apply conv + BN. \n    (Slightly modified from https://github.com/fchollet/keras/blob/master/keras/applications/inception_v3.py)\n    \"\"\"\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n    x = Convolution2D(nb_filter, (num_row, num_col),\n                      strides=strides,\n                      padding=padding,\n                      use_bias=use_bias,\n                      kernel_regularizer=regularizers.l2(0.00004),\n                      kernel_initializer=initializers.VarianceScaling(scale=2.0, mode='fan_in', distribution='normal', seed=None))(x)\n    x = BatchNormalization(axis=channel_axis, momentum=0.9997, scale=False)(x)\n    x = Activation('relu')(x)\n    return x\n\n\ndef block_inception_a(input):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    branch_0 = conv2d_bn(input, 96, 1, 1)\n\n    branch_1 = conv2d_bn(input, 64, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 96, 3, 3)\n\n    branch_2 = conv2d_bn(input, 64, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 96, 3, 3)\n    branch_2 = conv2d_bn(branch_2, 96, 3, 3)\n\n    branch_3 = AveragePooling2D((3,3), strides=(1,1), padding='same')(input)\n    branch_3 = conv2d_bn(branch_3, 96, 1, 1)\n\n    x = concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis)\n    return x\n\n\ndef block_reduction_a(input):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    branch_0 = conv2d_bn(input, 384, 3, 3, strides=(2,2), padding='valid')\n\n    branch_1 = conv2d_bn(input, 192, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 224, 3, 3)\n    branch_1 = conv2d_bn(branch_1, 256, 3, 3, strides=(2,2), padding='valid')\n\n    branch_2 = MaxPooling2D((3,3), strides=(2,2), padding='valid')(input)\n\n    x = concatenate([branch_0, branch_1, branch_2], axis=channel_axis)\n    return x\n\n\ndef block_inception_b(input):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    branch_0 = conv2d_bn(input, 384, 1, 1)\n\n    branch_1 = conv2d_bn(input, 192, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 224, 1, 7)\n    branch_1 = conv2d_bn(branch_1, 256, 7, 1)\n\n    branch_2 = conv2d_bn(input, 192, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 192, 7, 1)\n    branch_2 = conv2d_bn(branch_2, 224, 1, 7)\n    branch_2 = conv2d_bn(branch_2, 224, 7, 1)\n    branch_2 = conv2d_bn(branch_2, 256, 1, 7)\n\n    branch_3 = AveragePooling2D((3,3), strides=(1,1), padding='same')(input)\n    branch_3 = conv2d_bn(branch_3, 128, 1, 1)\n\n    x = concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis)\n    return x\n\n\ndef block_reduction_b(input):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    branch_0 = conv2d_bn(input, 192, 1, 1)\n    branch_0 = conv2d_bn(branch_0, 192, 3, 3, strides=(2, 2), padding='valid')\n\n    branch_1 = conv2d_bn(input, 256, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 256, 1, 7)\n    branch_1 = conv2d_bn(branch_1, 320, 7, 1)\n    branch_1 = conv2d_bn(branch_1, 320, 3, 3, strides=(2,2), padding='valid')\n\n    branch_2 = MaxPooling2D((3, 3), strides=(2, 2), padding='valid')(input)\n\n    x = concatenate([branch_0, branch_1, branch_2], axis=channel_axis)\n    return x\n\n\ndef block_inception_c(input):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    branch_0 = conv2d_bn(input, 256, 1, 1)\n\n    branch_1 = conv2d_bn(input, 384, 1, 1)\n    branch_10 = conv2d_bn(branch_1, 256, 1, 3)\n    branch_11 = conv2d_bn(branch_1, 256, 3, 1)\n    branch_1 = concatenate([branch_10, branch_11], axis=channel_axis)\n\n\n    branch_2 = conv2d_bn(input, 384, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 448, 3, 1)\n    branch_2 = conv2d_bn(branch_2, 512, 1, 3)\n    branch_20 = conv2d_bn(branch_2, 256, 1, 3)\n    branch_21 = conv2d_bn(branch_2, 256, 3, 1)\n    branch_2 = concatenate([branch_20, branch_21], axis=channel_axis)\n\n    branch_3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(input)\n    branch_3 = conv2d_bn(branch_3, 256, 1, 1)\n\n    x = concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis)\n    return x\n\n\ndef inception_v4_base(input):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    # Input Shape is 299 x 299 x 3 (th) or 3 x 299 x 299 (th)\n    net = conv2d_bn(input, 32, 3, 3, strides=(2,2), padding='valid')\n    net = conv2d_bn(net, 32, 3, 3, padding='valid')\n    net = conv2d_bn(net, 64, 3, 3)\n\n    branch_0 = MaxPooling2D((3,3), strides=(2,2), padding='valid')(net)\n\n    branch_1 = conv2d_bn(net, 96, 3, 3, strides=(2,2), padding='valid')\n\n    net = concatenate([branch_0, branch_1], axis=channel_axis)\n\n    branch_0 = conv2d_bn(net, 64, 1, 1)\n    branch_0 = conv2d_bn(branch_0, 96, 3, 3, padding='valid')\n\n    branch_1 = conv2d_bn(net, 64, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 64, 1, 7)\n    branch_1 = conv2d_bn(branch_1, 64, 7, 1)\n    branch_1 = conv2d_bn(branch_1, 96, 3, 3, padding='valid')\n\n    net = concatenate([branch_0, branch_1], axis=channel_axis)\n\n    branch_0 = conv2d_bn(net, 192, 3, 3, strides=(2,2), padding='valid')\n    branch_1 = MaxPooling2D((3,3), strides=(2,2), padding='valid')(net)\n\n    net = concatenate([branch_0, branch_1], axis=channel_axis)\n\n    # 35 x 35 x 384\n    # 4 x Inception-A blocks\n    for idx in range(4):\n    \tnet = block_inception_a(net)\n\n    # 35 x 35 x 384\n    # Reduction-A block\n    net = block_reduction_a(net)\n\n    # 17 x 17 x 1024\n    # 7 x Inception-B blocks\n    for idx in range(7):\n    \tnet = block_inception_b(net)\n\n    # 17 x 17 x 1024\n    # Reduction-B block\n    net = block_reduction_b(net)\n\n    # 8 x 8 x 1536\n    # 3 x Inception-C blocks\n    for idx in range(3):\n    \tnet = block_inception_c(net)\n\n    return net\n\n\ndef inception_v4(num_classes, dropout_keep_prob, weights, include_top):\n    '''\n    Creates the inception v4 network\n\n    Args:\n    \tnum_classes: number of classes\n    \tdropout_keep_prob: float, the fraction to keep before final layer.\n    \n    Returns: \n    \tlogits: the logits outputs of the model.\n    '''\n\n    # Input Shape is 299 x 299 x 3 (tf) or 3 x 299 x 299 (th)\n    if K.image_data_format() == 'channels_first':\n        inputs = Input((3, 299, 299))\n    else:\n        inputs = Input((299, 299, 3))\n\n    # Make inception base\n    x = inception_v4_base(inputs)\n\n\n    # Final pooling and prediction\n    if include_top:\n        # 1 x 1 x 1536\n        x = AveragePooling2D((8,8), padding='valid')(x)\n        x = Dropout(dropout_keep_prob)(x)\n        x = Flatten()(x)\n        # 1536\n        x = Dense(units=num_classes, activation='softmax')(x)\n\n    model = Model(inputs, x, name='inception_v4')\n\n    # load weights\n    if weights == 'imagenet':\n        if K.image_data_format() == 'channels_first':\n            if K.backend() == 'tensorflow':\n                warnings.warn('You are using the TensorFlow backend, yet you '\n                              'are using the Theano '\n                              'image data format convention '\n                              '(`image_data_format=\"channels_first\"`). '\n                              'For best performance, set '\n                              '`image_data_format=\"channels_last\"` in '\n                              'your Keras config '\n                              'at ~/.keras/keras.json.')\n        if include_top:\n            weights_path = get_file(\n                'inception-v4_weights_tf_dim_ordering_tf_kernels.h5',\n                WEIGHTS_PATH,\n                cache_subdir='models',\n                md5_hash='9fe79d77f793fe874470d84ca6ba4a3b')\n        else:\n            weights_path = get_file(\n                'inception-v4_weights_tf_dim_ordering_tf_kernels_notop.h5',\n                WEIGHTS_PATH_NO_TOP,\n                cache_subdir='models',\n                md5_hash='9296b46b5971573064d12e4669110969')\n        model.load_weights(weights_path, by_name=True)\n    return model\n\n\ndef create_model(num_classes=1001, dropout_prob=0.2, weights=None, include_top=False):\n    return inception_v4(num_classes, dropout_prob, weights, include_top)\n# ----------------------------------------------------------\nincept_model = create_model(num_classes=1001, dropout_prob=0.2, weights=None, include_top=False)\nincept_model.load_weights('../input/inceptionv4-weight-file-notop/inception-v4_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n\n\n\nfor l in incept_model.layers: \n    if l is not None: l.trainable = True \n        \nx = incept_model.output\nx = GlobalAveragePooling2D(data_format='channels_last')(x)\nx = BatchNormalization()(x)\n#x = Dense(1024, activation='relu')(x)\n#x = Dropout(0.2)(x)\nx = Dense(512, activation='relu')(x)\n#x = Dropout(0.2)(x)\npredictions = Dense(5, activation='softmax')(x)\n\nmodel = Model(inputs=incept_model.input, outputs=predictions)        \n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- call back functions ------------------\nearly_stop = EarlyStopping(monitor='val_loss', min_delta=0.0002, patience=10, verbose=1, mode='auto')\n# Reducing the Learning Rate if result is not improving. \nreduce_lr = ReduceLROnPlateau(monitor='val_loss', min_delta=0.0004, patience=5, factor=0.1, min_lr=1e-6, mode='auto', verbose=1)\n# ---------------------------------------------- ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\nBATCH_SIZE=32\nSAVED_MODEL_NAME=\"my_model.d5\"\nY_train = df_train.diagnosis.values\nclass_weight_ = class_weight.compute_class_weight('balanced',np.unique(Y_train),Y_train)\nprint(class_weight)\n# ---------------------------------------------- \ndef get_preds_and_labels(model, generator):\n    \"\"\"\n    Get predictions and labels from the generator\n    \"\"\"\n    preds = []\n    labels = []\n    for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):\n        x, y = next(generator)\n        preds.append(model.predict(x))\n        labels.append(y)\n    # Flatten list of numpy arrays\n    return np.concatenate(preds).ravel(), np.concatenate(labels).ravel()\n# ---------------------------------------------- \nclass Metrics(Callback):\n    \"\"\"\n    A custom Keras callback for saving the best model\n    according to the Quadratic Weighted Kappa (QWK) metric\n    \"\"\"\n    def on_train_begin(self, logs={}):\n        \"\"\"\n        Initialize list of QWK scores on validation data\n        \"\"\"\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \"\"\"\n        Gets QWK score on the validation data\n        \"\"\"\n        # Get predictions and convert to integers\n        y_pred, labels = get_preds_and_labels(model, valid_generator)\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        # We can use sklearns implementation of QWK straight out of the box\n        # as long as we specify weights as 'quadratic'\n        _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {round(_val_kappa, 4)}\")\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save(SAVED_MODEL_NAME)\n        return\n# ---------------------------------------------- \nkappa_metrics = Metrics()\n#from keras.utils import multi_gpu_model\n\ndf_train.id_code=df_train.id_code.apply(lambda x: x+\".png\")\ndf_train['diagnosis'] = df_train['diagnosis'].astype('str')\n\ndf_test.id_code=df_test.id_code.apply(lambda x: x+\".png\")\n\naug = ImageDataGenerator(rescale=1./255, validation_split=0.2, horizontal_flip=True, \n                         vertical_flip=True, rotation_range=120, zoom_range=0.1, \n                         width_shift_range=0.2, shear_range=0.15, fill_mode='nearest')\n\ntrain_generator=aug.flow_from_dataframe(dataframe=df_train, \n                                        #directory=\"../input/aptos2019-blindness-detection/train_images/\",\n                                        directory=\"../input/train_images/\",\n                                        x_col=\"id_code\", y_col=\"diagnosis\",\n                                        batch_size=BATCH_SIZE, class_mode=\"categorical\", target_size=(299, 299),\n                                        #preprocessing_function=circle_crop,\n                                        subset='training', shaffle=False, seed=SEED)\n\nvalid_generator=aug.flow_from_dataframe(dataframe=df_train, \n                                        #directory=\"../input/aptos2019-blindness-detection/train_images/\",\n                                        directory=\"../input/train_images/\",\n                                        x_col=\"id_code\", y_col=\"diagnosis\",\n                                        batch_size=BATCH_SIZE, class_mode=\"categorical\", target_size=(299, 299),\n                                        #preprocessing_function=circle_crop,\n                                        subset='validation', shaffle=False, seed=SEED)\n\nmodel.compile(loss='mse', optimizer=RAdam(lr=0.00005), metrics=['mse', 'acc'])\n\nH = model.fit_generator(generator=train_generator, validation_steps=50, \n                         validation_data=valid_generator,\n                         callbacks=[kappa_metrics, early_stop, reduce_lr],\n                         steps_per_epoch=100, \n                         epochs=40, \n                         verbose=1)\n#                         class_weight = class_weight_)\n# -------------------------------------------\n#tlog(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt \nplt.figure(figsize=(8, 8))\nplt.title(\"Learning curve\")\nplt.plot(H.history[\"loss\"], label=\"loss\")\nplt.plot(H.history[\"val_loss\"], label=\"val_loss\")\nplt.plot( np.argmin(H.history[\"val_loss\"]), np.min(H.history[\"val_loss\"]), marker=\"x\", color=\"r\", label=\"best model\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"log_loss\")\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accu = H.history['acc']\nval_acc = H.history['val_acc']\n\nplt.plot(accu,label=\"Accuracy\")\nplt.plot(val_acc)\nplt.plot(kappa_metrics.val_kappas)\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.legend(['Acc','val_acc','kappa'])\nplt.plot( np.argmax(H.history[\"val_acc\"]), np.max(H.history[\"val_acc\"]), marker=\"x\", color=\"r\", label=\"best model\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nfrom math import ceil\n\naug_test = ImageDataGenerator(rescale=1./255, validation_split=0.2, zoom_range=0.1,\n                              horizontal_flip=True, rotation_range=40)\n\ntest_generator=aug_test.flow_from_dataframe(dataframe=df_test, \n                                       #directory = \"../input/aptos2019-blindness-detection/test_images/\",\n                                       directory = \"../input/test_images/\",\n                                       x_col=\"id_code\",\n                                       target_size=(299, 299),batch_size=1,shuffle=False, \n                                       #preprocessing_function=circle_crop,\n                                       class_mode=None, seed=SEED)\n# -----------------------------------------------------    \nmodel.load_weights(SAVED_MODEL_NAME)\n\npreds_tta=[]\nfor i in tqdm(range(20)):\n    test_generator.reset()\n    preds = model.predict_generator(generator=test_generator,steps = ceil(df_test.shape[0]))\n    preds_tta.append(preds)\n# -----------------------------------------------------\n# print(preds_tta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_pred = np.mean(preds_tta, axis=0)\ny_class = np.argmax(final_pred, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.diagnosis = y_class\ndf_submit.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit['diagnosis'].value_counts().plot(kind='bar')\nplt.title('Test Samples Per Class')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tlog('Finished. Total time:'+str((timeit.default_timer()-start_time)/3600)+' hours.')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}