{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image, ImageOps\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy, categorical_crossentropy\nfrom keras.applications.resnet50 import preprocess_input\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score, accuracy_score\nfrom keras.utils import Sequence\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D)\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model\nimport os\nprint(os.listdir(\"../input\"))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nSIZE = 300\nNUM_CLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\nx = df_train['id_code']\ny = df_train['diagnosis']\n\nx, y = shuffle(x, y, random_state=8)\ny.hist()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.legacy import interfaces\nfrom keras.optimizers import Optimizer\nfrom keras import backend as K\n\n\nclass AdamAccumulate_v1(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')\n\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, dtype='int64')\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n\n        self.updates = [K.update(self.iterations, self.iterations + 1)]\n\n        flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)\n        flag = K.cast(flag, K.floatx())\n\n        self.updates.append(K.update(self.effective_iterations,\n                                     self.effective_iterations + K.cast(flag, 'int64')))\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.effective_iterations,\n                                                      K.dtype(self.decay))))\n\n        t = K.cast(self.effective_iterations, K.floatx()) + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n                     (1. - K.pow(self.beta_1, t)))\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):\n\n            gg_t = (1 - flag) * (gg + g)\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * (gg + flag * g) / K.cast(self.accum_iters, K.floatx())\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(\n                (gg + flag * g) / K.cast(self.accum_iters, K.floatx()))\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - flag * lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                p_t = p - flag * lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append((m, flag * m_t + (1 - flag) * m))\n            self.updates.append((v, flag * v_t + (1 - flag) * v))\n            self.updates.append((gg, gg_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n\nclass AdamAccumulate(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=2, **kwargs):\n        if accum_iters < 1:\n            raise ValueError('accum_iters must be >= 1')\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n        self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * completed_updates))\n\n        t = completed_updates + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n        update_switch = K.cast(update_switch, K.floatx())\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n            sum_grad = tg + g\n            avg_grad = sum_grad / self.accum_iters_float\n\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n            else:\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = to_categorical(y, num_classes=NUM_CLASSES)\n\ntrain_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,\n                                                      stratify=y, random_state=8)\nprint(train_x.shape)\nprint(train_y.shape)\nprint(valid_x.shape)\nprint(valid_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sometimes = lambda aug: iaa.Sometimes(0.5, aug)\nseq = iaa.Sequential([\n    sometimes(\n        iaa.OneOf([\n            iaa.Add((-10, 10), per_channel=0.5),\n            iaa.Multiply((0.9, 1.1), per_channel=0.5),\n            iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5)\n        ])\n    ),\n    iaa.Fliplr(0.5),\n    iaa.Crop(percent=(0, 0.1)),\n    iaa.Flipud(0.5)\n],random_order=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class My_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels,\n                 batch_size, is_train=False,\n                 mix=False, augment=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.batch_size = batch_size\n        self.is_train = is_train\n        self.is_augment = augment\n        if(self.is_train):\n            self.on_epoch_end()\n        self.is_mix = mix\n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        if(self.is_train):\n            return self.train_generate(batch_x, batch_y)\n        return self.valid_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_train):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n    \n    def mix_up(self, x, y):\n        lam = np.random.beta(0.2, 0.4)\n        ori_index = np.arange(int(len(x)))\n        index_array = np.arange(int(len(x)))\n        np.random.shuffle(index_array)        \n        \n        mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n        mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n        \n        return mixed_x, mixed_y\n\n    def train_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/aptos2019-blindness-detection/train_images/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            if(self.is_augment):\n                img = seq.augment_image(img)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        if(self.is_mix):\n            batch_images, batch_y = self.mix_up(batch_images, batch_y)\n        return batch_images, batch_y\n\n    def valid_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/aptos2019-blindness-detection/train_images/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        return batch_images, batch_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"function = \"softmax\"\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = ResNet50(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    base_model.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation=function, name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\n\nepochs = 30\nbatch_size = 32\ncheckpoint = ModelCheckpoint('../working/Resnet50.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, \n                                   verbose=1, mode='min', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=9)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n                       separator=',',\n                       append=True)\n\ntrain_generator = My_Generator(train_x, train_y, 128, is_train=True)\ntrain_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)\nvalid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)\n\nmodel = create_model(\n    input_shape=(SIZE,SIZE,3), \n    n_out=NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nclass QWKEvaluation(Callback):\n    def __init__(self, validation_data=(), batch_size=64, interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.batch_size = batch_size\n        self.valid_generator, self.y_val = validation_data\n        self.history = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict_generator(generator=self.valid_generator,\n                                                  steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)),\n                                                  workers=1, use_multiprocessing=True,\n                                                  verbose=1)\n            def flatten(y):\n                return np.argmax(y, axis=1).reshape(-1)\n                # return np.sum(y.astype(int), axis=1) - 1\n            \n            score = cohen_kappa_score(flatten(self.y_val),\n                                      flatten(y_pred),\n                                      labels=[0,1,2,3,4],\n                                      weights='quadratic')\n            print(\"\\n epoch: %d - QWK_score: %.6f \\n\" % (epoch+1, score))\n            self.history.append(score)\n            if score >= max(self.history):\n                print('save checkpoint: ', score)\n                self.model.save('../working/Resnet50_bestqwk.h5')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qwk = QWKEvaluation(validation_data=(valid_generator, valid_y),batch_size=batch_size, interval=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-5,0):\n    model.layers[i].trainable = True\n\nmodel.compile(\n    loss='categorical_crossentropy',\n    optimizer=Adam(1e-3))\n\nmodel.fit_generator(\n    train_generator,\n    steps_per_epoch=np.ceil(float(len(train_y)) / float(128)),\n    epochs=2,\n    workers=WORKERS, use_multiprocessing=True,\n    verbose=1,\n    callbacks=[qwk])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor layer in model.layers:\n    layer.trainable = True\n\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]\nmodel.compile(loss='categorical_crossentropy',optimizer=AdamAccumulate(lr=1e-4, accum_iters=2),metrics=['accuracy'])\n\nmodel.fit_generator(\n    train_mixup,\n    steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)),\n    validation_data=valid_generator,\n    validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)),\n    epochs=epochs,\n    verbose=1,\n    workers=1, use_multiprocessing=False,\n    callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nmodel.load_weights('../working/Resnet50_bestqwk.h5')\npredicted = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, name in tqdm(enumerate(submit['id_code'])):\n    path = os.path.join('../input/aptos2019-blindness-detection/test_images/', name+'.png')\n    image = cv2.imread(path)\n    image = cv2.resize(image, (SIZE, SIZE))\n    score_predict = model.predict((image[np.newaxis])/255)\n    label_predict = np.argmax(score_predict)\n    predicted.append(str(label_predict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['diagnosis'] = predicted\nsubmit.to_csv('submission.csv', index=False)\nsubmit.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}