{"cells":[{"metadata":{},"cell_type":"markdown","source":"* Update: changing the batch_size from 32 to 128 when warm up the model, adding Kappa loss from this kernel: https://www.kaggle.com/christofhenkel/weighted-kappa-loss-for-keras-tensorflow","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image, ImageOps\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy, categorical_crossentropy\nfrom keras.applications.resnet50 import preprocess_input\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score, accuracy_score\nfrom keras.utils import Sequence\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\n\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nSIZE = 300\nNUM_CLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ndf_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\n\nx = df_train['']\ny = df_train['diagnosis']\n\nx, y = shuffle(x, y, random_state=8)\ny.hist()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def to_multi_label(target):\n#     multi_label = np.zeros((len(target), NUM_CLASSES))\n#     for i in range(len(target)):\n#         j = target[i] + 1\n#         multi_label[i][:j] = 1\n#     return np.array(multi_label)\n# multi_y = to_multi_label(y)\n# for j in range(5):\n#     print('original: ', y[j])\n#     print('multi-label: ', multi_y[j])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.legacy import interfaces\nfrom keras.optimizers import Optimizer\nfrom keras import backend as K\n\n\nclass AdamAccumulate_v1(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')\n\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, dtype='int64')\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n\n        self.updates = [K.update(self.iterations, self.iterations + 1)]\n\n        flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)\n        flag = K.cast(flag, K.floatx())\n\n        self.updates.append(K.update(self.effective_iterations,\n                                     self.effective_iterations + K.cast(flag, 'int64')))\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.effective_iterations,\n                                                      K.dtype(self.decay))))\n\n        t = K.cast(self.effective_iterations, K.floatx()) + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n                     (1. - K.pow(self.beta_1, t)))\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):\n\n            gg_t = (1 - flag) * (gg + g)\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * (gg + flag * g) / K.cast(self.accum_iters, K.floatx())\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(\n                (gg + flag * g) / K.cast(self.accum_iters, K.floatx()))\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - flag * lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                p_t = p - flag * lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append((m, flag * m_t + (1 - flag) * m))\n            self.updates.append((v, flag * v_t + (1 - flag) * v))\n            self.updates.append((gg, gg_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n\nclass AdamAccumulate(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=2, **kwargs):\n        if accum_iters < 1:\n            raise ValueError('accum_iters must be >= 1')\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n        self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * completed_updates))\n\n        t = completed_updates + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n        # self.iterations incremented after processing a batch\n        # batch:              1 2 3 4 5 6 7 8 9\n        # self.iterations:    0 1 2 3 4 5 6 7 8\n        # update_switch = 1:        x       x    (if accum_iters=4)\n        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n        update_switch = K.cast(update_switch, K.floatx())\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n            sum_grad = tg + g\n            avg_grad = sum_grad / self.accum_iters_float\n\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n            else:\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = to_categorical(y, num_classes=NUM_CLASSES)\n\ntrain_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,\n                                                      stratify=y, random_state=8)\nprint(train_x.shape)\nprint(train_y.shape)\nprint(valid_x.shape)\nprint(valid_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://github.com/aleju/imgaug\nsometimes = lambda aug: iaa.Sometimes(0.5, aug)\nseq = iaa.Sequential([\n    sometimes(\n        iaa.OneOf([\n            iaa.Add((-10, 10), per_channel=0.5),\n            iaa.Multiply((0.9, 1.1), per_channel=0.5),\n            iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5)\n        ])\n    ),\n    iaa.Fliplr(0.5),\n    iaa.Crop(percent=(0, 0.1)),\n    iaa.Flipud(0.5)\n],random_order=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class My_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels,\n                 batch_size, is_train=False,\n                 mix=False, augment=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.batch_size = batch_size\n        self.is_train = is_train\n        self.is_augment = augment\n        if(self.is_train):\n            self.on_epoch_end()\n        self.is_mix = mix\n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        if(self.is_train):\n            return self.train_generate(batch_x, batch_y)\n        return self.valid_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_train):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n    \n    def mix_up(self, x, y):\n        lam = np.random.beta(0.2, 0.4)\n        ori_index = np.arange(int(len(x)))\n        index_array = np.arange(int(len(x)))\n        np.random.shuffle(index_array)        \n        \n        mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n        mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n        \n        return mixed_x, mixed_y\n\n    def train_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/siim-isic-melanoma-classification/train/'+sample+'.jpeg')\n            img = cv2.resize(img, (SIZE, SIZE))\n            if(self.is_augment):\n                img = seq.augment_image(img)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        if(self.is_mix):\n            batch_images, batch_y = self.mix_up(batch_images, batch_y)\n        return batch_images, batch_y\n\n    def valid_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/siim-isic-melanoma-classification/train/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        return batch_images, batch_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D)\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"function = \"softmax\"\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = ResNet50(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    base_model.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation=function, name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create callbacks list\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\n\nepochs = 30; batch_size = 32\ncheckpoint = ModelCheckpoint('../working/Resnet50.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, \n                                   verbose=1, mode='min', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=9)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n                       separator=',',\n                       append=True)\n# callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early]\n\ntrain_generator = My_Generator(train_x, train_y, 128, is_train=True)\ntrain_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)\nvalid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)\n\nmodel = create_model(\n    input_shape=(SIZE,SIZE,3), \n    n_out=NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nclass QWKEvaluation(Callback):\n    def __init__(self, validation_data=(), batch_size=64, interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.batch_size = batch_size\n        self.valid_generator, self.y_val = validation_data\n        self.history = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict_generator(generator=self.valid_generator,\n                                                  steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)),\n                                                  workers=1, use_multiprocessing=True,\n                                                  verbose=1)\n            def flatten(y):\n                return np.argmax(y, axis=1).reshape(-1)\n                # return np.sum(y.astype(int), axis=1) - 1\n            \n            score = cohen_kappa_score(flatten(self.y_val),\n                                      flatten(y_pred),\n                                      labels=[0,1,2,3,4],\n                                      weights='quadratic')\n#             print(flatten(self.y_val)[:5])\n#             print(flatten(y_pred)[:5])\n            print(\"\\n epoch: %d - QWK_score: %.6f \\n\" % (epoch+1, score))\n            self.history.append(score)\n            if score >= max(self.history):\n                print('save checkpoint: ', score)\n                self.model.save('../working/Resnet50_bestqwk.h5')\n\nqwk = QWKEvaluation(validation_data=(valid_generator, valid_y),\n                    batch_size=batch_size, interval=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# warm up model\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-5,0):\n    model.layers[i].trainable = True\n\nmodel.compile(\n    loss='categorical_crossentropy',\n    # loss='binary_crossentropy',\n    optimizer=Adam(1e-3))\n\nmodel.fit_generator(\n    train_generator,\n    steps_per_epoch=np.ceil(float(len(train_y)) / float(128)),\n    epochs=2,\n    workers=WORKERS, use_multiprocessing=True,\n    verbose=1,\n    callbacks=[qwk])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train all layers\nfor layer in model.layers:\n    layer.trainable = True\n\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]\nmodel.compile(loss='categorical_crossentropy',\n            # loss=kappa_loss,\n            # loss='binary_crossentropy',\n            # optimizer=Adam(lr=1e-4),\n            optimizer=AdamAccumulate(lr=1e-4, accum_iters=2),\n            metrics=['accuracy'])\n\nmodel.fit_generator(\n    train_mixup,\n    steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)),\n    validation_data=valid_generator,\n    validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)),\n    epochs=epochs,\n    verbose=1,\n    workers=1, use_multiprocessing=False,\n    callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\n# model.load_weights('../working/Resnet50.h5')\nmodel.load_weights('../working/Resnet50_bestqwk.h5')\npredicted = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, name in tqdm(enumerate(submit['id_code'])):\n    path = os.path.join('../input/siim-isic-melanoma-classification/test/', name+'.png')\n    image = cv2.imread(path)\n    image = cv2.resize(image, (SIZE, SIZE))\n    score_predict = model.predict((image[np.newaxis])/255)\n    label_predict = np.argmax(score_predict)\n    # label_predict = score_predict.astype(int).sum() - 1\n    predicted.append(str(label_predict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['diagnosis'] = predicted\nsubmit.to_csv('submission.csv', index=False)\nsubmit.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}