{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport cv2\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight,shuffle\nfrom imgaug import augmenters as iaa\nimport skimage.io\nimport imgaug as ia\nfrom keras.utils import Sequence\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate)\nfrom keras.applications.densenet import DenseNet121\nfrom keras.models import Model\nfrom keras.optimizers import Adam \nimport numpy as np\nfrom sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score\nfrom tqdm import tqdm\nfrom keras.callbacks import Callback\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\nimport tensorflow as tf\nfrom keras.losses import binary_crossentropy, categorical_crossentropy","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(\"../input\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = 5\nSIZE = 300","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = df_train['id_code']\ny = df_train['diagnosis']\n\nx, y = shuffle(x, y, random_state=8)\ny.hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x.shape)\nprint(y.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\ncollections.Counter(df_train['diagnosis'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = to_categorical(y, num_classes=NUM_CLASSES)\ntrain_x, valid_x,train_y, valid_y = train_test_split(x, y, test_size=0.15,\n                                                      stratify=y, random_state=8)\nprint(train_x.shape)\nprint(train_y.shape)\nprint(valid_x.shape)\nprint(valid_y.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from imblearn.over_sampling import RandomOverSampler\n#ros = RandomOverSampler(random_state=0)\n#train_x,train_y = ros.fit_resample(b_train_x.values.reshape(-1,1),b_train_y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_x.shape, train_y.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sometimes = lambda aug: iaa.Sometimes(0.5, aug)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq = iaa.Sequential(\n        [\n           \n            iaa.Fliplr(0.5),\n            iaa.Flipud(0.2), \n            sometimes(iaa.Affine(\n                scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n                translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n                rotate=(-10, 10), \n                shear=(-5, 5), \n                order=[0, 1],\n                cval=(0, 255),\n                mode=ia.ALL\n            )),\n            iaa.SomeOf((0, 5),\n                [\n                    sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))),\n                    iaa.OneOf([\n                        iaa.GaussianBlur((0, 1.0)),\n                        iaa.AverageBlur(k=(3, 5)),\n                        iaa.MedianBlur(k=(3, 5)),\n                    ]),\n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)),\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),\n                   \n                    iaa.SimplexNoiseAlpha(iaa.OneOf([\n                        iaa.EdgeDetect(alpha=(0.5, 1.0)),\n                        iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n                    ])),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5),\n                    iaa.OneOf([\n                        iaa.Dropout((0.01, 0.05), per_channel=0.5),\n                        iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2),\n                    ]),\n                    iaa.Invert(0.01, per_channel=True),\n                    iaa.Add((-2, 2), per_channel=0.5),\n                    iaa.AddToHueAndSaturation((-1, 1)),\n                    \n                    iaa.OneOf([\n                        iaa.Multiply((0.9, 1.1), per_channel=0.5),\n                        iaa.FrequencyNoiseAlpha(\n                            exponent=(-1, 0),\n                            first=iaa.Multiply((0.9, 1.1), per_channel=True),\n                            second=iaa.ContrastNormalization((0.9, 1.1))\n                        )\n                    ]),\n                    sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)),\n                    sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),\n                    sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Custom_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels,\n                 batch_size, is_train=True,\n                 mix=False, augment=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.batch_size = batch_size\n        self.is_train = is_train\n        self.is_augment = augment\n        if(self.is_train):\n            self.on_epoch_end()\n        self.is_mix = mix\n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        if(self.is_train):\n            return self.train_generate(batch_x, batch_y)\n        return self.valid_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_train):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n        else:\n            pass\n    \n    def train_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/aptos2019-blindness-detection/train_images/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            if(self.is_augment):\n                img = seq.augment_image(img)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        if(self.is_mix):\n            batch_images, batch_y = self.mix_up(batch_images, batch_y)\n        return batch_images, batch_y\n\n    def valid_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/aptos2019-blindness-detection/train_images/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        return batch_images, batch_y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this code reference from : https://www.kaggle.com/christofhenkel/weighted-kappa-loss-for-keras-tensorflow\ndef kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='kappa'):\n    \"\"\"A continuous differentiable approximation of discrete kappa loss.\n        Args:\n            y_pred: 2D tensor or array, [batch_size, num_classes]\n            y_true: 2D tensor or array,[batch_size, num_classes]\n            y_pow: int,  e.g. y_pow=2\n            N: typically num_classes of the model\n            bsize: batch_size of the training or validation ops\n            eps: a float, prevents divide by zero\n            name: Optional scope/name for op_scope.\n        Returns:\n            A tensor with the kappa loss.\"\"\"\n\n    with tf.name_scope(name):\n        y_true = tf.to_float(y_true)\n        repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))\n        repeat_op_sq = tf.square((repeat_op - tf.transpose(repeat_op)))\n        weights = repeat_op_sq / tf.to_float((N - 1) ** 2)\n    \n        pred_ = y_pred ** y_pow\n        try:\n            pred_norm = pred_ / (eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))\n        except Exception:\n            pred_norm = pred_ / (eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))\n    \n        hist_rater_a = tf.reduce_sum(pred_norm, 0)\n        hist_rater_b = tf.reduce_sum(y_true, 0)\n    \n        conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)\n    \n        nom = tf.reduce_sum(weights * conf_mat)\n        denom = tf.reduce_sum(weights * tf.matmul(\n            tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /\n                              tf.to_float(bsize))\n    \n        return nom*0.5 / (denom + eps) + categorical_crossentropy(y_true, y_pred)*0.5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = DenseNet121(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    base_model.load_weights(\"../input/densenet-121/DenseNet-BC-121-32-no-top.h5\")\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n    model = Model(input_tensor, final_output) \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = DenseNet121(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    base_model.load_weights(\"../input/dense-net/DenseNet-BC-121-32-no-top.h5\")\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n    model = Model(input_tensor, final_output) \n    #model = load_model('../input/pretrained-net/densenet_bestqwk.h5', custom_objects={'kappa_loss': kappa_loss})\n    model.load_weights('../input/pretrained-net/densenet_bestqwk.h5')\n    set_trainable = False\n    for layer in model.layers:\n        if layer.name.find(\"conv5\") != -1:\n            set_trainable = True\n        if set_trainable:\n            layer.trainable = True\n        else:\n            layer.trainable = False\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input/dense-net","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(\n    input_shape=(SIZE,SIZE,3), \n    n_out=NUM_CLASSES)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for lay in model.layers:\n    print(lay.name, lay.trainable)\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 20; batch_size = 32\ncheckpoint = ModelCheckpoint('../working/densenet_.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, \n                                   verbose=1, mode='auto', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=9)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n                       separator=',',\n                       append=True)\ntrain_mixup = Custom_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)\nvalid_generator = Custom_Generator(valid_x, valid_y, batch_size, is_train=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Quad_Weighted_Kappa(Callback):\n    def __init__(self, validation_data=(), batch_size=64, interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.batch_size = batch_size\n        self.valid_generator, self.y_val = validation_data\n        self.history = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict_generator(generator=self.valid_generator,\n                                                  steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)),\n                                                  workers=1, use_multiprocessing=False,\n                                                  verbose=1)\n            def flatten(y):\n                return np.argmax(y, axis=1).reshape(-1)\n            \n            score = cohen_kappa_score(flatten(self.y_val),\n                                      flatten(y_pred),\n                                      labels=[0,1,2,3,4],\n                                      weights='quadratic')\n            print(\"\\n epoch: %d - Quad_Wtd_Kappa_score: %.6f \\n\" % (epoch+1, score))\n            self.history.append(score)\n            if score >= max(self.history):\n                print('saving checkpoint: ', score)\n                self.model.save('../working/densenet_bestqwk.h5')\n\nquad_w_kappa = Quad_Weighted_Kappa(validation_data=(valid_generator, valid_y),\n                    batch_size=batch_size, interval=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = create_model(\n#    input_shape=(SIZE,SIZE,3), \n#    n_out=NUM_CLASSES)\n#categorical_crossentropy\ndef run_model():\n    for layer in model.layers:\n        layer.trainable = True\n    callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, quad_w_kappa]\n    model.compile(loss=kappa_loss\n                  ,optimizer=Adam(lr=1e-4))\n    model.fit_generator(\n        train_mixup,\n        steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)),\n        validation_data=valid_generator,\n        validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)),\n        epochs=epochs,\n        verbose=1,\n        workers=2, use_multiprocessing=True,\n        callbacks=callbacks_list)\nrun_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted = []\nsubmit = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\ndef predict(base_path):\n    model.load_weights('../working/densenet_bestqwk.h5')\n    for i, name in tqdm(enumerate(submit['id_code'])):\n        path = os.path.join(base_path+'test_images/', name+'.png')\n        image = cv2.imread(path)\n        image = cv2.resize(image, (SIZE, SIZE))\n        X = np.array((image[np.newaxis])/255)\n        score_predict=((model.predict(X).ravel()*model.predict(X[:, ::-1, :, :]).ravel()*model.predict(X[:, ::-1, ::-1, :]).ravel()*model.predict(X[:, :, ::-1, :]).ravel())**0.25).tolist()\n        label_predict = np.argmax(score_predict)\n        predicted.append(str(label_predict))\n\npredict('../input/aptos2019-blindness-detection/')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['diagnosis'] = predicted\nsubmit.to_csv('submission.csv', index=False)\nsubmit.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traing_log = pd.read_csv(\"../working/training_log.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nfrom matplotlib import pyplot as plt\n#history = model1.fit(train_x, train_y,validation_split = 0.1, epochs=50, batch_size=4)\nplt.plot(traing_log['loss'])\nplt.plot(traing_log['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}