{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel demonstrated idea how to use qwadratic kappa loss. I think use this loss \"as is\" is not good idea, u can use it simultaneously with another loss (CCE for instance) or for fitting pretrained model (after training CCE or MSE for instance). Moreover, the code is not optimal for kaggle kernels, the bottleneck in CPU, locally you can avoid this problem by resized and augmented images befor training, you can use memory_allocate=True in function \"get_data\" if have enough ram memory. But if you have powerful CPU and fast ssd in your local PC this code should work good out-of-the-box."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import train_test_split\n\nimport imgaug as ia\nimport imgaug.augmenters as iaa\n\nimport keras.backend as K\nfrom keras import models\nfrom keras import layers\nfrom keras import callbacks\nfrom keras import optimizers\nfrom keras.applications import DenseNet121","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create class for eyes"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Eye(object):\n    def __init__(self, path, target, memory_allocate=False, size=None):\n        self.path = path\n        self.size = size\n        self.target = target\n\n        if memory_allocate:\n            self.__image = self.__get_image()\n        else:\n            self.__image = None\n\n    def __get_image(self):\n        im = cv2.cvtColor(cv2.imread(self.path, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)\n        if self.size is not None:\n            return cv2.resize(im, self.size).astype(\"uint8\")\n        else:\n            return im.astype(\"uint8\")\n\n    @property\n    def image(self):\n        if self.__image is not None:\n            return self.__image\n        else:\n            return self.__get_image()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create train and test arrays."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(data_path, memory_allocate=False, image_size=None, extension=\"png\"):\n    def _get_data(pd_set, folder):\n        eyes = []\n        for im in tqdm(pd_set.iterrows(), total=pd_set.shape[0]):\n            im_path = os.path.join(data_path, folder, \"{}.{}\".format(im[1].id_code, extension))\n            if \"diagnosis\" in im[1].keys().values:\n                target = im[1].diagnosis\n            else:\n                target = None\n            eyes.append(Eye(path=im_path, memory_allocate=memory_allocate, size=image_size, target=target))\n        return eyes\n    train_df = pd.read_csv(os.path.join(data_path, \"train.csv\"))\n\n    test_eyes = None\n    train_eyes = _get_data(train_df, \"train_images\")\n    if os.path.exists(os.path.join(data_path, \"test.csv\")):\n        test_df = pd.read_csv(os.path.join(data_path, \"test.csv\"))\n        test_eyes = _get_data(test_df, \"test_images\")\n\n    return train_eyes, test_eyes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature extractor classes. Using it to flexibility changing parameters of input images. Can use different extractors for train and test (with different augmentation for instance."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"class SimpleExtractor(object):\n    def __init__(self, image_size: tuple=(224, 224), is_normalized=True, **kwargs):\n        self.size = image_size\n        self.is_normalized = is_normalized\n\n    @property\n    def shape(self):\n        return self.size + (3, )\n\n    def get_image(self, eye):\n        resized_image = cv2.resize(eye.image, self.size)\n        return resized_image\n\n    def normalize(self, image):\n        image = image.astype(\"float32\")\n        image -= np.mean(image, axis=(0, 1), keepdims=True)\n        image /= (np.std(image, axis=(0, 1), keepdims=True) + 1e-7)\n\n        return image\n\n    def __call__(self, eye):\n        im = self.get_image(eye=eye)\n        if self.is_normalized:\n            im = self.normalize(image=im)\n        return im[None, ...]\n\n\nclass AugmentExtractor(SimpleExtractor):\n    def __init__(self, augmentation=None, **kwargs):\n        super(AugmentExtractor, self).__init__(**kwargs)\n        self.augmentation = augmentation\n\n    def __call__(self, eye):\n        im = self.get_image(eye=eye)\n        if self.augmentation is not None:\n            im = self.augmentation.augment_image(im)\n\n        if self.is_normalized:\n            im = self.normalize(image=im)\n\n        return im[None, ...]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Keras sequential generator with ability balances or disbalances batch."},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.data_utils import Sequence\nclass SimpleGenerator(Sequence):\n    TRAIN = 0\n    TEST = 1\n\n    def __init__(self, eyes, extractor, mode, is_balances_batch=True, batch_size=32, one_hot=True):\n        assert mode in [self.TRAIN, self.TEST]\n\n        self.eyes = np.asarray(eyes)\n        self.extractor = extractor\n        self.mode = mode\n        self.one_hot = one_hot\n        self.is_balances_batch = is_balances_batch\n        self.batch_size = batch_size if self.mode == self.TRAIN else 1\n\n        self.targets = np.asarray([x.target for x in self.eyes])\n        self.unique_target = np.asarray(list(set(self.targets)))\n\n    @property\n    def n_classes(self):\n        return self.unique_target.shape[0]\n\n    def __len__(self):\n        return self.eyes.shape[0] // self.batch_size\n\n    def __getitem__(self, item):\n        if self.mode == self.TRAIN:\n            return self.get_train_batch()\n        else:\n            return self.get_test_batch(item)\n\n    def get_test_batch(self, item):\n        return self.extractor(self.eyes[item])\n\n    def get_train_batch(self):\n        X = np.empty((self.batch_size, ) + self.extractor.shape)\n        targets = np.random.choice(self.unique_target, self.batch_size)\n        Y = []\n        for n, t in enumerate(targets):\n            if self.is_balances_batch:\n                n_eye = np.random.choice(np.where(self.targets == t)[0])\n            else:\n                n_eye = np.random.randint(self.eyes.shape[0])\n            eye = self.eyes[n_eye]\n            X[n] = self.extractor(eye)\n            if self.one_hot:\n                Y.append(self.unique_target == eye.target)\n            else:\n                Y.append(eye.target)\n\n        Y = np.vstack(Y).astype(\"float64\")\n\n        return X, Y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"class QWK(callbacks.Callback):\n    def __init__(self, generator, save_path, monitor_length=5, net=None, one_hot=False):\n        self._not_improvement_epochs = 0\n        self.generator = generator\n        self.save_path = save_path\n        self.net = net\n        self.monitor_length = monitor_length\n        self.one_hot = one_hot\n        self.kappa_score = -1.0 * np.inf\n        super(QWK, self).__init__()\n\n    def on_epoch_end(self, epoch, logs={}):\n        if self.net is not None:\n            model = self.net\n        else:\n            model = self.model\n\n        y_val = self.generator.targets\n\n        y_pred = model.predict_generator(self.generator, verbose=1)\n        if self.one_hot:\n            y_pred = np.argmax(y_pred, axis=1)\n        else:\n            y_pred = np.round(y_pred)\n\n        qwk = cohen_kappa_score(y_val, y_pred, weights='quadratic')\n        if qwk > self.kappa_score:\n            self.model.save(os.path.join(self.save_path, \"final.h5\"), overwrite=True, include_optimizer=False)\n            self._not_improvement_epochs = 0\n        else:\n            self._not_improvement_epochs += 1\n        \n        if self._not_improvement_epochs > self.monitor_length:\n            self.model.stop_training = True\n        print(\"val_kappa: {:.4f}\".format(qwk))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create neural network. Densenet121 with imagenet pretrain in this kernel. U can change it ofc."},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_densenet121(input_shape=(224, 224, 3), classes=5, one_hot=True):\n    net = DenseNet121(\n                weights=None,\n                include_top=False,\n                input_shape=(224, 224, 3)\n                )\n    model = models.Sequential()\n    model.add(net)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(0.2))\n    if one_hot:\n        model.add(layers.Dense(classes, activation=\"softmax\"))\n    else:\n        model.add(layers.Dense(1, activation=\"relu\"))\n        model.add(layers.Lambda(lambda a: 4 - K.relu(-a + 4)))\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implementation QWK loss. Be careful, implementation have some changes for probabilistic output, confusion matrix compute with no rounded output. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def quadratic_kappa_coefficient(y_true, y_pred):\n    y_true = K.cast(y_true, \"float32\")\n    n_classes = K.cast(y_pred.shape[-1], \"float32\")\n    weights = K.arange(0, n_classes, dtype=\"float32\") / (n_classes - 1)\n    weights = (weights - K.expand_dims(weights, -1)) ** 2\n\n    hist_true = K.sum(y_true, axis=0)\n    hist_pred = K.sum(y_pred, axis=0)\n\n    E = K.expand_dims(hist_true, axis=-1) * hist_pred\n    E = E / K.sum(E, keepdims=False)\n\n    O = K.transpose(K.transpose(y_true) @ y_pred)  # confusion matrix\n    O = O / K.sum(O)\n\n    num = weights * O\n    den = weights * E\n\n    QWK = (1 - K.sum(num) / K.sum(den))\n    return QWK\n\ndef quadratic_kappa_loss(scale=2.0):\n    def _quadratic_kappa_loss(y_true, y_pred):\n        QWK = quadratic_kappa_coefficient(y_true, y_pred)\n        loss = -K.log(K.sigmoid(scale * QWK))\n        return loss\n        \n    return _quadratic_kappa_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## How it works:"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true   = np.asarray([[1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]])\ny_pred_1 = np.asarray([[0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1]])\ny_pred_2 = np.asarray([[0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0]])\n\nqwk_loss = quadratic_kappa_loss(scale=2.0)\nprint(\"for pred_1\")\nprint(\"sklearn QWK: {}\".format(cohen_kappa_score(np.argmax(y_true, axis=1), np.argmax(y_pred_1, axis=1), weights='quadratic')))\nprint(\"keras QWK: {}\".format(K.eval(quadratic_kappa_coefficient(K.variable(y_true), K.variable(y_pred_1)))))\nprint(\"keras QWK loss: {}\\n\".format(K.eval(qwk_loss(K.variable(y_true), K.variable(y_pred_1)))))\n\nprint(\"for pred_2\")\nprint(\"sklearn QWK for pred 2: {}\".format(cohen_kappa_score(np.argmax(y_true, axis=1), np.argmax(y_pred_2, axis=1), weights='quadratic')))\nprint(\"keras QWK for pred 2: {}\".format(K.eval(quadratic_kappa_coefficient(K.variable(y_true), K.variable(y_pred_2)))))\nprint(\"keras QWK loss for pred 2: {}\\n\".format(K.eval(qwk_loss(K.variable(y_true), K.variable(y_pred_2)))))\n\nprint(\"for fully correct output\")\nprint(\"sklearn QWK: {}\".format(cohen_kappa_score(np.argmax(y_true, axis=1), np.argmax(y_true, axis=1), weights='quadratic')))\nprint(\"keras QWK: {}\".format(K.eval(quadratic_kappa_coefficient(K.variable(y_true), K.variable(y_true)))))\nprint(\"keras QWK: {}\\n\".format(K.eval(qwk_loss(K.variable(y_true), K.variable(y_true)))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation recipe"},{"metadata":{"trusted":true},"cell_type":"code","source":"sometimes05 = lambda aug: iaa.Sometimes(0.5, aug)\nsometimes01 = lambda aug: iaa.Sometimes(0.1, aug)\n\naugment0 = iaa.Sequential(\n    [\n        # apply the following augmenters to most images\n        iaa.Fliplr(0.5), # horizontally flip 50% of all images\n        iaa.Flipud(0.5), # vertically flip 20% of all images\n        # crop images by -5% to 10% of their height/width\n        sometimes05(iaa.CropAndPad(\n            percent=(-0.05, 0.1),\n            pad_mode=ia.ALL,\n            pad_cval=(0, 255)\n        )),\n        sometimes01(iaa.Affine(\n            scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis\n            translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)\n            rotate=(-45, 45), # rotate by -45 to +45 degrees\n            shear=(-16, 16), # shear by -16 to +16 degrees\n            order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n            cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n            mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n        )),\n        sometimes01(iaa.Grayscale(alpha=(0.0, 1.0))),\n        iaa.OneOf([\n            iaa.Multiply((0.5, 1.5), per_channel=0.5),\n            iaa.FrequencyNoiseAlpha(\n                exponent=(-4, 0),\n                first=iaa.Multiply((0.5, 1.5), per_channel=True),\n                second=iaa.ContrastNormalization((0.5, 2.0))\n            )\n        ]),\n\n\n    ],\n    random_order=True\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(data_path, save_path, weights=None, initial_epoch=0, one_hot=True, is_balances_batch=True):\n    image_shape = (224, 224, 3)\n    \n    train_data, _ = get_data(data_path=data_path, memory_allocate=False)\n    train_eyes, validation_eyes = train_test_split(train_data, test_size=0.1)\n\n    train_extractor = AugmentExtractor(augmentation=augment0, image_size=image_shape[:-1], is_normalized=True)\n    test_extractor = AugmentExtractor(augmentation=None, image_size=image_shape[:-1], is_normalized=True)\n\n    train_generator = SimpleGenerator(eyes=train_eyes,\n                                      extractor=train_extractor,\n                                      batch_size=32,\n                                      is_balances_batch=is_balances_batch,\n                                      one_hot=one_hot,\n                                      mode=SimpleGenerator.TRAIN\n                                      )\n    validate_generator = SimpleGenerator(eyes=validation_eyes,\n                                         extractor=test_extractor,\n                                         batch_size=1,\n                                         one_hot=one_hot,\n                                         mode=SimpleGenerator.TEST\n                                         )\n    test_batch = train_generator[0]\n\n    net = build_densenet121(input_shape=image_shape, classes=5)\n\n    if one_hot:\n        loss = quadratic_kappa_loss(scale=2.0)\n        metrics=[\"accuracy\"]\n    else:\n        loss = \"MSE\"\n        metrics=[]\n        \n    optimizer = optimizers.Adam(lr=0.0001)\n    net.compile(loss=loss, optimizer=optimizer, metrics=metrics)\n\n    if weights is not None:\n        net.load_weights(weights, by_name=True, skip_mismatch=True)\n\n    callbacks = [QWK(validate_generator, one_hot=one_hot, save_path=save_path, monitor_length=4)]\n\n    net.fit_generator(train_generator,\n                      epochs=20,\n                      callbacks=callbacks,\n                      steps_per_epoch=None,\n                      initial_epoch=initial_epoch,\n                      use_multiprocessing=False,\n                      workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submit predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def submit(data_path, save_path, model_path, one_hot=True):\n    _, test_data = get_data(data_path=data_path, memory_allocate=False)\n    test_df = pd.read_csv(os.path.join(data_path, \"test.csv\"))\n\n    extractor = SimpleExtractor(image_size=(224, 224), is_normalized=True)\n    generator = SimpleGenerator(extractor=extractor,\n                                eyes=test_data,\n                                mode=SimpleGenerator.TEST)\n\n    test_batch = generator[0]\n    model = models.load_model(model_path)\n    pred = model.predict_generator(generator=generator, verbose=1)\n    if one_hot:\n        pred = np.argmax(pred, axis=-1)\n    else:\n        pred = np.round(pred).astype(\"int\")\n\n    test_df['diagnosis'] = pred\n    test_df.to_csv(os.path.join(save_path, 'submission.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(\"../input\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = os.path.join(\"../input\", \"aptos2019-blindness-detection\")\nweights = os.path.join(\"../input\", \"densenet121weights\", \"densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5\")\nsave_path = os.path.join(\"result\", \"densenet121\", \"qwk_loss\")\nmodel_path = os.path.join(save_path, \"final.h5\")\nos.makedirs(save_path, exist_ok=True)\n\nis_balances_batch = False\none_hot = True  # for QWK loss\n# one_hot = False  # for MSE loss\n\ntrain(data_path=data_path,\n      save_path=save_path,\n      weights=weights,\n      initial_epoch=0,\n      one_hot=one_hot, \n      is_balances_batch=is_balances_batch)\n\nsubmit(data_path=data_path, \n       save_path=save_path, \n       model_path=model_path, \n       one_hot=one_hot)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}