{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom random import randint\n\nimport PIL\nfrom PIL import Image\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nimport keras\nimport warnings\nfrom keras.utils import Sequence\nimport tensorflow as tf\nwarnings.filterwarnings(\"ignore\")\nSIZE = 299\nSEED = 777\nTHRESHOLD = 0.2","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Load dataset info\nDIR = '../input/'\n# data = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d95ed4ac4701525d165c6c4c274678f4712d8bb"},"cell_type":"code","source":"def getTrainDataset():\n    \n    path_to_train = DIR + '/train/'\n    data = pd.read_csv(DIR + '/train.csv')\n\n    paths = []\n    labels = []\n    \n    for name, lbl in zip(data['Id'], data['Target'].str.split(' ')):\n        y = np.zeros(28)\n        for key in lbl:\n            y[int(key)] = 1\n        paths.append(os.path.join(path_to_train, name))\n        labels.append(y)\n\n    return np.array(paths[:5000]), np.array(labels[:5000])\n\ndef getTestDataset():\n    \n    path_to_test = DIR + '/test/'\n    data = pd.read_csv(DIR + '/sample_submission.csv')\n\n    paths = []\n    labels = []\n    \n    for name in data['Id']:\n        y = np.ones(28)\n        paths.append(os.path.join(path_to_test, name))\n        labels.append(y)\n\n    return np.array(paths), np.array(labels)\n# paths, labels = getTrainDataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63f8c546a32b8db298c62dd06eb6ff23e1db360f"},"cell_type":"code","source":"# credits: https://github.com/keras-team/keras/blob/master/keras/utils/data_utils.py#L302\n# credits: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\nclass ProteinDataGenerator(keras.utils.Sequence):\n            \n    def __init__(self, paths, labels, batch_size, shape, channels = [], shuffle = False, use_cache = False, augmentor = False):\n        self.paths, self.labels = paths, labels\n        self.batch_size = batch_size\n        self.shape = shape\n        self.shuffle = shuffle\n        self.use_cache = use_cache\n        self.channels = channels\n        self.augmentor = augmentor\n        self.clahe = cv2.createCLAHE()\n        if use_cache == True:\n            self.cache = np.zeros((paths.shape[0], shape[0], shape[1], len(channels)))\n            self.is_cached = np.zeros((paths.shape[0]))\n        self.on_epoch_end()\n    \n    def __len__(self):\n        return int(np.ceil(len(self.paths) / float(self.batch_size)))\n    \n    def __getitem__(self, idx):\n        indexes = self.indexes[idx * self.batch_size : (idx+1) * self.batch_size]\n\n        paths = self.paths[indexes]\n        X = np.zeros((paths.shape[0], self.shape[0], self.shape[1], self.shape[2]))\n        # Generate data\n        if self.use_cache == True:\n            X = self.cache[indexes]\n            for i, path in enumerate(paths[np.where(self.is_cached[indexes] == 0)]):\n                image = self.__load_image(path)\n                self.is_cached[indexes[i]] = 1\n                self.cache[indexes[i]] = image\n                X[i] = image\n        else:\n            for i, path in enumerate(paths):\n                X[i] = self.__load_image(path)\n        if self.augmentor == True:\n            for i, item in enumerate(X):\n                X[i] = self.augment(item)\n        y = self.labels[indexes]\n        \n        return X, y\n    \n    def on_epoch_end(self):\n        \n        # Updates indexes after each epoch\n        self.indexes = np.arange(len(self.paths))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __iter__(self):\n        \"\"\"Create a generator that iterate over the Sequence.\"\"\"\n        for item in (self[i] for i in range(len(self))):\n            yield item\n            \n    def __load_image(self, path):\n        images = []\n        for channel in self.channels:\n            im = np.array(Image.open(path + '_' + channel + '.png'))\n            \n#             im = clahe.apply(im)\n            images.append(im)\n            \n        if len(self.channels) >= 2:\n            im = np.stack((\n                images\n            ), -1)\n            im = cv2.resize(im, (SIZE,SIZE))\n            im = np.divide(im, 255)\n\n        else:\n            im = images[0]\n            im = cv2.resize(im, (SIZE,SIZE))\n            im = np.divide(im, 255)\n            im = np.expand_dims(im, 2)\n        return im\n    def augment(self, image):\n        if randint(0,1) == 1:\n            augment_img = iaa.Sequential([\n                iaa.OneOf([\n                    iaa.Fliplr(0.5), # horizontal flips\n                    iaa.Flipud(0.5), # horizontal flips\n                    iaa.Crop(percent=(0, 0.1)), # random crops\n                    # Small gaussian blur with random sigma between 0 and 0.5.\n                    # But we only blur about 50% of all images.\n                    iaa.Sometimes(0.5,\n                        iaa.GaussianBlur(sigma=(0, 0.5))\n                    ),\n                    # Make some images brighter and some darker.\n                    # In 20% of all cases, we sample the multiplier once per channel,\n                    # which can end up changing the color of the images.\n                    iaa.Multiply((0.8, 1.2), per_channel=0.2),\n                    # Apply affine transformations to each image.\n                    # Scale/zoom them, translate/move them, rotate them and shear them.\n                    iaa.Affine(\n                        scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n                        translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n                        rotate=(-180, 180),\n                        shear=(-4, 4)\n                    )\n                ])], random_order=True)\n\n\n            image_aug = augment_img.augment_image(image)\n            return image_aug\n        else:\n            return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d7faf5a3f6e9107f487cc91c301edbacebb03b9"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D, BatchNormalization, Input, Conv2D, MaxPooling2D\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model\nfrom keras.utils import multi_gpu_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0ee25cf02d2981e4ff56b2a0a15213b2923f0b"},"cell_type":"code","source":"## Definition of loss functions -- the objective -- f1 and focal \n\ndef f1(y_true, y_pred):\n    \n    y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\n\ndef f1_loss(y_true, y_pred):\n\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1-K.mean(f1)\n\n\ndef focal_loss(gamma=2., alpha=.25):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        pt_1 = K.clip(pt_1, 1e-3, .999)\n        pt_0 = K.clip(pt_0, 1e-3, .999)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1))-K.sum((1-alpha) * K.pow( pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4861132d37a2190fc72fc8fe8dc89d84de60eb31"},"cell_type":"code","source":"def create_model(input_shape, n_out, channels):\n    input_tensor = Input(shape=(299,299,len(channels)))\n    # print(len(channels))\n    bn = BatchNormalization()(input_tensor)\n    base_model = InceptionV3(include_top=False, weights='imagenet')\n    x = base_model(bn)\n    x = Dropout(0.5)(x)\n    x = Conv2D(128, kernel_size=(3,3), activation='relu')(x)\n    x = Flatten()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    output = Dense(n_out, activation='sigmoid')(x)\n    model = Model(input_tensor, output)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f401707aa0ba02f615676ad4053aec2244da17fc"},"cell_type":"code","source":"## Load data\n\nSHAPE = (299, 299, 3)\nchannels = [\"green\", \"blue\", \"red\"]\n\n# create callbacks list\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\n\nepochs = 10; batch_size = 64; VAL_RATIO = .1; DEBUG = False\n# split data into train, valid\npaths, labels = getTrainDataset()\n\n# divide to \nkeys = np.arange(paths.shape[0], dtype=np.int)  \nnp.random.seed(SEED)\nnp.random.shuffle(keys)\nlastTrainIndex = int((1-VAL_RATIO) * paths.shape[0])\n\nif DEBUG == True:  # use only small subset for debugging, Kaggle's RAM is limited\n    pathsTrain = paths[0:256]\n    labelsTrain = labels[0:256]\n    pathsVal = paths[lastTrainIndex:lastTrainIndex+256]\n    labelsVal = labels[lastTrainIndex:lastTrainIndex+256]\n    use_cache = True\nelse:\n    pathsTrain = paths[0:lastTrainIndex]\n    labelsTrain = labels[0:lastTrainIndex]\n    pathsVal = paths[lastTrainIndex:]\n    labelsVal = labels[lastTrainIndex:]\n    use_cache = False\n\nuse_cache = False\n# print(paths.shape, labels.shape)\n# print(pathsTrain.shape, labelsTrain.shape, pathsVal.shape, labelsVal.shape)\n\ntg = ProteinDataGenerator(pathsTrain, labelsTrain, batch_size, SHAPE, channels, use_cache=use_cache)\nvg = ProteinDataGenerator(pathsVal, labelsVal, batch_size, SHAPE, channels, use_cache=use_cache)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee862682dedc369f217d91b29964d5bec44f9252"},"cell_type":"code","source":"checkpoint = ModelCheckpoint('../working/InceptionV3_3chan.h5', monitor='val_f1', verbose=1, \n                             save_best_only=True, mode='max', save_weights_only = False)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_f1', factor=0.5, patience=10, \n                                   verbose=1, mode='max', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_f1\", \n                      mode=\"max\", \n                      patience=20)\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"b86a3f548a5af61693c4610710f84e374d84014b"},"cell_type":"code","source":"# warm up model\nimport tensorflow as tf\nchannels = [\"green\", \"blue\", \"red\"]\n# with tf.device('/cpu:0'):\nmodel = create_model(\n    input_shape=(SIZE,SIZE,3), \n    n_out=28, channels = channels)\n\nfor layer in model.layers:\n    layer.trainable = False\n    \nfor i in range(-6,2):\n    model.layers[i].trainable = True\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a0a161dbada45228be8fc89de23a11b63cbba44"},"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\",\n            optimizer=Adam(lr=1e-4),\n            metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12b32de0c2283888b78112e43d98c7020b398d3b"},"cell_type":"code","source":"hist =  model.fit_generator(\n        tg,\n        steps_per_epoch=np.ceil(float(len(pathsTrain)) / float(batch_size))/2,\n        validation_data=vg,\n        validation_steps=np.ceil(float(len(pathsVal)) / float(batch_size))/2,\n        epochs=1, \n        verbose=1,\n        callbacks = callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f306dc8336466c069ef92c2d6fa69221a739707"},"cell_type":"code","source":"# Set all layers back to trainable\nfor layer in model.layers:\n    layer.trainable = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edb52d701b3dcdad6ca0a0c669dbe5dc15ee10bb"},"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\",\n            optimizer=Adam(lr=1e-4),\n            metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"2662c1164134dd3f2f41971b6b71d79e0db6a104"},"cell_type":"code","source":"batch_size = 12\n\ntg = ProteinDataGenerator(pathsTrain, labelsTrain, batch_size, SHAPE, channels, use_cache=use_cache)\nvg = ProteinDataGenerator(pathsVal, labelsVal, batch_size, SHAPE, channels, use_cache=use_cache)\n\nhist =  model.fit_generator(\n        tg,\n        steps_per_epoch=np.ceil(float(len(pathsTrain)) / float(batch_size))/2,\n        validation_data=vg,\n        validation_steps=np.ceil(float(len(pathsVal)) / float(batch_size))/2,\n        epochs=20, \n        verbose=1,\n        callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c76f74734b93361c94a21f46d9c1bef58831dfa9"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hist.epoch, hist.history[\"loss\"], label=\"Train loss\")\nax[0].plot(hist.epoch, hist.history[\"val_loss\"], label=\"Validation loss\")\nax[1].set_title('acc')\nax[1].plot(hist.epoch, hist.history[\"f1\"], label=\"Train F1\")\nax[1].plot(hist.epoch, hist.history[\"val_f1\"], label=\"Validation F1\")\nax[0].legend()\nax[1].legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab8fa73730129c64e36c76acbf7663bd17cd0c9f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}