{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nimport numpy as np\nimport keras\nfrom keras.utils import Sequence\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\nimport pandas as pd\nfrom tqdm import tqdm\nimport os\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3fd4b30584e8fdf544f23d1a56028fa5fa05d1f"},"cell_type":"code","source":"BATCH_SIZE = 32\nSEED = 777\nSHAPE = (512, 512, 4)\nDIR = '../input'\nVAL_RATIO = 0.1 # 10 % as validation\nTHRESHOLD = 0.5 # due to different cost of True Positive vs False Positive, this is the probability threshold to predict the class as 'yes'\n\nia.seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd1045b817e77ff540e789ea3b0eea22bcdb9a6f"},"cell_type":"code","source":"def getTrainDataset():\n    \n    path_to_train = DIR + '/train/'\n    data = pd.read_csv(DIR + '/train.csv')\n\n    paths = []\n    labels = []\n    \n    for name, lbl in zip(data['Id'], data['Target'].str.split(' ')):\n        y = np.zeros(28)\n        for key in lbl:\n            y[int(key)] = 1\n        paths.append(os.path.join(path_to_train, name))\n        labels.append(y)\n\n    return np.array(paths), np.array(labels)\n\ndef getTestDataset():\n    \n    path_to_test = DIR + '/test/'\n    data = pd.read_csv(DIR + '/sample_submission.csv')\n\n    paths = []\n    labels = []\n    \n    for name in data['Id']:\n        y = np.ones(28)\n        paths.append(os.path.join(path_to_test, name))\n        labels.append(y)\n\n    return np.array(paths), np.array(labels)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# credits: https://github.com/keras-team/keras/blob/master/keras/utils/data_utils.py#L302\n# credits: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\n\nclass ProteinDataGenerator(keras.utils.Sequence):\n            \n    def __init__(self, paths, labels, batch_size, shape, shuffle = False, use_cache = False, augment = False):\n        self.paths, self.labels = paths, labels\n        self.batch_size = batch_size\n        self.shape = shape\n        self.shuffle = shuffle\n        self.use_cache = use_cache\n        self.augment = augment\n        if use_cache == True:\n            self.cache = np.zeros((paths.shape[0], shape[0], shape[1], shape[2]), dtype=np.float16)\n            self.is_cached = np.zeros((paths.shape[0]))\n        self.on_epoch_end()\n    \n    def __len__(self):\n        return int(np.ceil(len(self.paths) / float(self.batch_size)))\n    \n    def __getitem__(self, idx):\n        indexes = self.indexes[idx * self.batch_size : (idx+1) * self.batch_size]\n\n        paths = self.paths[indexes]\n        X = np.zeros((paths.shape[0], self.shape[0], self.shape[1], self.shape[2]))\n        # Generate data\n        if self.use_cache == True:\n            X = self.cache[indexes]\n            for i, path in enumerate(paths[np.where(self.is_cached[indexes] == 0)]):\n                image = self.__load_image(path)\n                self.is_cached[indexes[i]] = 1\n                self.cache[indexes[i]] = image\n                X[i] = image\n        else:\n            for i, path in enumerate(paths):\n                X[i] = self.__load_image(path)\n\n        y = self.labels[indexes]\n                \n        if self.augment == True:\n            seq = iaa.Sequential([\n                iaa.OneOf([\n                    iaa.Fliplr(0.5), # horizontal flips\n                    iaa.Crop(percent=(0, 0.1)), # random crops\n                    # Small gaussian blur with random sigma between 0 and 0.5.\n                    # But we only blur about 50% of all images.\n                    iaa.Sometimes(0.5,\n                        iaa.GaussianBlur(sigma=(0, 0.5))\n                    ),\n                    # Strengthen or weaken the contrast in each image.\n                    iaa.ContrastNormalization((0.75, 1.5)),\n                    # Add gaussian noise.\n                    # For 50% of all images, we sample the noise once per pixel.\n                    # For the other 50% of all images, we sample the noise per pixel AND\n                    # channel. This can change the color (not only brightness) of the\n                    # pixels.\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5),\n                    # Make some images brighter and some darker.\n                    # In 20% of all cases, we sample the multiplier once per channel,\n                    # which can end up changing the color of the images.\n                    iaa.Multiply((0.8, 1.2), per_channel=0.2),\n                    # Apply affine transformations to each image.\n                    # Scale/zoom them, translate/move them, rotate them and shear them.\n                    iaa.Affine(\n                        scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n                        translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n                        rotate=(-180, 180),\n                        shear=(-8, 8)\n                    )\n                ])], random_order=True)\n\n            X = np.concatenate((X, seq.augment_images(X), seq.augment_images(X), seq.augment_images(X)), 0)\n            y = np.concatenate((y, y, y, y), 0)\n        \n        return X, y\n    \n    def on_epoch_end(self):\n        \n        # Updates indexes after each epoch\n        self.indexes = np.arange(len(self.paths))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __iter__(self):\n        \"\"\"Create a generator that iterate over the Sequence.\"\"\"\n        for item in (self[i] for i in range(len(self))):\n            yield item\n            \n    def __load_image(self, path):\n        R = Image.open(path + '_red.png')\n        G = Image.open(path + '_green.png')\n        B = Image.open(path + '_blue.png')\n        Y = Image.open(path + '_yellow.png')\n\n        im = np.stack((\n            np.array(R), \n            np.array(G), \n            np.array(B),\n            np.array(Y)), -1)\n        \n        im = cv2.resize(im, (SHAPE[0], SHAPE[1]))\n        im = np.divide(im, 255)\n        return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97a72f6757c7c2a0e6c8116eb604d185dc6c64ed"},"cell_type":"markdown","source":"# Using in Keras\nLet's try to test the multi_processing."},{"metadata":{"trusted":true,"_uuid":"31aff62537d634ee34cd7752025d98615b1fc8e9"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model, Model\nfrom keras.layers import Activation, Dropout, Flatten, Dense, Input, Conv2D, MaxPooling2D, BatchNormalization, Concatenate, ReLU, LeakyReLU, GlobalAveragePooling2D\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom keras import metrics\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import backend as K\nimport keras\nimport tensorflow as tf\n\nfrom tensorflow import set_random_seed\nset_random_seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5748b96367ce78ea9bce4a34a40a66dd5a49e945"},"cell_type":"code","source":"# credits: https://www.kaggle.com/guglielmocamporese/macro-f1-score-keras\n\ndef f1(y_true, y_pred):\n    #y_pred = K.round(y_pred)\n    #y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    #y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1-K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d9bbc05fdf72242f8f98422ef9c480a353f4fe3"},"cell_type":"code","source":"# some basic useless model\ndef create_model(input_shape):\n    \n    dropRate = 0.25\n    \n    init = Input(input_shape)\n    x = BatchNormalization(axis=-1)(init)\n    x = Conv2D(32, (3, 3))(x) #, strides=(2,2))(x)\n    x = ReLU()(x)\n\n    x = BatchNormalization(axis=-1)(x)\n    x = MaxPooling2D(pool_size=(2, 2))(x)\n    ginp1 = Dropout(dropRate)(x)\n    \n    x = BatchNormalization(axis=-1)(ginp1)\n    x = Conv2D(64, (3, 3), strides=(2,2))(x)\n    x = ReLU()(x)\n    x = BatchNormalization(axis=-1)(x)\n    x = Conv2D(64, (3, 3))(x)\n    x = ReLU()(x)\n    x = BatchNormalization(axis=-1)(x)\n    x = Conv2D(64, (3, 3))(x)\n    x = ReLU()(x)\n    \n    x = BatchNormalization(axis=-1)(x)\n    x = MaxPooling2D(pool_size=(2, 2))(x)\n    ginp2 = Dropout(dropRate)(x)\n    \n    x = BatchNormalization(axis=-1)(ginp2)\n    x = Conv2D(128, (3, 3))(x)\n    x = ReLU()(x)\n    x = BatchNormalization(axis=-1)(x)\n    x = Conv2D(128, (3, 3))(x)\n    x = ReLU()(x)\n    x = BatchNormalization(axis=-1)(x)\n    x = Conv2D(128, (3, 3))(x)\n    x = ReLU()(x)\n    ginp3 = Dropout(dropRate)(x)\n    \n    gap1 = GlobalAveragePooling2D()(ginp1)\n    gap2 = GlobalAveragePooling2D()(ginp2)\n    gap3 = GlobalAveragePooling2D()(ginp3)\n    \n    x = Concatenate()([gap1, gap2, gap3])\n    \n    x = BatchNormalization(axis=-1)(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(dropRate)(x)\n    \n    x = BatchNormalization(axis=-1)(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(0.1)(x)\n    \n    x = Dense(28)(x)\n    x = Activation('sigmoid')(x)\n    \n    model = Model(init, x)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4982e06ba3bb236e2bb10b0b08683c93149a61c"},"cell_type":"code","source":"model = create_model(SHAPE)\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=Adam(1e-03),\n    metrics=['acc',f1])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8831e442b8f5461dfc37da4292f66e1684c84c8"},"cell_type":"code","source":"paths, labels = getTrainDataset()\n\n# divide to \nkeys = np.arange(paths.shape[0], dtype=np.int)  \nnp.random.seed(SEED)\nnp.random.shuffle(keys)\nlastTrainIndex = int((1-VAL_RATIO) * paths.shape[0])\n\npathsTrain = paths[0:lastTrainIndex]\nlabelsTrain = labels[0:lastTrainIndex]\npathsVal = paths[lastTrainIndex:]\nlabelsVal = labels[lastTrainIndex:]\n\nprint(paths.shape, labels.shape)\nprint(pathsTrain.shape, labelsTrain.shape, pathsVal.shape, labelsVal.shape)\n\ntg = ProteinDataGenerator(pathsTrain, labelsTrain, BATCH_SIZE, SHAPE, use_cache=False, augment = False, shuffle = False)\nvg = ProteinDataGenerator(pathsVal, labelsVal, BATCH_SIZE, SHAPE, use_cache=False, shuffle = False)\n\n# https://keras.io/callbacks/#modelcheckpoint\ncheckpoint = ModelCheckpoint('./base.model', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='min', period=1)\nreduceLROnPlato = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1, mode='min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78b72de7c88f7845f92a0cc93eeea7a5498cdb72"},"cell_type":"code","source":"epochs = 20\n\nuse_multiprocessing = False # DO NOT COMBINE MULTIPROCESSING WITH CACHE! \nworkers = 1 # DO NOT COMBINE MULTIPROCESSING WITH CACHE! \n\nhist = model.fit_generator(\n    tg,\n    steps_per_epoch=len(tg),\n    validation_data=vg,\n    validation_steps=8,\n    epochs=epochs,\n    use_multiprocessing=use_multiprocessing,\n    workers=workers,\n    verbose=1,\n    callbacks=[checkpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9237d3586a1bba31a3a5ef7f8e81a91827d94743"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hist.epoch, hist.history[\"loss\"], label=\"Train loss\")\nax[0].plot(hist.epoch, hist.history[\"val_loss\"], label=\"Validation loss\")\nax[1].set_title('acc')\nax[1].plot(hist.epoch, hist.history[\"f1\"], label=\"Train F1\")\nax[1].plot(hist.epoch, hist.history[\"val_f1\"], label=\"Validation F1\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c52f627696b949a4dc04c451d867e3596b471b57"},"cell_type":"code","source":"# fine-tuning\n\nfor layer in model.layers:\n    layer.trainable = False\n    \nmodel.layers[-1].trainable = True\nmodel.layers[-2].trainable = True\nmodel.layers[-3].trainable = True\nmodel.layers[-4].trainable = True\nmodel.layers[-5].trainable = True\nmodel.layers[-6].trainable = True\nmodel.layers[-7].trainable = True\n\nmodel.compile(loss=f1_loss,\n            optimizer=Adam(lr=1e-4),\n            metrics=['accuracy', f1])\n\nmodel.fit_generator(\n    tg,\n    steps_per_epoch=len(tg),\n    validation_data=vg,\n    validation_steps=8,\n    epochs=1,\n    use_multiprocessing=use_multiprocessing, # you have to train the model on GPU in order to this to be benefitial\n    workers=workers, # you have to train the model on GPU in order to this to be benefitial\n    verbose=1,\n    max_queue_size=4\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5550c5553751b01fdfbdc3f3f310a819886d60aa"},"cell_type":"markdown","source":"# Full validation\nPerform validation on full validation dataset. Choose appropriate prediction threshold maximalizing the validation F1-score."},{"metadata":{"trusted":true,"_uuid":"0a8a8fbd700822b1a5ede79d05989ba28c9df588"},"cell_type":"code","source":"bestModel = load_model('./base.model', custom_objects={'f1': f1}) #, 'f1_loss': f1_loss})\n#bestModel = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f76abc9bb2cae0d85fc281c7582b3d642d2f1ff8"},"cell_type":"code","source":"fullValGen = vg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f03e6dcc106bad00e60cb6fefba20f84bf1f1be"},"cell_type":"code","source":"from sklearn.metrics import f1_score as off1\n\ndef getOptimalT(mdl, fullValGen):\n    \n    lastFullValPred = np.empty((0, 28))\n    lastFullValLabels = np.empty((0, 28))\n    for i in tqdm(range(len(fullValGen))): \n        im, lbl = fullValGen[i]\n        scores = mdl.predict(im)\n        lastFullValPred = np.append(lastFullValPred, scores, axis=0)\n        lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)\n    print(lastFullValPred.shape, lastFullValLabels.shape)\n    \n    rng = np.arange(0, 1, 0.001)\n    f1s = np.zeros((rng.shape[0], 28))\n    for j,t in enumerate(tqdm(rng)):\n        for i in range(28):\n            p = np.array(lastFullValPred[:,i]>t, dtype=np.int8)\n            #scoref1 = K.eval(f1_score(fullValLabels[:,i], p, average='binary'))\n            scoref1 = off1(lastFullValLabels[:,i], p, average='binary')\n            f1s[j,i] = scoref1\n            \n    print(np.max(f1s, axis=0))\n    print(np.mean(np.max(f1s, axis=0)))\n    \n    plt.plot(rng, f1s)\n    T = np.empty(28)\n    for i in range(28):\n        T[i] = rng[np.where(f1s[:,i] == np.max(f1s[:,i]))[0][0]]\n    #print('Choosing threshold: ', T, ', validation F1-score: ', max(f1s))\n    print(T)\n    \n    return T, np.mean(np.max(f1s, axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8160f53f010b7a5eed1e50f2f24be4f6906b1389"},"cell_type":"code","source":"fullValGen = ProteinDataGenerator(paths[lastTrainIndex:], labels[lastTrainIndex:], BATCH_SIZE, SHAPE)\nprint('Last model after fine-tuning')\nT1, ff1 = getOptimalT(model, fullValGen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6de70d7d65d4dd5b39af3221ceb4f89501ef2464"},"cell_type":"code","source":"print('Best save model')\nT2, ff2 = getOptimalT(bestModel, fullValGen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e48cd50cd31aac49f021d1266d5090228c054cda"},"cell_type":"code","source":"if ff1 > ff2:\n    T = T1\n    bestModel = model\nelse:\n    T = T2\n    bestModel = bestModel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"982c438bc6d8fa9208ba6050312abc9200f84799"},"cell_type":"code","source":"pathsTest, labelsTest = getTestDataset()\n\ntestg = ProteinDataGenerator(pathsTest, labelsTest, BATCH_SIZE, SHAPE)\nsubmit = pd.read_csv(DIR + '/sample_submission.csv')\nP = np.zeros((pathsTest.shape[0], 28))\nfor i in tqdm(range(len(testg))):\n    images, labels = testg[i]\n    score = bestModel.predict(images)\n    P[i*BATCH_SIZE:i*BATCH_SIZE+score.shape[0]] = score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f52ecf0f38c5b61e97c31698d2f0db69a6a48184"},"cell_type":"code","source":"PP = np.array(P)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"668fd597b69767521b17a4a23320d3455d70bc73"},"cell_type":"code","source":"prediction = []\n\nfor row in tqdm(range(submit.shape[0])):\n    \n    str_label = ''\n    \n    for col in range(PP.shape[1]):\n        if(PP[row, col] < T[col]):\n            str_label += ''\n        else:\n            str_label += str(col) + ' '\n    prediction.append(str_label.strip())\n    \nsubmit['Predicted'] = np.array(prediction)\nsubmit.to_csv('4channels_cnn_from_scratch.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe7fe7be3814319d6ccace15f05e4477e239c71f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}