{"cells":[{"metadata":{"_uuid":"fd780bdd4314e9a5628a4714c99970083d66b4fb"},"cell_type":"markdown","source":"# Sliding window source image breakdown\n\nThe input images are fairly large. Certainly larger then will fit into GPU memory with any modern image processing NN. One obvious way to solve that problem is to resize them. However, that loses a lot of information.\n\nOne interesting feature of these images though, is that they contain lot of similar elements (multiple cells per image). That means, if we cut them into pieces, we will still have lot of relevant info.\n\nThis notebook slices each 512x512 image into 4 299x299 images and feeds them to InceptionV3 network. This results in decent improvement at the expence of multiplying training time by 4.****"},{"metadata":{"_uuid":"e36a66744720b577857e6078befe67a0ffcae5a9","trusted":true},"cell_type":"code","source":"import cv2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44ebd91b9e21fd06ec4d5d91a947f339cfe498ea","trusted":true},"cell_type":"code","source":"import os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport PIL\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"823f34828bc5a0d9e2ddd5547ffbad66947c2146","trusted":true},"cell_type":"code","source":"import skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0ec3fb579bd3919cd5192dce2f52dbb2804faeb","trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08f3ce570e13c5f88b86872500baa12424078637","trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight, shuffle\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4972b93fcc952139ba8f65916bcdee0d0ce4ce52","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nWINDOW_SIZE = 299\nIMAGE_SIZE  = 512\nIMAGE_CHANNELS=3\nNUM_CLASSES=28","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08a3b4df8458a12b3c43913508616b215a9e9475"},"cell_type":"markdown","source":"## Dataset\n\nI use TensorFlow dataset API to feed our network, as it gives me slightly more flexibility at how to batch images. This will also allow for some performance improvements, if, for example, we convert the dataset into TF-Data format.\n\nThis will only work with recent versions of Keras though - 2.1.x didn't work for me, and I had to upgrade 2.2.4 locally (not something you need to worry about here on Kaggle)"},{"metadata":{"_uuid":"09b66099ed0509940f12a31e81bebb9a38ca10a3","trusted":true},"cell_type":"code","source":"\npath_to_train = '../input/human-protein-atlas-image-classification'\ndata = pd.read_csv(path_to_train+'/train.csv')\n\ntrain_dataset_info = []\nfor name, labels in zip(data['Id'], data['Target'].str.split(' ')):\n    train_dataset_info.append({\n        'path':os.path.join(path_to_train, name),\n        'labels':np.array([int(label) for label in labels])})\ntrain_dataset_info = np.array(train_dataset_info)\n\nclass data_generator:\n    def __init__(self, it):\n        self.it = it\n    def __call__(self):\n        return self.it\n\n    def get_dataset(dataset_info, batch_size, shape, augument=True):\n        gen = data_generator.create_train(dataset_info, batch_size, shape, augument)\n        gen = data_generator(gen)\n        types = (tf.float32, tf.float32)\n        shapes=(tf.TensorShape((WINDOW_SIZE, WINDOW_SIZE, IMAGE_CHANNELS)), tf.TensorShape([NUM_CLASSES]))\n        dataset = tf.data.Dataset.from_generator(\n            gen, types, shapes\n        )\n        #dataset = dataset.repeat()\n        dataset = dataset.batch(batch_size, drop_remainder=True).prefetch(batch_size*8)\n        return dataset\n\n    def create_train(dataset_info, batch_size, shape, augument=True):\n        assert shape[2] == 3\n        dataset_info = shuffle(dataset_info)\n        while True:\n            for xs, xe, ys, ye in data_generator.slice_images():\n                for idx in range(len(dataset_info)):\n                    #X_train_batch = dataset_info[start:end]\n                    batch_labels = np.zeros((NUM_CLASSES))\n                    image = data_generator.load_image(\n                            dataset_info[idx]['path'], shape)   \n                    if augument:\n                        image = data_generator.augment(image)\n                    #print(image)\n                    image=image/255.\n                    #print(image)\n                    batch_labels[dataset_info[idx]['labels']] = 1\n                    yield image[xs:xe, ys:ye, :], batch_labels\n\n    def load_image(path, shape):\n        image_red_ch = Image.open(path+'_red.png')\n        image_yellow_ch = Image.open(path+'_yellow.png')\n        image_green_ch = Image.open(path+'_green.png')\n        image_blue_ch = Image.open(path+'_blue.png')\n        image = np.stack((\n        np.array(image_red_ch), \n        np.array(image_green_ch), \n        np.array(image_blue_ch)), -1)\n        #image = cv2.resize(image, (shape[0], shape[1]))\n        return image\n\n    def augment(image):\n        augment_img = iaa.Sequential([\n            iaa.OneOf([\n                iaa.Affine(rotate=0),\n                iaa.Affine(rotate=90),\n                iaa.Affine(rotate=180),\n                iaa.Affine(rotate=270),\n                iaa.Fliplr(0.5),\n                iaa.Flipud(0.5),\n            ])], random_order=True)\n\n        image_aug = augment_img.augment_image(image)\n        return image_aug\n\n    def slice_images():\n        offset = int(IMAGE_SIZE%WINDOW_SIZE)\n        for i in range(2):\n            for j in range(2):\n                x_start=i*offset\n                x_end=x_start+WINDOW_SIZE\n\n                y_start=j*offset\n                y_end=y_start+WINDOW_SIZE\n                \n                yield x_start, x_end, y_start, y_end\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6617c42f8a7a929d69f29500c7a77b60ebdc93c","trusted":true},"cell_type":"code","source":"\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import Activation, Dropout, Flatten, Reshape, Dense, Concatenate, GlobalMaxPooling2D\nfrom keras.layers import BatchNormalization, Input, Conv2D, Lambda, Average\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam\nfrom keras import backend as K\nimport keras\nfrom keras.models import Model\n\n    \ndef create_model(n_out):\n    input_shape=(WINDOW_SIZE,WINDOW_SIZE, IMAGE_CHANNELS)\n    input_tensor = Input(shape=(WINDOW_SIZE, WINDOW_SIZE, IMAGE_CHANNELS))\n    base_model = InceptionV3(include_top=False,\n                             weights='imagenet',\n                             input_shape=input_shape\n                             #input_shape=(WINDOW_SIZE, WINDOW_SIZE, IMAGE_CHANNELS)\n                            )\n    bn = BatchNormalization()(input_tensor)\n    x = base_model(bn)\n    x = Conv2D(32, kernel_size=(1,1), activation='relu')(x)\n    x = Flatten()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    output = Dense(n_out, activation='sigmoid')(x)\n    model = Model(input_tensor, output)\n    \n    return model\n\n# warm up model\nmodel = create_model(n_out=NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2efc44adcd80fd3dbe7a274e7d169a694ab0af4","trusted":true},"cell_type":"code","source":"# create callbacks list\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\n\nepochs = 10; batch_size = 16\ncheckpoint = ModelCheckpoint('../working/InceptionV3.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, \n                                   verbose=1, mode='auto', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=6)\ncallbacks_list = [checkpoint, early, reduceLROnPlat]\n\n# split data into train, valid\nindexes = np.arange(train_dataset_info.shape[0])\nnp.random.shuffle(indexes)\ntrain_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)\n\n# create train and valid datagens\ntrain_generator = data_generator.get_dataset(\n    train_dataset_info[train_indexes], batch_size, (IMAGE_SIZE,IMAGE_SIZE,IMAGE_CHANNELS), augument=True)\nvalidation_generator = data_generator.get_dataset(\n    train_dataset_info[valid_indexes], 32, (IMAGE_SIZE,IMAGE_SIZE,IMAGE_CHANNELS), augument=False)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f661d60ee8fcdc8c8a21f757c4837a8e118bd8d","trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a1f35c66b6d4627fdb9ea3fd53c0aa28fbb7777","trusted":true},"cell_type":"code","source":"from IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\nprint(keras.__version__)\nSVG(model_to_dot(model).create(prog='dot', format='svg'))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"01750c613029089779d88fceaa15880fd8d6257a","trusted":true},"cell_type":"code","source":"\n\nfor layer in model.layers:\n    layer.trainable = False\nmodel.layers[-1].trainable = True\nmodel.layers[-2].trainable = True\nmodel.layers[-3].trainable = True\nmodel.layers[-4].trainable = True\nmodel.layers[-5].trainable = True\nmodel.layers[-6].trainable = True\n\nmodel.compile(\n    loss='binary_crossentropy', \n    optimizer=Adam(1e-03),\n    metrics=['acc'])\n# model.summary()\ntrain_images, train_labels = train_generator.make_one_shot_iterator().get_next()\nval_images, val_labels = validation_generator.make_one_shot_iterator().get_next()\n#model.fit(\n#    x=train_images, y=train_labels,\n#    steps_per_epoch=int(np.ceil(float(len(train_indexes)) / float(batch_size))*4),\n#    validation_data=(val_images, val_labels),\n#    validation_steps=int(np.ceil(float(len(valid_indexes)) / float(batch_size))*4),\n#    epochs=2, \n#    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb6ef7ff778b0b57c658bc3dfbcf7b930c66b3a3","trusted":true},"cell_type":"code","source":"# train all layers\nfor layer in model.layers:\n    layer.trainable = True\nmodel.compile(loss='binary_crossentropy',\n            optimizer=Adam(lr=1e-4),\n            metrics=['accuracy'])\n#model.fit(\n#    x=train_images, y=train_labels,\n#    steps_per_epoch=int(np.ceil(float(len(train_indexes)) / float(batch_size))*4),\n#    validation_data=(val_images, val_labels),\n#    validation_steps=int(np.ceil(float(len(valid_indexes)) / float(batch_size))*4),\n#    epochs=epochs,\n#    verbose=1,\n#    callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7ac0b026cacbf20b213c18b4b2c6756c571f425"},"cell_type":"markdown","source":"Since it takes so long to train, we load pre-trained weights, made on my local machine.\n\nIn this experiment I slice each image for prediction, run inference on all the slices, and combine the results to get list of classes.\nAnother two tests to do:\n1. Run inference on single slice\n2. Run inference on resized image"},{"metadata":{"_uuid":"c48f403b6410b1d7ac9df973081b8f3fe292fb73","trusted":true},"cell_type":"code","source":"# Create submit\nsubmit = pd.read_csv(path_to_train+'/sample_submission.csv')\npredicted = []\ndraw_predict = []\n#model.load_weights('../working/InceptionV3.h5')\nmodel.load_weights('../input/sliding-window-inception-v3-weights/InceptionV3.h5')\n\n# This function is used to generate images for inference\ndef iter_image_slices(image):\n    for x_start, x_end, y_start, y_end in data_generator.slice_images():\n        yield image[x_start:x_end, y_start:y_end, :]\n\nfor name in tqdm(submit['Id']):\n    path = os.path.join(path_to_train+'/test/', name)\n    image = data_generator.load_image(path, (IMAGE_SIZE,IMAGE_SIZE,3))/255.\n    classes=set()\n    for s in iter_image_slices(image):\n        score_predict = model.predict(s[np.newaxis])[0]\n        draw_predict.append(score_predict)\n        label_predict = np.arange(NUM_CLASSES)[score_predict>=0.2]\n        classes.update(label_predict)\n    str_predict_label = ' '.join(str(l) for l in classes)\n    predicted.append(str_predict_label)\n\nsubmit['Predicted'] = predicted\nsubmit.to_csv('submit_InceptionV3.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50bf6808331f8078cf2472329b738b12c193118a","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f9d8983fcf8ba12caded514e255b5250859641ad","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}