{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39ca85abc7a3b5061bef76e30aec009b20bc20f3"},"cell_type":"markdown","source":"## Loading data"},{"metadata":{"trusted":true,"_uuid":"73b8d95746be537999ae99912f9fa8660ae10dcb"},"cell_type":"code","source":"train_dir = \"../input/train/\"\ntest_dir = \"../input/test\"\n\ndf_train = pd.read_csv('../input/train_labels.csv')\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5d4950241469d7d56509fcbce21a22f2ff08edf"},"cell_type":"markdown","source":"## Use only 10k rows for experimentation and split up the dataset into train and test"},{"metadata":{"trusted":true,"_uuid":"6e6fd5d54f57e544892d1fe72ebdde6db3f5c430"},"cell_type":"code","source":"# taking 10000 sample so our model run fast (experimentation)\n\nfrom sklearn.model_selection import train_test_split\n\n# df = df_train.sample(n=10000, random_state=2018)\ndf = df_train # using full dataset\ntrain, valid = train_test_split(df,test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"def7414e73da910610b67724013549488b89bc6c"},"cell_type":"code","source":"# minimal preprocessing for experimentation. Will add more suitable preprocessing in the future\n\nfrom keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean()) / x.std() if x.std() > 0 else x,\n                                   horizontal_flip=True,\n                                   vertical_flip=True)\n\ntest_datagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean()) / x.std() if x.std() > 0 else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81f0fedc381b2638c7617a9b3ed79b55f0e79385"},"cell_type":"code","source":"# use flow_from_dataframe method to build train and valid generator\n# Only shuffle the train generator as we want valid generator to have the same structure as test\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe = train,\n    directory='../input/train/',\n    x_col='id',\n    y_col='label',\n    has_ext=False,\n#     subset='training',\n    batch_size=32,\n    seed=2018,\n    shuffle=True,\n    class_mode='binary',\n    target_size=(96,96))\n\nvalid_generator = test_datagen.flow_from_dataframe(\n    dataframe = valid,\n    directory='../input/train/',\n    x_col='id',\n    y_col='label',\n    has_ext=False,\n#     subset='validation',\n    batch_size=32,\n    seed=2018,\n    shuffle=False,\n    class_mode='binary',\n    target_size=(96,96)\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8eee1bebf4ab658b3f62307a7a9ad3c58d661c00"},"cell_type":"markdown","source":"## Building the model\n\nUse pretrained ResNet50 model with Adam optimizer and binary cross entropy loss. Freeze Resnet50 layer until \"res5a_branch2a\". Freezing whole layer will give really bad results as the image is really different compared to imagenet dataset which Resnet50 model trained on."},{"metadata":{"trusted":true,"_uuid":"4fd0e9832c867207ce4a93c18dc691756b4764cc"},"cell_type":"code","source":"from keras.applications.resnet50 import ResNet50\nfrom keras.models import Sequential\nfrom keras import layers\n\nIMG_SIZE = (96, 96)\nIN_SHAPE = (*IMG_SIZE, 3)\n\ndropout_dense=0.5\n\nconv_base = ResNet50(\n    weights='imagenet',\n    include_top=False,\n    input_shape=IN_SHAPE\n)\n       \nmodel = Sequential()\nmodel.add(conv_base)\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(256, use_bias=False))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Activation(\"relu\"))\nmodel.add(layers.Dropout(dropout_dense))\nmodel.add(layers.Dense(1, activation = \"sigmoid\"))\n\nconv_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6728122541e439c183f30a1188f9ff0e58704e35"},"cell_type":"code","source":"# freeze layer. unfreeze start at layer 5. if freeze everything, val acc will be really bad\n\nconv_base.Trainable=True\n\nset_trainable=False\nfor layer in conv_base.layers:\n    if layer.name == 'res5a_branch2a':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eebd402746c3ddbd8d118afab5e4e43ae893f786"},"cell_type":"code","source":"from keras import optimizers\n\n# conv_base.trainable = False\nmodel.compile(optimizers.Adam(0.01), loss = \"binary_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9c4ecabb737e486374d1676d93a9ff8df41098d"},"cell_type":"code","source":"# from keras.models import Sequential\n# from keras import layers\n\n# kernel_size=(3,3)\n# pool_size=(2,2)\n# first_filter=32\n# second_filter=64\n# third_filter=128\n\n# dropout_conv=0.3\n\n# model = Sequential()\n# model.add(layers.Conv2D(first_filter, kernel_size, activation='relu', input_shape= (96,96,3)))\n# model.add(layers.Conv2D(first_filter, kernel_size, use_bias=False))\n# model.add(layers.BatchNormalization())\n# model.add(layers.Activation('relu'))\n# model.add(layers.MaxPool2D(pool_size=pool_size))\n# model.add(layers.Dropout(dropout_conv))\n\n# model.add(layers.Conv2D(second_filter, kernel_size, use_bias=False))\n# model.add(layers.BatchNormalization())\n# model.add(layers.Activation('relu'))\n# model.add(layers.Conv2D(second_filter, kernel_size, use_bias=False))\n# model.add(layers.BatchNormalization())\n# model.add(layers.Activation(\"relu\"))\n# model.add(layers.MaxPool2D(pool_size = pool_size))\n# model.add(layers.Dropout(dropout_conv))\n\n# model.add(layers.Conv2D(third_filter, kernel_size, use_bias=False))\n# model.add(layers.BatchNormalization())\n# model.add(layers.Activation(\"relu\"))\n# model.add(layers.Conv2D(third_filter, kernel_size, use_bias=False))\n# model.add(layers.BatchNormalization())\n# model.add(layers.Activation(\"relu\"))\n# model.add(layers.MaxPool2D(pool_size = pool_size))\n# model.add(layers.Dropout(dropout_conv))\n\n# #model.add(GlobalAveragePooling2D())\n# model.add(layers.Flatten())\n# model.add(layers.Dense(256, use_bias=False))\n# model.add(layers.BatchNormalization())\n# model.add(layers.Activation(\"relu\"))\n# model.add(layers.Dropout(dropout_dense))\n# model.add(layers.Dense(1, activation = \"sigmoid\"))\n\n# # Compile the model\n# model.compile(optimizers.Adam(0.01), loss = \"binary_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fbf1d97f148034c46e4b61a3e8ae8ff26acae4a7"},"cell_type":"markdown","source":"## Training the model"},{"metadata":{"trusted":true,"_uuid":"249f559a2b2368b1bfb54b0fccb598ff37c84272"},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nSTEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size\n\nearlystopper = EarlyStopping(monitor='val_loss', patience=2, verbose=1, restore_best_weights=True)\nreducel = ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.1)\n\nhistory = model.fit_generator(train_generator, steps_per_epoch=STEP_SIZE_TRAIN, \n                    validation_data=valid_generator,\n                    validation_steps=STEP_SIZE_VALID,\n                    epochs=13,\n                   callbacks=[reducel, earlystopper])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a846b79913e27bf3de64d0aba76cc3d586fca8f2"},"cell_type":"markdown","source":"## Predict the test and submission\n\nThanks to @fmarazzi for the elegent code for predicting test and submission. https://www.kaggle.com/fmarazzi/baseline-keras-cnn-roc-fast-10min-0-925-lb"},{"metadata":{"trusted":true,"_uuid":"0cb51d527cd018efd101fc699ffabc94a45c9de0"},"cell_type":"code","source":"from glob import glob\nfrom skimage.io import imread\n\nbase_test_dir = '../input/test/'\ntest_files = glob(os.path.join(base_test_dir,'*.tif'))\nsubmission = pd.DataFrame()\nfile_batch = 5000\nmax_idx = len(test_files)\nfor idx in range(0, max_idx, file_batch):\n    print(\"Indexes: %i - %i\"%(idx, idx+file_batch))\n    test_df = pd.DataFrame({'path': test_files[idx:idx+file_batch]})\n    test_df['id'] = test_df.path.map(lambda x: x.split('/')[3].split(\".\")[0])\n    test_df['image'] = test_df['path'].map(imread)\n    K_test = np.stack(test_df[\"image\"].values)\n    K_test = (K_test - K_test.mean()) / K_test.std()\n    predictions = model.predict(K_test)\n    test_df['label'] = predictions\n    submission = pd.concat([submission, test_df[[\"id\", \"label\"]]])\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28892ddc3d5c41c022976d08b9c843913c4f5b26"},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False, header = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de21dc05b1cef517af935c5f3fd6e49a5825fe5c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}