{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":12836,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Baseline Keras CNN with 160k samples\nHeavily inspired by https://www.kaggle.com/hrmello/cnn-classification-80-accuracy\n\nThanks to @Marsh for https://www.kaggle.com/vbookshelf/cnn-how-to-use-160-000-images-without-crashing","metadata":{"_uuid":"a6de38c0ff88b6738c5c8751c722c9b489602d88"}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport shutil\nprint(os.listdir(\"../input\"))\n\nfrom glob import glob \nfrom skimage.io import imread\nimport gc\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load data","metadata":{"_uuid":"d06b98f87cfa19e2548b0d3a558128d563942e1b"}},{"cell_type":"code","source":"base_tile_dir = '../input/train/'\ndf = pd.DataFrame({'path': glob(os.path.join(base_tile_dir,'*.tif'))})\ndf['id'] = df.path.map(lambda x: x.split('/')[3].split(\".\")[0])\nlabels = pd.read_csv(\"../input/train_labels.csv\")\ndf_data = df.merge(labels, on = \"id\")\n\n# removing this image because it caused a training error previously\ndf_data = df_data[df_data['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']\n\n# removing this image because it's black\ndf_data = df_data[df_data['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']\ndf_data.head(3)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split X and y in train/test and build folders","metadata":{"_uuid":"0bfaeb9d367e1b720fd81623e9248c3ff7bb0b5e"}},{"cell_type":"code","source":"SAMPLE_SIZE = 80000 # load 80k negative examples\n\n# take a random sample of class 0 with size equal to num samples in class 1\ndf_0 = df_data[df_data['label'] == 0].sample(SAMPLE_SIZE, random_state = 101)\n# filter out class 1\ndf_1 = df_data[df_data['label'] == 1].sample(SAMPLE_SIZE, random_state = 101)\n\n# concat the dataframes\ndf_data = shuffle(pd.concat([df_0, df_1], axis=0).reset_index(drop=True))\n\n# train_test_split # stratify=y creates a balanced validation set.\ny = df_data['label']\ndf_train, df_val = train_test_split(df_data, test_size=0.10, random_state=101, stratify=y)\n\n# Create directories\ntrain_path = 'base_dir/train'\nvalid_path = 'base_dir/valid'\ntest_path = '../input/test'\nfor fold in [train_path, valid_path]:\n    for subf in [\"0\", \"1\"]:\n        os.makedirs(os.path.join(fold, subf))","metadata":{"trusted":true,"_uuid":"d60413a93a292379227e2b8979d2abeed70ed718"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set the id as the index in df_data\ndf_data.set_index('id', inplace=True)\ndf_data.head()","metadata":{"trusted":true,"_uuid":"e85d943cfc261cca54b34550554549244f947400"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for image in df_train['id'].values:\n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.tif'\n    label = str(df_data.loc[image,'label']) # get the label for a certain image\n    src = os.path.join('../input/train', fname)\n    dst = os.path.join(train_path, label, fname)\n    shutil.copyfile(src, dst)\n\nfor image in df_val['id'].values:\n    fname = image + '.tif'\n    label = str(df_data.loc[image,'label']) # get the label for a certain image\n    src = os.path.join('../input/train', fname)\n    dst = os.path.join(valid_path, label, fname)\n    shutil.copyfile(src, dst)\n","metadata":{"trusted":true,"_uuid":"c6683c986522a604dc1a715687b1d1c621628e94"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\nIMAGE_SIZE = 96\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 64\nval_batch_size = 64\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)\n\ndatagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean()) / x.std() if x.std() > 0 else x,\n                            horizontal_flip=True,\n                            vertical_flip=True)\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='binary')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='binary')\n\n# Note: shuffle=False causes the test dataset to not be shuffled\ntest_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='binary',\n                                        shuffle=False)","metadata":{"trusted":true,"_uuid":"145ef9177524908eb5656fd1deb55a82aeb01973"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define the model \n**Model structure (optimizer: Adam):**\n\n* In \n* [Conv2D*3 -> MaxPool2D -> Dropout] x3 --> (filters = 16, 32, 64)\n* Flatten \n* Dense (256) \n* Dropout \n* Out","metadata":{"_uuid":"037766b4d6d71232ff280ee04aee45671db93b7a"}},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation\nfrom keras.layers import Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop, Adam\n\nkernel_size = (3,3)\npool_size= (2,2)\nfirst_filters = 32\nsecond_filters = 64\nthird_filters = 128\n\ndropout_conv = 0.3\ndropout_dense = 0.5\n\nmodel = Sequential()\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu', input_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)))\nmodel.add(Conv2D(first_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#model.add(GlobalAveragePooling2D())\nmodel.add(Flatten())\nmodel.add(Dense(256, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(1, activation = \"sigmoid\"))\n\n# Compile the model\nmodel.compile(Adam(0.01), loss = \"binary_crossentropy\", metrics=[\"accuracy\"])","metadata":{"trusted":true,"_uuid":"1d252eb588aaf888171ff82b332efd4a0880cab3"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train","metadata":{"_uuid":"56a7cce49809eae68d588a542620e16368e4fd1a"}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\nearlystopper = EarlyStopping(monitor='val_loss', patience=2, verbose=1, restore_best_weights=True)\nreducel = ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.1)\nhistory = model.fit_generator(train_gen, steps_per_epoch=train_steps, \n                    validation_data=val_gen,\n                    validation_steps=val_steps,\n                    epochs=1,\n                   callbacks=[reducel, earlystopper])","metadata":{"trusted":true,"_uuid":"a6dc621f8fc4f4558cb22289986a4886fe9e64c8"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, roc_auc_score\nimport matplotlib.pyplot as plt\n\n# make a prediction\ny_pred_keras = model.predict_generator(test_gen, steps=len(df_val), verbose=1)\nfpr_keras, tpr_keras, thresholds_keras = roc_curve(test_gen.classes, y_pred_keras)\nauc_keras = auc(fpr_keras, tpr_keras)\nauc_keras","metadata":{"trusted":true,"_uuid":"8c8099fc4994ae9792c74477711794c83db3aeee"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Plot ROC Curve","metadata":{"_uuid":"f755d935752d007d21a59ac111cd0d604880c5fa"}},{"cell_type":"code","source":"plt.figure(1)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_keras, tpr_keras, label='area = {:.3f}'.format(auc_keras))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()","metadata":{"trusted":true,"_uuid":"2397d1c4956ea8615140b9cc40bb857e8a5deeae"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load test data and predict\nI could not find a smart way to do this without crashing the Kernel (due to MemoryError). So I just load the test files in batches, predict, and concatenate the results.","metadata":{"_uuid":"d52bccbbd7d4a8114b5bc9f53ce26db2601fa238"}},{"cell_type":"code","source":"base_test_dir = '../input/test/'\ntest_files = glob(os.path.join(base_test_dir,'*.tif'))\nsubmission = pd.DataFrame()\nfile_batch = 5000\nmax_idx = len(test_files)\nfor idx in range(0, max_idx, file_batch):\n    print(\"Indexes: %i - %i\"%(idx, idx+file_batch))\n    test_df = pd.DataFrame({'path': test_files[idx:idx+file_batch]})\n    test_df['id'] = test_df.path.map(lambda x: x.split('/')[3].split(\".\")[0])\n    test_df['image'] = test_df['path'].map(imread)\n    K_test = np.stack(test_df[\"image\"].values)\n    K_test = (K_test - K_test.mean()) / K_test.std()\n    predictions = model.predict(K_test)\n    test_df['label'] = predictions\n    submission = pd.concat([submission, test_df[[\"id\", \"label\"]]])\nsubmission.head()","metadata":{"trusted":true,"_uuid":"1ccdefc4770fa3b4d57c56990105173161471420"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#submission\n# Delete the test_dir directory we created to prevent a Kaggle error.\n# Kaggle allows a max of 500 files to be saved.\n\nshutil.rmtree(train_path)\nshutil.rmtree(valid_path)\nsubmission.to_csv(\"submission.csv\", index = False, header = True)","metadata":{"trusted":true,"_uuid":"c0c095619c41eb1996612d96e1ec3b147ebbe601"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv(\"submission.csv\")","metadata":{"trusted":true,"_uuid":"357d239024be4067fa88d87ca0dc0202a59b9d1b"},"outputs":[],"execution_count":null}]}