{"cells":[{"metadata":{"_uuid":"a6de38c0ff88b6738c5c8751c722c9b489602d88"},"cell_type":"markdown","source":"# Basic Pipeline custom CNN\nBased on https://www.kaggle.com/fmarazzi/baseline-keras-cnn-roc-fast-5min-0-8253-lb"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport shutil\nfrom tqdm import tqdm\nfrom PIL import Image\n\nDATA_PATH = \"../input/\"\nTRAIN_PATH = DATA_PATH + 'train_images/'\n\nprint(os.listdir(DATA_PATH))\n\nfrom glob import glob \nfrom skimage.io import imread\nimport gc\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d06b98f87cfa19e2548b0d3a558128d563942e1b"},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"base_tile_dir = DATA_PATH + 'train_images/'\n\ndf = pd.DataFrame({'path': glob(base_tile_dir +'/*.png')})\n\ndf['id'] = df.path.map(lambda x: x.split('/')[3].split(\".\")[0])\n\nlabels = pd.read_csv(DATA_PATH + \"train.csv\")\nlabels = labels.rename(index=str, columns={'id_code':'id', 'diagnosis':'label'})\n\ndf_data = df.merge(labels, on = \"id\")\n\ndf_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bfaeb9d367e1b720fd81623e9248c3ff7bb0b5e"},"cell_type":"markdown","source":"# Split X and y in train/test and build folders"},{"metadata":{"_uuid":"d60413a93a292379227e2b8979d2abeed70ed718","trusted":true},"cell_type":"code","source":"SAMPLE_SIZE = 150 # load 80k negative examples\n\n# take a random sample of class 0 with size equal to num samples in class 1\ndf_0 = df_data[df_data['label'] == 0].sample(1805, random_state = 101)\n# filter out class 1\ndf_1 = df_data[df_data['label'] == 1].sample(370, random_state = 101)\n# filter out class 2\ndf_2 = df_data[df_data['label'] == 2].sample(999, random_state = 101)\n# filter out class 3\ndf_3 = df_data[df_data['label'] == 3].sample(193, random_state = 101)\n# filter out class 4\ndf_4 = df_data[df_data['label'] == 4].sample(295, random_state = 101)\n\n# concat the dataframes\ndf_data = shuffle(pd.concat([df_0, df_1, df_2, df_3, df_4], axis=0).reset_index(drop=True))\n\nprint(df_data.head())\nprint(df_data.label.value_counts())\n\n# train_test_split # stratify=y creates a balanced validation set.\ny = df_data['label']\ndf_train, df_val = train_test_split(df_data, test_size=0.10, random_state=101, stratify=y)\n\n# Create directories\ntrain_path = 'base_dir/train'\nvalid_path = 'base_dir/valid'\ntest_path = '../input/test'\nfor fold in [train_path, valid_path]:\n    for subf in [\"0\", \"1\",\"2\",\"3\",\"4\"]:\n        os.makedirs(os.path.join(fold, subf))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e85d943cfc261cca54b34550554549244f947400","trusted":true},"cell_type":"code","source":"# Set the id as the index in df_data\ndf_data.set_index('id', inplace=True)\ndf_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6683c986522a604dc1a715687b1d1c621628e94","trusted":true},"cell_type":"code","source":"\nIMAGE_SIZE = 192\n\nfor image in tqdm(df_train['id'].values):\n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.png'\n    label = str(df_data.loc[image,'label']) # get the label for a certain image\n    src = os.path.join(TRAIN_PATH, fname)\n    dst = os.path.join(train_path, label, fname)\n    \n    pil_im = Image.open(src)\n    resized_image = pil_im.resize((IMAGE_SIZE, IMAGE_SIZE))\n    resized_image.save(dst)\n\nfor image in tqdm(df_val['id'].values):\n    fname = image + '.png'\n    label = str(df_data.loc[image,'label']) # get the label for a certain image\n    src = os.path.join(TRAIN_PATH, fname)\n    dst = os.path.join(valid_path, label, fname)\n    \n    pil_im = Image.open(src)\n    resized_image = pil_im.resize((IMAGE_SIZE, IMAGE_SIZE))\n    resized_image.save(dst)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"145ef9177524908eb5656fd1deb55a82aeb01973","trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 32\nval_batch_size = 32\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)\n\ndatagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean()) / x.std() if x.std() > 0 else x,\n                            horizontal_flip=True,\n                            vertical_flip=True)\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='categorical')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='categorical')\n\n# Note: shuffle=False causes the test dataset to not be shuffled\ntest_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='categorical',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"037766b4d6d71232ff280ee04aee45671db93b7a"},"cell_type":"markdown","source":"# Define the model \n**Model structure (optimizer: Adam):**\n\n* In \n* [Conv2D*3 -> MaxPool2D -> Dropout] x3 --> (filters = 16, 32, 64)\n* Flatten \n* Dense (256) \n* Dropout \n* Out"},{"metadata":{"_uuid":"1d252eb588aaf888171ff82b332efd4a0880cab3","trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation\nfrom keras.layers import Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop, Adam\n\nkernel_size = (3,3)\npool_size= (2,2)\nini_filters = 32\nfirst_filters = 32\nsecond_filters = 64\nthird_filters = 128\n\ndropout_conv = 0.3\ndropout_dense = 0.5\n\nmodel = Sequential()\n\nmodel.add(Conv2D(ini_filters, kernel_size, activation = 'relu', input_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)))\nmodel.add(Conv2D(ini_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu'))\nmodel.add(Conv2D(first_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#model.add(GlobalAveragePooling2D())\nmodel.add(Flatten())\nmodel.add(Dense(256, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(5, activation = \"sigmoid\"))\n\n# Compile the model\nmodel.compile(Adam(0.01), loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])\n\nprint(\"Done !\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56a7cce49809eae68d588a542620e16368e4fd1a"},"cell_type":"markdown","source":"# Train"},{"metadata":{"_uuid":"a6dc621f8fc4f4558cb22289986a4886fe9e64c8","trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\nearlystopper = EarlyStopping(monitor='val_loss', patience=2, verbose=1, restore_best_weights=True)\nreducel = ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.1)\nhistory = model.fit_generator(train_gen, steps_per_epoch=train_steps, \n                    validation_data=val_gen,\n                    validation_steps=val_steps,\n                    epochs=30,\n                   callbacks=[reducel, earlystopper])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c8099fc4994ae9792c74477711794c83db3aeee","trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, roc_auc_score\nimport matplotlib.pyplot as plt\n\n# make a prediction\ny_pred_keras = model.predict_generator(test_gen, steps=len(df_val), verbose=1)\n\nprint(y_pred_keras)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d52bccbbd7d4a8114b5bc9f53ce26db2601fa238"},"cell_type":"markdown","source":"# Load test data and predict\nI could not find a smart way to do this without crashing the Kernel (due to MemoryError). So I just load the test files in batches, predict, and concatenate the results."},{"metadata":{"_uuid":"1ccdefc4770fa3b4d57c56990105173161471420","trusted":true},"cell_type":"code","source":"base_test_dir = '../input/test_images/'\n\ntest_files = glob(os.path.join(base_test_dir,'*.png'))\n\nos.makedirs('valid/')\n\nfor image in tqdm(test_files):\n    fname = image.split('/')[-1]\n    \n    src = os.path.join(base_test_dir, fname)\n    dst = os.path.join(\"valid/\",fname)\n    \n    pil_im = Image.open(src)\n    resized_image = pil_im.resize((IMAGE_SIZE, IMAGE_SIZE))\n    resized_image.save(dst)\n    \ntest_files = glob(os.path.join('valid','*.png'))\n\n\nsubmission = pd.DataFrame()\nfile_batch = 20\nmax_idx = len(test_files)\nfor idx in range(0, max_idx, file_batch):\n    print(\"Indexes: %i - %i\"%(idx, idx+file_batch))\n    test_df = pd.DataFrame({'path': test_files[idx:idx+file_batch]})\n    test_df['id_code'] = test_df.path.map(lambda x: x.split('/')[1].split(\".\")[0])\n    test_df['image'] = test_df['path'].map(imread)\n    K_test = np.stack(test_df[\"image\"].values)\n    K_test = (K_test - K_test.mean()) / K_test.std()\n    predictions = model.predict(K_test)\n    \n    pred = []\n    \n    for l in predictions:\n        pred.append(np.argmax(l))\n    \n    \n    test_df['diagnosis'] = pred\n    submission = pd.concat([submission, test_df[[\"id_code\", \"diagnosis\"]]])\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c0c095619c41eb1996612d96e1ec3b147ebbe601","trusted":true},"cell_type":"code","source":"#submission\n# Delete the test_dir directory we created to prevent a Kaggle error.\n# Kaggle allows a max of 500 files to be saved.\nsubmission.to_csv(\"submission.csv\", index = False, header = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"357d239024be4067fa88d87ca0dc0202a59b9d1b","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"submission.csv\")\nprint(df[\"diagnosis\"].value_counts())\n\nprint(predictions)\nprint(pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree(train_path)\nshutil.rmtree(valid_path)\nshutil.rmtree('valid/')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}