{"cells":[{"metadata":{"_uuid":"aa8401d73c7a19e1a43fdd6a992ea9dcb60039a2"},"cell_type":"markdown","source":"# Overview\nHere we use a pretrained model and transfer learning to try and identify the different types of proteins present in the image.\n\n## Beyond\nThe model currently just uses the green channel of the image (arbitrary) and includes all labels although some are exceedingly rare, better image usage and stratification would definitely help"},{"metadata":{"_uuid":"a6cd9d5ad61ffe3b8858769f20a5f9493f024a56"},"cell_type":"markdown","source":"## Model Parameters\nWe might want to adjust these later (or do some hyperparameter optimizations). It is slightly easier to keep track of parallel notebooks with different parameters if they are all at the beginning in a clear (machine readable format, see Kaggling with Kaggle (https://www.kaggle.com/kmader/kaggling-with-kaggle)."},{"metadata":{"trusted":true,"_uuid":"301a5d939c566d1487a049bb2554d09b592b18b1"},"cell_type":"code","source":"GAUSSIAN_NOISE = 0.1\nUPSAMPLE_MODE = 'SIMPLE'\n# number of validation images to use\nVALID_IMG_COUNT = 1000\n# maximum number of training images\nMAX_TRAIN_IMAGES = 15000 \nBASE_MODEL='RESNET52' # ['VGG16', 'RESNET52', 'InceptionV3', 'Xception', 'DenseNet169', 'DenseNet121']\nIMG_SIZE = (299, 299) # [(224, 224), (384, 384), (512, 512), (640, 640)]\nBATCH_SIZE = 32 # [1, 8, 16, 24]\nDROPOUT = 0.5\nDENSE_COUNT = 128\nLEARN_RATE = 1e-4\nEPOCHS = 10\nRGB_FLIP = 1 # should rgb be flipped when rendering images","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util import montage\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\nbase_dir = '../input'\ntrain_image_dir = os.path.join(base_dir, 'train')\ntest_image_dir = os.path.join(base_dir, 'test')\nimport gc; gc.enable() # memory is tight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ca7119188fbb4c6540d9df55f5833b55435287e"},"cell_type":"code","source":"image_df = pd.read_csv(os.path.join('../input/',\n                                 'train.csv'))\nprint(image_df.shape[0], 'masks found')\nprint(image_df['Id'].value_counts().shape[0])\n# just use green for now\nimage_df['green_path'] = image_df['Id'].map(lambda x: os.path.join(train_image_dir, '{}_green.png'.format(x)))\nimage_df['target_list'] = image_df['Target'].map(lambda x: [int(a) for a in x.split(' ')])\nimage_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01cee5ea869f2954d3de6f4446ee95fe4cabfc03"},"cell_type":"code","source":"from itertools import chain\nfrom collections import Counter\nall_labels = list(chain.from_iterable(image_df['target_list'].values))\nc_val = Counter(all_labels)\nn_keys = c_val.keys()\nmax_idx = max(n_keys)\nfig, ax1 = plt.subplots(1,1, figsize = (10, 5))\nax1.bar(n_keys, [c_val[k] for k in n_keys])\nfor k,v in c_val.items():\n    print(k, 'count:', v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"888df3c45ff058205bd75a5005ad08ffce10202c"},"cell_type":"code","source":"# create a categorical vector\nimage_df['target_vec'] = image_df['target_list'].map(lambda ck: [i in ck for i in range(max_idx+1)])\nimage_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40cb72e241c0c3d8bc245b4e3c663b4a835b0011"},"cell_type":"markdown","source":"# Split into training and validation groups\nWe stratify by the number of boats appearing so we have nice balances in each set"},{"metadata":{"trusted":true,"_uuid":"871720221ac25f7f9408bfe01aeb4ccb95edbd1f"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, valid_df = train_test_split(image_df, \n                 test_size = 0.3, \n                  # hack to make stratification work                  \n                 stratify = image_df['Target'].map(lambda x: x[:3] if '27' not in x else '0'))\nprint(train_df.shape[0], 'training masks')\nprint(valid_df.shape[0], 'validation masks')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c21d5bff04bf9180463969ac120379345745ed03"},"cell_type":"markdown","source":"### Examine Number of Ship Images\nHere we examine how often ships appear and replace the ones without any ships with 0"},{"metadata":{"trusted":true,"_uuid":"fa2154d3621497e7497731e5ef6c6c72f27ad483"},"cell_type":"code","source":"train_df = train_df.sample(min(MAX_TRAIN_IMAGES, train_df.shape[0])) # limit size of training set (otherwise it takes too long)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2612fa47c7e9fdcaa7aa720c4e15fc86fd65d69a"},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (10, 5))\ntrain_sum_vec = np.sum(np.stack(train_df['target_vec'].values, 0), 0)\nvalid_sum_vec = np.sum(np.stack(valid_df['target_vec'].values, 0), 0)\nax1.bar(n_keys, [train_sum_vec[k] for k in n_keys])\nax1.set_title('Training Distribution')\nax2.bar(n_keys, [valid_sum_vec[k] for k in n_keys])\nax2.set_title('Validation Distribution')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8f65e7942816fb75b687a549dc1d5cc48d00e21"},"cell_type":"markdown","source":"# Augment Data"},{"metadata":{"trusted":true,"_uuid":"ccbc1747ffb0f2942cc99bc27e4afc3262ba9f94"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nif BASE_MODEL=='VGG16':\n    from keras.applications.vgg16 import VGG16 as PTModel, preprocess_input\nelif BASE_MODEL=='RESNET52':\n    from keras.applications.resnet50 import ResNet50 as PTModel, preprocess_input\nelif BASE_MODEL=='InceptionV3':\n    from keras.applications.inception_v3 import InceptionV3 as PTModel, preprocess_input\nelif BASE_MODEL=='Xception':\n    from keras.applications.xception import Xception as PTModel, preprocess_input\nelif BASE_MODEL=='DenseNet169': \n    from keras.applications.densenet import DenseNet169 as PTModel, preprocess_input\nelif BASE_MODEL=='DenseNet121':\n    from keras.applications.densenet import DenseNet121 as PTModel, preprocess_input\nelse:\n    raise ValueError('Unknown model: {}'.format(BASE_MODEL))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ndg_args = dict(featurewise_center = False, \n                  samplewise_center = False,\n                  rotation_range = 45, \n                  width_shift_range = 0.1, \n                  height_shift_range = 0.1, \n                  shear_range = 0.01,\n                  zoom_range = [0.9, 1.25],  \n                  brightness_range = [0.5, 1.5],\n                  horizontal_flip = True, \n                  vertical_flip = True,\n                  fill_mode = 'reflect',\n                   data_format = 'channels_last',\n              preprocessing_function = preprocess_input)\nvalid_args = dict(fill_mode = 'reflect',\n                   data_format = 'channels_last',\n                  preprocessing_function = preprocess_input)\n\ncore_idg = ImageDataGenerator(**dg_args)\nvalid_idg = ImageDataGenerator(**valid_args)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5cc31e3117fdfa923b2acb1e6542a7007f4f955"},"cell_type":"code","source":"def flow_from_dataframe(img_data_gen, in_df, path_col, y_col, **dflow_args):\n    base_dir = os.path.dirname(in_df[path_col].values[0])\n    print('## Ignore next message from keras, values are replaced anyways')\n    df_gen = img_data_gen.flow_from_directory(base_dir, \n                                     class_mode = 'sparse',\n                                    **dflow_args)\n    df_gen.filenames = in_df[path_col].values\n    df_gen.classes = np.stack(in_df[y_col].values)\n    df_gen.samples = in_df.shape[0]\n    df_gen.n = in_df.shape[0]\n    df_gen._set_index_array()\n    df_gen.directory = '' # since we have the full path\n    print('Reinserting dataframe: {} images'.format(in_df.shape[0]))\n    return df_gen","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67136e1743dbbc4e07dba0d69f79231603f31d93"},"cell_type":"code","source":"train_gen = flow_from_dataframe(core_idg, train_df, \n                             path_col = 'green_path',\n                            y_col = 'target_vec', \n                            target_size = IMG_SIZE,\n                             color_mode = 'rgb',\n                            batch_size = BATCH_SIZE)\n\n# used a fixed dataset for evaluating the algorithm\nvalid_x, valid_y = next(flow_from_dataframe(valid_idg, \n                               valid_df, \n                             path_col = 'green_path',\n                            y_col = 'target_vec', \n                            target_size = IMG_SIZE,\n                             color_mode = 'rgb',\n                            batch_size = VALID_IMG_COUNT)) # one big batch\nprint(valid_x.shape, valid_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6122ccb9e58bfac6fa5e11c86121e78d9e5151b1"},"cell_type":"code","source":"t_x, t_y = next(train_gen)\nprint('x', t_x.shape, t_x.dtype, t_x.min(), t_x.max())\nprint('y', t_y.shape, t_y.dtype, t_y.min(), t_y.max())\nfig, (ax1) = plt.subplots(1, 1, figsize = (10, 10))\nax1.imshow(montage_rgb((t_x-t_x.min())/(t_x.max()-t_x.min()))[:, :, ::RGB_FLIP], cmap='gray')\nax1.set_title('images')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba08494eb9736ec3556b7c879143cdcdea89febf"},"cell_type":"markdown","source":"# Build a Model\nWe build the pre-trained top model and then use a global-max-pooling (we are trying to detect any ship in the image and thus max is better suited than averaging (which would tend to favor larger ships to smaller ones). "},{"metadata":{"trusted":true,"_uuid":"9fb62d14ec059f7f92d82e07b35822169c77112d"},"cell_type":"code","source":"base_pretrained_model = PTModel(input_shape =  t_x.shape[1:], \n                              include_top = False, weights = 'imagenet')\nbase_pretrained_model.trainable = True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bc7c0ee177697838d22aed4de046e7542027a10"},"cell_type":"markdown","source":"## Setup the Subsequent Layers\nHere we setup the rest of the model which we will actually be training"},{"metadata":{"trusted":true,"_uuid":"2687377309d3cbbab1197f4eccd2b50ab996f5a6"},"cell_type":"code","source":"from keras import models, layers\nfrom keras.optimizers import Adam\nimg_in = layers.Input(t_x.shape[1:], name='Image_RGB_In')\nimg_noise = layers.GaussianNoise(GAUSSIAN_NOISE)(img_in)\npt_features = base_pretrained_model(img_noise)\npt_depth = base_pretrained_model.get_output_shape_at(0)[-1]\nbn_features = layers.BatchNormalization()(pt_features)\nfeature_dropout = layers.SpatialDropout2D(DROPOUT)(bn_features)\ngmp_dr = layers.GlobalMaxPooling2D()(feature_dropout)\ndr_steps = layers.Dropout(DROPOUT)(layers.Dense(DENSE_COUNT, activation = 'relu')(gmp_dr))\nout_layer = layers.Dense(t_y.shape[1], activation = 'sigmoid')(dr_steps)\n\nprotein_model = models.Model(inputs = [img_in], outputs = [out_layer], name = 'full_model')\n\nprotein_model.compile(optimizer = Adam(lr=LEARN_RATE), \n                   loss = 'binary_crossentropy',\n                   metrics = ['binary_accuracy'])\n\nprotein_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7282d18de3aff1cee12ff89b7d511a391702814f"},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('boat_detector')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=15) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b67d808c0b8c7e28bff41e6d3858ff6f09dd626","scrolled":false},"cell_type":"code","source":"train_gen.batch_size = BATCH_SIZE\nprotein_model.fit_generator(train_gen, \n                            steps_per_epoch = train_gen.samples//BATCH_SIZE,\n                      validation_data = (valid_x, valid_y), \n                      epochs = EPOCHS, \n                      callbacks = callbacks_list,\n                      workers = 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a168c8b1af446b800f6129104906003ededd61c4"},"cell_type":"code","source":"protein_model.load_weights(weight_path)\nprotein_model.save('full_protein_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"422ac411191203ee639829ad82fffcdcc5050b79"},"cell_type":"code","source":"for k, v in zip(protein_model.metrics_names, \n        protein_model.evaluate(valid_x, valid_y)):\n    if k!='loss':\n        print('{:40s}:\\t{:2.1f}%'.format(k, 100*v))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17edb177402ae51651692511827a7e9d60646533"},"cell_type":"markdown","source":"# Run the test data\nWe use the sample_submission file as the basis for loading and running the images."},{"metadata":{"trusted":true,"_uuid":"4911811f267f9f3397a58902da9e75c6f261ad40"},"cell_type":"code","source":"test_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')\nsubmission_df = pd.read_csv('../input/sample_submission.csv')\nsubmission_df['green_path'] = submission_df['Id'].map(lambda x: \n                                                      os.path.join(test_image_dir, '{}_green.png'.format(x)))\nsubmission_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b3eea954bd883d598c6ac80167dfd4353b0f558"},"cell_type":"markdown","source":"# Setup Test Data Generator\nWe use the same generator as before to read and preprocess images"},{"metadata":{"trusted":true,"_uuid":"f75595679ba8606fd1ac15645b8612e117db0b30"},"cell_type":"code","source":"test_gen = flow_from_dataframe(valid_idg, \n                               submission_df, \n                             path_col = 'green_path',\n                            y_col = 'Predicted', \n                            target_size = IMG_SIZE,\n                             color_mode = 'rgb',\n                            batch_size = BATCH_SIZE, \n                              shuffle = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73ef7b3b2a74bf64968c79b4005075d4f0e23143"},"cell_type":"code","source":"fig, m_axs = plt.subplots(3, 1, figsize = (20, 30))\nfor (ax1), (t_x, c_img_names) in zip(m_axs, test_gen):\n    t_y = protein_model.predict(t_x)\n    t_stack = ((t_x-t_x.min())/(t_x.max()-t_x.min()))[:, :, :, ::RGB_FLIP]\n    ax1.imshow(montage_rgb(t_stack))\n    ax1.set_title('images')\nfig.savefig('test_predictions.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22f5f7e53cb82ff54bfb0e4ee818315d05c1e1af"},"cell_type":"markdown","source":"# Prepare Submission\nProcess all images (batchwise) and keep the score at the end"},{"metadata":{"trusted":true,"_uuid":"b103e0ce6ccae69dbf82a55067dc693efaeadf8f"},"cell_type":"code","source":"BATCH_SIZE = BATCH_SIZE*2 # we can use larger batches for inference\ntest_gen = flow_from_dataframe(valid_idg, \n                               submission_df, \n                             path_col = 'green_path',\n                            y_col = 'Id', \n                            target_size = IMG_SIZE,\n                             color_mode = 'rgb',\n                            batch_size = BATCH_SIZE, \n                              shuffle = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a38d343b2654f87934a88524ebc14a5759e07cb"},"cell_type":"code","source":"from tqdm import tqdm_notebook\nall_scores = dict()\nfor _, (t_x, t_names) in zip(tqdm_notebook(range(test_gen.n//BATCH_SIZE+1)),\n                            test_gen):\n    t_y = protein_model.predict(t_x)\n    for c_id, c_score in zip(t_names, t_y):\n        all_scores[c_id] = ' '.join([str(i) for i,s in enumerate(c_score) if s>0.5])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a942dbb4a939d73526dbb35745402b35785fdf4"},"cell_type":"markdown","source":"# Show the Scores\nHere we see the scores and we have to decide about a cut-off for counting an image as ship or not. We can be lazy and pick 0.5 but some more rigorous cross-validation would definitely improve this process."},{"metadata":{"trusted":true,"_uuid":"41ab9326407f59c983f3991ff736da8dd822605c"},"cell_type":"code","source":"submission_df['Predicted'] = submission_df['Id'].map(lambda x: all_scores.get(x, '0'))\nsubmission_df['Predicted'].value_counts()[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b534a825f5a1997a935aa3e3f338da657024bc19"},"cell_type":"code","source":"out_df = submission_df[['Id', 'Predicted']]\nout_df.to_csv('submission.csv', index=False)\nout_df.head(20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}