{"cells":[{"metadata":{"_uuid":"e45969575c29e8e2ececd36b8b152e88b1ae36b0"},"cell_type":"markdown","source":"# Introduction\n* This is part 3 for this competitions submit"},{"metadata":{"_uuid":"3e75027dfabe5bbd6494f7e0c0df9e52364e73d1"},"cell_type":"markdown","source":"##  Part1 - Simple tansfer learning to detect ship exist\n* https://www.kaggle.com/super13579/simple-transfer-learning-detect-ship-exist-keras"},{"metadata":{"_uuid":"a26725fb30a17dbb9e21dcb1165af5f995339d57"},"cell_type":"markdown","source":"## Part2 - U-net base on ResNet34 transfer learning \n* https://www.kaggle.com/super13579/u-net-base-on-resnet34-transfer-learning-keras"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util.montage import montage2d as montage\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\nship_dir = '../input/airbus-ship-detection/'\ntrain_image_dir = os.path.join(ship_dir, 'train_v2')\ntest_image_dir = os.path.join(ship_dir, 'test_v2')\nimport gc; gc.enable() # memory is tight\n\nfrom skimage.morphology import label\ndef multi_rle_encode(img):\n    labels = label(img[:, :, 0])\n    return [rle_encode(labels==k) for k in np.unique(labels[labels>0])]\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    # Take the individual ship masks and create a single mask array for all ships\n    all_masks = np.zeros((768, 768), dtype = np.int16)\n    #if isinstance(in_mask_list, list):\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks += rle_decode(mask)\n    return np.expand_dims(all_masks, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df2ca59de8503d9dfc1a4c6febb705b80d2083cb"},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b0de3d4a89c4f50e1704d15f4adba7dc2a2959e"},"cell_type":"code","source":"from keras import models, layers\nfullres_model = models.load_model('../input/u-net-base-on-resnet34-transfer-learning-keras/seg_model.h5', compile=False)\nseg_in_shape = fullres_model.get_input_shape_at(0)[1:3]\nseg_out_shape = fullres_model.get_output_shape_at(0)[1:3]\nprint(seg_in_shape, '->', seg_out_shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"92f008d2e160de454164fc996af40f98cf5ff68e"},"cell_type":"markdown","source":"## Plot Test Image by part-2 trained model"},{"metadata":{"trusted":true,"_uuid":"1b48f4f6ab4f66737d502ffe76b981942e23e3c5"},"cell_type":"code","source":"test_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fd73463d835cdd31a1eb677c991ab83b90a18a2","scrolled":false},"cell_type":"code","source":"fig, m_axs = plt.subplots(8, 2, figsize = (10, 40))\nfor (ax1, ax2), c_img_name in zip(m_axs, test_paths):\n    c_path = os.path.join(test_image_dir, c_img_name)\n    c_img = imread(c_path)\n    first_img = np.expand_dims(c_img, 0)/255.0\n    first_seg = fullres_model.predict(first_img)\n    ax1.imshow(first_img[0])\n    ax1.set_title(c_img_name)\n    ax2.imshow(first_seg[0, :, :, 0], vmin = 0, vmax = 1)\n    ax2.set_title('Prediction')\nfig.savefig('test_predictions.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cff6f78c34c4e3a1a26547fa8cfc938ce1c05d65"},"cell_type":"code","source":"#Debug use\n\"\"\"\nfrom skimage.morphology import binary_opening, disk\nc_path = os.path.join(test_image_dir, '8a56c9bdd.jpg')\nc_img = imread(c_path)\nfirst_img = np.expand_dims(c_img, 0)/255.0\nfirst_seg = fullres_model.predict(first_img)[0]\nfirst_seg = binary_opening(first_seg>0.5, np.expand_dims(disk(2), -1))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"951defeb9e0d4d5d3d2a18e013d92274c6020082"},"cell_type":"markdown","source":"## Use Part-1 result to seperate have ship and no ship data"},{"metadata":{"trusted":true,"_uuid":"f13c3dc5e3cf865df17067df171f16b4c3115f30"},"cell_type":"code","source":"have_ship= pd.read_csv(\"../input/simple-transfer-learning-detect-ship-exist-keras/Have_ship_or_not.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0341c4965ec6382e4acf3e64b37c2a4618ba0b90"},"cell_type":"code","source":"have_ship.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"866680c9c959d7b8b4c4c25662a15813db8e744e"},"cell_type":"code","source":"test_names = have_ship.loc[have_ship['Have_ship'] > 0.5, ['ImageId']]['ImageId'].values.tolist()\ntest_names_nothing = have_ship.loc[have_ship['Have_ship'] <= 0.5, ['ImageId']]['ImageId'].values.tolist()\nlen(test_names), len(test_names_nothing)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac40b6197a700dd583a570d9dd9918d21149af5f"},"cell_type":"markdown","source":"## No ship data to submission file"},{"metadata":{"trusted":true,"_uuid":"52a2e91ed5c87b8ed5b8a5e45339b7e94cb1e281"},"cell_type":"code","source":"ship_list_dict = []\nfor name in test_names_nothing:\n    ship_list_dict.append({'ImageId':name,'EncodedPixels':None})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18d6957b699e78bf56f5ee0518b232840ed131e2"},"cell_type":"markdown","source":"## Have ship data to submission file"},{"metadata":{"trusted":true,"_uuid":"fdcd2c18af8049c259bb463503baea83e2eb693d"},"cell_type":"code","source":"from tqdm import tqdm_notebook\nfrom skimage.morphology import binary_opening, disk\nfor c_img_name in tqdm_notebook(test_paths):\n    if c_img_name in test_names:\n        c_path = os.path.join(test_image_dir, c_img_name)\n        c_img = imread(c_path)\n        c_img = np.expand_dims(c_img, 0)/255.0\n        cur_seg = fullres_model.predict(c_img)[0]\n        cur_seg = binary_opening(cur_seg>0.5, np.expand_dims(disk(2), -1))\n        cur_rles = multi_rle_encode(cur_seg)\n        if len(cur_rles)>0:\n            for c_rle in cur_rles:\n                ship_list_dict += [{'ImageId': c_img_name, 'EncodedPixels': c_rle}]\n        else:\n            ship_list_dict += [{'ImageId': c_img_name, 'EncodedPixels': None}]\n    gc.collect()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2252ecba1732eddd63e7f4546c0db35cc79ad0fb"},"cell_type":"code","source":"submission_df = pd.DataFrame(ship_list_dict)[['ImageId', 'EncodedPixels']]\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.sample(3)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}