{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## Airbus Ship Detection Project"},{"metadata":{"_uuid":"3e3e1ab2ad1228069f5c8de837a44ba3c7cccaa6"},"cell_type":"markdown","source":"Dataset:\n\n- Load CSV files\n    - Label Dataset: find how many unique images with and without ships\n    - Count Ships: count how many ships in images\n- Show sample images\n    - show area given by EncodedPixels in a bounding box\n\nThings considered:\n\n- check if there are duplicates of 'ImageId'\n- do not use 'ImageId' as index, it is not unique\n- image may have several copies and each one may have a partial set of ship pixels\n- an image copy may show one or more ships\n- check if there are duplicates of 'EncodedPixels', whether the whole string or a tuple"},{"metadata":{"_uuid":"985008392932bf11c19fa141698dcea62c53e4d4"},"cell_type":"markdown","source":"## Data Analysis and Visualization"},{"metadata":{"_uuid":"991c4fc7fafa11db6d2a045a5727d66a89a6424a"},"cell_type":"markdown","source":"### Load CSV Files"},{"metadata":{"trusted":true,"_uuid":"2856d8f268b74aef2e59ed7d11daa81d307c15f5"},"cell_type":"code","source":"import matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport random\nimport csv\nimport cv2\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e04a569a32d69af697357819621e4dcc8af31c97"},"cell_type":"code","source":"basedir = '../input/train_v2/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"231e84ff14043f0e9b05501739c144b98e59cdfe"},"cell_type":"markdown","source":"### TRAIN IMAGES"},{"metadata":{"trusted":true,"_uuid":"a6c87e6008de91decc59d85de1744c82718576d7"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train_ship_segmentations_v2.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"585ef6405c4522ba0bba7518dc19233d59609e7f"},"cell_type":"markdown","source":"### TRAIN IMAGES WITH LABELS"},{"metadata":{"trusted":true,"_uuid":"4e87043e3c739a5a8c89fabd38a9ba60e5dff81c"},"cell_type":"code","source":"train_df[\"GotShip\"] = 0\ntrain_df.loc[train_df[\"EncodedPixels\"].notnull(), \"GotShip\"] = 1\n# train_df['GotShips'] = np.where(train_df['EncodedPixels'].isnull(), 0, 1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0c14e9f95005569dc66d49f52baea8f25edbb48"},"cell_type":"markdown","source":"### TRAIN IMAGES WITHOUT SHIPS"},{"metadata":{"trusted":true,"_uuid":"de7b864438ad1027655b57decdd84686b69d19be"},"cell_type":"code","source":"print('Number of images without ships in train log: ', train_df.ImageId[train_df['GotShip'] == 0].nunique())\n\n# train_df.to_csv(\"./dataset/train/ships.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3913daab9efceb6273b4d4662cab86f653edb873"},"cell_type":"code","source":"noship = train_df[train_df['GotShip'] == 0]\nnoship.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b76966e3d227363e34da97e57a366f1a830c25e5"},"cell_type":"markdown","source":"### SAMPLE IMAGES WITHOUT SHIPS"},{"metadata":{"trusted":true,"_uuid":"c8b3f3ff08060d603bce908e7685b0d293601dfb"},"cell_type":"code","source":"def show_samples(imagedata, no_of_images, no_of_rows=4, no_of_cols=4):\n    i = 0\n    ship_sx = random.sample(range(0, len(imagedata)), no_of_images)\n    samples = imagedata.iloc[ship_sx]\n    fig = plt.figure(1, figsize = (20,20))\n    for index, row in samples.iterrows():\n        i = i + 1\n        image = mpimg.imread(basedir + row['ImageId'])\n        img = image.copy()\n        rszImg = cv2.resize(img, (200, 200), cv2.INTER_AREA)\n\n        ax = fig.add_subplot(no_of_rows, no_of_cols, i)\n        ax.set_title(index)\n        ax.imshow(rszImg)\n        fig.tight_layout()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53db803d29807dd194764e28681f89a8e73251f8"},"cell_type":"code","source":"show_samples(noship, 16)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d707badb85b8753b2bd0ecf6be28df57230c0b91"},"cell_type":"markdown","source":"### TEST IMAGES WITH SHIPS "},{"metadata":{"trusted":true,"_uuid":"3be8f21705c8a95885475cc7121ab560fe4eced8"},"cell_type":"code","source":"print('Number of images with ships in train log: ', train_df.ImageId[train_df['GotShip'] != 0].size)\nprint('Number of unique images with ships in train log: ', train_df.ImageId[train_df['GotShip'] != 0].nunique())\n\n# train_df.to_csv(\"./dataset/train/ships.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98ed6764a8c7e24d9d824f6747852948c9806884"},"cell_type":"code","source":"ship = train_df[train_df['GotShip'] != 0]\nship.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fbe15b79f39b46e09f99a980eba866e825baa34"},"cell_type":"markdown","source":"### SAMPLE IMAGES WITH SHIPS"},{"metadata":{"_uuid":"46300263c7bbfa12be034bba3add44c7530a7fd4","trusted":true},"cell_type":"code","source":"show_samples(ship, 15)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a31cebc46bffba96aa000a710359b2c776a29455"},"cell_type":"markdown","source":"Several images with ships have similar ImageId but different EncodedPixels. An example is given below."},{"metadata":{"trusted":true,"_uuid":"f8e8ceac81e84a94a7130d89f000e614a67edbec"},"cell_type":"code","source":"x = train_df[train_df[\"ImageId\"] == \"000194a2d.jpg\"]\nx","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b5cc3eb8c6e905bdec040377b87037adc051446"},"cell_type":"markdown","source":"### SAMPLE IMAGES WITH SHIPS, SIMILAR IMAGEID"},{"metadata":{"trusted":true,"_uuid":"1123619b4548366e91045fde51867a06b950e221"},"cell_type":"code","source":"show_samples(x, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd61870f7a753c2e14877a6ac6b0e95314127a86"},"cell_type":"code","source":"# CHECK THAT NO DUPLICATE ENCODEDPIXELS ARE LISTED\nduped_ship = ship.drop_duplicates(\"EncodedPixels\")\nprint (len(duped_ship))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"245f2cac7f0db9d2fa319810eb914ede43aad18d"},"cell_type":"markdown","source":"## Count Ships"},{"metadata":{"_uuid":"3427a6c9043c35e5ccad6eabc848415e45d8998f"},"cell_type":"markdown","source":"### IMAGES WITH/WITHOUT SHIPS DISTRIBUTION"},{"metadata":{"trusted":true,"_uuid":"96bb8148a2a0615874b04ca2d0294344b2fcd3e1"},"cell_type":"code","source":"df1 = pd.DataFrame({'':['Ship', 'No Ship'], 'Image Count':[len(ship), len(noship)]})\ndf1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bde00d8f615e80ee622ade11230090df05fe1e17"},"cell_type":"code","source":"df1.plot.bar(x='', y='Image Count', rot=0, color='b', legend=None, title=\"Ship Count Distribution\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a94813aefe70355d59b87da50c7f057183bf7a65"},"cell_type":"markdown","source":"### IMAGES WITH DUPLICATES"},{"metadata":{"trusted":true,"_uuid":"58ec96f7025056725be80cff68accffaf4c132f6"},"cell_type":"code","source":"# COUNT THE NUMBER OF DUPLICATES EACH IMAGE HAS\nunique_ship = ship['ImageId'].value_counts().reset_index()\nunique_ship.columns = ['ImageId', 'NumberOfDuplicates']\nunique_ship.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5417e0e9e2e4735e673d7f2e153471a6a82ea117"},"cell_type":"code","source":"# COUNT THE NUMBER OF IMAGES vs NUMBER OF DUPLICATES \ndupeship = unique_ship.groupby('NumberOfDuplicates').count()\ndupeship","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b4f73b69e013d6795b0288c65036173c5fb8cf1"},"cell_type":"code","source":"plt.figure()\ndf2 = pd.DataFrame(dupeship, columns=['NumberOfDuplicates', 'ImageId'])\nax = df2.plot.bar(color='r', legend=None, title=\"Ship Duplicates Distribution\")\nax.set_xlabel(\"Number of Duplicates\")\nax.set_ylabel(\"Number of Images\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14a2c23bccd72555e9c95d2c2446637bbd1f21cb"},"cell_type":"markdown","source":"### NUMBER OF SHIPS PER IMAGE DISTRIBUTION"},{"metadata":{"trusted":true,"_uuid":"2091f5b0118a145def8ad92750d68926d4343a04"},"cell_type":"code","source":"# SAMPLE\nidx = random.sample(range(0, len(ship)), 1)\nsx_one = ship.iloc[idx]\nencodedpixels = sx_one['EncodedPixels'].values\nsx_image = sx_one['ImageId'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3f90a36e1fd57b48c18dae58345362cef6f2bb7"},"cell_type":"code","source":"sx = sx_image[0]\nsx_base = basedir + sx\nsx_base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3092cca87306e54a01dfec20e74876ecfb7f977a"},"cell_type":"code","source":"sample_data = ship[ship['ImageId'] == sx]\nsample_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4720a3a5c6a3a1e92ac6cb60895727d26190f446"},"cell_type":"code","source":"unique_ship.NumberOfDuplicates[unique_ship['ImageId'] == sx_image[0]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"30e713dab09b35686e45f5a980289f13ce51dd3a"},"cell_type":"markdown","source":"##### THE MASK"},{"metadata":{"trusted":true,"_uuid":"b946a6fc45aa82142e49cf3c0dc167058976aec0"},"cell_type":"code","source":"# CREATE AN IMAGE MASK\nmask = np.zeros((768, 768))\n\n# UNRAVEL MASK INTO ARRAY\nmask = mask.ravel()\nmask","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a37721c32946200d903c437ca7073d381c2ed014"},"cell_type":"markdown","source":"##### THE ENCODED PIXELS"},{"metadata":{"trusted":true,"_uuid":"dab05add7dc110a490a44eba92d411b9b29097fb"},"cell_type":"code","source":"# CREATE SHIP MASK\ndef encode_rle(encodedpixels, n=2):\n    # SPLIT ENCODED PIXELS STRING\n    shipmask = encodedpixels.split()\n    # CONVERT LIST TO TUPLES\n    shipmask = zip(*[iter(shipmask)]*n)\n    # CONVERT STRING TO INT\n    rle = [(int(start), int(start) + int(length)) for start, length in shipmask]\n    return rle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13b9046bf848787ba1481682960d9e5ab6846034"},"cell_type":"code","source":"rle_data = sample_data['EncodedPixels'].apply(encode_rle)\nrle_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f58d54a8e2e345d480708133be52017b8ef102e"},"cell_type":"markdown","source":"##### IMAGE MASK AND ENCODEDPIXELS COMBINED"},{"metadata":{"trusted":true,"_uuid":"77b9dac5680f1af57702bdbd64495db89604653e"},"cell_type":"code","source":"def total_mask(rle_data, mask):\n    for rle in rle_data:\n        for start,end in rle:\n            print (start, end)\n            mask[start:end] = 1\n    mask = mask.reshape(768,768).T\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"92816cbf9a3b71b4f4df363a9244c8e6f5d7f907"},"cell_type":"code","source":"mask = total_mask(rle_data, mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db0ab36fbc9a195e7151d6cad350a27c1ef46477"},"cell_type":"code","source":"img_mask = np.dstack((mask, mask, mask))\nfig = plt.figure()\nax = fig.add_subplot(1, 1, 1)\nax.imshow(img_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afd6948c86908f3e0d4ea9807ce3a9d7e255d51a"},"cell_type":"code","source":"# SHOW MASK IMAGE\nfig = plt.figure()\n\nax = fig.add_subplot(1, 2, 1)\nax.set_title(\"RLE Masking\")\nax.imshow(img_mask)\n\norig_image = mpimg.imread(sx_base)\nax = fig.add_subplot(1, 2, 2)\nax.set_title(\"Orig Image\")\nax.imshow(orig_image)\nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53d74a78647d267c5d99a623beb45b050236a33e"},"cell_type":"code","source":"x = range(1200)\nfig, ax = plt.subplots(1, figsize = (50,50))\nax.imshow(orig_image, extent=[0, 1200, 0, 1200])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7662c8fb31a50be2f1e720b0b10baefcfa3f418c"},"cell_type":"code","source":"poly = np.ascontiguousarray(mask, dtype=np.uint8)\n(flags, contours, h) = cv2.findContours(poly, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f72d2c48ee36cd994ddf675a49194f1dc629bc0"},"cell_type":"code","source":"contour_image = orig_image.copy()\ncv2.drawContours(contour_image, contours, -1, (0,255,0), 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9772366b1594b02611cf4f93126cc456a5b32b0b"},"cell_type":"code","source":"x = range(1200)\nfig, ax = plt.subplots(1, figsize = (50,50))\nax.imshow(contour_image, extent=[0, 1200, 0, 1200])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}