{"cells":[{"metadata":{"_uuid":"3155253862a45097415d6f39266d17201196d2a7"},"cell_type":"markdown","source":"### A simple take on Kaggle's Airbus Ship Detection Challenge\n\nFor more detailed explanations, please refer to Julián Peller excellent kernel series at:\nhttps://www.kaggle.com/julian3833/3-basic-exploratory-analysis"},{"metadata":{"trusted":true,"_uuid":"0fe21a78f0a817992d9a16f8997bfb65029b0684"},"cell_type":"code","source":"# Import libraries\nimport os, PIL\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deb03b2fd86a2bb92c9936d6ad04a5b0ef16e3a8"},"cell_type":"code","source":"# Set visualization style\nplt.rcParams[\"patch.force_edgecolor\"] = True\nplt.rc('xtick', labelsize=20) \nplt.rc('ytick', labelsize=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2a760b9cadb24e15d4881b3ea3716d6e24e47e1"},"cell_type":"markdown","source":"List Files in Directory"},{"metadata":{"trusted":true,"_uuid":"fe4422bdf73ca3ee753d61f986fb39dcacd52d65"},"cell_type":"code","source":"ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce94fd570db4c0728a680875e49ec8c269fe8980"},"cell_type":"code","source":"# Read Data\ndf = pd.read_csv('../input/airbus-ship-detection/train_ship_segmentations.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b02584c85204df113c45681779e4f474c463c3b0"},"cell_type":"code","source":"# Features Engineering - We will use EncodedPixels in a second DataFrame bellow\ndf['Ships'] = df['EncodedPixels'].notnull()\ndf = df.groupby('ImageId').sum().reset_index()\ndf['ShipPresent'] = df['Ships'] > 0\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"847a68c75f0bc4a2c518a9bf29edb5d4c30d1903"},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ec576b5296640ef37a1acf5dae673926b710dd6"},"cell_type":"code","source":"# Features Engineering - Second DataFrame with EncodedPixels and only images with ships\n\ndf_box = pd.read_csv('../input/airbus-ship-detection/train_ship_segmentations.csv')\ndf_box = df_box.dropna().groupby(\"ImageId\")[['EncodedPixels']].agg(lambda rle_code: ' '.join(rle_code)).reset_index()\ndf_box['Path'] = df_box['ImageId'].apply(lambda filename: os.path.join('../input/airbus-ship-detection/train/', filename))\ndf_box.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00b5d68c635d8f2fb635e04f1e4268189703fb32"},"cell_type":"code","source":"df_box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21ff42159b2363c952555c52fffd1e34ef4dbef2"},"cell_type":"code","source":"def rle_to_pixels(rle_code):\n    ''' This function decodes Run Lenght Encoding into pixels '''\n    rle_code = [int(i) for i in rle_code.split()]\n    \n    pixels = [(pixel_position % 768, pixel_position // 768) \n              for start, length in list(zip(rle_code[0:-1:2], rle_code[1::2])) \n              for pixel_position in range(start, start + length)]\n        \n    return pixels\n\ndef apply_mask(image, mask):\n    ''' This function saturates the Red and Green RGB colors in the image \n        where the coordinates match the mask'''\n    for x, y in mask:\n        image[x, y, [0, 1, 2]] = (255, 255, 0)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"a7b06e7f6110dd6522da9b77a4597e7d3c09084c"},"cell_type":"code","source":"# Plots with masked ships on random images from the dataset\n\nh, w = 3, 3\nload_img = lambda path: np.array(PIL.Image.open(path))\nfig, axes_list = plt.subplots(h, w, figsize=(4*h, 4*w))\n\nfor axes in axes_list:\n    for ax in axes:\n        ax.axis(\"off\")\n        path = np.random.choice(df_box['Path'])\n        img = apply_mask(load_img(path), \\\n                rle_to_pixels(df_box[df_box['Path'] == path]['EncodedPixels'].iloc[0]))\n        ax.imshow(img)\n        ax.set_title(df_box[df_box['Path'] == path]['ImageId'].iloc[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bb5fd025b8b386994ba2ee8c30b407ab583392fa"},"cell_type":"code","source":"df_box.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d371505f5bf23e235905fc88fdd8813b4c7d6b9"},"cell_type":"code","source":"# Imbalanced Dataset | Ship/No-Ship Ratio\n\ntotal_images = len(df)\nships = df['Ships'].sum()\nships_images = len(df[df['Ships'] > 0])\nno_ship = total_images - ships_images\n\nprint(f\"Images: {total_images} \\nShips:  {ships}\")\nprint(f\"Images with ships:    {round(ships_images/total_images,2)} ({ships_images})\")\nprint(f\"Images with no ships: {round(no_ship/total_images,2)} ({no_ship})\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14cc32339cb6acd20736536cf7455dd70aa1e306"},"cell_type":"code","source":"# Engineering Features for the graphs\n\nship_ratio = df['ShipPresent'].value_counts()/total_images\nship_ratio = ship_ratio.rename(index={True:'Ship', False:'No Ship'})\n\ntotal_ship_distribution = df['Ships'].value_counts()[1::]/ships_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5678366a567acd0ff996cde7b0e0ec1dfcedac1"},"cell_type":"code","source":"# Plotting\nfig, axes = plt.subplots(nrows=1, ncols=2, figsize=(30, 12), gridspec_kw={'width_ratios':[1,5]})\n\nship_ratio.plot.bar(ax=axes[0], title=\"Ship/No-Ship distribution\")\ntotal_ship_distribution.plot.bar(ax=axes[1], title=\"Total Ship Distribution\")\n\naxes[0].title.set_size(30)\naxes[1].title.set_size(30)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"fcc4759fc64a0f6b6f7d5c0662c8fb6a93c1fe56"},"cell_type":"code","source":"# The operation bellow is expensive, if possible just load the pre-calculated dataset\n\n# df_box['Pixels'] = df_box['EncodedPixels'].apply(rle_to_pixels).str.len() # EXPENSIVE\n# df_box.to_csv('train_box_pixels.csv', encoding='utf-8', index=False)\ndf_box = pd.read_csv('../input/airbus-challenge/train_box_pixels.csv')\ndf_box.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10e40590174fbbb1f6e505a2ab0e37ae3211ff59"},"cell_type":"code","source":"# Imbalanced Dataset | Ship/No-Ship Pixels Ratio\n# Due to the heavy imbalance of the dataset, we'll conduct our analysis only with ship images\n\nn_images = df_box['ImageId'].nunique()\nship_pixels = df_box['Pixels'].sum()\ntotal_pixels = n_images * 768 * 768\nratio = ship_pixels/total_pixels\n\nprint(f'Ship Pixels:   {round(ratio, 3)*100}%    ({ship_pixels})')\nprint(f'Total Pixels: {round(1 - ratio, 3)*100}% ({total_pixels - ship_pixels})')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}