{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9988,"databundleVersionId":868324,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport random\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n\nimport numpy as np\nimport pandas as pd\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-18T18:17:52.864945Z","iopub.execute_input":"2024-07-18T18:17:52.866031Z","iopub.status.idle":"2024-07-18T18:17:52.872097Z","shell.execute_reply.started":"2024-07-18T18:17:52.865991Z","shell.execute_reply":"2024-07-18T18:17:52.870690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"train_path = '/kaggle/input/airbus-ship-detection/train_v2'\ntrain_segmentation_path = '/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv'\n\ntrain_segmentations = pd.read_csv(train_segmentation_path)\ntrain_segmentations.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:52.874450Z","iopub.execute_input":"2024-07-18T18:17:52.874812Z","iopub.status.idle":"2024-07-18T18:17:53.638707Z","shell.execute_reply.started":"2024-07-18T18:17:52.874782Z","shell.execute_reply":"2024-07-18T18:17:53.637225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_segmentations['EncodedPixels'][24])\ntrain_segmentations.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:53.640715Z","iopub.execute_input":"2024-07-18T18:17:53.641078Z","iopub.status.idle":"2024-07-18T18:17:53.654201Z","shell.execute_reply.started":"2024-07-18T18:17:53.641045Z","shell.execute_reply":"2024-07-18T18:17:53.652885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('isnull:\\n', train_segmentations.isnull().sum())\nprint('isna:\\n',  train_segmentations.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:53.655817Z","iopub.execute_input":"2024-07-18T18:17:53.656278Z","iopub.status.idle":"2024-07-18T18:17:53.761489Z","shell.execute_reply.started":"2024-07-18T18:17:53.656236Z","shell.execute_reply":"2024-07-18T18:17:53.760238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_segmentations.nunique() # id -> pixels is not 'onto' mapping wtf?","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:53.764496Z","iopub.execute_input":"2024-07-18T18:17:53.764855Z","iopub.status.idle":"2024-07-18T18:17:53.907534Z","shell.execute_reply.started":"2024-07-18T18:17:53.764824Z","shell.execute_reply":"2024-07-18T18:17:53.906466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Description\nEncodedPixels - a list of pixels for ship segmentation in a compressed format (in run-length encoding format) (RLE).\n\nEncodedPixels сonsists of pairs of values that contain a start position and a run length. E.g. '1 3' implies starting at pixel 1 and running a total of 3 pixels (1,2,3).\n\nThe pixels are one-indexed and numbered from top to bottom, then left to right: 1 is pixel (1,1), 2 is pixel (2,1), etc.\n\nA prediction of of \"no ship in image\" have a blank value in the EncodedPixels column.\n\nObject segments do not overlap.","metadata":{}},{"cell_type":"markdown","source":"# Some useful functions","metadata":{}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(768, 768)) -> np.array:\n    \"\"\"\n    decode run-length encoded segmentation mask\n    Assumed all images aRe 768x768 (and ThereforE have the saMe shape)\n    \"\"\"\n    \n    # if no segmentation mask (nan) return matrix of zeros\n    if not mask_rle or pd.isna(mask_rle):\n        return np.zeros(shape, dtype=np.uint8)\n\n    # RLE sequence str split to and map to int\n    s = list(map(int, mask_rle.split()))\n\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n\n    # indices: 2k - starts, 2k+1 lengths\n    starts, lengths = s[0::2], s[1::2]\n    for start, length in zip(starts, lengths):\n        img[start:start + length] = 1\n\n    return img.reshape(shape).T","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:53.908952Z","iopub.execute_input":"2024-07-18T18:17:53.909369Z","iopub.status.idle":"2024-07-18T18:17:53.918748Z","shell.execute_reply.started":"2024-07-18T18:17:53.909337Z","shell.execute_reply":"2024-07-18T18:17:53.917482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"some_image_mask = rle_decode(train_segmentations['EncodedPixels'][2])\nplt.imshow(some_image_mask, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:53.920083Z","iopub.execute_input":"2024-07-18T18:17:53.920544Z","iopub.status.idle":"2024-07-18T18:17:54.425938Z","shell.execute_reply.started":"2024-07-18T18:17:53.920498Z","shell.execute_reply":"2024-07-18T18:17:54.424754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_with_mask(df: pd.DataFrame, index: int) -> None:\n    fig, ax = plt.subplots(ncols=2)\n    \n    img_name = df['ImageId'][index]\n    img_rle_seqs = df.loc[df['ImageId'] == img_name]['EncodedPixels']  # get rle seqence for each object on the image\n    \n    img = cv2.imread(os.path.join(\n        '/kaggle/input/airbus-ship-detection/train_v2', img_name))\n    \n    # sum masks of all objects\n    img_mask = np.zeros((768, 768), dtype=np.uint8)\n    \n    for rle in img_rle_seqs:\n        img_mask += rle_decode(rle)\n    \n    plt.grid(False)\n    ax[0].imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n    ax[1].imshow(img_mask, cmap='gray')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:54.427523Z","iopub.execute_input":"2024-07-18T18:17:54.427896Z","iopub.status.idle":"2024-07-18T18:17:54.437718Z","shell.execute_reply.started":"2024-07-18T18:17:54.427863Z","shell.execute_reply":"2024-07-18T18:17:54.436473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_image_with_mask(train_segmentations, 666)\nplot_image_with_mask(train_segmentations, 3)","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:54.439482Z","iopub.execute_input":"2024-07-18T18:17:54.439958Z","iopub.status.idle":"2024-07-18T18:17:55.187717Z","shell.execute_reply.started":"2024-07-18T18:17:54.439916Z","shell.execute_reply":"2024-07-18T18:17:55.186413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"ship_count_by_id = train_segmentations\\\n    .groupby('ImageId', as_index=False)\\\n    .count()\\\n    .rename(columns={'EncodedPixels': 'nObjects'})\n\nship_counts = ship_count_by_id\\\n    .groupby('nObjects', as_index=False)\\\n    .count()\\\n    .rename(columns={'ImageId': 'nCounts'})","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:55.189316Z","iopub.execute_input":"2024-07-18T18:17:55.189736Z","iopub.status.idle":"2024-07-18T18:17:55.500202Z","shell.execute_reply.started":"2024-07-18T18:17:55.189686Z","shell.execute_reply":"2024-07-18T18:17:55.499088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ship_count_by_id.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:55.504166Z","iopub.execute_input":"2024-07-18T18:17:55.504563Z","iopub.status.idle":"2024-07-18T18:17:55.515921Z","shell.execute_reply.started":"2024-07-18T18:17:55.504531Z","shell.execute_reply":"2024-07-18T18:17:55.514551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ship_counts.T","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:55.517771Z","iopub.execute_input":"2024-07-18T18:17:55.518251Z","iopub.status.idle":"2024-07-18T18:17:55.535720Z","shell.execute_reply.started":"2024-07-18T18:17:55.518207Z","shell.execute_reply":"2024-07-18T18:17:55.534463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all numbers of images with certain amount of ships on them\n\nsns.set(style=\"whitegrid\")\n\nplt.figure(figsize=(10, 4))\nsns.barplot(x='nObjects', y='nCounts', data=ship_counts, palette='Blues_d')\nplt.xlabel('Number of Ships')\nplt.ylabel('Number of Images')\nplt.title('Number of Images with a Certain Number of Ships')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:55.537597Z","iopub.execute_input":"2024-07-18T18:17:55.538063Z","iopub.status.idle":"2024-07-18T18:17:56.018677Z","shell.execute_reply.started":"2024-07-18T18:17:55.538019Z","shell.execute_reply":"2024-07-18T18:17:56.017396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(8, 8))\n# plt.pie(ship_counts['nCounts'], labels=ship_counts['nObjects'], autopct='%1.1f%%', startangle=140)\n# plt.title('Distribution of nCounts by nObjects')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:24:01.413773Z","iopub.execute_input":"2024-07-18T18:24:01.414211Z","iopub.status.idle":"2024-07-18T18:24:01.419713Z","shell.execute_reply.started":"2024-07-18T18:24:01.414148Z","shell.execute_reply":"2024-07-18T18:24:01.418444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the same as preveious but exclude no-ship images\n\nplt.figure(figsize=(10, 4))\nsns.barplot(x='nObjects', y='nCounts', data=ship_counts[1:], palette='Blues_d')\nplt.xlabel('Number of Ships')\nplt.ylabel('Number of Images')\nplt.title('Number of Images with a Certain Number of Ships')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:56.020221Z","iopub.execute_input":"2024-07-18T18:17:56.020581Z","iopub.status.idle":"2024-07-18T18:17:56.520578Z","shell.execute_reply.started":"2024-07-18T18:17:56.020550Z","shell.execute_reply":"2024-07-18T18:17:56.519440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(8, 8))\n# plt.pie(ship_counts[1:]['nCounts'], labels=ship_counts[1:]['nObjects'], autopct='%1.1f%%', startangle=140)\n# plt.title('Distribution of nCounts by nObjects')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:23:56.437298Z","iopub.execute_input":"2024-07-18T18:23:56.437736Z","iopub.status.idle":"2024-07-18T18:23:56.443578Z","shell.execute_reply.started":"2024-07-18T18:23:56.437700Z","shell.execute_reply":"2024-07-18T18:23:56.442224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_segmentations['Area'] = train_segmentations['EncodedPixels'].apply(lambda rle: rle_decode(rle).sum())","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:17:56.522120Z","iopub.execute_input":"2024-07-18T18:17:56.522570Z","iopub.status.idle":"2024-07-18T18:19:39.334340Z","shell.execute_reply.started":"2024-07-18T18:17:56.522531Z","shell.execute_reply":"2024-07-18T18:19:39.332873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_segmentations","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:19:39.335955Z","iopub.execute_input":"2024-07-18T18:19:39.336501Z","iopub.status.idle":"2024-07-18T18:19:39.350793Z","shell.execute_reply.started":"2024-07-18T18:19:39.336445Z","shell.execute_reply":"2024-07-18T18:19:39.349652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by unique images and perform aggregation: count, sum\nship_stats_by_id = train_segmentations\\\n    .groupby('ImageId', as_index=False)\\\n    .agg({'EncodedPixels': 'count', 'Area': 'sum'})\\\n    .rename(columns={'EncodedPixels': 'nObjects'})\n\nship_stats_by_id['avgShipArea'] = (ship_stats_by_id['Area'] / ship_stats_by_id['nObjects']).fillna(0)\n\nship_stats_by_id.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:19:39.352122Z","iopub.execute_input":"2024-07-18T18:19:39.352516Z","iopub.status.idle":"2024-07-18T18:19:39.697553Z","shell.execute_reply.started":"2024-07-18T18:19:39.352483Z","shell.execute_reply":"2024-07-18T18:19:39.696360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group images by ship count, \n\nship_stats_by_counts = ship_stats_by_id\\\n    .groupby('nObjects', as_index=False)\\\n    .agg({'nObjects': 'count', 'Area': 'sum'})\n\nship_stats_by_counts['avgArea'] = (ship_stats_by_counts['Area'] / ship_stats_by_counts['nObjects'])\n\nship_stats_by_counts.T.astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:19:39.699023Z","iopub.execute_input":"2024-07-18T18:19:39.699439Z","iopub.status.idle":"2024-07-18T18:19:39.728750Z","shell.execute_reply.started":"2024-07-18T18:19:39.699408Z","shell.execute_reply":"2024-07-18T18:19:39.727491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 4))\nsns.barplot(x=ship_stats_by_counts.index, y='avgArea', data=ship_stats_by_counts, palette='Blues_d')\nplt.xlabel('Number of Ships')\nplt.ylabel('Average Area')\nplt.title('Segmented area by Number of Ships per Image')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-18T18:19:39.730200Z","iopub.execute_input":"2024-07-18T18:19:39.730557Z","iopub.status.idle":"2024-07-18T18:19:40.265973Z","shell.execute_reply.started":"2024-07-18T18:19:39.730524Z","shell.execute_reply":"2024-07-18T18:19:40.264738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}