{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm_notebook as tqdm\nimport shutil","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"ships = pd.read_csv(\"/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/airbus-ship-detection/sample_submission_v2.csv\")\nships.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are few images for which there is no Encoded Pixel values which means there is no ship for these images. \nThere are few images for which there are multiple rows of Encoded Pixel values which means there are multiple ships for these images"},{"metadata":{},"cell_type":"markdown","source":"### Finding the Uniq images for the data set"},{"metadata":{"trusted":true},"cell_type":"code","source":"ships[\"Ship\"] = ships[\"EncodedPixels\"].map(lambda x:1 if isinstance(x,str) else 0)\nship_unique = ships[[\"ImageId\",\"Ship\"]].groupby(\"ImageId\").agg({\"Ship\":\"sum\"}).reset_index()\nship_unique.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Defining Function to encode the bounding boxes for the Encoded Pixels"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2bbox(rle, shape):\n    \n    a = np.fromiter(rle.split(), dtype=np.uint)\n    a = a.reshape((-1, 2))  # an array of (start, length) pairs\n    a[:,0] -= 1  # `start` is 1-indexed\n    \n    y0 = a[:,0] % shape[0]\n    y1 = y0 + a[:,1]\n    if np.any(y1 > shape[0]):\n        # got `y` overrun, meaning that there are a pixels in mask on 0 and shape[0] position\n        y0 = 0\n        y1 = shape[0]\n    else:\n        y0 = np.min(y0)\n        y1 = np.max(y1)\n    \n    x0 = a[:,0] // shape[0]\n    x1 = (a[:,0] + a[:,1]) // shape[0]\n    x0 = np.min(x0)\n    x1 = np.max(x1)\n    \n    if x1 > shape[1]:\n        # just went out of the image dimensions\n        raise ValueError(\"invalid RLE or image dimensions: x1=%d > shape[1]=%d\" % (\n            x1, shape[1]\n        ))\n\n    xc = (x0+x1)/(2*768)\n    yc = (y0+y1)/(2*768)\n    w = np.abs(x1-x0)/768\n    h = np.abs(y1-y0)/768\n    return [xc, yc, h, w]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding the Bounding boxes from encoded pixels Normalized\nships[\"Bbox\"] = ships[\"EncodedPixels\"].apply(lambda x:rle2bbox(x,(768,768)) if isinstance(x,str) else np.NaN)\nships.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Droping the Encoded Pixels from the dataset\nships.drop(\"EncodedPixels\", axis =1, inplace =True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ships[\"BboxArea\"]=ships[\"Bbox\"].map(lambda x:x[2]*768*x[3]*768 if x==x else 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plotting the distribution of the bounding box areas to check the ship sizes\n\narea = ships[ships.Ship>0]\n\nplt.figure(figsize = (12,5))\nplt.subplot(1,2,1)\nsns.boxplot(area[\"BboxArea\"])\nplt.title(\"Areas of Bounding boxes for ships\")\nplt.xscale(\"log\")\nplt.subplot(1,2,2)\nsns.distplot(area[\"BboxArea\"], bins=50)\nplt.xscale(\"log\")\nplt.title(\"Distribution of Bounding boxe area\")\nplt.tight_layout()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.percentile(area[\"BboxArea\"],[1,5,25,50,75,95,99])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are close to 1% of bounding boxes with area <2 pixcels and some are 0. Removing these tiny boxes"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing boxes which are less than 1 percentile\nships = ships[ships[\"BboxArea\"]>np.percentile(ships[\"BboxArea\"],1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding the distribution of no of ships\nship_unique[\"Hasship\"]= [1 if x>0 else 0 for x in ship_unique[\"Ship\"]]\nplt.figure(figsize = (12,12))\nplt.subplot(2,2,1)\nsns.countplot(ship_unique[\"Hasship\"])\nplt.title(\"Images with ship vs Without Ship\")\nplt.subplot(2,2,2)\nsns.countplot(ship_unique[\"Ship\"])\nplt.title(\"No of Ships count\")\nwithship = ship_unique[ship_unique[\"Hasship\"]==1]\nplt.subplot(2,2,3)\nsns.countplot(withship[\"Ship\"])\nplt.title(\"Count of no of ships(Excluding no ship)\")\nplt.subplot(2,2,4)\nsns.boxplot(withship[\"Ship\"])\nplt.title(\"Distribution of no of ships(Excluding no ship)\")\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Most of the ships are very small in sizes although there are few images with significantly large ship sizes.\n* Most of the images with ships contains 1 ship.\n* there is a data imbalance for the no of ships in the Images."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Since the data set volumn is very high, we are downsampling to select 1000 images from each of the classes(Where more than 1000)\nbalanced_df = ship_unique.groupby(\"Ship\").apply(lambda x:x.sample(1000) if len(x)>=1000 else x.sample(len(x)))\nbalanced_df.reset_index(drop=True,inplace=True)\nbalanced_df[\"Ship\"].hist(bins=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(balanced_df.sample(5))\nprint(balanced_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Approx. 8000 images were selected from the total dataset for training the Yolov4 network"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating dataframe for Bounding boxes for the images in Balanced_df\nbalanced_bbox = ships.merge(balanced_df[[\"ImageId\"]], how =\"inner\", on = \"ImageId\")\nbalanced_bbox.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualizing the bounding boxes and images\npath =\"../input/airbus-ship-detection/train_v2/\"\nplt.figure(figsize =(20,20))\nfor i in range(15):\n    imageid = balanced_df[balanced_df.Ship ==i].iloc[0][0]\n    image = np.array(cv2.imread(path+imageid)[:,:,::-1])\n    if i>0:\n        bbox = balanced_bbox[balanced_bbox.ImageId==imageid][\"Bbox\"]\n        \n        for items in bbox:\n            Xmin  = int((items[0]-items[3]/2)*768)\n            Ymin  = int((items[1]-items[2]/2)*768)\n            Xmax  = int((items[0]+items[3]/2)*768)\n            Ymax  = int((items[1]+items[2]/2)*768)\n            cv2.rectangle(image,\n                          (Xmin,Ymin),\n                          (Xmax,Ymax),\n                          (255,0,0),\n                          thickness = 2)\n    plt.subplot(4,4,i+1)\n    plt.imshow(image)\n    plt.title(\"No of ships = {}\".format(i))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are some case where the ships are very small Images . We can remove images which are very small.We can decide to remove the Bboxes where the area is botton 5 percentile after checking the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating New folder to place the image and Text files\nimport os\nif not os.path.exists(\"train_data_yolo\"):\n    os.makedirs(\"train_data_yolo\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creatint Text Files for the images in balanced dataframe\nfolder_location = \"train_data_yolo\"\nfor img_id in tqdm(balanced_df[\"ImageId\"]):\n    \n#     k+1# loop through all unique image ids. Remove the slice to do all images\n    bbox = ships[[\"ImageId\",\"Ship\",\"Bbox\"]][ships.ImageId==img_id]\n    all_boxes = bbox.Bbox.values\n    img_id = img_id.split(\".\")[0]\n    file_name = \"{}/{}.txt\".format(folder_location,img_id) \n    s = \"0 %s %s %s %s \\n\" \n    with open(file_name, 'a') as file: # append lines to file\n        if bbox.Ship.iloc[0]>0:\n            for i in all_boxes:\n                new_line = (s % tuple(i))\n                file.write(new_line)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transfering the images to the newly created folder where text files were created\ndirectory = \"../input/airbus-ship-detection/train_v2/\" \nfor img_id in tqdm(balanced_df[\"ImageId\"]):\n    image_id = directory+img_id\n    shutil.copy(image_id, \"train_data_yolo\")\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}