{"cells":[{"metadata":{"_uuid":"ce3a0a5397b9706ad94df8417b5260ea7d086829"},"cell_type":"markdown","source":"# SHIP DETECTION CHALLENGE"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\nfrom skimage.measure import label, regionprops","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e71ec9d67669af31349bb613e2d9f315fda0f1b5"},"cell_type":"markdown","source":"## TOOLS : CUSTOM FUNCTIONS AND CLASSES"},{"metadata":{"trusted":true,"_uuid":"f01b3db62fcc8e3ba23234c4a57aca08c8f76595","collapsed":true},"cell_type":"code","source":"class SatelliteImage(object):       \n    def load_file(filePath):\n        matrix = cv2.imread(filePath)\n        matrix = cv2.cvtColor(matrix, cv2.COLOR_BGR2RGB)\n        \n        instance = SatelliteImage()\n        instance.image = matrix\n        instance.height = matrix.shape[0]\n        instance.width = matrix.shape[1]\n        instance.shape = (instance.height, instance.width)\n        \n        return instance\n    \n    def __init__(self):\n        self.title = \"\"\n        self.image = None\n        self.width = 0\n        self.height = 0\n        self.shape = (0, 0)\n        self.objects_mask_image = None\n    \n    def set_title(self, title):\n        self.title = title\n    \n    def add_objects(self, rle_masks):   \n        masks = np.zeros(self.shape, dtype = np.uint8)\n        \n        for rle_mask in rle_masks:\n            if isinstance(rle_mask, str):\n                masks += RDE.decode(rle_mask, self.shape)\n        \n        self.objects_mask_image = np.expand_dims(masks, -1)\n    \n    def show(self, with_object_rectangles = False):\n        plt.axis(\"off\")\n        plt.title(self.title)\n        \n        if with_object_rectangles:\n            copy = self.image.copy()\n            for prop in regionprops(label(self.objects_mask_image)): \n                cv2.rectangle(\n                    img = copy, \n                    pt1 = (prop.bbox[1], prop.bbox[0]), \n                    pt2 = (prop.bbox[3], prop.bbox[2]), \n                    color = (250, 0, 0), \n                    thickness = 2\n                )\n            plt.imshow(copy)\n            \n        else:\n            plt.imshow(self.image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"da43e80cbdd55071c9711cda626914b3ccae1da0"},"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\nclass RDE(object):\n    def encode(image):\n        pixels = image.T.flatten()\n        pixels = np.concatenate([[0], pixels, [0]])\n        runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n        runs[1::2] -= runs[::2]\n        \n        return ' '.join(str(x) for x in runs)\n    \n    def decode(encoding, shape):\n        strings = encoding.split()\n        starts, lengths = [np.asarray(x, dtype = int) for x in (strings[0:][::2], strings[1:][::2])]\n        starts -= 1\n        ends = starts + lengths\n        image = np.zeros(shape[0] * shape[1], dtype = np.uint8)\n        \n        for lo, hi in zip(starts, ends):\n            image[lo:hi] = 1\n        \n        return image.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf72809e16805a9649a4c8d6609284e313712521"},"cell_type":"markdown","source":"## IMPORTING DATA"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"PROJECT_DIR = os.path.join(\"..\", \"input\")\nSHIP_MASKS_DATASET_FILE = os.path.join(PROJECT_DIR, \"train_ship_segmentations.csv\")\nIMAGES_PROCESSED_DIR = os.path.join(PROJECT_DIR, \"train\")\nIMAGES_UNPROCESSED_DIR = os.path.join(PROJECT_DIR, \"test\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0878a5c06a7216082669194402e669bbaa9f5545"},"cell_type":"code","source":"def create_and_populate_images_dataSet(images_dir):\n    images_dataSet = pd.DataFrame(os.listdir(images_dir), columns = [\"filename\"])\n    images_dataSet[\"id\"] = images_dataSet[\"filename\"].str.split(\".\").str[0].astype(str)\n    images_dataSet[\"filePath\"] = images_dir + \"/\" + images_dataSet[\"filename\"]\n    images_dataSet.set_index([\"id\"], inplace = True)\n    \n    return images_dataSet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c00f92ae2218a992d57572a250c3ef2cecd904ec","collapsed":true},"cell_type":"code","source":"IMAGES_PROCESSED_DATASET = create_and_populate_images_dataSet(IMAGES_PROCESSED_DIR)\nIMAGES_UNPROCESSED_DATASET = create_and_populate_images_dataSet(IMAGES_UNPROCESSED_DIR)\n\nIMAGES_PROCESSED_COUNT = IMAGES_PROCESSED_DATASET.shape[0]\nIMAGES_UNPROCESSED_COUNT = IMAGES_UNPROCESSED_DATASET.shape[0]\nIMAGES_COUNT = IMAGES_PROCESSED_COUNT + IMAGES_UNPROCESSED_COUNT\n\nprint(\"{:,d} images | {:,d} processed images ({:.2f}%) | {:,d} unprocessed images ({:.2f}%)\".format(\n    IMAGES_COUNT,\n    IMAGES_PROCESSED_COUNT, (IMAGES_PROCESSED_COUNT / IMAGES_COUNT) * 100,\n    IMAGES_UNPROCESSED_COUNT, (IMAGES_UNPROCESSED_COUNT / IMAGES_COUNT) * 100\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfb3a0fb9b8960b8110e26ed8d23d6004a949eef","collapsed":true},"cell_type":"code","source":"SHIP_MASKS_DATASET = pd.read_csv(SHIP_MASKS_DATASET_FILE, names = [\"filename\", \"mask\"], header = 0)\nSHIP_MASKS_DATASET[\"imageId\"] = SHIP_MASKS_DATASET[\"filename\"].str.split(\".\").str[0].astype(str)\nSHIP_MASKS_DATASET = SHIP_MASKS_DATASET.drop(\"filename\", axis = 1)\nSHIP_MASKS_DATASET.set_index([\"imageId\"], inplace = True)\n\nIMAGES_WITH_SHIPS_COUNT = SHIP_MASKS_DATASET.groupby(\"imageId\").count().astype(bool).sum().values[0]\nIMAGES_WITHOUT_SHIPS_COUNT = IMAGES_PROCESSED_COUNT - IMAGES_WITH_SHIPS_COUNT\n\nprint(\"{:,d} processed images | {:,d} images with ships ({:.2f}%) | {:,d} images without ships ({:.2f}%)\".format(\n    IMAGES_PROCESSED_COUNT,\n    IMAGES_WITH_SHIPS_COUNT, (IMAGES_WITH_SHIPS_COUNT / IMAGES_PROCESSED_COUNT) * 100,\n    IMAGES_WITHOUT_SHIPS_COUNT, (IMAGES_WITHOUT_SHIPS_COUNT / IMAGES_PROCESSED_COUNT) * 100\n))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"163c33f120e34f00cd0101d52d8dd645c923e706"},"cell_type":"markdown","source":"## DATA ANALYSIS"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9a486cc64523cb2c75e251642402e8e2aa1548bb"},"cell_type":"code","source":"def create_SatelliteImage_from_sample(index, sample):\n    satelliteImage = SatelliteImage.load_file(sample[\"filePath\"])\n    satelliteImage.set_title(\"#{:s} ({:d} ships)\".format(index.upper(), sample[\"shipsCount\"]))\n    satelliteImage.add_objects(pd.Series(SHIP_MASKS_DATASET.loc[index][\"mask\"]))\n    \n    return satelliteImage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e7bb449a4750b9272cca4673b158c79c2152667b"},"cell_type":"code","source":"def display_SatelliteImages(satelliteImages, cols = 4, figsize = (5, 5)):\n    w, h = figsize\n    rows = len(satelliteImages) * 1 // cols + 1\n    plt.figure(figsize = (cols * w, rows * h))\n\n    i = 1    \n    for satelliteImage in satelliteImages:\n        plt.subplot(rows, cols, i)\n        satelliteImage.show(True)\n        i += 1\n\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2f75ee8b402ddeba0199c64cc7661e208aa1d8a","collapsed":true},"cell_type":"code","source":"SHIPS_BY_IMAGE = SHIP_MASKS_DATASET.groupby(\"imageId\").count().astype(int)\nSHIPS_BY_IMAGE = SHIPS_BY_IMAGE.rename(columns = {\"mask\" : \"shipsCount\"})\nIMAGES_PROCESSED_DATASET = IMAGES_PROCESSED_DATASET.join(SHIPS_BY_IMAGE, how = \"inner\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0ab2a5a834c17f56684938b7f3ebfe9f031eb3b"},"cell_type":"markdown","source":"### DATA ANALYSIS : SHIPS FREQUENCY"},{"metadata":{"trusted":true,"_uuid":"7a42ca0c6f555a4f2356c1ae72993fb833952fc6","collapsed":true},"cell_type":"code","source":"SHIPS_BY_IMAGE_COUNT = len(SHIPS_BY_IMAGE)\n\nSHIPS_BY_IMAGE_MEAN = SHIPS_BY_IMAGE[\"shipsCount\"].mean()\nSHIPS_BY_IMAGE_STD = SHIPS_BY_IMAGE[\"shipsCount\"].std()\nSHIPS_BY_IMAGE_LOWER = int(max(0, SHIPS_BY_IMAGE_MEAN - 3 * SHIPS_BY_IMAGE_STD))\nSHIPS_BY_IMAGE_UPPER = int(min(SHIPS_BY_IMAGE_MEAN + 3 * SHIPS_BY_IMAGE_STD, SHIPS_BY_IMAGE[\"shipsCount\"].max()))\n\nSHIPS_BY_IMAGE_OUTLIERS = SHIPS_BY_IMAGE[(SHIPS_BY_IMAGE[\"shipsCount\"] < SHIPS_BY_IMAGE_LOWER) | (SHIPS_BY_IMAGE[\"shipsCount\"] > SHIPS_BY_IMAGE_UPPER)]\nSHIPS_BY_IMAGE_OUTLIERS_COUNT = SHIPS_BY_IMAGE_OUTLIERS.shape[0]\n\nprint(\"{:.2f} ± {:.2f} ships by image ({:.2f}%) | {:,d} outlier images ({:.2f}%)\".format(\n    SHIPS_BY_IMAGE_MEAN, 3 * SHIPS_BY_IMAGE_STD, ((SHIPS_BY_IMAGE_COUNT - SHIPS_BY_IMAGE_OUTLIERS_COUNT) / SHIPS_BY_IMAGE_COUNT) * 100,\n    SHIPS_BY_IMAGE_OUTLIERS_COUNT, (SHIPS_BY_IMAGE_OUTLIERS_COUNT / SHIPS_BY_IMAGE_COUNT) * 100\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09eb0328d4e7fc1add0f73b1b3689de30aa98692","collapsed":true},"cell_type":"code","source":"def display_shipsCount_frequency(shipsCounts):\n    plt.figure(figsize = (23, 8))\n    ax = sns.countplot(data = shipsCounts, x = \"shipsCount\")\n    \n    for p in ax.patches:\n        x = p.get_bbox().get_points()[:,0]\n        y = p.get_bbox().get_points()[1,1]\n        ax.annotate(\"{:.2f}%\".format(y / len(shipsCounts) * 100), (x.mean(), y), ha = \"center\", va = \"bottom\")\n    \n    plt.title(\"Ships Frequency Distribution\")\n    plt.ylabel(\"Frequency\")\n    plt.xlabel(\"Ships count\")\n    plt.show()\n    \ndisplay_shipsCount_frequency(SHIPS_BY_IMAGE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95b96b730274f52c8130b031cdb80fa8ea503b9d"},"cell_type":"markdown","source":"#### Insights :\n* Most of the images are empty."},{"metadata":{"_uuid":"0b8ac2127e26264ad58a669b381c45ba20d8f9eb"},"cell_type":"markdown","source":"### DATA ANALYSIS : IMAGES WITHOUT SHIP"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"373e76e324b88ab268a38288036c28e724a0ebb2"},"cell_type":"code","source":"SAMPLES = IMAGES_PROCESSED_DATASET[IMAGES_PROCESSED_DATASET[\"shipsCount\"] == 0].sample(32)\nSATELLITE_IMAGES = [create_SatelliteImage_from_sample(index, sample) for index, sample in SAMPLES.iterrows()]\ndisplay_SatelliteImages(SATELLITE_IMAGES)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3e9de891ae3dc66de7103ccf610feb5f1cbf3b6"},"cell_type":"markdown","source":"### DATA ANALYSIS : IMAGES WITH SHIPS"},{"metadata":{"trusted":true,"_uuid":"cdb42a999c20d8390428a66889bd25e727be42d6","scrolled":false,"collapsed":true},"cell_type":"code","source":"SAMPLES = IMAGES_PROCESSED_DATASET[(IMAGES_PROCESSED_DATASET[\"shipsCount\"] > 0) & (IMAGES_PROCESSED_DATASET[\"shipsCount\"] < SHIPS_BY_IMAGE_UPPER)].sample(32)\nSATELLITE_IMAGES = [create_SatelliteImage_from_sample(index, sample) for index, sample in SAMPLES.iterrows()]\ndisplay_SatelliteImages(SATELLITE_IMAGES)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e2afa1b7567e22f2963b708493bf383f99e8d169"},"cell_type":"markdown","source":"#### Insights :\n* Some images are corrupted.\n* There are 4 kinds of image : all sea, no sea, shore and sea, cloud and sea.\n* Images are not take all at the same zoom level."},{"metadata":{"_uuid":"6b15787f7c9b79e9d4cf8e485fc6513ec2dd2662"},"cell_type":"markdown","source":"### DATA ANALYSIS : IMAGES WITH MANY SHIPS (OUTLIERS)"},{"metadata":{"trusted":true,"_uuid":"d981fd5e6cc84e52fe74af6e21a8db2025c928fd","scrolled":false,"collapsed":true},"cell_type":"code","source":"SAMPLES = IMAGES_PROCESSED_DATASET[IMAGES_PROCESSED_DATASET[\"shipsCount\"] > SHIPS_BY_IMAGE_UPPER].sample(32)\nSATELLITE_IMAGES = [create_SatelliteImage_from_sample(index, sample) for index, sample in SAMPLES.iterrows()]\ndisplay_SatelliteImages(SATELLITE_IMAGES)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"433f73330331b58d85dec3401b7c2365a779ba55"},"cell_type":"markdown","source":"#### Insights :"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}