{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e25c6d6db03de9bbc49c9e8eb441b099f70afe7"},"cell_type":"code","source":"df = pd.read_csv(\"../input/train_ship_segmentations_v2.csv\")\ndfWtShipOnly = df.drop( df.index[df.EncodedPixels.apply(lambda x: not isinstance(x, str)).tolist()])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fde82074b9b5a076e25124e5ff5b2bc9db9275c"},"cell_type":"markdown","source":"# Creating the clean dataset (with boat only)"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Use rle and its position in the image as identifier of the image\ndfWtShipOnly[\"rleAndPosition\"] = dfWtShipOnly.EncodedPixels.apply(lambda x: ' '.join(x.split(\" \")[1::2]) + ' ' + ' '.join([ str(int(hor) % 256) for hor in x.split(\" \")[0::2]]) if (isinstance(x, str)) else x)\ndfWtShipOnly.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab08a71f5ca8ac02d8dcfbdd1e4826f28dc9a8fe"},"cell_type":"code","source":"#List in a new column all the ImageId where the 'rleAndPosition' occurs.\ndfWtShipOnly[\"allSameRle\"] = dfWtShipOnly[\"rleAndPosition\"].apply(lambda x: dfWtShipOnly.ImageId[dfWtShipOnly[\"rleAndPosition\"] == x].tolist())\ndfWtShipOnly.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4533ee614836906c23b9f5309ca46cad76243fb6"},"cell_type":"code","source":"# Verify that 'rleAndPosition' values only occurs mainly a few times \ndfWtShipOnly[\"rleAndPosition\"].value_counts().describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36311c03d4d766c1c55eb1e9d4882a84164daa65"},"cell_type":"code","source":"# Plot \"allSameRle\"\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport cv2\nfrom PIL import Image\n\ndef get_image_data(image_id, image_type, **kwargs):\n    img = _get_image_data_opencv(image_id, image_type, **kwargs)\n    img = img.astype('uint8')\n    return img\n\ndef _get_image_data_opencv(image_id, image_type, **kwargs):\n    fname = \"../input/train_v2/\" + image_id\n    img = cv2.imread(fname)\n    assert img is not None, \"Failed to read image : %s, %s\" % (image_id, image_type)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img\n\nmaxRepetition = 8\nplt.figure(figsize=(20,20))\nImgNumber = 10\ni = 0\nfor y in range(ImgNumber) :\n    image_id = dfWtShipOnly[\"ImageId\"].tolist()[np.random.randint(0,len(dfWtShipOnly))]\n    img = get_image_data(image_id, 'Train')\n    ID_list = dfWtShipOnly[dfWtShipOnly[\"ImageId\"] == image_id][\"allSameRle\"].tolist()[0]\n    if len(ID_list) > maxRepetition: ID_list = ID_list[0:maxRepetition-1]\n    for ID in ID_list :\n        i += 1\n        img = get_image_data(ID, 'Train')\n        plt.subplot(ImgNumber,maxRepetition,i)\n        plt.imshow(img, cmap='binary')\n    i = maxRepetition * (y+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"791102676344cc482dce5bc044df5343dcbab2b5"},"cell_type":"code","source":"# Group the 'rleAndPosition' by ImageId  \ndfWtShipOnlyUnique = dfWtShipOnly.groupby('ImageId')['allSameRle'].apply(lambda x: set(x.sum())) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c258f0f63c3a83e0c4a84243aa6e6b3ca4e13ae"},"cell_type":"code","source":"print(len(df))\nprint(len(dfWtShipOnly))\nprint(len(dfWtShipOnlyUnique))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46e58350c625640c9a69b8192f272f16a5e2d5db","scrolled":true},"cell_type":"code","source":"print(len(dfWtShipOnlyUnique))\nalreadyDropped = []\ndfWtShipOnlyUniqueCopy = dfWtShipOnlyUnique\nfor itemKeeped in dfWtShipOnlyUnique.iteritems() :\n    if not itemKeeped[0] in alreadyDropped :\n        for itemToCheck in dfWtShipOnlyUnique.iteritems() :\n            if itemToCheck[0] in itemKeeped[1] and not itemToCheck[0] in alreadyDropped and itemToCheck[0] != itemKeeped[0]:\n                dfWtShipOnlyUnique = dfWtShipOnlyUnique.drop(itemToCheck[0])  \n                alreadyDropped = alreadyDropped + [itemToCheck[0]]\nprint(len(dfWtShipOnlyUnique))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe09db59a4963e03b7beb17814f14ced4a835f0"},"cell_type":"code","source":"# Splitting\ntrainDfWtShipOnlyUnique=dfWtShipOnlyUnique.sample(frac=0.9,random_state=768)\nvalidationDfWtShipOnlyUnique=dfWtShipOnlyUnique.drop(trainDfWtShipOnlyUnique.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5abcbcc3458016a2878409bc73be6956e1582620"},"cell_type":"code","source":"# Save the labels\nallUniqLabels = dfWtShipOnly.loc[[True if ID in dfWtShipOnlyUnique.index else False for ID in dfWtShipOnly[\"ImageId\"]]]\nallUniqLabels.to_csv('/kaggle/working/uniqueAllLabels.csv', index=True)\nprint(len(allUniqLabels))\ntrainUniqLabels = dfWtShipOnly.loc[[True if ID in trainDfWtShipOnlyUnique.index else False for ID in dfWtShipOnly[\"ImageId\"]]]\ntrainUniqLabels.to_csv('/kaggle/working/uniqueTrainLabels.csv', index=True)\nprint(len(trainUniqLabels))\nvalidationUniqLabels = dfWtShipOnly.loc[[True if ID in validationDfWtShipOnlyUnique.index else False for ID in dfWtShipOnly[\"ImageId\"]]]\nvalidationUniqLabels.to_csv('/kaggle/working/uniqueValidationLabels.csv', index=True)\nprint(len(validationUniqLabels))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}