{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nimport os\nimport json\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Description"},{"metadata":{"trusted":true},"cell_type":"code","source":"json_data=open(\"../input/label_descriptions.json\").read()\nlabel_descriptions = json.loads(json_data)\n\n# dataset info\nlabel_descriptions['info']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset categories\ncategories_label_df = pd.DataFrame(label_descriptions['categories'])\nprint(\"The number of categories : \",len(categories_label_df))\ndisplay(categories_label_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# supercategory class\ncategories_label_df.groupby('supercategory')['name'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset attributes\nattributes_label_df = pd.DataFrame(label_descriptions['attributes'])\nprint(\"The number of attributes : \",len(attributes_label_df))\ndisplay(attributes_label_df.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Train Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\nprint(train_df.shape)\nprint(\"number of unique images :\",len(set(train_df['ImageId'])))\ntrain_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**There are 333,415 total data. **\n\n**Considering only one unique image, the number of images is 45,625 total.**\n\n**That is, there are multiple ClassIds in one image**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# split classid to category & attributes ..\ndef split_ClassId(data):\n    \n    data_c = data.copy()\n\n    class_list = []\n    attribute_list = []\n\n    for i in range(len(data_c)):\n        classid = data_c.iloc[i,4]\n\n        # IF attribute exists, \n        if len(classid) > 2:\n            class_attribute_list = classid.split(\"_\")\n\n            class_list.append(class_attribute_list[0])\n            attribute_list.append(class_attribute_list[1:])\n\n        else:\n            class_list.append(classid)\n            attribute_list.append(\"\")\n\n    data_c[\"class_\"] = class_list\n    data_c[\"attributes\"] = attribute_list\n    \n    return data_c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split classid to category & attributes\ntrain_df['Category'] = train_df['ClassId'].apply(lambda x: int(x.split(\"_\")[0]))\ntrain_df['Attributes'] = train_df['ClassId'].apply(lambda x: list(map(int, x.split(\"_\")[1:])))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"groupby_category = train_df.groupby('Category')['ImageId'].count()\ngroupby_category.index = map(int, groupby_category.index)\ngroupby_category = groupby_category.sort_index()\ngroupby_category[:5]\n\nfig = plt.figure(figsize=(10, 4))\nx = groupby_category.index\ny = groupby_category.values\n\nsns.barplot(x,y)\nplt.title(\"Number of images by category\", fontsize=20)\nplt.xlabel(\"Category\", fontsize=20)\nplt.ylabel(\"# of images\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"groupby_category = train_df[['ImageId','Category']].groupby('ImageId').count()\nprint(\"Average number of categories per unique image: \",np.mean(groupby_category['Category'][::]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## categories w and w/o attributes"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_Having_attributes_df = train_df[train_df['Attributes'].apply(len) > 0]\nprint(\"the number of images having attributes:\",len(train_Having_attributes_df['ImageId']))\nprint(\"the number of unique images having attributes:\",len(set(train_Having_attributes_df['ImageId'])))\n#print(\"---------------------------------------------\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The Number of images with Attributes by Category\ngroupby_category_Having_attributes = train_Having_attributes_df[['ImageId','Category']].groupby('Category').count()\ngroupby_category_Having_attributes.columns = ['# of imgs']\ndisplay(groupby_category_Having_attributes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Having_attributes = list(groupby_category_Having_attributes.index)\nprint(\"categories having attributes are: \\n\", Having_attributes)\nprint()\nprint(\"categories not having attributes are: \\n\", list(categories_label_df.id[~categories_label_df.id.isin(Having_attributes)]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_category_df = train_Having_attributes_df[[\"Category\",\"Attributes\"]].groupby(by = \"Category\").sum().reset_index()\ngroup_category_df = pd.merge(group_category_df,categories_label_df,left_on = 'Category',right_on='id')\n\n# kind of Attributes per category\ngroup_category_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**category id 0 - 12 : evenly distributed attributes **\n\n**category id 27 28 33 : only 20, 61 attribute****"},{"metadata":{"trusted":true},"cell_type":"code","source":"# the distribution of # of attributes per images, among images with attributes\ntrain_Having_attributes_desc = train_Having_attributes_df.iloc[:,6].apply(len).describe()\ntrain_Having_attributes_desc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The 1,1540 images having attributes have an average of 6-7 attributes**"},{"metadata":{},"cell_type":"markdown","source":"---"},{"metadata":{},"cell_type":"markdown","source":"## Unique Image EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The number of ClassId(Category) per unique Image\ngroup_unique_df = train_df[[\"ImageId\", \"Height\"]].groupby(by = \"ImageId\").count().reset_index()\ngroup_unique_df.columns = ['ImageId', 'Num_of_ClassId']\ndisplay(group_unique_df.head(3))\nsns.boxplot(group_unique_df['Num_of_ClassId'])\nprint(group_unique_df.describe(percentiles = np.arange(0.5, 1.0, 0.05)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_over20_data_df = group_unique_df[group_unique_df['Num_of_ClassId'] > 20]\nprint(class_over20_data_df.head())\nprint(class_over20_data_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" group_unique_df[group_unique_df['Num_of_ClassId']==74]\ntrain_df[train_df.ImageId=='361cc7654672860b1b7c85fe8e92b38a.jpg'].drop_duplicates('ClassId')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**There are many images with duplicated ClassId.**"},{"metadata":{},"cell_type":"markdown","source":"# EDA on Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_img(IMG_FILE):\n    I = cv2.imread(\"../input/train/\" + IMG_FILE, cv2.IMREAD_COLOR)\n    I = cv2.cvtColor(I, cv2.COLOR_BGR2RGB)\n    plt.imshow(I) \n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## make sample data\n- Category : 0, 4, 6, 28, 29, 31, 32\n- no Attributes\n- ImageId == '00000663ed1ff0c4e0132b9b9ac53f6e.jpg'"},{"metadata":{"trusted":true},"cell_type":"code","source":"show_img('00000663ed1ff0c4e0132b9b9ac53f6e.jpg')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**'1 3 10 5' implies pixels 1,2,3,10,11,12,13,14 are to be included in the mask. ****"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_mask(IMG_FILE):\n    df = train_df[train_df.ImageId == IMG_FILE].reset_index(drop = True)\n    H = df.iloc[0,2]\n    W = df.iloc[0,3]\n    \n    print(\"Correct Category :\", sorted(set((list(df.Category)))))\n    # 1d mask \n    mask = np.full(H*W,dtype='int',fill_value = -1)\n    \n    for line in df[['EncodedPixels','Category']].iterrows():\n        EncodedPixels = line[1][0]\n        Category = line[1][1]\n        \n        pixel_loc = list(map(int,EncodedPixels.split(' ')[0::2]))\n        iter_num =  list(map(int,EncodedPixels.split(' ')[1::2]))\n        for p,i in zip(pixel_loc,iter_num):\n            mask[p:(p+i)] = Category\n    \n    print(\"Output :\",sorted(set(list(mask))))\n    #rle\n    mask = mask.reshape(W,H).T\n    \n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Category : 0, 4, 6, 28, 29, 31, 32\nmask = make_mask('00000663ed1ff0c4e0132b9b9ac53f6e.jpg')\nplt.imshow(mask,cmap='jet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Masking_Image(IMG_FILE):\n        \n    I = cv2.imread(\"../input/train/\" + IMG_FILE, cv2.IMREAD_COLOR)\n    I = cv2.cvtColor(I, cv2.COLOR_BGR2RGB)\n    mask = make_mask(IMG_FILE)\n\n    fig, ax = plt.subplots(nrows = 1, ncols = 3,figsize = (18,6))\n    \n    ax[0].imshow(I)\n    ax[1].imshow(mask, cmap = 'jet')\n    ax[2].imshow(I,interpolation = 'none')\n    ax[2].imshow(mask,cmap = 'jet', interpolation = 'none', alpha = 0.6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Masking_Image('00000663ed1ff0c4e0132b9b9ac53f6e.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Masking_Image(train_df.iloc[132,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Masking_Image(train_df.iloc[2838,0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ** 2 Issues **\n\n**1.  Missing Category**\n    - Maybe because the mask between categories overlaps.\n**2. Separate category id 0 and mask's default 0**"},{"metadata":{},"cell_type":"markdown","source":"# Binary mask \n**Solve Problem 1**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_binary_mask(IMG_FILE):\n    df = train_df[train_df.ImageId == IMG_FILE].reset_index(drop = True)\n    H = df.iloc[0,2]\n    W = df.iloc[0,3]\n    binary_mask_list = []\n    \n    print(\"Correct Category :\", sorted(set((list(df.Category)))))\n    for line in df[['EncodedPixels','Category']].iterrows():\n    \n        binary_mask = np.zeros(H*W,dtype='int')\n        EncodedPixels = line[1][0]\n        Category = line[1][1]\n        print(Category,categories_label_df.iloc[int(Category),2])\n        \n        pixel_loc = list(map(int,EncodedPixels.split(' ')[0::2]))\n        iter_num =  list(map(int,EncodedPixels.split(' ')[1::2]))\n        for p,i in zip(pixel_loc,iter_num):\n            binary_mask[p:(p+i)] = 1\n            \n        binary_mask = binary_mask.reshape(W,H).T\n        binary_mask_list.append(binary_mask)\n    \n    return binary_mask_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary_mask_list = make_binary_mask('00000663ed1ff0c4e0132b9b9ac53f6e.jpg')\nplt.imshow(binary_mask_list[7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}