{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport cv2\nfrom PIL import Image as pil_image\nfrom tqdm import tqdm\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\n\nimport albumentations as A","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:26:29.412804Z","iopub.execute_input":"2022-03-25T22:26:29.413677Z","iopub.status.idle":"2022-03-25T22:26:30.389524Z","shell.execute_reply.started":"2022-03-25T22:26:29.413623Z","shell.execute_reply":"2022-03-25T22:26:30.388535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PROJECT_FOLDER = '../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/'","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:36.071465Z","iopub.execute_input":"2022-03-25T22:18:36.071878Z","iopub.status.idle":"2022-03-25T22:18:36.0757Z","shell.execute_reply.started":"2022-03-25T22:18:36.071836Z","shell.execute_reply":"2022-03-25T22:18:36.074861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_im_with_channels(image):\n    channels = [\"Red\", \"Green\", \"Blue\"]\n    \n    fig, ax = plt.subplots(1, 4, figsize=(22,6))\n    ax[0].imshow(image)\n    for i in range(3):\n        ax[i+1].imshow(image[:, :, i])\n        ax[i+1].set_title(channels[i])\n        \ndef open_img_as_array(im_path):\n    return np.array(pil_image.open(im_path)).astype(np.uint8)\n\ndef pad_and_resize(img, img_size):\n    w, h, c = np.shape(img)\n    if w > h:\n        pad = int((w - h) / 2)\n        img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_CONSTANT, value=0)\n    else:\n        pad = int((h - w) / 2)\n        img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_CONSTANT, value=0)\n        \n    img = cv2.resize(img, (img_size, img_size))\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:36.076981Z","iopub.execute_input":"2022-03-25T22:18:36.077269Z","iopub.status.idle":"2022-03-25T22:18:36.094011Z","shell.execute_reply.started":"2022-03-25T22:18:36.077237Z","shell.execute_reply":"2022-03-25T22:18:36.092952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Occlusions in test dataset\n\nUnlike last year's competition the test set images in this competition are all partially masked to emulate the added challenge of having a person in the blocking part of the view of the room.","metadata":{}},{"cell_type":"code","source":"test_image = open_img_as_array(PROJECT_FOLDER + '/test_images/abc.jpg')\nshow_im_with_channels(test_image)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:36.095975Z","iopub.execute_input":"2022-03-25T22:18:36.096216Z","iopub.status.idle":"2022-03-25T22:18:37.548776Z","shell.execute_reply.started":"2022-03-25T22:18:36.096188Z","shell.execute_reply":"2022-03-25T22:18:37.548003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Provided ~~train~~ test masks\n\n**train_masks** - Occlusions like the ones that will be present in the images in the test set.\n\n[Abby Stylianou [Competition Host]](https://www.kaggle.com/competitions/hotel-id-to-combat-human-trafficking-2022-fgvc9/discussion/313547#1732937)\n> The mask files are available for your convenience -- if there's a query image called 0001.jpg and a mask called 0001.png, the mask is simply a PNG that includes the exact mask from the query JPG. You can use these for processing of the query images in case it's easier than detecting the mask from the JPG, and also may also use any of the masks for whatever purpose in your training (as some other comments have suggested). You aren't required to use the PNG masks for anything -- like I said, they're just there for convenience.","metadata":{}},{"cell_type":"code","source":"mask_image = open_img_as_array('../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_masks/00000.png')\nshow_im_with_channels(mask_image)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:37.550191Z","iopub.execute_input":"2022-03-25T22:18:37.550644Z","iopub.status.idle":"2022-03-25T22:18:38.69217Z","shell.execute_reply.started":"2022-03-25T22:18:37.550611Z","shell.execute_reply":"2022-03-25T22:18:38.690878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mapping masks to images?\n> if there's a query image called 0001.jpg and a mask called 0001.png, the mask is simply a PNG that includes the exact mask from the query JPG","metadata":{}},{"cell_type":"code","source":"# load image and mask names\nMASK_FOLDER = PROJECT_FOLDER + 'train_masks/'\nmask_df = pd.DataFrame(data={\"mask_id\": os.listdir(MASK_FOLDER)}).sort_values(by=\"mask_id\")\ntrain_df = pd.read_csv('../input/hotelid-2022-train-images-256x256/train.csv')\n\nprint(\"Images:\", train_df[\"image_id\"].sort_values().head().values)\nprint(\"Masks:\", mask_df[\"mask_id\"].sort_values().head().values)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:38.694491Z","iopub.execute_input":"2022-03-25T22:18:38.695176Z","iopub.status.idle":"2022-03-25T22:18:39.255799Z","shell.execute_reply.started":"2022-03-25T22:18:38.69512Z","shell.execute_reply":"2022-03-25T22:18:39.254627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# replace file extension for easy mapping\ntrain_id = train_df[\"image_id\"].str.replace('.jpg', '', regex=False)\nmask_id = mask_df[\"mask_id\"].str.replace('.png', '', regex=False)\n\nprint(\"Image ids:\", train_id.sort_values().head().values)\nprint(\"Mask ids:\", mask_id.sort_values().head().values)\n\n# find if any mask id appear in train dataset ids\nprint(\"Number of mask ids matching image ids:\", mask_id.isin(train_id).sum())","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:39.256978Z","iopub.execute_input":"2022-03-25T22:18:39.257233Z","iopub.status.idle":"2022-03-25T22:18:39.389439Z","shell.execute_reply.started":"2022-03-25T22:18:39.257204Z","shell.execute_reply":"2022-03-25T22:18:39.38797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"~~But **there are no masks matching any images in train dataset**.~~","metadata":{}},{"cell_type":"markdown","source":"~~## No matching masks and images. Try to match it and display result\nThe name format is not the same so maybe if we prepand the mask name with additional 0s to match training image names it might work. Lets try to display some examples.\nThe resolution of mask and image is not always the same so they probably don't belong together.~~","metadata":{}},{"cell_type":"code","source":"# def load_and_display_pair(image_id, hotel_id, mask_id):\n#     train_image = open_img_as_array(f\"{PROJECT_FOLDER}train_images/{hotel_id}/{image_id}\")\n#     mask_image = open_img_as_array(f\"{MASK_FOLDER}{mask_id}\")\n\n#     fig, ax = plt.subplots(1,2, figsize=(16, 4))\n#     ax[0].imshow(train_image)\n#     ax[0].set_title(f\"{image_id} {np.shape(train_image)}\")\n#     ax[1].imshow(mask_image)\n#     ax[1].set_title(f\"{mask_id} {np.shape(mask_image)}\")\n\n# load_and_display_pair(\"000000000.jpg\", \"95500\", \"00000.png\")\n# load_and_display_pair(\"000001694.jpg\", \"209817\", \"01694.png\")\n# load_and_display_pair(\"000005235.jpg\", \"204287\", \"05235.png\")","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:39.390874Z","iopub.execute_input":"2022-03-25T22:18:39.391112Z","iopub.status.idle":"2022-03-25T22:18:39.396287Z","shell.execute_reply.started":"2022-03-25T22:18:39.391083Z","shell.execute_reply":"2022-03-25T22:18:39.395051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[Abby Stylianou [Competition Host]](https://www.kaggle.com/competitions/hotel-id-to-combat-human-trafficking-2022-fgvc9/discussion/313547#1734687)\n> Oh! I have figured out the source of the confusion. **There was a mixup on the host end -- the \"train_masks\" folder should be named \"test_masks\"** (I've asked the kaggle team to update this). There are no training masks provided. This matches the real world setting, where the \"test\" images (from investigations) have occlusions in the region of the image where the victim is located.\n>\n> Training images, on the other hand, are not (by default) occluded. Competitors may choose to include occlusions in their training process, but we do not dictate that (or any other approach). If a competitor chose to incorporate masks, they could either generate their own, or repurpose the ones that match the test images (resizing them as necessary).","metadata":{}},{"cell_type":"markdown","source":"## Plot occlusion areas and image size\nWe can calculate what part of test images is covered by provided occlusions so we can simulate it better during training.","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 256\n\nx_dim_array = [] # image size X\ny_dim_array = [] # image size Y\nocc_area = [] # percentage of occlusion in image\nocc_img = np.zeros((IMG_SIZE, IMG_SIZE, 3))\n\nmask_files = os.listdir(MASK_FOLDER)\nfor mask_id in tqdm(mask_files):\n    mask_image = open_img_as_array(MASK_FOLDER + mask_id)\n    X, Y, C = np.shape(mask_image)\n    occ = (mask_image[:, :, 0] > 0).mean()\n    x_dim_array.append(X)\n    y_dim_array.append(Y)\n    occ_area.append(occ)\n    \n    occ_img += pad_and_resize(mask_image[:, :, :3], IMG_SIZE)\n    \nocc_img /= len(mask_files)\nocc_img = occ_img.astype(int)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:32:38.744885Z","iopub.execute_input":"2022-03-25T22:32:38.745241Z","iopub.status.idle":"2022-03-25T22:33:01.02215Z","shell.execute_reply.started":"2022-03-25T22:32:38.745163Z","shell.execute_reply":"2022-03-25T22:33:01.020817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(occ_area, nbins=50, marginal=\"box\", height=350)\nfig.update_layout(title=\"Distribution of occlusion coverage in test images\")\nfig.update_traces(hovertemplate=\"Image count: %{y} <br>Occlusion coverage: %{x}\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:33:03.330728Z","iopub.execute_input":"2022-03-25T22:33:03.33103Z","iopub.status.idle":"2022-03-25T22:33:03.433903Z","shell.execute_reply.started":"2022-03-25T22:33:03.330994Z","shell.execute_reply":"2022-03-25T22:33:03.432642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mean over all masks to show most common position.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6,6))\nplt.imshow(occ_img)\nplt.suptitle('Projection of all masks')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test images should have the same size as masks so we can plot dimensions of images in test dataset.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Box(x=y_dim_array, name=\"Y\", boxpoints=\"all\"))\nfig.add_trace(go.Box(x=x_dim_array, name=\"X\", boxpoints=\"all\"))\nfig.update_yaxes(title=\"Axis\")\nfig.update_xaxes(title=\"Pixels\")\nfig.update_layout(title=f\"Box plots - dimensions of images in test dataset\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:33:07.173479Z","iopub.execute_input":"2022-03-25T22:33:07.173769Z","iopub.status.idle":"2022-03-25T22:33:07.195238Z","shell.execute_reply.started":"2022-03-25T22:33:07.173734Z","shell.execute_reply":"2022-03-25T22:33:07.193919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Simulating occlusions using albumentation CoarseDropout\nTo simulate the masks we can use CoarseDropout from [Albumentation](https://github.com/albumentations-team/albumentations) library.","metadata":{}},{"cell_type":"code","source":"train_image = open_img_as_array('../input/hotelid-2022-train-images-256x256/images/000000000.jpg')\nshow_im_with_channels(train_image)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:39.502914Z","iopub.status.idle":"2022-03-25T22:18:39.503321Z","shell.execute_reply.started":"2022-03-25T22:18:39.503098Z","shell.execute_reply":"2022-03-25T22:18:39.503125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use albumentations CoarseDropout with fill_value=(255,0,0) to simulate red rectangle oclussion in dataset.","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 256\ndropout = A.CoarseDropout(p=1., max_holes=1, \n                          min_height=IMG_SIZE//4, max_height=IMG_SIZE//2,\n                          min_width=IMG_SIZE//4,  max_width=IMG_SIZE//2, \n                          fill_value=(255,0,0))\n\n\ntrain_image = dropout(image=train_image)[\"image\"]\nshow_im_with_channels(train_image)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:39.505194Z","iopub.status.idle":"2022-03-25T22:18:39.505753Z","shell.execute_reply.started":"2022-03-25T22:18:39.505472Z","shell.execute_reply":"2022-03-25T22:18:39.505502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The result looks similar to test image but it's not perfect. The location of occlusion in test masks is mainly center while with CoarseDropout it will be (should be) evenly distributed in the image.","metadata":{}},{"cell_type":"code","source":"test_image = open_img_as_array(PROJECT_FOLDER + '/test_images/abc.jpg')\nshow_im_with_channels(test_image)","metadata":{"execution":{"iopub.status.busy":"2022-03-25T22:18:39.507638Z","iopub.status.idle":"2022-03-25T22:18:39.508136Z","shell.execute_reply.started":"2022-03-25T22:18:39.507869Z","shell.execute_reply":"2022-03-25T22:18:39.507895Z"},"trusted":true},"execution_count":null,"outputs":[]}]}