{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport random\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom plotly.subplots import make_subplots","metadata":{"execution":{"iopub.status.busy":"2022-05-23T21:01:26.735474Z","iopub.execute_input":"2022-05-23T21:01:26.736248Z","iopub.status.idle":"2022-05-23T21:01:26.743927Z","shell.execute_reply.started":"2022-05-23T21:01:26.736205Z","shell.execute_reply":"2022-05-23T21:01:26.742854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PAD = True\nWIDTH = 256\nHEIGHT = 256\n\nTARGET_SAMPLE_COUNT = 20\n\nONE_UNCROPPED_SAMPLE = False\nMIN_DIM_MIN_CROP = 0.5\nMAX_DIM_MIN_CROP = 0.9","metadata":{"execution":{"iopub.status.busy":"2022-05-23T21:01:26.782965Z","iopub.execute_input":"2022-05-23T21:01:26.784055Z","iopub.status.idle":"2022-05-23T21:01:26.791293Z","shell.execute_reply.started":"2022-05-23T21:01:26.784008Z","shell.execute_reply":"2022-05-23T21:01:26.790372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\ntrain_folder = os.path.join(data_folder, 'train_images')\nchain_names = os.listdir(train_folder)\n\n#chain_names = chain_names[:3]\n\nprint(os.listdir(data_folder))\nprint(len(chain_names))","metadata":{"papermill":{"duration":0.225587,"end_time":"2021-04-16T01:53:14.982769","exception":false,"start_time":"2021-04-16T01:53:14.757182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-23T21:01:26.824925Z","iopub.execute_input":"2022-05-23T21:01:26.825252Z","iopub.status.idle":"2022-05-23T21:01:26.837545Z","shell.execute_reply.started":"2022-05-23T21:01:26.825217Z","shell.execute_reply":"2022-05-23T21:01:26.836849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_image(img):\n    w, h, c = np.shape(img)\n    if w > h:\n        pad = int((w - h) / 2)\n        img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_CONSTANT, value=0)\n    else:\n        pad = int((h - w) / 2)\n        img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_CONSTANT, value=0)\n        \n    return img\n\ndef get_random_crop(image, crop_height, crop_width):\n    max_x = image.shape[1] - crop_width\n    max_y = image.shape[0] - crop_height\n\n    x = np.random.randint(0, max_x)\n    y = np.random.randint(0, max_y)\n\n    crop = image[y: y + crop_height, x: x + crop_width]\n\n    return crop\n\ndef crop_randomly(img):\n    img_size = img.shape\n    crop_height = img_size[0]\n    crop_width = img_size[1]\n    if crop_height > crop_width:\n        crop_height = np.random.randint(img_size[0]*MIN_DIM_MIN_CROP, img_size[0])\n        crop_width = np.random.randint(img_size[1]*MAX_DIM_MIN_CROP, img_size[1])\n    else:\n        crop_height = np.random.randint(img_size[0]*MAX_DIM_MIN_CROP, img_size[0])\n        crop_width = np.random.randint(img_size[1]*MIN_DIM_MIN_CROP, img_size[1])\n    return get_random_crop(img, crop_height, crop_width)\n\n\ndef open_and_preprocess_image(image_folder, image_name, random_crop=True):\n    img = cv2.imread(os.path.join(image_folder, image_name))\n    \n    if random_crop:\n        img = crop_randomly(img)\n    \n    if PAD:\n        img = pad_image(img)\n    \n    img = cv2.resize(img, (WIDTH, HEIGHT))\n    img = cv2.transpose(img)\n    \n    return img\n\n\ndef save_image(path, img):\n    cv2.imwrite(path, img)\n\n\ndef process_chain(data_folder, chain_name, train_df, valid_df):\n    chain_folder = os.path.join(data_folder, chain_name)\n    image_names = os.listdir(chain_folder)\n    original_sample_count = len(image_names)\n    \n    validPopI = np.random.randint(0, len(image_names))\n    image_name = image_names.pop(validPopI)\n    #img = cv2.imread(os.path.join(chain_folder, image_name))\n    img = open_and_preprocess_image(chain_folder, image_name, random_crop=False)\n    save_image(\"valid_images/{}.jpg\".format(image_name), img)\n    valid_df = valid_df.append({'image_id': image_name, 'hotel_id': chain_name}, ignore_index=True)\n    \n    if original_sample_count <= 1:\n        image_names = os.listdir(chain_folder)\n    \n    nr = 0\n    images = image_names.copy()\n    reset_images = False\n    target_count = int(TARGET_SAMPLE_COUNT*0.5 + len(images)*0.5 + 0.5)\n    for i in range(target_count):\n        if len(images) <= 0:\n            images = image_names.copy()\n            reset_images = True\n\n        popI = np.random.randint(0, len(images))\n        image_name = images.pop(popI)\n        image_nr = int(image_name.split(\".\")[0])\n\n        save_name = \"1{}{:07d}\".format(image_nr, nr)\n        img = open_and_preprocess_image(chain_folder, image_name, random_crop=(reset_images or not ONE_UNCROPPED_SAMPLE))\n        save_image(\"train_images/{}.jpg\".format(save_name), img)\n        train_df = train_df.append({'image_id': save_name, 'hotel_id': chain_name}, ignore_index=True)\n        nr += 1\n\n    return train_df, valid_df","metadata":{"papermill":{"duration":0.01604,"end_time":"2021-04-16T01:53:15.003615","exception":false,"start_time":"2021-04-16T01:53:14.987575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-23T21:01:26.862576Z","iopub.execute_input":"2022-05-23T21:01:26.863245Z","iopub.status.idle":"2022-05-23T21:01:26.886216Z","shell.execute_reply.started":"2022-05-23T21:01:26.863203Z","shell.execute_reply":"2022-05-23T21:01:26.885255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isdir(\"train_images\"):\n    os.mkdir(\"train_images\")\nif not os.path.isdir(\"valid_images\"):\n    os.mkdir(\"valid_images\")","metadata":{"execution":{"iopub.status.busy":"2022-05-23T21:01:26.888916Z","iopub.execute_input":"2022-05-23T21:01:26.8893Z","iopub.status.idle":"2022-05-23T21:01:26.90735Z","shell.execute_reply.started":"2022-05-23T21:01:26.889256Z","shell.execute_reply":"2022-05-23T21:01:26.906194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#dfs_proc = Parallel(n_jobs=4, prefer='threads')(delayed(process_chain)(train_folder, chain_names[i]) for i in range(0, len(chain_names)))\n\ntrain_df = pd.DataFrame(columns={'image_id', 'hotel_id'})\nvalid_df = pd.DataFrame(columns={'image_id', 'hotel_id'})\n\nfor hotel_id in tqdm(chain_names):\n    train_df, valid_df = process_chain(train_folder, hotel_id, train_df, valid_df)","metadata":{"papermill":{"duration":2514.619147,"end_time":"2021-04-16T02:35:09.627391","exception":false,"start_time":"2021-04-16T01:53:15.008244","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-23T21:01:26.912876Z","iopub.execute_input":"2022-05-23T21:01:26.913323Z","iopub.status.idle":"2022-05-23T21:01:30.483135Z","shell.execute_reply.started":"2022-05-23T21:01:26.91329Z","shell.execute_reply":"2022-05-23T21:01:30.481969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Output count train:\", len(os.listdir(\"train_images\")))\nprint(\"Output count valid:\", len(os.listdir(\"valid_images\")))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T21:01:30.485181Z","iopub.execute_input":"2022-05-23T21:01:30.48543Z","iopub.status.idle":"2022-05-23T21:01:30.492711Z","shell.execute_reply.started":"2022-05-23T21:01:30.485398Z","shell.execute_reply":"2022-05-23T21:01:30.491305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_df = train_df.groupby([\"hotel_id\"]).size().to_frame(\"image_count\").sort_values(\"image_count\")[::-1].reset_index()\n\n# top and low\nlow_df = group_df.iloc[-50:]\ntop_df = group_df.iloc[:50]\n\nfig = make_subplots(rows=2, cols=2, \n                    specs=[[{\"colspan\": 2}, None], [{}, {}]],\n                    horizontal_spacing=0.02, vertical_spacing=0.2, \n                    shared_yaxes=False,\n                    subplot_titles=(\"\", \"Top 50\", \"Bottom 50\"))\n\n\nfig.add_trace(go.Scatter(x=group_df[\"hotel_id\"], y=group_df[\"image_count\"], showlegend = False), 1, 1)\nfig.add_trace(go.Bar(x=top_df[\"hotel_id\"], y=top_df[\"image_count\"], showlegend = False), 2, 1)\nfig.add_trace(go.Bar(x=low_df[\"hotel_id\"], y=low_df[\"image_count\"], showlegend = False), 2, 2)\n\nfig.update_yaxes(title_text=\"Image count\", row=1, col=1)\nfig.update_yaxes(title_text=\"Image count\", row=2, col=1)\nfig.update_xaxes(type=\"category\", visible=False, row=1, col=1)\nfig.update_xaxes(title_text=\"Hotel ID\", type=\"category\", row=2, col=1)\nfig.update_xaxes(title_text=\"Hotel ID\", type=\"category\", row=2, col=2)\n\nfig.update_layout(title=\"Image count per hotel in training df\", height=550)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T21:01:30.493732Z","iopub.execute_input":"2022-05-23T21:01:30.49398Z","iopub.status.idle":"2022-05-23T21:01:30.577078Z","shell.execute_reply.started":"2022-05-23T21:01:30.493949Z","shell.execute_reply":"2022-05-23T21:01:30.57604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!cd /kaggle/working/images/ & zip -jqr images.zip .\n#!find . -name \"*.jpg\" -delete\nprint(train_df)\ntrain_df.to_csv('train.csv', index=False)\n\nprint(valid_df)\nvalid_df.to_csv('valid.csv', index=False)","metadata":{"papermill":{"duration":401.161771,"end_time":"2021-04-16T02:41:50.796691","exception":false,"start_time":"2021-04-16T02:35:09.63492","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-23T21:01:30.579207Z","iopub.execute_input":"2022-05-23T21:01:30.579522Z","iopub.status.idle":"2022-05-23T21:01:30.595045Z","shell.execute_reply.started":"2022-05-23T21:01:30.57948Z","shell.execute_reply":"2022-05-23T21:01:30.593768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"from PIL import Image as pil_image\n\nclass HotelTrainDataset:\n    def __init__(self, data, transform=None, data_path=\"train_images/\"):\n        self.data = data\n        self.data_path = data_path\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        record = self.data.iloc[idx]\n        image_path = \"\"\n        if isinstance(record[\"image_id\"], str):\n            image_path = \"{}{}.jpg\".format(self.data_path, record[\"image_id\"])\n        else:\n            image_path = \"{}{:07d}.jpg\".format(self.data_path, record[\"image_id\"])\n        image = np.array(pil_image.open(image_path).resize((HEIGHT, WIDTH))).astype(np.uint8)\n\n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n        \n        return {\n            \"image\" : image,\n            \"target\" : 0,\n        }\n\ndef show_images(ds, title_text, n_images=5):\n    fig, ax = plt.subplots(1,5, figsize=(22,8))\n    \n    ax[0].set_ylabel(title_text)\n    \n    for i in range(5):\n        d = ds.__getitem__(i)\n        ax[i].imshow(d[\"image\"])\n    \ntrain_dataset = HotelTrainDataset(train_df, None, data_path=\"train_images/\")\nshow_images(train_dataset, 'Training Images')","metadata":{"execution":{"iopub.status.busy":"2022-05-23T21:18:14.931898Z","iopub.execute_input":"2022-05-23T21:18:14.932228Z","iopub.status.idle":"2022-05-23T21:18:15.93643Z","shell.execute_reply.started":"2022-05-23T21:18:14.932184Z","shell.execute_reply":"2022-05-23T21:18:15.935336Z"}}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}