{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\nprint(\"hello\")","metadata":{"execution":{"iopub.status.busy":"2022-05-20T07:47:03.380942Z","iopub.execute_input":"2022-05-20T07:47:03.381426Z","iopub.status.idle":"2022-05-20T07:47:03.800955Z","shell.execute_reply.started":"2022-05-20T07:47:03.381335Z","shell.execute_reply":"2022-05-20T07:47:03.800164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PAD = True\nWIDTH = 512\nHEIGHT = 512","metadata":{"execution":{"iopub.status.busy":"2022-05-20T07:47:03.802301Z","iopub.execute_input":"2022-05-20T07:47:03.80252Z","iopub.status.idle":"2022-05-20T07:47:03.806581Z","shell.execute_reply.started":"2022-05-20T07:47:03.802486Z","shell.execute_reply":"2022-05-20T07:47:03.805675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"data_folder = \"/kaggle/input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\ntrain_folder = os.path.join(data_folder, 'train_images')\nchain_names = os.listdir(train_folder)\n\nprint(os.listdir(data_folder))\nprint(len(chain_names))","metadata":{"papermill":{"duration":0.225587,"end_time":"2021-04-16T01:53:14.982769","exception":false,"start_time":"2021-04-16T01:53:14.757182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-20T07:47:11.869085Z","iopub.execute_input":"2022-05-20T07:47:11.86941Z","iopub.status.idle":"2022-05-20T07:47:11.989048Z","shell.execute_reply.started":"2022-05-20T07:47:11.869373Z","shell.execute_reply":"2022-05-20T07:47:11.988185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame(columns={'image_id', 'hotel_id'})\nfor hotel_id in tqdm(chain_names):\n    for image_id in os.listdir(os.path.join(train_folder, hotel_id)):\n        train_df = train_df.append({'image_id': image_id, 'hotel_id': hotel_id}, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-20T07:47:12.892841Z","iopub.execute_input":"2022-05-20T07:47:12.893442Z","iopub.status.idle":"2022-05-20T07:47:53.979596Z","shell.execute_reply.started":"2022-05-20T07:47:12.893391Z","shell.execute_reply":"2022-05-20T07:47:53.978158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df['hotel_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-05-20T07:47:53.980636Z","iopub.status.idle":"2022-05-20T07:47:53.980951Z","shell.execute_reply.started":"2022-05-20T07:47:53.980791Z","shell.execute_reply":"2022-05-20T07:47:53.980809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_info = []\nfor i in train_df['hotel_id'].unique():\n    class_info.append({'class_len': len(train_df[train_df['hotel_id'] == i]), 'class_name': i})","metadata":{"execution":{"iopub.status.busy":"2022-05-20T07:47:53.982033Z","iopub.status.idle":"2022-05-20T07:47:53.982318Z","shell.execute_reply.started":"2022-05-20T07:47:53.982176Z","shell.execute_reply":"2022-05-20T07:47:53.982192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = pd.DataFrame(class_info)\nc = c.sort_values(['class_len'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-20T07:47:53.983493Z","iopub.status.idle":"2022-05-20T07:47:53.983772Z","shell.execute_reply.started":"2022-05-20T07:47:53.983629Z","shell.execute_reply":"2022-05-20T07:47:53.983645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n\nfrom numpy import expand_dims\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2022-05-19T16:21:11.545801Z","iopub.execute_input":"2022-05-19T16:21:11.545992Z","iopub.status.idle":"2022-05-19T16:21:19.577527Z","shell.execute_reply.started":"2022-05-19T16:21:11.545966Z","shell.execute_reply":"2022-05-19T16:21:19.57605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fd(num, hotel_id, df):\n    df = df.sample(num, replace=True)\n    df['image_id'].apply(lambda x: aug_img(f'../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images/{hotel_id}/{x}', hotel_id))","metadata":{"execution":{"iopub.status.busy":"2022-05-19T16:21:19.579116Z","iopub.execute_input":"2022-05-19T16:21:19.579312Z","iopub.status.idle":"2022-05-19T16:21:19.585293Z","shell.execute_reply.started":"2022-05-19T16:21:19.579285Z","shell.execute_reply":"2022-05-19T16:21:19.584405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aug_img(image_id, hotel_id):\n#     Original_Image = Image.open(f'../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images/{hotel_id}/{image_id}')\n    img = load_img(image_id)\n    # convert to numpy array\n    data = img_to_array(img)\n\n    # expand dimension to one sample\n    samples = expand_dims(data, 0)\n\n    # create image data augmentation generator\n    datagen = ImageDataGenerator(\n        featurewise_center=True,\n        rotation_range=(0-30),\n        width_shift_range=0.2,                \n        height_shift_range=0.2,\n        brightness_range=[0.5,1.5],\n        shear_range=0.2, \n        zoom_range=0.2,\n        channel_shift_range=0.2,\n        horizontal_flip=True, \n        vertical_flip=True,\n        fill_mode='nearest',\n    )\n    # prepare iterator\n    it = datagen.flow(samples, batch_size=1, save_to_dir='.', save_prefix=f'{hotel_id}-aug-',\n                                            save_format='jpg')\n    \n    # generate samples and plot\n    batch = it.next()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T16:21:19.586398Z","iopub.execute_input":"2022-05-19T16:21:19.586655Z","iopub.status.idle":"2022-05-19T16:21:19.608455Z","shell.execute_reply.started":"2022-05-19T16:21:19.586623Z","shell.execute_reply":"2022-05-19T16:21:19.607729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random.sample(the_list, 50)\n# train_df\nglobal count\ncount = 0\n\nglobal below\nbelow = []\nglobal after\nafter = []\ndef cl(x):\n    global count\n    count = count + 1\n    print(count)\n    if x['class_len'] > 40:\n        class_imgs = train_df[train_df['hotel_id'] == x['class_name']]\n        global below\n        below.extend(class_imgs.sample(40).to_dict('records'))\n    else:\n        class_imgs = train_df[train_df['hotel_id'] == x['class_name']]\n        fd(40-len(class_imgs), x['class_name'], class_imgs)\n# c.apply(lambda x: cl(x), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T16:28:44.193194Z","iopub.execute_input":"2022-05-19T16:28:44.19349Z","iopub.status.idle":"2022-05-19T16:28:44.203676Z","shell.execute_reply.started":"2022-05-19T16:28:44.193454Z","shell.execute_reply":"2022-05-19T16:28:44.202575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndfs_proc =Parallel(n_jobs=4, prefer='threads')(delayed(cl)(i) for v, i in c.iterrows())","metadata":{"execution":{"iopub.status.busy":"2022-05-19T16:28:44.927465Z","iopub.execute_input":"2022-05-19T16:28:44.928078Z","iopub.status.idle":"2022-05-19T16:28:50.44492Z","shell.execute_reply.started":"2022-05-19T16:28:44.928048Z","shell.execute_reply":"2022-05-19T16:28:50.443946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nglobal top\ntop = []\ndef get_csv(x):\n    name = x['class_name']\n    jpgFilenamesList = glob.glob(f'{name}*.jpg')\n    for i in jpgFilenamesList:\n        top.append({'hotel_id': name, 'image_id': i})\n\nc.apply(lambda x: get_csv(x), axis=1)\ntop.extend(below)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_image(img):\n    w, h, c = np.shape(img)\n    if w > h:\n        pad = int((w - h) / 2)\n        img = cv2.copyMakeBorder(img, 0, 0, pad, pad, cv2.BORDER_CONSTANT, value=0)\n    else:\n        pad = int((h - w) / 2)\n        img = cv2.copyMakeBorder(img, pad, pad, 0, 0, cv2.BORDER_CONSTANT, value=0)\n        \n    return img\n\n\ndef open_and_preprocess_image(img):\n    print(os.path.join('.', img))\n    img = cv2.imread(os.path.join('.', img))\n    if PAD:\n        img = pad_image(img)\n    \n    return cv2.resize(img, (WIDTH, HEIGHT))\n\n\ndef save_image(image_name, img):\n    cv2.imwrite(image_name, img)\n    \n    \ndef process_chain(img):\n#     chain_folder = os.path.join(data_folder, chain_name)\n    name = img['image_id']\n    img = open_and_preprocess_image(img['image_id'])\n    save_image(name, img)","metadata":{"papermill":{"duration":0.01604,"end_time":"2021-04-16T01:53:15.003615","exception":false,"start_time":"2021-04-16T01:53:14.987575","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndfs_proc = Parallel(n_jobs=4, prefer='threads')(delayed(process_chain)(top[i]) for i in range(0, len(top)))","metadata":{"papermill":{"duration":2514.619147,"end_time":"2021-04-16T02:35:09.627391","exception":false,"start_time":"2021-04-16T01:53:15.008244","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cd /kaggle/working/ & zip -jqr images.zip .\n!find . -name \"*.jpg\" -delete\ntop.to_csv('train.csv', index=False)","metadata":{"papermill":{"duration":401.161771,"end_time":"2021-04-16T02:41:50.796691","exception":false,"start_time":"2021-04-16T02:35:09.63492","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}