{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":29762,"databundleVersionId":2541532,"sourceType":"competition"},{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":36414,"databundleVersionId":3974967,"sourceType":"competition"},{"sourceId":1572891,"sourceType":"datasetVersion","datasetId":929774}],"dockerImageVersionId":30235,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport scipy.io\nimport copy\nimport glob\nimport os\nimport gc\n\n\ngc.enable()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-30T13:24:11.111133Z","iopub.execute_input":"2022-08-30T13:24:11.111545Z","iopub.status.idle":"2022-08-30T13:24:11.141954Z","shell.execute_reply.started":"2022-08-30T13:24:11.111514Z","shell.execute_reply":"2022-08-30T13:24:11.140928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Google Landmark Recognition 2021\n\n#### Category: Landmarks\n#### Total images: 1590815","metadata":{}},{"cell_type":"code","source":"def preprocess_google_landmark_reconginition_2021_data_frame(data_frame, directory, columns=None):\n    def get_folder_path(id, directory):\n        folder_path = \"/\".join([_ for _ in id[:3]])\n        folder_path = os.path.join(directory, folder_path)\n        return folder_path\n        \n    data_frame = copy.deepcopy(data_frame)\n    \n    data_frame[\"folder_path\"] = data_frame[\"id\"].apply(lambda id: get_folder_path(id=id, directory=directory))\n    data_frame[\"file\"] = data_frame[\"id\"].apply(lambda id: f\"{id}.jpg\")\n    data_frame[\"image_path\"] = data_frame.apply(lambda row: os.path.join(row[\"folder_path\"], row[\"file\"]), axis=1)\n    \n    new_columns = {\n        \"landmark_id\": \"label\",\n    }\n    data_frame = data_frame.rename(columns=new_columns)\n    \n    if columns is not None:\n        data_frame = data_frame[columns]\n    \n    return data_frame","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:09:18.415729Z","iopub.execute_input":"2022-08-30T13:09:18.416189Z","iopub.status.idle":"2022-08-30T13:09:18.42578Z","shell.execute_reply.started":"2022-08-30T13:09:18.416155Z","shell.execute_reply":"2022-08-30T13:09:18.424941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pathes\ngoogle_landmark_reconginition_2021_directory = \"../input/landmark-recognition-2021/\"\ngoogle_landmark_reconginition_2021_train_path = os.path.join(google_landmark_reconginition_2021_directory, \"train.csv\")\ngoogle_landmark_reconginition_2021_train_directory = os.path.join(google_landmark_reconginition_2021_directory, \"train/\")\ngoogle_landmark_reconginition_2021_test_directory = os.path.join(google_landmark_reconginition_2021_directory, \"test/\")\n\n# loading\ngoogle_landmark_reconginition_2021_train = pd.read_csv(google_landmark_reconginition_2021_train_path)\ngoogle_landmark_reconginition_2021_columns = [\"id\", \"image_path\", \"label\"]\n\n# train dataset pre-processing\ngoogle_landmark_reconginition_2021_train = preprocess_google_landmark_reconginition_2021_data_frame(\n    data_frame=google_landmark_reconginition_2021_train, \n    directory=google_landmark_reconginition_2021_train_directory,\n    columns=google_landmark_reconginition_2021_columns,\n)\n\n# test dataset pre-processing\ngoogle_landmark_reconginition_2021_test_filenames_format = os.path.join(google_landmark_reconginition_2021_test_directory, \"*/*/*/*.jpg\")\ngoogle_landmark_reconginition_2021_test_filenames = glob.glob(google_landmark_reconginition_2021_test_filenames_format)\n\ngoogle_landmark_reconginition_2021_test = pd.DataFrame({\n    \"image_path\": google_landmark_reconginition_2021_test_filenames,\n    \"label\": np.nan,\n})\n\ngoogle_landmark_reconginition_2021_get_id_from_path = lambda path: path.split(\".\")[-2].split(\"/\")[-1]\ngoogle_landmark_reconginition_2021_test[\"id\"] = google_landmark_reconginition_2021_test[\"image_path\"].apply(google_landmark_reconginition_2021_get_id_from_path)\ngoogle_landmark_reconginition_2021_test = google_landmark_reconginition_2021_test[google_landmark_reconginition_2021_columns]\n\n# concatenating\ngoogle_landmark_reconginition_2021_datasets = [google_landmark_reconginition_2021_train, google_landmark_reconginition_2021_test]\ngoogle_landmark_reconginition_2021 = pd.concat(google_landmark_reconginition_2021_datasets, axis=0)\n\n# saving\ngoogle_landmark_reconginition_2021.to_csv(\"google_landmark_reconginition_2021.csv\", index=False)\n\n# deleting\ndel google_landmark_reconginition_2021, google_landmark_reconginition_2021_test, google_landmark_reconginition_2021_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:15:04.92349Z","iopub.execute_input":"2022-08-30T13:15:04.923944Z","iopub.status.idle":"2022-08-30T13:15:38.635925Z","shell.execute_reply.started":"2022-08-30T13:15:04.923892Z","shell.execute_reply":"2022-08-30T13:15:38.63463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# H&M Personalized Fashion Recommendations\n#### Category: Apparel & Accessories\n#### Total images: 105542","metadata":{}},{"cell_type":"code","source":"def adjust_id(x):\n    '''\n    Adjusts article ID code.\n    https://www.kaggle.com/code/andradaolteanu/h-m-eda-rapids-and-similarity-recommenders\n    '''\n    x = str(x)\n    if len(x) == 9:\n        x = \"0\"+x\n    \n    return x","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:57:59.722441Z","iopub.execute_input":"2022-08-30T13:57:59.723503Z","iopub.status.idle":"2022-08-30T13:57:59.729628Z","shell.execute_reply.started":"2022-08-30T13:57:59.723464Z","shell.execute_reply":"2022-08-30T13:57:59.728453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pathes\nhandm_personalised_fashion_recommendations_directory = \"../input/h-and-m-personalized-fashion-recommendations\"\nhandm_personalised_fashion_recommendations_images_directory = os.path.join(handm_personalised_fashion_recommendations_directory, \"images/\")\nhandm_personalised_fashion_recommendations_articles_path = os.path.join(handm_personalised_fashion_recommendations_directory, \"articles.csv\")\nhandm_personalised_fashion_recommendations_customers_path = os.path.join(handm_personalised_fashion_recommendations_directory, \"customers.csv\")\nhandm_personalised_fashion_recommendations_train_transactions_path = os.path.join(handm_personalised_fashion_recommendations_directory, \"transactions_train.csv\")\n\n# loading\nhandm_personalised_fashion_recommendations_articles = pd.read_csv(handm_personalised_fashion_recommendations_articles_path)\n\n# pre-processing\nhandm_personalised_fashion_recommendations_articles[\"article_id\"] = handm_personalised_fashion_recommendations_articles[\"article_id\"].apply(lambda x: adjust_id(x))\nhandm_personalised_fashion_recommendations_articles[\"product_code\"] = handm_personalised_fashion_recommendations_articles[\"article_id\"].apply(lambda x: x[:3])\nhandm_personalised_fashion_recommendations_articles_get_image_path = lambda row: os.path.join(handm_personalised_fashion_recommendations_images_directory, row[\"product_code\"], f\"{row['article_id']}.jpg\")\nhandm_personalised_fashion_recommendations_articles[\"image_path\"] = handm_personalised_fashion_recommendations_articles.apply(handm_personalised_fashion_recommendations_articles_get_image_path, axis=1)\n\nhandm_personalised_fashion_recommendations_articles_columns = [\"article_id\", \"image_path\", \"product_type_name\"]\nhandm_personalised_fashion_recommendations_articles = handm_personalised_fashion_recommendations_articles[handm_personalised_fashion_recommendations_articles_columns]\n\nhandm_personalised_fashion_recommendations_articles_new_columns = {\n    \"article_id\": \"id\",\n    \"product_type_name\": \"label\",\n}\n\nhandm_personalised_fashion_recommendations_articles = handm_personalised_fashion_recommendations_articles.rename(columns=handm_personalised_fashion_recommendations_articles_new_columns)\n\n# saving\nhandm_personalised_fashion_recommendations_articles.to_csv(\"handm_personalised_fashion_recommendations.csv\", index=False)\n\n# deleting\ndel handm_personalised_fashion_recommendations_articles\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-30T14:06:28.617436Z","iopub.execute_input":"2022-08-30T14:06:28.617873Z","iopub.status.idle":"2022-08-30T14:06:32.086152Z","shell.execute_reply.started":"2022-08-30T14:06:28.617837Z","shell.execute_reply":"2022-08-30T14:06:32.084963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clothing dataset (full, high resolution)\n#### Category: Apparel & Accessories\n#### Total images: 5403","metadata":{}},{"cell_type":"code","source":"# pathes\nclothing_dataset_full_directory = \"../input/clothing-dataset-full\"\nclothing_dataset_full_images_path = os.path.join(clothing_dataset_full_directory, \"images.csv\")\nclothing_dataset_full_images_directory = os.path.join(clothing_dataset_full_directory, \"images_original/\")\n\n# loading\nclothing_dataset_full_images = pd.read_csv(clothing_dataset_full_images_path)\n\n# pre-processing\nclothing_dataset_full_images_new_columns = {\"image\": \"id\"}\nclothing_dataset_full_images = clothing_dataset_full_images.rename(columns=clothing_dataset_full_images_new_columns)\nclothing_dataset_full_images[\"image_path\"] = clothing_dataset_full_images[\"id\"].apply(lambda id: os.path.join(clothing_dataset_full_images_directory, f\"{id}.jpg\"))\n \nclothing_dataset_full_images_columns = [\"id\", \"image_path\", \"label\"]\nclothing_dataset_full_images = clothing_dataset_full_images[clothing_dataset_full_images_columns]\n\n# saving\nclothing_dataset_full_images.to_csv(\"clothing_dataset_full.csv\", index=False)\n\n# deleting\ndel clothing_dataset_full_images\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-30T14:30:31.03402Z","iopub.execute_input":"2022-08-30T14:30:31.03448Z","iopub.status.idle":"2022-08-30T14:30:31.085485Z","shell.execute_reply.started":"2022-08-30T14:30:31.034441Z","shell.execute_reply":"2022-08-30T14:30:31.084168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# General statistics\n#### Landmarks: 1590815\n#### Apparel & Accessories: 110945","metadata":{}}]}