{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Google Landmark Recognition - Basic EDA","metadata":{}},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import os\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-28T14:01:54.558714Z","iopub.execute_input":"2021-08-28T14:01:54.559265Z","iopub.status.idle":"2021-08-28T14:01:55.011524Z","shell.execute_reply.started":"2021-08-28T14:01:54.559137Z","shell.execute_reply":"2021-08-28T14:01:55.010438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utility Functions","metadata":{}},{"cell_type":"code","source":"BASE_PATH = '../input/landmark-recognition-2021'\n\n@np.vectorize\ndef get_train_filepath(data_id):\n    return os.path.join(BASE_PATH, 'train', data_id[0], data_id[1], data_id[2], data_id + '.jpg')\n\n@np.vectorize\ndef get_test_filepath(data_id):\n    return os.path.join(BASE_PATH, 'test', data_id[0], data_id[1], data_id[2], data_id + '.jpg')\n\n@np.vectorize\ndef get_img_dim(filepath):\n    return Image.open(filepath).size","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:01:55.012877Z","iopub.execute_input":"2021-08-28T14:01:55.013226Z","iopub.status.idle":"2021-08-28T14:01:55.020076Z","shell.execute_reply.started":"2021-08-28T14:01:55.013196Z","shell.execute_reply":"2021-08-28T14:01:55.019097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/landmark-recognition-2021/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:01:55.022006Z","iopub.execute_input":"2021-08-28T14:01:55.022363Z","iopub.status.idle":"2021-08-28T14:01:56.281755Z","shell.execute_reply.started":"2021-08-28T14:01:55.02233Z","shell.execute_reply":"2021-08-28T14:01:56.280545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:01:56.284131Z","iopub.execute_input":"2021-08-28T14:01:56.284586Z","iopub.status.idle":"2021-08-28T14:01:56.291519Z","shell.execute_reply.started":"2021-08-28T14:01:56.284535Z","shell.execute_reply":"2021-08-28T14:01:56.290642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_df[\"filepath\"] = get_train_filepath(train_df['id'])\ntrain_df[['width', 'height']] = np.array(get_img_dim(train_df[\"filepath\"])).T\ntrain_df['aspect_ratio'] = train_df['width'] / train_df['height']\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:01:56.303682Z","iopub.execute_input":"2021-08-28T14:01:56.303973Z","iopub.status.idle":"2021-08-28T14:09:31.241905Z","shell.execute_reply.started":"2021-08-28T14:01:56.303944Z","shell.execute_reply":"2021-08-28T14:09:31.241017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:31.243434Z","iopub.execute_input":"2021-08-28T14:09:31.243869Z","iopub.status.idle":"2021-08-28T14:09:31.278611Z","shell.execute_reply.started":"2021-08-28T14:09:31.243837Z","shell.execute_reply":"2021-08-28T14:09:31.277804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"width\", \"height\", \"aspect_ratio\"]].describe().round(2)","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:31.279917Z","iopub.execute_input":"2021-08-28T14:09:31.280561Z","iopub.status.idle":"2021-08-28T14:09:31.335224Z","shell.execute_reply.started":"2021-08-28T14:09:31.280514Z","shell.execute_reply":"2021-08-28T14:09:31.334256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 8))\nsns.kdeplot(data = train_df[[\"width\", \"height\"]], common_norm = False)\nplt.xlabel(\"Pixels\", fontsize = 16)\nplt.ylabel(\"Density\", fontsize = 16)\nplt.legend([\"Width\", \"Height\"], loc = \"upper left\", fontsize = 16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:31.337987Z","iopub.execute_input":"2021-08-28T14:09:31.338306Z","iopub.status.idle":"2021-08-28T14:09:33.226044Z","shell.execute_reply.started":"2021-08-28T14:09:31.338276Z","shell.execute_reply":"2021-08-28T14:09:33.225042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 8))\nsns.kdeplot(data = train_df[\"aspect_ratio\"], common_norm = False)\nplt.xlabel(\"Aspect Ratio\", fontsize = 16)\nplt.ylabel(\"Density\", fontsize = 16)\nplt.xticks(np.arange(int(min(train_df[\"aspect_ratio\"])), int(max(train_df[\"aspect_ratio\"])) + 1, 1.0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:19:17.535456Z","iopub.execute_input":"2021-08-28T14:19:17.535974Z","iopub.status.idle":"2021-08-28T14:19:18.474359Z","shell.execute_reply.started":"2021-08-28T14:19:17.535925Z","shell.execute_reply":"2021-08-28T14:19:18.473229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_vc = train_df[[\"landmark_id\"]].value_counts()\nprint(f\"Total unique landmarks: {train_df[['landmark_id']].nunique().values[0]}\")\nprint(\"\\nLandmarks with most samples:\\n\")\nlandmark_vc.sort_values(ascending = False)[:10]","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:33.939514Z","iopub.execute_input":"2021-08-28T14:09:33.939833Z","iopub.status.idle":"2021-08-28T14:09:33.96219Z","shell.execute_reply.started":"2021-08-28T14:09:33.939802Z","shell.execute_reply":"2021-08-28T14:09:33.961407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\nLandmarks with least samples:\\n\")\nlandmark_vc.sort_values()[:10]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = landmark_vc >= len(train_df) * 0.001\nprint(f\"Count of landmarks with at least 0.1% of total samples: {landmark_vc[mask].count()}\")\nprint(f\"Count of landmarks with less than 0.1% of total samples: {landmark_vc[~mask].count()}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:33.963331Z","iopub.execute_input":"2021-08-28T14:09:33.963754Z","iopub.status.idle":"2021-08-28T14:09:33.970767Z","shell.execute_reply.started":"2021-08-28T14:09:33.963717Z","shell.execute_reply":"2021-08-28T14:09:33.970086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 8))\nsns.kdeplot(data = landmark_vc, common_norm = False)\nplt.xlabel(\"Count\", fontsize = 16)\nplt.ylabel(\"Number of Samples\", fontsize = 16)\nplt.title(\"Most landmarks have very few samples\", fontsize = 20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:33.971853Z","iopub.execute_input":"2021-08-28T14:09:33.97217Z","iopub.status.idle":"2021-08-28T14:09:34.210935Z","shell.execute_reply.started":"2021-08-28T14:09:33.972132Z","shell.execute_reply":"2021-08-28T14:09:34.210224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing Data","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('../input/landmark-recognition-2021/sample_submission.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:34.211937Z","iopub.execute_input":"2021-08-28T14:09:34.212332Z","iopub.status.idle":"2021-08-28T14:09:34.233431Z","shell.execute_reply.started":"2021-08-28T14:09:34.212302Z","shell.execute_reply":"2021-08-28T14:09:34.23245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:34.234601Z","iopub.execute_input":"2021-08-28T14:09:34.234888Z","iopub.status.idle":"2021-08-28T14:09:34.241403Z","shell.execute_reply.started":"2021-08-28T14:09:34.234861Z","shell.execute_reply":"2021-08-28T14:09:34.240197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntest_df[\"filepath\"] = get_test_filepath(test_df['id'])\ntest_df[['width', 'height']] = np.array(get_img_dim(test_df[\"filepath\"])).T\ntest_df['aspect_ratio'] = test_df['width'] / test_df['height']\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:09:34.242641Z","iopub.execute_input":"2021-08-28T14:09:34.243012Z","iopub.status.idle":"2021-08-28T14:10:24.753169Z","shell.execute_reply.started":"2021-08-28T14:09:34.24298Z","shell.execute_reply":"2021-08-28T14:10:24.7523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:10:24.75438Z","iopub.execute_input":"2021-08-28T14:10:24.754852Z","iopub.status.idle":"2021-08-28T14:10:24.76722Z","shell.execute_reply.started":"2021-08-28T14:10:24.754805Z","shell.execute_reply":"2021-08-28T14:10:24.766164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[[\"width\", \"height\", \"aspect_ratio\"]].describe().round(2)","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:10:24.768535Z","iopub.execute_input":"2021-08-28T14:10:24.769116Z","iopub.status.idle":"2021-08-28T14:10:24.809606Z","shell.execute_reply.started":"2021-08-28T14:10:24.769071Z","shell.execute_reply":"2021-08-28T14:10:24.80869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 8))\nsns.kdeplot(data = test_df[[\"width\", \"height\"]], common_norm = False)\nplt.xlabel(\"Pixels\", fontsize = 16)\nplt.ylabel(\"Density\", fontsize = 16)\nplt.legend([\"Width\", \"Height\"], loc = \"upper left\", fontsize = 16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:10:24.810959Z","iopub.execute_input":"2021-08-28T14:10:24.8115Z","iopub.status.idle":"2021-08-28T14:10:25.288727Z","shell.execute_reply.started":"2021-08-28T14:10:24.811456Z","shell.execute_reply":"2021-08-28T14:10:25.28789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12, 8))\nsns.kdeplot(data = test_df[\"aspect_ratio\"], common_norm = False)\nplt.xlabel(\"Aspect Ratio\", fontsize = 16)\nplt.ylabel(\"Density\", fontsize = 16)\nplt.xticks(np.arange(int(min(test_df[\"aspect_ratio\"])), int(max(test_df[\"aspect_ratio\"])) + 1, 1.0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:19:46.09638Z","iopub.execute_input":"2021-08-28T14:19:46.096821Z","iopub.status.idle":"2021-08-28T14:19:46.424377Z","shell.execute_reply.started":"2021-08-28T14:19:46.096783Z","shell.execute_reply":"2021-08-28T14:19:46.423332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving the outputs","metadata":{}},{"cell_type":"code","source":"train_df.to_csv(\"train_df.csv\", index = False)\ntest_df.to_csv(\"test_df.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2021-08-28T14:10:25.621705Z","iopub.execute_input":"2021-08-28T14:10:25.622405Z","iopub.status.idle":"2021-08-28T14:10:26.49635Z","shell.execute_reply.started":"2021-08-28T14:10:25.622364Z","shell.execute_reply":"2021-08-28T14:10:26.495423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}