{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        continue\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Content\n!ls /kaggle/input/landmark-retrieval-2020/\n!ls /kaggle/input/landmark-retrieval-2020/train\n!ls /kaggle/input/landmark-retrieval-2020/test\n!ls /kaggle/input/landmark-retrieval-2020/index\nprint('')\n\n# Size of data\n#!du -hs /kaggle/input/landmark-retrieval-2020/train/  # 101 GB\n#!du -hs /kaggle/input/landmark-retrieval-2020/test/   # 0.07 GB\n#!du -hs /kaggle/input/landmark-retrieval-2020/index/  # 4.9 GB","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## File structure\n    \nThree folders with images are available:    \n- `train`  \n- `test`  \n- `index`  \n\nLandmark labels are placed in `train.csv`  \n\nEach image has a unique **id** and is placed within **3** subfolders according to the first **3** characters of the image id:  \n    For example, `abcdef.jpg` is placed in `a/b/c/abcdef.jpg`.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\n\ndata_path = Path('/kaggle/input/landmark-retrieval-2020/')\ntrain_path = data_path / 'train'\ntest_path = data_path / 'test'\nindex_path = data_path / 'index'\n\nprint('training_path =', train_path)\nprint('test_path     =', test_path)\nprint('index_path    =', index_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\ntrain_list = glob.glob('../input/landmark-retrieval-2020/train/*/*/*/*')\ntest_list = glob.glob('../input/landmark-retrieval-2020/test/*/*/*/*')\nindex_list = glob.glob('../input/landmark-retrieval-2020/index/*/*/*/*')\nprint( 'train_images :', len(train_list))\nprint( 'test_images  :', len(test_list))\nprint( 'index_images :', len(index_list))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explore train data. CSV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/landmark-retrieval-2020/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot landmark labels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(figsize=(16,6))\n\nax = sns.distplot(train['landmark_id'])\n\nplt.savefig('landmark_id.png', dpi=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(figsize=(16,6))\n\nax = sns.distplot(train['landmark_id'], \n                  rug=False, \n                  #rug_kws={\"color\": \"g\"}, \n                  kde_kws={\"color\": \"k\", \"lw\": 3, \"label\": \"KDE\"}, \n                  hist_kws={\"histtype\": \"step\", \"linewidth\": 3, \"alpha\": 1, \"color\": \"g\"})\n\nplt.savefig('landmark_id_2.png', dpi=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['landmark_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train['landmark_id'].value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Top 20 by popularity within dataset\n\nfig, axs = plt.subplots(figsize=(12,8))\n\ntrain['landmark_id'].value_counts(sort=True, ascending=False)[:30].plot(kind='barh')\n\nplt.title('Top 50 by popularity')\nplt.xlabel('Number of images', fontsize=14)\nplt.ylabel('Landmark', fontsize=14)\n\nplt.savefig('landmark_id_top30.png', dpi=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_count  = train['landmark_id'].value_counts(sort=True, ascending=False)\nimage_count = image_count[:30,]\n\nplt.figure(figsize=(23,6))\n\nsns.barplot(image_count.index, image_count.values, alpha=0.8)\n\nplt.title('Images per landmark')\nplt.ylabel('Number of images', fontsize=14)\nplt.xlabel('Landmark', fontsize=14)\n\nplt.savefig('landmark_id_top30_vs_images.png', dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_count_all  = train['landmark_id'].value_counts(sort=True, ascending=False)\n\nprint('landmark_id:')\n\nprint(image_count_all)\n\nprint()\nprint ('Max images for a given landmark_id: {}'.format(image_count_all.max()))\nprint ('Min images for a given landmark_id: {}'.format(image_count_all.min()))\n\nprint()\nprint('Number of unique landmark IDs: {}'.format(len(image_count_all))) # len(image_count_all.index.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Landmarks \n- Maximum number of images=6272 (landmark_id='138982')\n- Minimum number of images=2 (landmark_id='197219')\n    - Totally 4750 are with 2 images only","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_image_count_all = pd.DataFrame(image_count_all.reset_index().values, columns=['landmark_id', 'Number' ])\ndf_image_count_all_ind = df_image_count_all.sort_index(axis = 0, ascending=True)\ndf_image_count_all_ind","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 4750 are with 2 images only","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_image_count_all[df_image_count_all['Number']<3].count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 44646 are with 10 or less images only, almost 50% of total landmarks","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_image_count_all[df_image_count_all['Number']<11].count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 81306 are with 1000 or less images only (out of 81313)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_image_count_all[df_image_count_all['Number']<1001].count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 81306 are with 1000 or less images only (out of 81313), only 7 with >1000 images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_image_count_all[df_image_count_all['Number']>1001].count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot distributions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_list = list(df_image_count_all['Number'])\n\nfig, axs = plt.subplots(figsize=(23,6))\nplt.hist(num_list, density=False, bins=6272, color='green', alpha=0.5)  \n\naxs.set_xscale('log')\n\nplt.ylabel('Counts') # Probability if scaled\nplt.xlabel('Number of images per Landmark ID')\n\nplt.savefig('images_per_landmark_id.png', dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_list = list(df_image_count_all['Number'])\n\nfig, axs = plt.subplots(figsize=(23,6))\nplt.hist(num_list, density=False, bins=6272, color='green', alpha=0.5)  \n\naxs.set_xscale('log')\naxs.set_yscale('log')\n\nplt.ylabel('Counts') # Probability if scaled\nplt.xlabel('Number of images per Landmark ID')\n\nplt.savefig('images_per_landmark_id_2.png', dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.figure(figsize=(23,6))\n\nfig, axs = plt.subplots(2,1, figsize=(23,12))\n\n# Normal version\nsns.lineplot(x=df_image_count_all_ind['landmark_id'], y=df_image_count_all_ind['Number'], data=df_image_count_all_ind, color='green', alpha=0.7, ax=axs[0])\n# Log version\nplott = sns.lineplot(x=df_image_count_all_ind['landmark_id'], y=df_image_count_all_ind['Number'], data=df_image_count_all_ind, color='green', alpha=0.5, ax=axs[1])\nplott.set(yscale=\"log\")\n\nplt.savefig('landmark_id_distribution_nonlog_log.png', dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\n\nplt.rcParams[\"axes.grid\"] = False\nfig, axs = plt.subplots(4, 4, figsize=(23, 23))\n\ncurr_row = 0\n\nfor i in range(0,16):\n    example = cv2.imread(test_list[i])\n    example = example[:,:,::-1]\n    \n    col = i%4\n    \n    ## Add title information\n    axs[i//4, i%4].set_title('{}/{}/{}/{}'.format(\n        test_list[i].split('/')[-4],\n        test_list[i].split('/')[-3],\n        test_list[i].split('/')[-2],\n        test_list[i].split('/')[-1]))\n    \n    ## Plot n x m images\n    axs[col, curr_row].imshow(example)\n    \n    if col == 3:\n        curr_row += 1\n    \nplt.savefig('some_data_images.png', dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}