{"cells":[{"metadata":{},"cell_type":"markdown","source":"I will be performing exploratory data analysis on the landmark training data in hopes of finding something useful for training a model later on. I have used this notebook as a starting reference: \n\n- https://www.kaggle.com/codename007/a-very-extensive-landmark-exploratory-analysis\n- https://github.com/jamesdietle/Kaggle2019/blob/master/GoogleLandmarkRecognition/GoogleLandmarkRecognition.ipynb","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/landmark-retrieval-2020/train.csv')\nprint(\"Size of training data:\", train_data.shape)\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of unique landmark ids:\", len(train_data['landmark_id'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Looking for missing data\n\ntrain_data.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mostly recorded landmarks\ntemp = pd.DataFrame(train_data.landmark_id.value_counts().head(10))\ntemp.reset_index(inplace=True)\ntemp.columns = ['landmark_id','count']\n\n# Plot the most frequent landmark_ids\nplt.figure(figsize = (9, 8))\nplt.title('Most frequent landmarks')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"landmark_id\", y=\"count\", data=temp,\n            label=\"Count\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Class distribution\nplt.figure(figsize = (10, 8))\nplt.title('Category Distribuition')\nsns.distplot(train_data['landmark_id'])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of classes under 20 occurences\",(train_data['landmark_id'].value_counts() <= 20).sum(),'out of total number of categories',len(train_data['landmark_id'].unique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image visualization","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Visualizing the images of the most observed landmarks id in the training data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_landmark(landmark_id=0):\n\n    # Filtering the dataframe for most repeated landmark\n    plotting_data = train_data[train_data['landmark_id']==train_data.landmark_id.value_counts().head(10).index[landmark_id]].reset_index()\n    number_of_subplots = 3\n    fig, axs = plt.subplots(number_of_subplots,figsize=(20,20))\n    for i in range(number_of_subplots):\n\n        # Creating file path\n        image_name = plotting_data['id'][i] + '.jpg'\n        first_initial, second_initial, third_initial = image_name[0]+'/', image_name[1]+'/', image_name[2]+'/'\n        image_path = \"../input/landmark-retrieval-2020/train/\"+ first_initial + second_initial + third_initial + image_name\n\n        # Displaying the image\n        img = plt.imread(image_path)\n        axs[i].imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_landmark(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"All of these seem to be random landmarks therefore they may be in huge numbers in the dataset.\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Visualizing another landmark id that is lower on the top 10 list.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_landmark(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"These landmarks look like castles.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Data Pre-processing","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"There are 81313 unique landmark ids and many of them do not have a lot of images. It is better to falsely label these images rather than taking it into account in the assumption that the test set has these images in low numbers.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train_data.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# How many unique landmarks are there?\ngroups=df.groupby('landmark_id')['id'].nunique()\ngroups.sort_values()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Make a sample of everything under 22 images\n# under22=df.groupby('landmark_id')['id'].nunique()\n# under22=under22.where(under22 < 22)\n# under22=under22.dropna(how='any')\n# under22=under22.index.tolist()\n\n# # Change them into landmark id 99999999\n# changed=df.replace([under22],99999999)\n\n# # save this\n# changed.to_csv(\"under22.csv\", encoding='utf-8',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Creating subset of top 20000 categories","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the top X categories\nlst = df.groupby('landmark_id')['id'].nunique()\n\n# Get the X largest categories\ncategories = 20000\ntop = lst.nlargest(categories)\n\n# Create df subset\nsamplelocations = list(top.index.values)\n\n#Receive files for subset\ndfsubset=df[df['landmark_id'].isin(samplelocations)]\n\n#Make Top 10\n#dfsubset.to_csv(\"/home/jd/data/google/256/t10labels.csv\", encoding='utf-8',index=False)\ndfsubset.to_csv(\"t20000labels.csv\", encoding='utf-8',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}