{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Exploring Superclass Content with Clustering\n[Google Landmark Recognition 2021\n](https://www.kaggle.com/c/landmark-recognition-2021)\n\nThis notebook explores the target landmark ID class imbalance in this challenge and uses k-means clustering on images within a target superclass (i.e., target class with superhigh number of images in the training set) to determine how superclasses are organized.","metadata":{}},{"cell_type":"code","source":"import math, os, re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as img\nfrom keras.preprocessing.image import load_img \nfrom keras.preprocessing.image import img_to_array \nfrom keras.applications.vgg16 import preprocess_input \nfrom keras.applications.vgg16 import VGG16 \nfrom keras.models import Model\nfrom sklearn.cluster import KMeans\nfrom sklearn import metrics\nfrom sklearn.decomposition import PCA\n\nprint(os.listdir('../input/landmark-recognition-2021'))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-20T00:48:37.355372Z","iopub.execute_input":"2021-08-20T00:48:37.355816Z","iopub.status.idle":"2021-08-20T00:48:42.629123Z","shell.execute_reply.started":"2021-08-20T00:48:37.355731Z","shell.execute_reply":"2021-08-20T00:48:42.627664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/landmark-recognition-2021'\nos.listdir(path)\ntrain_images = f'{path}/train'\ntrain_df = pd.read_csv(f'{path}/train.csv')\ntrain_df['path'] = train_df['id'].apply(lambda f: os.path.join('../input/landmark-recognition-2021/train',f[0], f[1], f[2], f + '.jpg'))\ntest_images = f'{path}/test'\ntest_df = pd.read_csv(f'{path}/sample_submission.csv')\ntest_df['path'] = test_df['id'].apply(lambda f: os.path.join('../input/landmark-recognition-2021/test',f[0], f[1], f[2], f + '.jpg'))\n\nnum_classes = train_df['landmark_id'].nunique()\nprint('number of target classes:', num_classes)\nprint('number of images in training set:', len(train_df))","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:48:42.630594Z","iopub.execute_input":"2021-08-20T00:48:42.630926Z","iopub.status.idle":"2021-08-20T00:48:49.435163Z","shell.execute_reply.started":"2021-08-20T00:48:42.630889Z","shell.execute_reply":"2021-08-20T00:48:49.434253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With such a high number of unique landmark IDs given the size of the training set, there is likely going to be imbalanced classsification issues. Viewing the distribution of class counts confirms imbalance.","metadata":{}},{"cell_type":"code","source":"counts = train_df['landmark_id'].value_counts()\ncounts.describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:48:49.436932Z","iopub.execute_input":"2021-08-20T00:48:49.437257Z","iopub.status.idle":"2021-08-20T00:48:49.478496Z","shell.execute_reply.started":"2021-08-20T00:48:49.437219Z","shell.execute_reply":"2021-08-20T00:48:49.477461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's take the landmark ID with 5th highest image count\nsuperclass = train_df[train_df['landmark_id'] == counts.iloc[[5]].index[0]]\nimages = superclass.path.to_list()\nlen(images)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:48:49.480233Z","iopub.execute_input":"2021-08-20T00:48:49.480602Z","iopub.status.idle":"2021-08-20T00:48:49.648062Z","shell.execute_reply.started":"2021-08-20T00:48:49.480564Z","shell.execute_reply":"2021-08-20T00:48:49.647254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looking at a sample of images from a superclass shows a lot of diversity. The same landmark ID covers different buildings and landscapes.","metadata":{}},{"cell_type":"code","source":"sample = superclass.sample(n=12, replace=False)\n\nplt.subplots(3, 4, figsize=(160, 160))\nfor i in range(len(sample)):\n    plt.subplot(3, 4, i + 1)\n    plt.axis('Off')\n    image = img.imread(sample.iloc[i][2])\n    plt.imshow(image)\n    plt.title(f'landmark id:{sample.iloc[i][1]} ', fontsize=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:48:49.649290Z","iopub.execute_input":"2021-08-20T00:48:49.649638Z","iopub.status.idle":"2021-08-20T00:49:07.756456Z","shell.execute_reply.started":"2021-08-20T00:48:49.649603Z","shell.execute_reply":"2021-08-20T00:49:07.755474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Landmarks with a lot of images in the training set have a lot of different representations contained therein. Clustering can group images based on similar feature vectors.\nSteps:\n1. preprocess images into arrays with batch dimension \n2. extract features vector using VGG model prediction\n3. PCA to select most important components\n4. K-means clustering to group images (n clusters chosen with SSE elbow method)\n5. viewing images in clusters to compare","metadata":{}},{"cell_type":"markdown","source":"### Image preprocessing and feature extraction","metadata":{}},{"cell_type":"code","source":"model = VGG16()\nmodel = Model(inputs = model.inputs, outputs = model.layers[-2].output)\n\ndef extract_features(file, model):\n    # VGG expects 224x224 arrays\n    img = load_img(file, target_size=(224,224))\n    img = np.array(img) \n    # reshape the data for the model reshape(batch size, height pixel, channels)\n    reshaped_img = img.reshape(1,224,224,3) \n    # prepare image for model\n    imgx = preprocess_input(reshaped_img)\n    # get the feature vector\n    features = model.predict(imgx, use_multiprocessing=True)\n    return features","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:49:07.757750Z","iopub.execute_input":"2021-08-20T00:49:07.758120Z","iopub.status.idle":"2021-08-20T00:49:13.532504Z","shell.execute_reply.started":"2021-08-20T00:49:07.758083Z","shell.execute_reply":"2021-08-20T00:49:13.531685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extract_features(images[0], model)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:49:13.533911Z","iopub.execute_input":"2021-08-20T00:49:13.534243Z","iopub.status.idle":"2021-08-20T00:49:20.144871Z","shell.execute_reply.started":"2021-08-20T00:49:13.534209Z","shell.execute_reply":"2021-08-20T00:49:20.143981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeatures = {}\n\n# loop through each image in the dataset\nfor image in images:\n    # extract features and update dictionary (filepath=key)\n    feature = extract_features(image, model)\n    features[image] = feature\n       \n# get a list of the filenames\nfilenames = np.array(list(features.keys()))\n\n# get a list of just the features\nfeat = np.array(list(features.values()))\nfeat.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:49:20.147817Z","iopub.execute_input":"2021-08-20T00:49:20.148098Z","iopub.status.idle":"2021-08-20T00:50:18.289384Z","shell.execute_reply.started":"2021-08-20T00:49:20.148068Z","shell.execute_reply":"2021-08-20T00:50:18.288373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reshape to sample size count of 4096 vectors\nfeat = feat.reshape(-1,4096)\nfeat.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:50:18.290976Z","iopub.execute_input":"2021-08-20T00:50:18.291235Z","iopub.status.idle":"2021-08-20T00:50:18.299788Z","shell.execute_reply.started":"2021-08-20T00:50:18.291208Z","shell.execute_reply":"2021-08-20T00:50:18.298675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PCA to identify important components\nMax n_components must be less than or equal to the number of images in the superclass as the square covariance matrix is sized according to the number of images","metadata":{}},{"cell_type":"code","source":"# PCA to select most important of 4,096 dimensions\nif len(images) >= 4096:\n    pca = PCA(n_components=len(images)//100, random_state=888)\nelse:\n    pca = PCA(n_components=len(images)//60, random_state=888)\n    \npca.fit(feat)\nx = pca.transform(feat)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:50:18.301217Z","iopub.execute_input":"2021-08-20T00:50:18.301747Z","iopub.status.idle":"2021-08-20T00:50:18.652574Z","shell.execute_reply.started":"2021-08-20T00:50:18.301712Z","shell.execute_reply":"2021-08-20T00:50:18.651415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# k-means clustering\n# ID good number of clusters using SSE distance to cluster center 'elbow method'\nnp.random.seed(888)\ns = np.zeros(50)\n\nfor k in range(0, 50):\n    est = KMeans(n_clusters=k+2)\n    est.fit(x)\n    s[k] = est.inertia_\n    \n\nplt.plot(range(0,50), s)\nplt.xlabel('cluster counts')\nplt.ylabel('distortion')\nplt.title('');","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:50:18.657296Z","iopub.execute_input":"2021-08-20T00:50:18.657678Z","iopub.status.idle":"2021-08-20T00:50:49.475888Z","shell.execute_reply.started":"2021-08-20T00:50:18.657640Z","shell.execute_reply":"2021-08-20T00:50:49.475228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is no clear elbow in the cluster count plot, so I will choose 15 clusters to balance complexity in the model and the SSE cost function\n","metadata":{}},{"cell_type":"code","source":"kmeans = KMeans(n_clusters=15, random_state=888).fit(x)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:50:49.478856Z","iopub.execute_input":"2021-08-20T00:50:49.480365Z","iopub.status.idle":"2021-08-20T00:50:49.897103Z","shell.execute_reply.started":"2021-08-20T00:50:49.480331Z","shell.execute_reply":"2021-08-20T00:50:49.896405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = {}\nfor file, cluster in zip(images,kmeans.labels_):\n    if cluster not in groups.keys():\n        groups[cluster] = []\n        groups[cluster].append(file)\n    else:\n        groups[cluster].append(file)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:50:49.900881Z","iopub.execute_input":"2021-08-20T00:50:49.903370Z","iopub.status.idle":"2021-08-20T00:50:49.913749Z","shell.execute_reply.started":"2021-08-20T00:50:49.903335Z","shell.execute_reply":"2021-08-20T00:50:49.913107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view_images_in_cluster(cluster):\n    plt.figure(figsize = (25,25));\n    files = groups[cluster]\n    # view <=50 images in cluster\n    if len(files) > 50:\n        print(f'showing 50 of {len(files)} images in cluster')\n        files = files[:49]\n    for index, file in enumerate(files):\n        plt.subplot(10,10,index+1);\n        img = load_img(file)\n        img = np.array(img)\n        plt.imshow(img)\n        plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:50:49.918569Z","iopub.execute_input":"2021-08-20T00:50:49.921001Z","iopub.status.idle":"2021-08-20T00:50:49.931556Z","shell.execute_reply.started":"2021-08-20T00:50:49.920960Z","shell.execute_reply":"2021-08-20T00:50:49.930732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_images_in_cluster(2)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:53:13.894500Z","iopub.execute_input":"2021-08-20T00:53:13.894877Z","iopub.status.idle":"2021-08-20T00:53:18.957032Z","shell.execute_reply.started":"2021-08-20T00:53:13.894845Z","shell.execute_reply":"2021-08-20T00:53:18.952577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_images_in_cluster(4)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:53:28.118376Z","iopub.execute_input":"2021-08-20T00:53:28.118735Z","iopub.status.idle":"2021-08-20T00:53:33.256137Z","shell.execute_reply.started":"2021-08-20T00:53:28.118703Z","shell.execute_reply":"2021-08-20T00:53:33.255363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_images_in_cluster(12)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T00:51:00.408055Z","iopub.execute_input":"2021-08-20T00:51:00.408380Z","iopub.status.idle":"2021-08-20T00:51:05.869081Z","shell.execute_reply.started":"2021-08-20T00:51:00.408346Z","shell.execute_reply":"2021-08-20T00:51:05.862644Z"},"trusted":true},"execution_count":null,"outputs":[]}]}