{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

DBSCAN + EfficientNet baseline

\n\n

\n Density-Based Spacial Clustering of Applications with noise. It is an efficient clustering algoritm designed to be fast and effective. It is better than Kmeans clustering as it does not require us to select the number of clusters.

\n The Best case run time complexity of DBScan is O(nlogn) whereas for Kmeans, its O(n2) . It can even find arbitrary shaped clusters, for example a cluster that is surrounded by another cluster, but not connected to it. KMeans cannot do the same.
\n
\n In this notebook, we extract ImageNet EfficientNet embeddings and cluster them using DBSCAN .\n \n

\n\n\n
\n

\nACKNOWLEDGMENTS\n

\n\n

\n Making this wouldn't have been possible without the help of following notebooks, do check them out!\n\n

\n\n\n \n","metadata":{}},{"cell_type":"markdown","source":"### LOADING LIBRARIES AND DATA","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master\")\n\n# utils\nimport numpy as np \nimport pandas as pd \nimport math\nimport random \nimport gc\nimport os \nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\n# image utils\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport cv2\n\n# DBSCAN and sklearn\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MaxAbsScaler\nfrom cuml.cluster import DBSCAN\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom cuml.metrics.pairwise_distances import pairwise_distances\nimport cupy\n\nfrom sklearn.model_selection import StratifiedKFold\n\n# deep learning & models\nimport torch\nfrom torch.utils.data import Dataset\nimport torch.nn as nn\nfrom efficientnet_pytorch import EfficientNet","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:38.886724Z","iopub.execute_input":"2022-02-15T08:34:38.887294Z","iopub.status.idle":"2022-02-15T08:34:45.260003Z","shell.execute_reply.started":"2022-02-15T08:34:38.887258Z","shell.execute_reply":"2022-02-15T08:34:45.259218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\ntest = pd.read_csv(\"../input/happy-whale-and-dolphin/sample_submission.csv\")\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:45.261768Z","iopub.execute_input":"2022-02-15T08:34:45.262030Z","iopub.status.idle":"2022-02-15T08:34:45.405093Z","shell.execute_reply.started":"2022-02-15T08:34:45.262000Z","shell.execute_reply":"2022-02-15T08:34:45.404439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:45.406447Z","iopub.execute_input":"2022-02-15T08:34:45.406824Z","iopub.status.idle":"2022-02-15T08:34:45.412485Z","shell.execute_reply.started":"2022-02-15T08:34:45.406790Z","shell.execute_reply":"2022-02-15T08:34:45.411675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SET CONFIG","metadata":{}},{"cell_type":"code","source":"class Config:\n \n # models\n efficientnet_name = 'efficientnet-b2' # 0 for b0, 1 for b1, etc\n \n # image size\n image_size = 256\n \n seed = 2022\n batch_size = 16\n device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n \n # dbscan metric - cosine/euclidean\n dbscan_metric = 'euclidean'\n ","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:45.414835Z","iopub.execute_input":"2022-02-15T08:34:45.415271Z","iopub.status.idle":"2022-02-15T08:34:45.421003Z","shell.execute_reply.started":"2022-02-15T08:34:45.415235Z","shell.execute_reply":"2022-02-15T08:34:45.420226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DEFINING DATASET AND DATALOADER\n\n","metadata":{}},{"cell_type":"code","source":"# transform for pytorch\ntransforms = A.Compose([\n A.Resize(Config.image_size, Config.image_size),\n A.Normalize(\n mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225], \n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.)\n\n\n# Dataset class\nclass HappyWhaleDataset(Dataset):\n def __init__(self, df, transforms, test=False):\n self.df = df\n self.transforms = transforms\n self.test = test\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, idx):\n \n if self.test: \n impath = f\"../input/happy-whale-and-dolphin/test_images/{self.df.loc[idx, 'image']}\"\n else:\n impath = f\"../input/happy-whale-and-dolphin/train_images/{self.df.loc[idx, 'image']}\"\n \n img = cv2.imread(impath)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n \n img = self.transforms(image=img)[\"image\"]\n \n return img\n\n\n# map@5\n# from - https://www.kaggle.com/pestipeti/explanation-of-map5-scoring-metric\ndef map_per_image(label, predictions): \n try:\n return 1 / (predictions[:5].index(label) + 1)\n except ValueError:\n return 0.0\n\ndef map_per_set(labels, predictions):\n return np.mean([map_per_image(l, p) for l,p in zip(labels, predictions)])\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:45.422391Z","iopub.execute_input":"2022-02-15T08:34:45.422653Z","iopub.status.idle":"2022-02-15T08:34:45.436031Z","shell.execute_reply.started":"2022-02-15T08:34:45.422621Z","shell.execute_reply":"2022-02-15T08:34:45.435047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"whale_dt = HappyWhaleDataset(train, transforms)\nwhale_dataloader = torch.utils.data.DataLoader(whale_dt, batch_size = Config.batch_size,\n num_workers = 2, pin_memory=True, shuffle=False)\n\nwhale_test_dt = HappyWhaleDataset(test, transforms, \"test\")\nwhale_test_dataloader = torch.utils.data.DataLoader(whale_test_dt, batch_size = Config.batch_size,\n num_workers = 2, pin_memory=True, shuffle=False)\n\n\nlen(whale_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:45.437345Z","iopub.execute_input":"2022-02-15T08:34:45.438012Z","iopub.status.idle":"2022-02-15T08:34:45.449985Z","shell.execute_reply.started":"2022-02-15T08:34:45.437970Z","shell.execute_reply":"2022-02-15T08:34:45.449147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DEFINING THE EFFICIENTNET","metadata":{}},{"cell_type":"code","source":"class WhaleModel(nn.Module):\n def __init__(self):\n super(WhaleModel, self).__init__()\n \n self.model = EfficientNet.from_name(Config.efficientnet_name, include_top=False)\n self.model.load_state_dict(torch.load(glob(f\"../input/efficientnet-pytorch/{Config.efficientnet_name}*\")[0]), \n strict=False)\n print(f\"{Config.efficientnet_name} weights loaded\")\n \n def forward(self, img):\n \n return self.model.forward(img)\n \nmodel = WhaleModel()\nmodel = model.to(Config.device)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:45.451472Z","iopub.execute_input":"2022-02-15T08:34:45.452045Z","iopub.status.idle":"2022-02-15T08:34:52.675661Z","shell.execute_reply.started":"2022-02-15T08:34:45.451997Z","shell.execute_reply":"2022-02-15T08:34:52.674931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EXTRACT EMBEDDINGS","metadata":{}},{"cell_type":"code","source":"# The embedding generation takes around 50 mins, I have generate the embeddings beforehand and uploaded it in a Dataset.\n# You can uncomment the below code to generate embeddings for some other efficientnet.\n\n# image_embs = np.load(\"../input/happywhaleeffnetembs/effb0-train.npy\")\n\n# Uncomment to generate embeddings\nmodel.eval()\n\nembeddings = []\n\nfor bind, img in enumerate(tqdm(whale_dataloader)):\n img = img.to(Config.device)\n embs = model(img)\n \n embeddings.append(embs.cpu().detach().numpy()) \n \n del img, embs\n gc.collect()\n\nimage_embs = np.concatenate(embeddings).squeeze()\n\ndel embeddings\ngc.collect()\n\nimage_embs.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:34:52.678984Z","iopub.execute_input":"2022-02-15T08:34:52.679198Z","iopub.status.idle":"2022-02-15T09:29:02.083333Z","shell.execute_reply.started":"2022-02-15T08:34:52.679174Z","shell.execute_reply":"2022-02-15T09:29:02.082350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TEST IMAGES EMBEDDINGS\n\n# test_image_embs = np.load(\"../input/happywhaleeffnetembs/effb0-test.npy\")\n\n# uncomment to generate embeddings manually\nembeddings = []\n\nfor bind, img in enumerate(tqdm(whale_test_dataloader)):\n img = img.to(Config.device)\n embs = model(img)\n \n embeddings.append(embs.cpu().detach().numpy()) \n \n del img, embs\n gc.collect()\n\ntest_image_embs = np.concatenate(embeddings).squeeze()\n\ndel model, embeddings\ngc.collect()\n\ntest_image_embs.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-15T09:29:02.084738Z","iopub.execute_input":"2022-02-15T09:29:02.085130Z","iopub.status.idle":"2022-02-15T09:58:33.239995Z","shell.execute_reply.started":"2022-02-15T09:29:02.085085Z","shell.execute_reply":"2022-02-15T09:58:33.239058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### FIT DBSCAN ","metadata":{}},{"cell_type":"code","source":"def train_dbscan(eps, min_samples, image_embs, verbose=True, metric = Config.dbscan_metric):\n \n db = DBSCAN(eps = eps, \n min_samples = min_samples, \n metric = 'euclidean', \n calc_core_sample_indices = True,\n verbose = 0)\n\n # trick to use cosine by normalizing euclidean\n # https://medium.com/ai-for-real/relationship-between-cosine-similarity-and-euclidean-distance-7e283a277dff\n if metric == 'cosine':\n norms = cupy.linalg.norm(cupy.array(image_embs), ord=2, axis=1)\n image_embs = image_embs/norms.get().reshape(-1, 1)\n \n db.fit(image_embs)\n \n if verbose:\n print(f\"Generated {len(db.core_sample_indices_)} clusters from {image_embs.shape[0]} data points\")\n \n return db, len(db.core_sample_indices_)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T09:58:33.467422Z","iopub.status.idle":"2022-02-15T09:58:33.468027Z","shell.execute_reply.started":"2022-02-15T09:58:33.467776Z","shell.execute_reply":"2022-02-15T09:58:33.467812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CALCULATE TRAIN CV SCORE","metadata":{}},{"cell_type":"markdown","source":"

\n eps \n \n It is the most important parameter in DBSCAN. It is the maximum distance between two samples for one to be considered as in the neighborhood of the other.\n \n We will tune this parameter, by maximizing validation mAP score. The total number of individual ids in train set must be the number of clusters in the train set. Let us sweep different values of eps to select the best one.\n \n

","metadata":{}},{"cell_type":"code","source":"# trying out all values from 2 to 8 with steps of 0.1\n\nnumclus = []\nfor eps in tqdm(np.arange(2, 8, 0.1)): \n numclus.append(train_dbscan(eps, 2, image_embs, verbose=False)[1])\n \n \nplt.plot(np.arange(2, 8, 0.1), numclus)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T09:58:33.469180Z","iopub.status.idle":"2022-02-15T09:58:33.469757Z","shell.execute_reply.started":"2022-02-15T09:58:33.469499Z","shell.execute_reply":"2022-02-15T09:58:33.469525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train set has {len(train.individual_id.value_counts())} different ids\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-15T04:54:49.091656Z","iopub.execute_input":"2022-02-15T04:54:49.092008Z","iopub.status.idle":"2022-02-15T04:54:49.115002Z","shell.execute_reply.started":"2022-02-15T04:54:49.091967Z","shell.execute_reply":"2022-02-15T04:54:49.114135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n \n From the graph we can see that the best eps should be 5.5 (approx) , Let us train with this eps and check validation score\n \n

","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:35:58.828446Z","iopub.execute_input":"2022-02-14T16:35:58.828855Z","iopub.status.idle":"2022-02-14T16:35:59.014488Z","shell.execute_reply.started":"2022-02-14T16:35:58.828816Z","shell.execute_reply":"2022-02-14T16:35:59.013797Z"}}},{"cell_type":"code","source":"def dbscan_predict(db, train_embs, test_embs, metric = Config.dbscan_metric):\n \n core_samples_embs = train_embs[db.core_sample_indices_, :]\n print(\"Core image embeddings shape \", core_samples_embs.shape)\n \n predictions = []\n \n print(f\"Predicting on {test_embs.shape[0]} test embeddings\")\n for i in tqdm(range(test_embs.shape[0])):\n pred = []\n \n dists = pairwise_distances(core_samples_embs, test_embs[[i,], :], metric = metric)\n top5 = sorted(list(zip(dists, db.core_sample_indices_)))[:5]\n \n vals = [x[0] for x in top5]\n inds = [x[1] for x in top5]\n indids = [train.loc[x, 'individual_id'] for x in inds]\n \n if vals[0] > db.eps:\n pred.append(-1)\n pred.extend(indids[:4])\n else:\n pred.extend(indids)\n \n predictions.append(pred)\n \n print(f\"Generated {len(predictions)} predictions for the test set\")\n \n return predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-15T04:54:49.116352Z","iopub.execute_input":"2022-02-15T04:54:49.117061Z","iopub.status.idle":"2022-02-15T04:54:49.127795Z","shell.execute_reply.started":"2022-02-15T04:54:49.117001Z","shell.execute_reply":"2022-02-15T04:54:49.126904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dbscan_model, _ = train_dbscan(eps = 5.5,\n min_samples = 2,\n image_embs = image_embs)\n \n\n# takes too much time to check complete train set - increase this if you want to check for more\nCHECK_SIZE = 2500\n\npredictions = dbscan_predict(dbscan_model, image_embs, image_embs[:CHECK_SIZE])\nscore = map_per_set(train.individual_id.tolist()[:CHECK_SIZE], predictions)\n\nprint(f\"CV score - \", score)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T04:54:49.129097Z","iopub.execute_input":"2022-02-15T04:54:49.12957Z","iopub.status.idle":"2022-02-15T05:07:03.565246Z","shell.execute_reply.started":"2022-02-15T04:54:49.129528Z","shell.execute_reply":"2022-02-15T05:07:03.564515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TEST PREDICTION","metadata":{}},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-02-15T05:07:03.566577Z","iopub.execute_input":"2022-02-15T05:07:03.566995Z","iopub.status.idle":"2022-02-15T05:07:03.578712Z","shell.execute_reply.started":"2022-02-15T05:07:03.566956Z","shell.execute_reply":"2022-02-15T05:07:03.578047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = dbscan_predict(dbscan_model, image_embs, test_image_embs)\n\n\ndef process_preds(pred):\n s = \" \"\n if pred[0] == -1:\n s += \"new_individual \"\n else:\n s += str(pred[0]) + \" \"\n\n s += \" \".join([str(x) for x in pred[1:]])\n\n s.strip()\n\n return s\n\ntest.loc[:, 'predlist'] = predictions\ntest.loc[:, 'predictions'] = test['predlist'].apply(process_preds)\n\ntest[['image', 'predictions']].to_csv(\"submission.csv\", index=False)\npd.read_csv(\"submission.csv\").head()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T05:07:03.57994Z","iopub.execute_input":"2022-02-15T05:07:03.580353Z","iopub.status.idle":"2022-02-15T05:12:12.312575Z","shell.execute_reply.started":"2022-02-15T05:07:03.5803Z","shell.execute_reply":"2022-02-15T05:12:12.311403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\nPOSSIBLE IMPROVEMENTS\n

\n\n\n\n

\n Show your appreciation by a simple upvote! 😊\n\n

","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}