{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\n

- Full Training Pipeline - Step by Step Guide with Explanation -

\n\n> 🐳 **Competition Goal:** Identify and group all images that contain the same individual through time.\n\n> 🙏 **Inspiration**: A super huge thank you to [Debarshi Chanda](https://www.kaggle.com/debarshichanda) and his very [clean comprehensive notebook](https://www.kaggle.com/debarshichanda/pytorch-arcface-gem-pooling-starter) that helped me put a start to this competition.\n\n### ⬇ Libraries\n\n🐳 **What is `timm`?** - It is a library that gathers all **PyTorch Image Models**, for ease of access and convenience. The [full documentation can be found here](https://rwightman.github.io/pytorch-image-models/models/).\n* **Feature Extraction** - All of the [models in timm](https://rwightman.github.io/pytorch-image-models/feature_extraction/) have consistent mechanisms for obtaining various types of features from the model for tasks besides classification. This will help us when extracting the embeddings from the pretrained algorithms (aka the backbone).\n* **Models** - I am also leaving here a [link](https://rwightman.github.io/pytorch-image-models/results/) to all the models and *their names* that are currently available within the library.","metadata":{"execution":{"iopub.status.busy":"2022-03-04T05:35:49.566746Z","iopub.execute_input":"2022-03-04T05:35:49.567272Z","iopub.status.idle":"2022-03-04T05:35:49.578091Z","shell.execute_reply.started":"2022-03-04T05:35:49.567227Z","shell.execute_reply":"2022-03-04T05:35:49.576383Z"}}},{"cell_type":"code","source":"# Helpful Installs\n!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-12T04:46:30.218262Z","iopub.execute_input":"2022-03-12T04:46:30.218821Z","iopub.status.idle":"2022-03-12T04:46:40.070406Z","shell.execute_reply.started":"2022-03-12T04:46:30.218785Z","shell.execute_reply":"2022-03-12T04:46:40.069573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Libraries\nimport os\nimport gc\nimport wandb\nimport time\nimport random\nimport math\nfrom scipy import spatial\nfrom tqdm import tqdm\nimport warnings\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom numpy import dot, sqrt\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.patches as patches\nimport matplotlib.pyplot as plt\nfrom IPython.display import display_html\n\nfrom sklearn.model_selection import StratifiedKFold\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim import Adam, lr_scheduler\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\n\nfrom albumentations.pytorch import transforms\nimport albumentations\nimport timm\n\n# Environment check\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"WANDB_SILENT\"] = \"true\"\nCONFIG = {'competition': 'happywhale', '_wandb_kernel': 'aot'}\n\n# 🐝 Secrets\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\n#secret_value_0 = user_secrets.get_secret(\"wandb\")\n\n#! wandb login $secret_value_0\n! wandb login '731732fef357694e773d793056c1a7fde4130b51'\n# Custom colors\nclass clr:\n S = '\\033[1m' + '\\033[96m'\n E = '\\033[0m'\n \nmy_colors = [\"#21295C\", \"#1F4E78\", \"#1C7293\", \"#73ABAF\", \"#C9E4CA\", \"#87BBA2\", \"#618E83\", \"#3B6064\"]\nprint(clr.S+\"Notebook Color Scheme:\"+clr.E)\nsns.palplot(sns.color_palette(my_colors))\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-12T04:46:40.073087Z","iopub.execute_input":"2022-03-12T04:46:40.073362Z","iopub.status.idle":"2022-03-12T04:46:46.702691Z","shell.execute_reply.started":"2022-03-12T04:46:40.073325Z","shell.execute_reply":"2022-03-12T04:46:46.701858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ⬇ Helper Functions","metadata":{}},{"cell_type":"code","source":"def plot_loss_graph(train_losses, valid_losses, epoch, fold):\n '''Lineplot of the training/validation losses.'''\n \n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 2.5))\n fig.suptitle(f\"Fold {fold} | Epoch {epoch}\", fontsize=12, y=1.05)\n axes = [ax1, ax2]\n data = [train_losses, valid_losses]\n sns.lineplot(y=train_losses, x=range(len(train_losses)),\n lw=2.3, ls=\":\", color=my_colors[1], ax=ax1)\n sns.lineplot(y=valid_losses, x=range(len(valid_losses)),\n lw=2.3, ls=\"-\", color=my_colors[6], ax=ax2)\n for ax, t, d in zip(axes, [\"Train\", \"Valid\"], data):\n ax.set_title(f\"{t} Evolution\", size=12, weight='bold')\n ax.set_xlabel(\"Iteration\", weight='bold', size=9)\n ax.set_ylabel(\"Loss\", weight='bold', size=9)\n ax.tick_params(labelsize=9)\n plt.show()\n \n \ndef show_values_on_bars(axs, h_v=\"v\", space=0.4):\n '''Plots the value at the end of the a seaborn barplot.\n axs: the ax of the plot\n h_v: weather or not the barplot is vertical/ horizontal'''\n \n def _show_on_single_plot(ax):\n if h_v == \"v\":\n for p in ax.patches:\n _x = p.get_x() + p.get_width() / 2\n _y = p.get_y() + p.get_height()\n value = int(p.get_height())\n ax.text(_x, _y, format(value, ','), ha=\"center\") \n elif h_v == \"h\":\n for p in ax.patches:\n _x = p.get_x() + p.get_width() + float(space)\n _y = p.get_y() + p.get_height()\n value = int(p.get_width())\n ax.text(_x, _y, format(value, ','), ha=\"left\")\n\n if isinstance(axs, np.ndarray):\n for idx, ax in np.ndenumerate(axs):\n _show_on_single_plot(ax)\n else:\n _show_on_single_plot(axs)\n\n\n# === 🐝 W&B ===\ndef save_dataset_artifact(run_name, artifact_name, path):\n '''Saves dataset to W&B Artifactory.\n run_name: name of the experiment\n artifact_name: under what name should the dataset be stored\n path: path to the dataset'''\n \n run = wandb.init(project='happywhale', \n name=run_name, \n config=CONFIG, anonymous=\"allow\")\n artifact = wandb.Artifact(name=artifact_name, \n type='dataset')\n artifact.add_file(path)\n\n wandb.log_artifact(artifact)\n wandb.finish()\n print(\"Artifact has been saved successfully.\")\n \n \ndef create_wandb_plot(x_data=None, y_data=None, x_name=None, y_name=None, title=None, log=None, plot=\"line\"):\n '''Create and save lineplot/barplot in W&B Environment.\n x_data & y_data: Pandas Series containing x & y data\n x_name & y_name: strings containing axis names\n title: title of the graph\n log: string containing name of log'''\n \n data = [[label, val] for (label, val) in zip(x_data, y_data)]\n table = wandb.Table(data=data, columns = [x_name, y_name])\n \n if plot == \"line\":\n wandb.log({log : wandb.plot.line(table, x_name, y_name, title=title)})\n elif plot == \"bar\":\n wandb.log({log : wandb.plot.bar(table, x_name, y_name, title=title)})\n elif plot == \"scatter\":\n wandb.log({log : wandb.plot.scatter(table, x_name, y_name, title=title)})\n \n \ndef create_wandb_hist(x_data=None, x_name=None, title=None, log=None):\n '''Create and save histogram in W&B Environment.\n x_data: Pandas Series containing x values\n x_name: strings containing axis name\n title: title of the graph\n log: string containing name of log'''\n \n data = [[x] for x in x_data]\n table = wandb.Table(data=data, columns=[x_name])\n wandb.log({log : wandb.plot.histogram(table, x_name, title=title)})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-12T04:51:38.415727Z","iopub.execute_input":"2022-03-12T04:51:38.415995Z","iopub.status.idle":"2022-03-12T04:51:38.437999Z","shell.execute_reply.started":"2022-03-12T04:51:38.415965Z","shell.execute_reply":"2022-03-12T04:51:38.437095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 🌱 Setting the Seed & Device","metadata":{}},{"cell_type":"code","source":"def set_seed(seed = 1234):\n '''\n 🌱src:https://www.kaggle.com/andradaolteanu/melanoma-competiton-aug-resnet-effnet-lb-0-91\n Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n\n# Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(clr.S+'Device available now:'+clr.E, device)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:41.173976Z","iopub.execute_input":"2022-03-12T04:51:41.174379Z","iopub.status.idle":"2022-03-12T04:51:41.183289Z","shell.execute_reply.started":"2022-03-12T04:51:41.174343Z","shell.execute_reply":"2022-03-12T04:51:41.182442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data & Parameters\n\n🐳 **The Data:** I am using my preprocessed `.csv` file for training and test data (for more info on how I made them [check out this notebook](https://www.kaggle.com/andradaolteanu/whales-dolphins-effnet-embedding-cos-distance)). I am also using the [images dataset](https://www.kaggle.com/phalanx/whale2-cropped-dataset) that has been resized and cropped by [phalanx](https://www.kaggle.com/phalanx) using the Detic methodology ([Discussion and Explanation here](https://www.kaggle.com/c/happy-whale-and-dolphin/discussion/305503))","metadata":{}},{"cell_type":"code","source":"# --------- INITIAL PARAMETERS ---------\nTRAIN_FOLDER = \"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"\nTEST_FOLDER = \"../input/whale2-cropped-dataset/cropped_test_images/cropped_test_images/\"\n\n# Set some parameters for sanity checks & experimenting\nN_SPLITS = 5\nBATCH_SIZE = 16\nMODEL_NAME = 'efficientnet_b4'\nNUM_CLASSES = 15587\nNO_NEURONS = 512\nEMBEDDING_SIZE = 512\n# -------------------------------------","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:41.593711Z","iopub.execute_input":"2022-03-12T04:51:41.593966Z","iopub.status.idle":"2022-03-12T04:51:41.608901Z","shell.execute_reply.started":"2022-03-12T04:51:41.593937Z","shell.execute_reply":"2022-03-12T04:51:41.604395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the data\ntrain = pd.read_csv(\"../input/happywhale-2022/train.csv\")\ntest = pd.read_csv(\"../input/happywhale-2022/test.csv\")\n#print(train)\n# Update path to new image folders\ntrain[\"path\"] = TRAIN_FOLDER + train[\"image\"]\ntest[\"path\"] = TEST_FOLDER + test[\"image\"]\n\nprint(clr.S+\"TRAIN:\"+clr.E)\ndisplay_html(train.head())\nprint(\"\\n\", clr.S+\"TEST:\"+clr.E)\ndisplay_html(test.head())\n#print(train)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:41.801487Z","iopub.execute_input":"2022-03-12T04:51:41.801784Z","iopub.status.idle":"2022-03-12T04:51:42.339124Z","shell.execute_reply.started":"2022-03-12T04:51:41.801750Z","shell.execute_reply":"2022-03-12T04:51:42.338380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. PyTorch Dataset","metadata":{}},{"cell_type":"code","source":"class HappyWhaleDataset(Dataset):\n \n def __init__(self, csv, trainFlag):\n '''Module to create the PyTorch Dataset.\n csv: full dataframe (train or test)\n trainFlag: True if csv is a training/validation dataset, False otherwise\n return: image and class target if trainFlag, otherwise only image'''\n \n self.csv = csv\n self.trainFlag = trainFlag\n if self.trainFlag:\n self.transform = albumentations.Compose([\n albumentations.Resize(128, 128),\n albumentations.HorizontalFlip(),\n albumentations.VerticalFlip(),\n albumentations.Rotate(),\n albumentations.Normalize(),\n # B&W?\n ])\n else:\n self.transform = albumentations.Compose([\n albumentations.Normalize()\n ])\n\n \n def __len__(self):\n return self.csv.shape[0]\n\n \n def __getitem__(self, index):\n # Get data\n row = self.csv.iloc[index]\n # Retrieve the target group\n if self.trainFlag: \n target = torch.tensor(row.individual_key)\n else:\n target = torch.tensor(1)\n # Read and transform the image\n image = cv2.imread(row.path)\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n# image = image[:, :, ::-1]\n \n transformed_img = self.transform(image=image)['image'].astype(np.float32)\n image = transformed_img.transpose(2, 0, 1)\n image = torch.tensor(image)\n\n if self.trainFlag:\n return image, target\n else:\n return image, target","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:42.340910Z","iopub.execute_input":"2022-03-12T04:51:42.341391Z","iopub.status.idle":"2022-03-12T04:51:42.351488Z","shell.execute_reply.started":"2022-03-12T04:51:42.341341Z","shell.execute_reply":"2022-03-12T04:51:42.350848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Example\n\n🐳 In the below example we have:\n* 4 batches in total for 12 images, each batch having a size of 3 images => 3 images/batch\n* The output per each batch:\n * a 4D tensor of 3 images with size 3 channels x 128 width x 128 height\n * a 1D tensor of size 3 -> one target class per each image","metadata":{}},{"cell_type":"code","source":"# Example for the Dataset data\nexample_dataset = HappyWhaleDataset(test.head(12), trainFlag=0)\nexample_loader = DataLoader(example_dataset, batch_size=3)\n\nfor k, (image,target)in enumerate(example_loader):\n print(clr.S+f\"--- Batch {k} ---\"+clr.E)\n print(\"Image Shape:\", image.shape)\n print(\"Target:\", target, \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:42.752009Z","iopub.execute_input":"2022-03-12T04:51:42.752674Z","iopub.status.idle":"2022-03-12T04:51:43.131297Z","shell.execute_reply.started":"2022-03-12T04:51:42.752637Z","shell.execute_reply":"2022-03-12T04:51:43.129953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. PyTorch Helper Functions\n\n## 3.1 Generalized Mean (or GeM)\n\n🐳 There is a great article [here](https://amaarora.github.io/2020/08/30/gempool.html) from Aman Arora that explains in detail the mathematics behind GeM Pooling. A notebook comparison between GeM Pooling and Average Pooling can also be found [here](https://github.com/amaarora/amaarora.github.io/blob/master/nbs/GeM%20Pooling.ipynb).\n\nIn short, an image has 3 dimensions: `K x H x W`, where:\n* K: the number of channels\n* H: the image height\n* W: the image width\n\nLet `Xk` be the **spatial feature map activation**, then the difference between *Max Pooling*, *Average Pooling* and *GeM Pooling* is the following:\n\n
\n\n🐳 The pooling parameter `pk` can be *set* or *learned*, since this operation can be learned during back-propagation. In other words, *GeM Pooling* can also be trainable.","metadata":{}},{"cell_type":"code","source":"# src: https://amaarora.github.io/2020/08/30/gempool.html\n\nclass GeM(nn.Module):\n def __init__(self, p=3, eps=1e-6):\n super(GeM,self).__init__()\n self.p = nn.Parameter(torch.ones(1)*p)\n self.eps = eps\n\n def forward(self, x):\n return self.gem(x, p=self.p, eps=self.eps)\n \n def gem(self, x, p=3, eps=1e-6):\n # Applies 2D average-pooling operation in kH * kW regions by step size\n return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n \n def __repr__(self):\n return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:43.738882Z","iopub.execute_input":"2022-03-12T04:51:43.739470Z","iopub.status.idle":"2022-03-12T04:51:43.748538Z","shell.execute_reply.started":"2022-03-12T04:51:43.739423Z","shell.execute_reply":"2022-03-12T04:51:43.747657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Additive Angular Margin Loss (or ArcFace)\n\n🐳 **ArcFace**, or [Additive Angular Margin Loss](https://paperswithcode.com/method/arcface#:~:text=ArcFace%2C%20or%20Additive%20Angular%20Margin,traditionally%20used%20in%20these%20tasks.), *is a loss function* used in face recognition tasks.\n\nThe `softmax` is traditionally used in these tasks. However, the softmax loss function does not *explicitly optimise* the feature embedding to enforce **higher similarity for intraclass samples** and **diversity for inter-class samples** - in other words? We want the ambeddings that are super similar to be VERY CLOSE to each-other and the embeddings that are different to be VERY FAR from each-other:\n\n
","metadata":{"execution":{"iopub.status.busy":"2022-03-04T05:36:47.853212Z","iopub.execute_input":"2022-03-04T05:36:47.853472Z","iopub.status.idle":"2022-03-04T05:36:47.861917Z","shell.execute_reply.started":"2022-03-04T05:36:47.853442Z","shell.execute_reply":"2022-03-04T05:36:47.860911Z"}}},{"cell_type":"code","source":"# src: https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/blob/master/src/modeling/metric_learning.py\n\nclass ArcMarginProduct(nn.Module):\n def __init__(self, in_features, out_features, s=30.0, \n m=0.50, easy_margin=False, ls_eps=0.0):\n '''\n in_features: dimension of the input\n out_features: dimension of the last layer (in our case the classification)\n s: norm of input feature\n m: margin\n ls_eps: label smoothing'''\n \n super(ArcMarginProduct, self).__init__()\n self.in_features, self.out_features = in_features, out_features\n self.s = s\n self.m = m\n self.ls_eps = ls_eps\n self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n # Fills the input `Tensor` with values according to the method described in\n # `Understanding the difficulty of training deep feedforward neural networks`\n # Glorot, X. & Bengio, Y. (2010)\n # using a uniform distribution.\n nn.init.xavier_uniform_(self.weight)\n\n self.easy_margin = easy_margin\n self.cos_m, self.sin_m = math.cos(m), math.sin(m)\n self.th = math.cos(math.pi - m)\n self.mm = math.sin(math.pi - m) * m\n\n def forward(self, input, label):\n # --------------------------- cos(theta) & phi(theta) ---------------------\n cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n phi = cosine * self.cos_m - sine * self.sin_m\n if self.easy_margin:\n phi = torch.where(cosine > 0, phi, cosine)\n else:\n phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n # --------------------------- convert label to one-hot ---------------------\n one_hot = torch.zeros(cosine.size()).to(device)\n one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n if self.ls_eps > 0:\n one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n output *= self.s\n\n return output","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:44.579278Z","iopub.execute_input":"2022-03-12T04:51:44.579833Z","iopub.status.idle":"2022-03-12T04:51:44.592079Z","shell.execute_reply.started":"2022-03-12T04:51:44.579793Z","shell.execute_reply":"2022-03-12T04:51:44.591363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. The PyTorch Model","metadata":{}},{"cell_type":"code","source":"class HappyWhaleModel(nn.Module):\n def __init__(self, modelName, numClasses, noNeurons, embeddingSize):\n \n super(HappyWhaleModel, self).__init__()\n # Retrieve pretrained weights\n self.backbone = timm.create_model(modelName, pretrained=True)\n # Save the number features from the backbone\n ### different models have different numbers e.g. EffnetB3 has 1536\n backbone_features = self.backbone.classifier.in_features\n self.backbone.classifier = nn.Identity() # ?????\n self.backbone.global_pool = nn.Identity() # ?????\n self.gem = GeM()\n # Embedding layer (what we actually need)\n self.embedding = nn.Sequential(nn.Linear(backbone_features, noNeurons),\n nn.BatchNorm1d(noNeurons),\n nn.ReLU(),\n nn.Dropout(p=0.2),\n \n nn.Linear(noNeurons, embeddingSize),\n nn.BatchNorm1d(embeddingSize),\n nn.ReLU(),\n nn.Dropout(p=0.2))\n self.arcface = ArcMarginProduct(in_features=embeddingSize, \n out_features=numClasses,\n s=30.0, m=0.3, easy_margin=False, ls_eps=0.0)\n \n \n def forward(self, image, target, prints=False):\n features = self.backbone(image)\n # flatten transforms from e.g.: [3, 1536, 1, 1] to [3, 1536]\n gem_pool = self.gem(features).flatten(1)\n embedding = self.embedding(gem_pool)\n out = self.arcface(embedding, target)\n \n if prints:\n print(clr.S+\"0. IN:\", \"image shape:\"+clr.E, image.shape, \"target:\", target)\n print(clr.S+\"1. Backbone Output:\"+clr.E, features.shape)\n print(clr.S+\"2. GeM Pool Output:\"+clr.E, gem_pool.shape)\n print(clr.S+\"3. Embedding Output:\"+clr.E, embedding.shape)\n print(clr.S+\"4. ArcFace Output:\"+clr.E, out.shape)\n \n return out, embedding","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:45.390772Z","iopub.execute_input":"2022-03-12T04:51:45.391292Z","iopub.status.idle":"2022-03-12T04:51:45.402795Z","shell.execute_reply.started":"2022-03-12T04:51:45.391255Z","shell.execute_reply":"2022-03-12T04:51:45.402134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Example\n\n> 🐳 **Note**: Below you can find a simple schema on what the `HappyWhaleModel()` does:\n
","metadata":{}},{"cell_type":"code","source":"# Create an example model - Effnet\nmodel_example = HappyWhaleModel(MODEL_NAME, NUM_CLASSES, NO_NEURONS, EMBEDDING_SIZE).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:46.756915Z","iopub.execute_input":"2022-03-12T04:51:46.757462Z","iopub.status.idle":"2022-03-12T04:51:53.892699Z","shell.execute_reply.started":"2022-03-12T04:51:46.757419Z","shell.execute_reply":"2022-03-12T04:51:53.889085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Criterion\ncriterion_example = nn.CrossEntropyLoss()\n\n# We'll use previous datasets & dataloader\nfor k, (image, target) in enumerate(example_loader):\n print(clr.S+f\"=== Batch {k} ===\"+clr.E)\n image, target = image.to(device), target.to(device)\n out, _ = model_example(image, target, prints=True)\n loss = criterion_example(out, target)\n print(clr.S+'--- LOSS ---'+clr.E, loss.item(), \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:53.900745Z","iopub.execute_input":"2022-03-12T04:51:53.904836Z","iopub.status.idle":"2022-03-12T04:51:59.987059Z","shell.execute_reply.started":"2022-03-12T04:51:53.904797Z","shell.execute_reply":"2022-03-12T04:51:59.986309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model_example\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:51:59.988455Z","iopub.execute_input":"2022-03-12T04:51:59.989904Z","iopub.status.idle":"2022-03-12T04:52:00.154474Z","shell.execute_reply.started":"2022-03-12T04:51:59.989856Z","shell.execute_reply":"2022-03-12T04:52:00.153782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Training Classifier Model\n\nThe problem at hand could be divided into 3 steps:\n1. Train a `classification` model on the data\n2. Extract the `embedding layer` right before the classification layer\n3. Use these embeddings in order to group the individuals together\n\n# 5.1 Training Functions\n\n### I. Data Loaders","metadata":{}},{"cell_type":"code","source":"def get_loaders(df, train_i, valid_i):\n '''\n df: the full initial dataframe\n train_i, valid_i: list of indexes for train and validation split\n VALID_PERC: percentage of how much of valid data to preserve - leave 1 for full dataset\n return: train_loader and valid_loader\n '''\n \n train_df = df.iloc[train_i, :]\n # To go quicker through validation\n valid_df = df.iloc[valid_i, :].sample(int(len(valid_i)*VALID_PERC), random_state=23)\n\n # Datasets & Dataloader\n train_dataset = HappyWhaleDataset(train_df, trainFlag=True)\n valid_dataset = HappyWhaleDataset(valid_df, trainFlag=True)\n\n train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n valid_loader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)\n \n return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:52:00.156272Z","iopub.execute_input":"2022-03-12T04:52:00.156928Z","iopub.status.idle":"2022-03-12T04:52:00.188796Z","shell.execute_reply.started":"2022-03-12T04:52:00.156888Z","shell.execute_reply":"2022-03-12T04:52:00.188059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### II. Model, Optimizer, Criterion\n\n🐳 **How to adjust learning rate?**: `torch.optim.lr_scheduler` provides [several methods](https://pytorch.org/docs/stable/optim.html) to adjust the learning rate based on the number of epochs.\n\nHere is a full list of all PyTorch schedulers: https://pytorch.org/docs/stable/optim.html","metadata":{}},{"cell_type":"code","source":"def get_model_optimizer_criterion():\n \n model = HappyWhaleModel(MODEL_NAME, NUM_CLASSES, NO_NEURONS, EMBEDDING_SIZE).to(device)\n optimizer = Adam(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY, amsgrad=False)\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=T_MAX, eta_min=MIN_LR)\n criterion = nn.CrossEntropyLoss()\n #\n criterion = nn.SmoothL1Loss(reduction='mean')\n \n return model, optimizer, scheduler, criterion","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:52:00.190305Z","iopub.execute_input":"2022-03-12T04:52:00.190684Z","iopub.status.idle":"2022-03-12T04:52:00.199848Z","shell.execute_reply.started":"2022-03-12T04:52:00.190647Z","shell.execute_reply":"2022-03-12T04:52:00.199222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### III. Training Function\n\n> Below there is a complete step by step schema of what the `train_pipeline()` function does:\n
","metadata":{}},{"cell_type":"code","source":"def train_pipeline(train):\n '''\n train: the full training dataframe (to be split in train data & valid data)\n '''\n\n s = time.time()\n \n # 🐝 W&B Initialize \n RUN_CONFIG = CONFIG.copy()\n params = dict(model=MODEL_NAME, epochs=EPOCHS, split=N_SPLITS, \n batch=BATCH_SIZE, lr=LR, weight_decay=WEIGHT_DECAY,\n t_max=T_MAX, min_lr=MIN_LR)\n RUN_CONFIG.update(params)\n run = wandb.init(project='happywhale', name=RUN_NAME, config=RUN_CONFIG, anonymous=\"allow\")\n\n \n # === CV Split ===\n skf = StratifiedKFold(n_splits=N_SPLITS)\n skf_splits = skf.split(X=train, y=train[\"individual_key\"])\n\n\n for fold, (train_i, valid_i) in enumerate(skf_splits):\n\n print(\"~\"*25)\n print(\"~\"*8, clr.S+f\"FOLD {fold}\"+clr.E, \"~\"*8)\n print(\"~\"*25)\n\n # Retrieve data loaders\n train_loader, valid_loader = get_loaders(train, train_i, valid_i)\n\n # Model/ Optimizer/ Scheduler/ Criterion\n model, optimizer, scheduler, criterion = get_model_optimizer_criterion()\n # Hooks into the torch model to collect gradients and the topology\n wandb.watch(model, log_freq=100)\n #model.load_state_dict(torch.load('./EffNetB0_fold_0_loss_15.422.pt'))\n # Run Training\n BEST_SCORE = 9999\n\n for epoch in range(EPOCHS):\n print(\"~\"*8, clr.S+f\"Epoch {epoch}\"+clr.E, \"~\"*8)\n\n # === TRAIN ===\n model.train()\n train_losses = []\n\n for images, targets in tqdm(train_loader, desc = 'TRAIN'):\n images, targets = images.to(device), targets.to(device)\n\n # Clear gradients BEFORE prediction\n optimizer.zero_grad()\n # Make predictions\n out, _ = model(images, targets)\n # Compute Loss and Optimize\n loss = criterion(out, targets) \n loss.backward()\n optimizer.step()\n\n train_losses.append(loss.cpu().detach().numpy().tolist())\n\n # Adjust Learning Rate\n scheduler.step()\n\n mean_train_loss = np.mean(train_losses)\n print(clr.S+\"Mean Train Loss:\"+clr.E, mean_train_loss)\n wandb.log({\"mean_train_loss\": np.float(mean_train_loss)}, step=epoch)\n\n\n # === EVAL ===\n model.eval()\n valid_losses, valid_preds, valid_targets = [], [], []\n with torch.no_grad():\n for images, targets in valid_loader:\n valid_targets.append(targets)\n images, targets = images.to(device), targets.to(device)\n\n out, _ = model(images, targets)\n loss = criterion(out, targets)\n\n valid_preds.append(out)\n valid_losses.append(loss.cpu().detach().numpy().tolist())\n\n mean_valid_loss = np.mean(valid_losses)\n print(clr.S+\"Mean Valid Loss:\"+clr.E, mean_valid_loss)\n wandb.log({\"mean_valid_loss\": np.float(mean_valid_loss)}, step=epoch)\n gc.collect()\n\n plot_loss_graph(train_losses, valid_losses, epoch, fold)\n create_wandb_plot(x_data=range(len(train_losses)), y_data=train_losses,\n x_name=\"Iterations\", y_name=\"Loss\", title=\"Train Loss\",\n log=\"train_loss\", plot=\"line\")\n\n # === UPDATES ===\n\n if mean_valid_loss < BEST_SCORE: \n print(\"! Saving model in fold {} | epoch {} ...\".format(fold, epoch), \"\\n\")\n torch.save(model.state_dict(), f\"EffNetB4_fold_{fold}_loss_{round(mean_valid_loss, 3)}.pt\")\n\n BEST_SCORE = mean_valid_loss\n\n # Clean memory before next fold\n del model, optimizer, scheduler, criterion, images, targets, \\\n train_losses, valid_losses, valid_preds, valid_targets\n torch.cuda.empty_cache()\n gc.collect()\n\n\n print(clr.S+f\"Time to run: {round((time.time() - s)/60, 2)} minutes\"+clr.E)\n wandb.finish()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-12T04:52:00.202169Z","iopub.execute_input":"2022-03-12T04:52:00.202440Z","iopub.status.idle":"2022-03-12T04:52:00.224280Z","shell.execute_reply.started":"2022-03-12T04:52:00.202385Z","shell.execute_reply":"2022-03-12T04:52:00.223569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.2 Train Experiments\n\n🐳 **A few things to keep in mind:**\n* `NUM_CLASSES`: target labels need to start from 0 and have consecutive numbers, otherwise PyTorch will get upset\n * e.g.: bad labels example: [1, 2, 5, 10]\n * e.g.: good labels example: [0, 1, 2, 3, 4, ......100]\n* `BATCH_SIZE`: if set too hight the notebook might get a memory load. This also applies to `IMAGE_SIZE` and the model used (I have tried de EffNet B3 with 512x512 size and I couldn't run the training pipeline)\n* `VALID_PERC`: this I use so the pipeline goes faster through the validation part (so the notebook commits faster); you can set it to `1` to run the full validation dataset.","metadata":{}},{"cell_type":"code","source":"# --------- GLOBAL PARAMETERS ---------\nNUM_CLASSES = 15587\nN_SPLITS = 3\nBATCH_SIZE = 128\nMODEL_NAME = 'efficientnet_b4'\nRUN_NAME = \"effnetB4_baseline\"\nEPOCHS = 5\nVALID_PERC = 0.1\nNO_NEURONS = 2048\nEMBEDDING_SIZE =512\n# -> Optimizer\nLR = 0.001#0.0001\nWEIGHT_DECAY = 0.00001#0.000001\n# -> Scheduler\nT_MAX = 500 # Maximum number of iterations\nMIN_LR = 0.000001 # Minimum learning rate. Default: 0\n# ------------------------------------","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:52:00.225645Z","iopub.execute_input":"2022-03-12T04:52:00.226181Z","iopub.status.idle":"2022-03-12T04:52:00.233003Z","shell.execute_reply.started":"2022-03-12T04:52:00.226146Z","shell.execute_reply":"2022-03-12T04:52:00.232115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pipeline(train)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:52:00.235917Z","iopub.execute_input":"2022-03-12T04:52:00.236568Z","iopub.status.idle":"2022-03-12T06:32:36.537362Z","shell.execute_reply.started":"2022-03-12T04:52:00.236533Z","shell.execute_reply":"2022-03-12T06:32:36.536255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# 6. Model Embeddings\n\nNow that we have let the model see the images, we can take the **parameters** from the second to last layer (not the one that creates the output, but the one before that) and use it to **create the image embeddings**.\n\n🐳 **Image embeddings** are the *juice* of an image, the very core of it's *information*. This layer has learned everything there is to know about each image and each individual. Moreover, as the model becomes more accurate, the embeddings become more precise too, not only in classifying the individuals, but also being able to recognize the differences between them.\n\n## 6.1 Retrieve the Embeddings\n\nWe will use `torch.load()` to load into a model the pretrained weights & biases that we have created during the Classification task.","metadata":{}},{"cell_type":"code","source":"#pretrained_name = \"EffNetB0_fold_0_loss_14.979\"\n\n# Path to trained model parameters (i.e. weights and biases)\n#classif_model_path = f\"../input/happywhale-2022/{pretrained_name}.pt\"\nclassif_model_path='./EffNetB4_fold_0_loss_11.334.pt'\n# Load the model and append learned params\nmodel = HappyWhaleModel(MODEL_NAME, NUM_CLASSES, NO_NEURONS, EMBEDDING_SIZE).to(device)\nmodel.load_state_dict(torch.load(classif_model_path))","metadata":{"execution":{"iopub.status.busy":"2022-03-12T06:46:08.568871Z","iopub.execute_input":"2022-03-12T06:46:08.569143Z","iopub.status.idle":"2022-03-12T06:46:09.351943Z","shell.execute_reply.started":"2022-03-12T06:46:08.569113Z","shell.execute_reply":"2022-03-12T06:46:09.351255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T06:47:23.626794Z","iopub.execute_input":"2022-03-12T06:47:23.627102Z","iopub.status.idle":"2022-03-12T06:47:23.863164Z","shell.execute_reply.started":"2022-03-12T06:47:23.627062Z","shell.execute_reply":"2022-03-12T06:47:23.862486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataLoader\ndataset = HappyWhaleDataset(train, trainFlag=False)\ndataloader = DataLoader(dataset, batch_size=64, shuffle=False)\n\n# Retrieve all embeddings for each image\nall_embeddings = []\n\nmodel.eval()\nwith torch.no_grad():\n for image,target in tqdm(dataloader): \n image ,target= image.to(device), target.to(device)\n _, embedding = model(image,target)\n embedding = embedding.detach().cpu().numpy()\n all_embeddings.append(embedding)\n \n# Concatenate batches together\nimage_embeddings = np.concatenate(all_embeddings)\n\n# Save embeddings and corresponding image\n#np.save('../input/happywhale-2022/effnet_test_embeddings.npy', image_embeddings)\n# Save embeddings and corresponding image\nnp.save('effnet_train_512embeddings.npy', image_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T06:48:04.877201Z","iopub.execute_input":"2022-03-12T06:48:04.877468Z","iopub.status.idle":"2022-03-12T06:48:05.779054Z","shell.execute_reply.started":"2022-03-12T06:48:04.877437Z","shell.execute_reply":"2022-03-12T06:48:05.777860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-03-05T13:23:23.703925Z","iopub.execute_input":"2022-03-05T13:23:23.704528Z","iopub.status.idle":"2022-03-05T13:23:23.720659Z","shell.execute_reply.started":"2022-03-05T13:23:23.704491Z","shell.execute_reply":"2022-03-05T13:23:23.719785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 🐝Save embeddings to W&B\nsave_dataset_artifact(run_name=\"EffNetB0_fold_1_loss_15\", \n artifact_name=\"EffNetB0_fold_1_loss_15\", \n path=\"../input/happywhale-2022/EffNetB0_fold_1_loss_15.216.npy\")","metadata":{"execution":{"iopub.status.busy":"2022-03-04T08:40:45.060021Z","iopub.execute_input":"2022-03-04T08:40:45.06028Z","iopub.status.idle":"2022-03-04T08:40:50.623044Z","shell.execute_reply.started":"2022-03-04T08:40:45.060252Z","shell.execute_reply":"2022-03-04T08:40:50.619754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. RAPIDS Clustering\n\nLast step is to create the clusters. The `k_neighbors` will be set to 5, as this is the maximum number to predict ([as stated in competition guidelines](https://www.kaggle.com/c/happy-whale-and-dolphin/overview/evaluation)).","metadata":{}},{"cell_type":"code","source":"# RAPIDS Libraries\nimport numpy as np\nfrom tqdm import tqdm\nfrom cuml.neighbors import NearestNeighbors\nimage_embeddings=np.load('../input/happywhale-2022/effnet_image_embeddings.npy')\nimage_embeddings=np.load('../input/happywhale-2022/EffNetB0_fold_1_loss_15.216.npy')","metadata":{"execution":{"iopub.status.busy":"2022-03-05T08:55:35.228765Z","iopub.execute_input":"2022-03-05T08:55:35.229225Z","iopub.status.idle":"2022-03-05T08:55:35.248461Z","shell.execute_reply.started":"2022-03-05T08:55:35.229187Z","shell.execute_reply":"2022-03-05T08:55:35.247787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === CLUSTERING ===\nknn_model = NearestNeighbors(n_neighbors=5)\n# Train the model\nknn_model.fit(image_embeddings)\n\n# Infer on the training data\n# distances - the distance between each point in the group\n# indices - the index row of each image\ndistances, indices = knn_model.kneighbors(image_embeddings)\nprint(image_embeddings.shape)\nprint(distances)\nprint(indices)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T08:55:39.647901Z","iopub.execute_input":"2022-03-05T08:55:39.648185Z","iopub.status.idle":"2022-03-05T08:55:39.976186Z","shell.execute_reply.started":"2022-03-05T08:55:39.64815Z","shell.execute_reply":"2022-03-05T08:55:39.975364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === PREDICTION ===\n# Create the grouped predictions based on distances & indices\npredictions = {\"images\": [], \"embeddings\": []}\nfor i in tqdm(range(len(image_embeddings))):\n index = np.where(distances[i, ] < 6.0)[0]\n split = indices[i, index]\n \n grouped_images = train.iloc[split][\"image\"].values\n grouped_embeddings = image_embeddings[split]\n\n predictions[\"images\"].append(grouped_images)\n predictions[\"embeddings\"].append(grouped_embeddings)\n#print(predictions[\"images\"]) ","metadata":{"execution":{"iopub.status.busy":"2022-03-05T09:31:12.462094Z","iopub.execute_input":"2022-03-05T09:31:12.462348Z","iopub.status.idle":"2022-03-05T09:31:27.515645Z","shell.execute_reply.started":"2022-03-05T09:31:12.462319Z","shell.execute_reply":"2022-03-05T09:31:27.514819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compute Cos Distance\n\nBelow you can see a few examples of **clustered train embeddings** and the cos distance similarity between them.","metadata":{}},{"cell_type":"code","source":"def get_cosine_similarity(embeddings):\n '''Compute cos distance between n embedding vector and itself.'''\n similarity_matrix = []\n \n for embed1 in embeddings:\n similarity_row = []\n for embed2 in embeddings:\n similarity_row.append(1 - spatial.distance.cosine(embed1, embed2))\n similarity_matrix.append(similarity_row)\n \n return np.array(similarity_matrix, dtype=\"float32\")\n\n\ndef plot_heatmap(example_paths, cos_matrix, mask):\n '''Computes cos distance heatmap.'''\n \n # Plots\n fig = plt.figure(figsize=(12, 12))\n ax1 = plt.subplot2grid(shape=(6, 6), loc=(5, 1), colspan=1)\n ax2 = plt.subplot2grid(shape=(6, 6), loc=(5, 2), colspan=1)\n ax3 = plt.subplot2grid(shape=(6, 6), loc=(5, 3), colspan=1)\n ax4 = plt.subplot2grid(shape=(6, 6), loc=(5, 4), colspan=1)\n ax5 = plt.subplot2grid(shape=(6, 6), loc=(5, 5), colspan=1)\n h_axes = [ax1, ax2, ax3, ax4, ax5]\n\n ax6 = plt.subplot2grid(shape=(6, 6), loc=(0, 0), colspan=1)\n ax7 = plt.subplot2grid(shape=(6, 6), loc=(1, 0), colspan=1)\n ax8 = plt.subplot2grid(shape=(6, 6), loc=(2, 0), colspan=1)\n ax9 = plt.subplot2grid(shape=(6, 6), loc=(3, 0), colspan=1)\n ax10 = plt.subplot2grid(shape=(6, 6), loc=(4, 0), colspan=1)\n v_axes = [ax6, ax7, ax8, ax9, ax10]\n\n ax11 = plt.subplot2grid(shape=(6, 6), loc=(0, 1), colspan=5, rowspan=5)\n\n fig.suptitle('- Cosine Distance -', size = 21, color = my_colors[7], weight='bold')\n for k, ax in enumerate(h_axes):\n ax.imshow(plt.imread(example_paths[k]))\n ax.set_axis_off()\n\n for k, ax in enumerate(v_axes):\n ax.imshow(plt.imread(example_paths[k]))\n ax.set_axis_off()\n\n sns.heatmap(cos_matrix, ax=ax11, fmt=\".5\",\n cbar=False, annot=True, linewidths=0.5, mask=mask, square=True, cmap=\"winter_r\")\n\n plt.tight_layout()\n plt.show();","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-05T09:31:28.849284Z","iopub.execute_input":"2022-03-05T09:31:28.849555Z","iopub.status.idle":"2022-03-05T09:31:28.868933Z","shell.execute_reply.started":"2022-03-05T09:31:28.849513Z","shell.execute_reply":"2022-03-05T09:31:28.86818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a clustered group\ngroup = 7\n\nexample_paths = [\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"+img \\\n for img in predictions[\"images\"][group]]\nexample_embeds = predictions[\"embeddings\"][group]\n# Compute similarity matrix\ncos_matrix = get_cosine_similarity(example_embeds)\nmask = np.zeros_like(cos_matrix)\nmask[np.triu_indices_from(mask)] = True\nprint(cos_matrix)\nplot_heatmap(example_paths, cos_matrix, mask)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T09:34:15.54904Z","iopub.execute_input":"2022-03-05T09:34:15.549306Z","iopub.status.idle":"2022-03-05T09:34:16.56817Z","shell.execute_reply.started":"2022-03-05T09:34:15.549275Z","shell.execute_reply":"2022-03-05T09:34:16.567485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a clustered group\ngroup = 24\n\nexample_paths = [\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"+img \\\n for img in predictions[\"images\"][group]]\nexample_embeds = predictions[\"embeddings\"][group]\n\n# Compute similarity matrix\ncos_matrix = get_cosine_similarity(example_embeds)\nmask = np.zeros_like(cos_matrix)\nmask[np.triu_indices_from(mask)] = True\n\nplot_heatmap(example_paths, cos_matrix, mask)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T08:50:07.292719Z","iopub.execute_input":"2022-03-04T08:50:07.29299Z","iopub.status.idle":"2022-03-04T08:50:08.566114Z","shell.execute_reply.started":"2022-03-04T08:50:07.292961Z","shell.execute_reply":"2022-03-04T08:50:08.562156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a clustered group\ngroup = 100\n\nexample_paths = [\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"+img \\\n for img in predictions[\"images\"][group]]\nexample_embeds = predictions[\"embeddings\"][group]\n\n# Compute similarity matrix\ncos_matrix = get_cosine_similarity(example_embeds)\nmask = np.zeros_like(cos_matrix)\nmask[np.triu_indices_from(mask)] = True\n\nplot_heatmap(example_paths, cos_matrix, mask)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T08:50:32.776556Z","iopub.execute_input":"2022-03-04T08:50:32.777299Z","iopub.status.idle":"2022-03-04T08:50:33.870029Z","shell.execute_reply.started":"2022-03-04T08:50:32.777257Z","shell.execute_reply":"2022-03-04T08:50:33.869339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. Inference\n\n### WIP","metadata":{}},{"cell_type":"code","source":"# RAPIDS Libraries\nimport numpy as np\nfrom tqdm import tqdm\nfrom cuml.neighbors import NearestNeighbors\ntrain_embeddings=np.load('../input/happywhale-2022/EffNetB0_fold_1_loss_15.216.npy')\ntest_embeddings=np.load('effnet_test_embeddings.npy')\n\n# === CLUSTERING ===\nknn_model = NearestNeighbors(n_neighbors=5)\n# Train the model\nknn_model.fit(train_embeddings)\n# Infer on the training data\n# distances - the distance between each point in the group\n# indices - the index row of each image\ndistances, indices = knn_model.kneighbors(test_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T13:47:16.371331Z","iopub.execute_input":"2022-03-05T13:47:16.371586Z","iopub.status.idle":"2022-03-05T13:47:30.843915Z","shell.execute_reply.started":"2022-03-05T13:47:16.371556Z","shell.execute_reply":"2022-03-05T13:47:30.843209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the data\nfrom IPython.display import display_html\nimport matplotlib.pyplot as plt\n\ntrain = pd.read_csv(\"../input/happywhale-2022/train.csv\")\ntest = pd.read_csv(\"../input/happywhale-2022/test.csv\")\n#display_html(test.head())\n#display_html(train.head())\ngroup=10010\ntrain_paths = [\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"\\\n +img for img in train.iloc[indices[group]][\"image\"]]\ntest_path=[\"../input/whale2-cropped-dataset/cropped_test_images/cropped_test_images/\"\\\n +test.iloc[group][\"image\"]]\nprint(test_path)\nfig1 = plt.figure(figsize=(12, 12))\nax1 = plt.subplot2grid(shape=(6, 6), loc=(0, 0), colspan=1)\nax2 = plt.subplot2grid(shape=(6, 6), loc=(1, 0), colspan=1)\nax3 = plt.subplot2grid(shape=(6, 6), loc=(2, 0), colspan=1)\nax4 = plt.subplot2grid(shape=(6, 6), loc=(3, 0), colspan=1)\nax5 = plt.subplot2grid(shape=(6, 6), loc=(4, 0), colspan=1)\nax0 = plt.subplot2grid(shape=(6, 6), loc=(0, 1), colspan=5, rowspan=5)\nh_axes = [ax1, ax2, ax3, ax4, ax5]\nfor k, ax in enumerate(h_axes):\n ax.imshow(plt.imread(train_paths[k]))\n ax.set_axis_off()\nax0.imshow(plt.imread(test_path[0]))\nax0.set_axis_off()\nplt.tight_layout()\nplt.show();\n","metadata":{"execution":{"iopub.status.busy":"2022-03-05T15:20:01.38188Z","iopub.execute_input":"2022-03-05T15:20:01.382571Z","iopub.status.idle":"2022-03-05T15:20:02.600989Z","shell.execute_reply.started":"2022-03-05T15:20:01.382532Z","shell.execute_reply":"2022-03-05T15:20:02.600339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nindividual_id= pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\nsubmission='image,predictions\\n'\nfor k in tqdm(range(len(test))):\n path=test.iloc[k][\"image\"]\n ids=individual_id.iloc[indices[k]][\"individual_id\"].tolist()\n predictions_id=''\n for i in range(4):\n predictions_id+=ids[i]+' '\n predictions_id+='new_individual'+'\\n'\n submission=submission+path+\",\"+predictions_id\n#print(submission)\nwith open(\"submission.csv\", \"w\", encoding='utf-8') as f:\n f.write(submission)\n f.close()","metadata":{"execution":{"iopub.status.busy":"2022-03-05T16:48:43.349021Z","iopub.execute_input":"2022-03-05T16:48:43.350094Z","iopub.status.idle":"2022-03-05T16:48:58.202214Z","shell.execute_reply.started":"2022-03-05T16:48:43.350044Z","shell.execute_reply":"2022-03-05T16:48:58.201448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n##临时保存\nsimilarity=[]\nindex=[]\nrows_in_slice = 10000\nslice_start = 0\nslice_end = slice_start + rows_in_slice\nwhile slice_end <= test_embeddings.shape[0]:\n start=time.time()\n S,I=get_cos_similar_matrix(test_embeddings[slice_start:slice_end,:],train_embeddings)\n similarity.append(S)\n index.append(I)\n slice_start += rows_in_slice\n slice_end = slice_start + rows_in_slice\n print(\"计算花费时间\",str(time.time()-start),'时间 到达',str(slice_end))\nsimilarity = np.concatenate(similarity)\nindex = np.concatenate(index)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T16:09:45.172222Z","iopub.execute_input":"2022-03-05T16:09:45.172778Z","iopub.status.idle":"2022-03-05T16:09:45.18065Z","shell.execute_reply.started":"2022-03-05T16:09:45.172738Z","shell.execute_reply":"2022-03-05T16:09:45.179898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n### 🐝 W&B Dashboard\n\n> My [W&B Dashboard](https://wandb.ai/andrada/happywhale?workspace=user-andrada).\n\n
\n\n
\n\n### My Specs\n\n* 🖥 Z8 G4 Workstation\n* 💾 2 CPUs & 96GB Memory\n* 🎮 NVIDIA Quadro RTX 8000\n* 💻 Zbook Studio G7 on the go","metadata":{}},{"cell_type":"markdown","source":"# 8.1 余弦间距推断","metadata":{}},{"cell_type":"code","source":"# RAPIDS Libraries\nimport numpy as np\nfrom tqdm import tqdm\n#from cuml.neighbors import NearestNeighbors\nfrom scipy import spatial\nimport time\n","metadata":{"execution":{"iopub.status.busy":"2022-03-11T08:52:14.189908Z","iopub.execute_input":"2022-03-11T08:52:14.190469Z","iopub.status.idle":"2022-03-11T08:52:14.195296Z","shell.execute_reply.started":"2022-03-11T08:52:14.190430Z","shell.execute_reply":"2022-03-11T08:52:14.193724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings=np.load('../input/myhappywhale/effnet_train_256embeddings.npy')\ntest_embeddings=np.load('../input/myhappywhale/effnet_test_256embeddings.npy')\n\ndef get_cosine_similarity(test_embeddings,train_embeddings):\n '''Compute cos distance between n embedding vector and itself.'''\n similarity_matrix = []\n similarity_index_mat=[]\n for embed1 in tqdm(test_embeddings):\n similarity_row = []\n #start=time.time()\n for embed2 in train_embeddings:\n similarity_row.append(1 - spatial.distance.cosine(embed1, embed2))\n #mid=time.time()\n sorted_id = sorted(range(len(similarity_row)), key=lambda k: similarity_row[k], reverse=True)[0:5]\n #end=time.time()\n similarity_index_mat.append(sorted_id)\n similarity_matrix.append([similarity_row[i] for i in sorted_id])\n #print(\"计算\",str(mid-start),\"排序：\",str(end-mid))\n return np.array(similarity_matrix, dtype=\"float32\"),np.array(similarity_index_mat, dtype=\"float32\")\ndef get_cos_similar_matrix(v1, v2):\n num = np.dot(v1, np.array(v2).T) # 向量点乘\n denom = np.linalg.norm(v1, axis=1).reshape(-1, 1) * np.linalg.norm(v2, axis=1) # 求模长的乘积\n res = num / denom\n #res[np.isneginf(res)] = 0\n index=np.argsort(res, axis=1)[:,::-1][:,0:5]#按行排序得到索引 取前5列\n res=np.sort(res,axis=1)[:,::-1][:,0:5]\n return res,index\nsimilarity,index=get_cos_similar_matrix(test_embeddings[:1000,:],train_embeddings[:,:])\nprint(similarity.shape,index.shape) \n#similarity,index=get_cosine_similarity(test_embeddings[:2,:],train_embeddings[:100,:])\n#print(similarity,index)\n#np.save(\"train_similarity_10000npy\",similarity)\n#np.save(\"train_index_10000.npy\",index)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T08:52:34.413972Z","iopub.execute_input":"2022-03-11T08:52:34.414270Z","iopub.status.idle":"2022-03-11T08:52:42.745875Z","shell.execute_reply.started":"2022-03-11T08:52:34.414237Z","shell.execute_reply":"2022-03-11T08:52:42.744044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.2余弦相似度导入","metadata":{}},{"cell_type":"code","source":"index_10000=np.load('../input/whales-dolphins-effnet-train-rapids-clusters/index_10000.npy')\nindex_20000=np.load('../input/whales-dolphins-effnet-train-rapids-clusters/index_20000.npy')\nindex_27956=np.load('../input/whales-dolphins-effnet-train-rapids-clusters/index_27956.npy')\nindex=np.concatenate((index_10000,index_20000,index_27956),axis=0)\nsimilarity_10000=np.load('../input/whales-dolphins-effnet-train-rapids-clusters/similarity_10000.npy')\nsimilarity_20000=np.load('../input/whales-dolphins-effnet-train-rapids-clusters/similarity_20000.npy')\nsimilarity_27956=np.load('../input/whales-dolphins-effnet-train-rapids-clusters/similarity_27956.npy')\nsimilarity=np.concatenate((similarity_10000,similarity_20000,similarity_27956),axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T01:07:19.45473Z","iopub.execute_input":"2022-03-08T01:07:19.455076Z","iopub.status.idle":"2022-03-08T01:07:19.624684Z","shell.execute_reply.started":"2022-03-08T01:07:19.455041Z","shell.execute_reply":"2022-03-08T01:07:19.623958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the data\nfrom IPython.display import display_html\nimport matplotlib.pyplot as plt\nimport pandas as pd\ntrain = pd.read_csv(\"../input/happywhale-2022/train.csv\")\ntest = pd.read_csv(\"../input/happywhale-2022/test.csv\")\n#display_html(test.head())\n#display_html(train.head())\ngroup=0\ntrain_paths = [\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"\\\n +img for img in train.iloc[index[group,:]][\"image\"]]\ntest_path=[\"../input/whale2-cropped-dataset/cropped_test_images/cropped_test_images/\"\\\n +test.iloc[group][\"image\"]]\nindividual_id= pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\nids=individual_id.iloc[index[group,:]][\"individual_id\"].tolist()\nprint(train_paths)\nprint(\"余弦相似度：\",similarity[group,:])\n#print(ids)\nfig1 = plt.figure(figsize=(12, 12))\nax1 = plt.subplot2grid(shape=(6, 6), loc=(0, 0), colspan=1)\nax2 = plt.subplot2grid(shape=(6, 6), loc=(1, 0), colspan=1)\nax3 = plt.subplot2grid(shape=(6, 6), loc=(2, 0), colspan=1)\nax4 = plt.subplot2grid(shape=(6, 6), loc=(3, 0), colspan=1)\nax5 = plt.subplot2grid(shape=(6, 6), loc=(4, 0), colspan=1)\nax0 = plt.subplot2grid(shape=(6, 6), loc=(0, 1), colspan=5, rowspan=5)\nh_axes = [ax1, ax2, ax3, ax4, ax5]\nfor k, ax in enumerate(h_axes):\n ax.imshow(plt.imread(train_paths[k]))\n ax.set_title(\"ID:\"+ids[k])\n ax.set_axis_off()\nax0.imshow(plt.imread(test_path[0]))\nax0.set_title(str(test_path))\nax0.set_axis_off()\nplt.tight_layout()\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-03-11T09:56:02.583554Z","iopub.execute_input":"2022-03-11T09:56:02.584102Z","iopub.status.idle":"2022-03-11T09:56:03.953705Z","shell.execute_reply.started":"2022-03-11T09:56:02.584062Z","shell.execute_reply":"2022-03-11T09:56:03.952981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the data\nfrom IPython.display import display_html\nimport matplotlib.pyplot as plt\nimport pandas as pd\nsimilarity=np.load('./train_similarity_10000npy.npy')\nindex=np.load('./train_index_10000.npy')\ntrain = pd.read_csv(\"../input/happywhale-2022/train.csv\")\ntest = pd.read_csv(\"../input/happywhale-2022/train.csv\")\n#display_html(test.head())\n#display_html(train.head())\ngroup=48\ntrain_paths = [\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"\\\n +img for img in train.iloc[index[group,:]][\"image\"]]\ntest_path=[\"../input/whale2-cropped-dataset/cropped_train_images/cropped_train_images/\"\\\n +test.iloc[group][\"image\"]]\ntest_id=test.iloc[group][\"individual_id\"]\nindividual_id= pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\nids=individual_id.iloc[index[group,:]][\"individual_id\"].tolist()\nprint(train_paths)\nprint(\"余弦相似度：\",similarity[group,:])\nprint(test_path)\n#print(ids)\nfig1 = plt.figure(figsize=(12, 12))\nax1 = plt.subplot2grid(shape=(6, 6), loc=(0, 0), colspan=1)\nax2 = plt.subplot2grid(shape=(6, 6), loc=(1, 0), colspan=1)\nax3 = plt.subplot2grid(shape=(6, 6), loc=(2, 0), colspan=1)\nax4 = plt.subplot2grid(shape=(6, 6), loc=(3, 0), colspan=1)\nax5 = plt.subplot2grid(shape=(6, 6), loc=(4, 0), colspan=1)\nax0 = plt.subplot2grid(shape=(6, 6), loc=(0, 1), colspan=5, rowspan=5)\nh_axes = [ax1, ax2, ax3, ax4, ax5]\nfor k, ax in enumerate(h_axes):\n ax.imshow(plt.imread(train_paths[k]))\n ax.set_title(\"ID:\"+ids[k])\n ax.set_axis_off()\nax0.imshow(plt.imread(test_path[0]))\nax0.set_title(str(test_id))\nax0.set_axis_off()\nplt.tight_layout()\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-03-11T08:28:59.169421Z","iopub.execute_input":"2022-03-11T08:28:59.169751Z","iopub.status.idle":"2022-03-11T08:29:00.794279Z","shell.execute_reply.started":"2022-03-11T08:28:59.169717Z","shell.execute_reply":"2022-03-11T08:29:00.793593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.3 保存结果文件","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\nindividual_id= pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\nsubmission='image,predictions\\n'\nsubmissionlist=[]\nfor k in tqdm(range(len(test))):\n path=test.iloc[k][\"image\"]\n ids=individual_id.iloc[index[k]][\"individual_id\"].tolist()\n predictions_id=''\n for i in range(5):\n if similarity[k,i]>=0.990:\n predictions_id+=ids[i]+' '\n else :\n predictions_id+=\"new_individual\"+' '\n predictions_id+='\\n'\n submissionlist.append(path+\",\"+predictions_id)\nsubmissionlist.sort()\n#print(submissionlist[:10])\nsubmission += ''.join(submissionlist)\nwith open(\"submission.csv\", \"w\", encoding='utf-8') as f:\n f.write(submission)\n f.close()","metadata":{"execution":{"iopub.status.busy":"2022-03-11T10:01:11.651149Z","iopub.execute_input":"2022-03-11T10:01:11.651433Z","iopub.status.idle":"2022-03-11T10:01:25.268343Z","shell.execute_reply.started":"2022-03-11T10:01:11.651403Z","shell.execute_reply":"2022-03-11T10:01:25.267568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9 KNN实现推理","metadata":{"execution":{"iopub.status.busy":"2022-03-08T01:33:37.654153Z","iopub.execute_input":"2022-03-08T01:33:37.654778Z","iopub.status.idle":"2022-03-08T01:33:37.672873Z","shell.execute_reply.started":"2022-03-08T01:33:37.654736Z","shell.execute_reply":"2022-03-08T01:33:37.671955Z"}}},{"cell_type":"code","source":"# RAPIDS Libraries\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.neighbors import NearestNeighbors\nfrom scipy import spatial\nimport pandas as pd\nimport time\ntrain_embeddings=np.load('../input/myhappywhale/effnet_train_256embeddings.npy')\ntest_embeddings=np.load('../input/myhappywhale/effnet_test_256embeddings.npy')\ntrain_id=pd.read_csv('../input/happy-whale-and-dolphin/train.csv')\nneigh = NearestNeighbors(n_neighbors=5,metric='cosine')\nneigh.fit(train_embeddings)\nGROUP=48\ndistances,idxs=neigh.kneighbors(test_embeddings[:][:].reshape(-1,256), 100, return_distance=True)\nprint(1-distances)\nprint(idxs.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T09:54:35.028653Z","iopub.execute_input":"2022-03-11T09:54:35.028937Z","iopub.status.idle":"2022-03-11T09:55:03.972156Z","shell.execute_reply.started":"2022-03-11T09:54:35.028904Z","shell.execute_reply":"2022-03-11T09:55:03.971394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"similarity=1-distances\nindex=idxs","metadata":{"execution":{"iopub.status.busy":"2022-03-11T10:00:57.313036Z","iopub.execute_input":"2022-03-11T10:00:57.313779Z","iopub.status.idle":"2022-03-11T10:00:57.319545Z","shell.execute_reply.started":"2022-03-11T10:00:57.313732Z","shell.execute_reply":"2022-03-11T10:00:57.318597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GROUP=0\nprint(idxs.tolist()[0])\ntrain_id=pd.read_csv('../input/happy-whale-and-dolphin/train.csv')\nprint(train_id.iloc[GROUP])\ntemp=train_id.iloc[(idxs).tolist()[0],:]#[\"individual_id\"].value_counts()\nprint(temp.head(20))\n","metadata":{"execution":{"iopub.status.busy":"2022-03-11T09:59:53.159383Z","iopub.execute_input":"2022-03-11T09:59:53.160158Z","iopub.status.idle":"2022-03-11T09:59:53.577848Z","shell.execute_reply.started":"2022-03-11T09:59:53.160116Z","shell.execute_reply":"2022-03-11T09:59:53.577100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}