{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\n

๐Ÿณ Part I. Data Understanding ๐Ÿฌ

\n\n> **Competition Goal:** Identify and group all images that contain the same individual through time.\n\n### โฌ‡ Libraries","metadata":{}},{"cell_type":"code","source":"!pip install imagesize","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-01T16:55:33.462054Z","iopub.execute_input":"2022-03-01T16:55:33.463144Z","iopub.status.idle":"2022-03-01T16:55:40.823000Z","shell.execute_reply.started":"2022-03-01T16:55:33.463102Z","shell.execute_reply":"2022-03-01T16:55:40.822303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Libraries\nimport os\nimport sys\nimport wandb\nimport time\nimport random\nfrom tqdm import tqdm\nimport warnings\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.patches as patches\nimport matplotlib.pyplot as plt\nfrom IPython.display import display_html\nimport imagesize\nfrom sklearn.model_selection import StratifiedKFold\n\n# Environment check\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"WANDB_SILENT\"] = \"true\"\nCONFIG = {'competition': 'happywhale', '_wandb_kernel': 'aot'}\n\n# Custom colors\nclass clr:\n S = '\\033[1m' + '\\033[96m'\n E = '\\033[0m'\n \nmy_colors = [\"#21295C\", \"#1F4E78\", \"#1C7293\", \"#73ABAF\", \"#C9E4CA\", \"#87BBA2\", \"#618E83\", \"#3B6064\"]\nprint(clr.S+\"Notebook Color Scheme:\"+clr.E)\nsns.palplot(sns.color_palette(my_colors))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:55:40.824582Z","iopub.execute_input":"2022-03-01T16:55:40.824839Z","iopub.status.idle":"2022-03-01T16:55:40.917488Z","shell.execute_reply.started":"2022-03-01T16:55:40.824796Z","shell.execute_reply":"2022-03-01T16:55:40.916586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ๐Ÿ W&B Fork & Run\n\nIn order to run this notebook you will need to input your own **secret API key** within the `! wandb login $secret_value_0` line. \n\n๐Ÿ**How do you get your own API key?**\n\nSuper simple! Go to **https://wandb.ai/site** -> Login -> Click on your profile in the top right corner -> Settings -> Scroll down to API keys -> copy your very own key (for more info check [this amazing notebook for ML Experiment Tracking on Kaggle](https://www.kaggle.com/ayuraj/experiment-tracking-with-weights-and-biases)).\n\n
","metadata":{}},{"cell_type":"code","source":"# ๐Ÿ Secrets\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb\")\n\n! wandb login $secret_value_0","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:55:40.919109Z","iopub.execute_input":"2022-03-01T16:55:40.919414Z","iopub.status.idle":"2022-03-01T16:55:43.481132Z","shell.execute_reply.started":"2022-03-01T16:55:40.919375Z","shell.execute_reply":"2022-03-01T16:55:43.480214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### โฌ‡ Helper Functions","metadata":{}},{"cell_type":"code","source":"def show_values_on_bars(axs, h_v=\"v\", space=0.4):\n '''Plots the value at the end of the a seaborn barplot.\n axs: the ax of the plot\n h_v: weather or not the barplot is vertical/ horizontal'''\n \n def _show_on_single_plot(ax):\n if h_v == \"v\":\n for p in ax.patches:\n _x = p.get_x() + p.get_width() / 2\n _y = p.get_y() + p.get_height()\n value = int(p.get_height())\n ax.text(_x, _y, format(value, ','), ha=\"center\") \n elif h_v == \"h\":\n for p in ax.patches:\n _x = p.get_x() + p.get_width() + float(space)\n _y = p.get_y() + p.get_height()\n value = int(p.get_width())\n ax.text(_x, _y, format(value, ','), ha=\"left\")\n\n if isinstance(axs, np.ndarray):\n for idx, ax in np.ndenumerate(axs):\n _show_on_single_plot(ax)\n else:\n _show_on_single_plot(axs)\n\n\n# === ๐Ÿ W&B ===\ndef save_dataset_artifact(run_name, artifact_name, path):\n '''Saves dataset to W&B Artifactory.\n run_name: name of the experiment\n artifact_name: under what name should the dataset be stored\n path: path to the dataset'''\n \n run = wandb.init(project='happywhale', \n name=run_name, \n config=CONFIG)\n artifact = wandb.Artifact(name=artifact_name, \n type='dataset')\n artifact.add_file(path)\n\n wandb.log_artifact(artifact)\n wandb.finish()\n print(\"Artifact has been saved successfully.\")\n \n \ndef create_wandb_plot(x_data=None, y_data=None, x_name=None, y_name=None, title=None, log=None, plot=\"line\"):\n '''Create and save lineplot/barplot in W&B Environment.\n x_data & y_data: Pandas Series containing x & y data\n x_name & y_name: strings containing axis names\n title: title of the graph\n log: string containing name of log'''\n \n data = [[label, val] for (label, val) in zip(x_data, y_data)]\n table = wandb.Table(data=data, columns = [x_name, y_name])\n \n if plot == \"line\":\n wandb.log({log : wandb.plot.line(table, x_name, y_name, title=title)})\n elif plot == \"bar\":\n wandb.log({log : wandb.plot.bar(table, x_name, y_name, title=title)})\n elif plot == \"scatter\":\n wandb.log({log : wandb.plot.scatter(table, x_name, y_name, title=title)})\n \n \ndef create_wandb_hist(x_data=None, x_name=None, title=None, log=None):\n '''Create and save histogram in W&B Environment.\n x_data: Pandas Series containing x values\n x_name: strings containing axis name\n title: title of the graph\n log: string containing name of log'''\n \n data = [[x] for x in x_data]\n table = wandb.Table(data=data, columns=[x_name])\n wandb.log({log : wandb.plot.histogram(table, x_name, title=title)})","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T16:55:43.484322Z","iopub.execute_input":"2022-03-01T16:55:43.484660Z","iopub.status.idle":"2022-03-01T16:55:43.500517Z","shell.execute_reply.started":"2022-03-01T16:55:43.484619Z","shell.execute_reply":"2022-03-01T16:55:43.499750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Metadata Cleaning\n\n๐Ÿฌ **Species column adjustement:**\n* `bottlenose_dolpin` -> `bottlenose_dolphin`\n* `kiler_whale` -> `killer_whale`\n* `beluga` -> `beluga_whale`\n* `globis` & `pilot_whale` -> `short_finned_pilot_whale` (due to extreme similarities [according to this discussion](https://www.kaggle.com/c/happy-whale-and-dolphin/discussion/305909))","metadata":{}},{"cell_type":"code","source":"# Import the training data\ntrain = pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\n\n# Adjust typos in \"species\" column\ntrain[\"species\"] = train[\"species\"].replace([\"bottlenose_dolpin\", \"kiler_whale\",\n \"beluga\", \n \"globis\", \"pilot_whale\"],\n [\"bottlenose_dolphin\", \"killer_whale\",\n \"beluga_whale\", \n \"short_finned_pilot_whale\", \"short_finned_pilot_whale\"])\n# Create a \"class\" column\ntrain[\"class\"] = train[\"species\"].apply(lambda x: x.split(\"_\")[-1])\n\n# Create path to train images\nTRAIN_PATH = \"../input/happy-whale-and-dolphin/train_images/\"\ntrain[\"path\"] = TRAIN_PATH + train[\"image\"]\n\n\n# --- Inspect ---\nprint(clr.S+\"--- TEST ---\"+clr.E)\nprint(clr.S+\"Total Number of Samples:\"+clr.E, len(os.listdir(\"../input/happy-whale-and-dolphin/test_images\")), \"\\n\")\nprint(clr.S+\"--- TRAIN ---\"+clr.E)\nprint(clr.S+\"Number of Missing Values:\"+clr.E, train.isna().sum().sum())\nprint(clr.S+\"Data Shape:\"+clr.E, train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:55:43.502496Z","iopub.execute_input":"2022-03-01T16:55:43.502854Z","iopub.status.idle":"2022-03-01T16:55:44.144020Z","shell.execute_reply.started":"2022-03-01T16:55:43.502821Z","shell.execute_reply":"2022-03-01T16:55:44.143316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Individual Analysis\n\n### ๐Ÿฌ Things to be noted:\n* There is an individual with 400 aparitions, which is an extreme outlier - a `minke_whale`\n* ~95% of inidividuals have less than 10 apparitions within the dataset\n* there are 60% \"new appearances\" - meaning that the ID appears only once","metadata":{}},{"cell_type":"code","source":"# ๐Ÿ W&B Experiment\nrun = wandb.init(project='happywhale', name='IndividualAnalysis', config=CONFIG)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:55:44.145149Z","iopub.execute_input":"2022-03-01T16:55:44.145473Z","iopub.status.idle":"2022-03-01T16:55:51.798955Z","shell.execute_reply.started":"2022-03-01T16:55:44.145445Z","shell.execute_reply":"2022-03-01T16:55:51.798198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_indivs = train[\"individual_id\"].value_counts().reset_index()\nindividuals = train[\"individual_id\"].value_counts().reset_index().head(1000)\nprint(clr.S+\"Total Unique IDs:\"+clr.E, all_indivs.shape[0])\nprint(clr.S+\"Max. Number of Apparitions:\"+clr.E, all_indivs[\"individual_id\"].max())\nprint(clr.S+\"Min. Number of Apparitions:\"+clr.E, all_indivs[\"individual_id\"].min())\n\n# Plots\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))\nfig.suptitle('- Individuals Analysis -', size = 26, color = my_colors[7], weight='bold')\naxs = [ax1, ax2]\n\nsns.histplot(data=individuals, x=\"individual_id\", ax=ax1, color=my_colors[1])\nax1.set_title(\"Distribution of appearances per individual - top 1000 in descending order\",\n size = 15, color = my_colors[6], weight='bold')\nax1.set_xlabel(\"Individual Numbers\", size = 13, color = my_colors[6], weight='bold')\nax1.set_ylabel(\"Count\", size = 13, color = my_colors[6], weight='bold')\nax1.arrow(x=270, y=90, dx=0, dy=-80, \n width = 0.05, head_width = 8, head_length=7, color=my_colors[6])\nax1.text(140, 100, '~95% have less than 10 apparitions', \n size=13, color=my_colors[6], weight='bold')\n\nsns.barplot(data=individuals.head(30), x=\"individual_id\", y=\"index\", ax=ax2, palette=\"Blues_r\")\nshow_values_on_bars(ax2, h_v=\"h\", space=0.4)\nax2.set_title(\"Top 30 IDs with most appearances\",\n size = 15, color = my_colors[6], weight='bold')\nax2.set_ylabel(\"Individual ID\", size = 13, color = my_colors[6], weight='bold')\nax2.set_xlabel(\"Frequency\", size = 13, color = my_colors[6], weight='bold')\nax2.set_xticks([])\nax2.axhspan(-0.5, 10.5, color=my_colors[4], alpha=0.3)\nax2.text(200, 5, 'Individuals with high', \n size=13, color=my_colors[6], weight='bold')\nax2.text(200, 6, 'number of apparitions', \n size=13, color=my_colors[6], weight='bold')\nax2.yaxis.set_tick_params(labelsize=12)\n\nsns.despine(left=True, bottom=True)\nplt.subplots_adjust(left=None, bottom=None, right=None, top=0.86, wspace=0.3, hspace=None);","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T16:55:51.800386Z","iopub.execute_input":"2022-03-01T16:55:51.800637Z","iopub.status.idle":"2022-03-01T16:55:53.678695Z","shell.execute_reply.started":"2022-03-01T16:55:51.800598Z","shell.execute_reply":"2022-03-01T16:55:53.678098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ๐Ÿ Log plots into W&B Dashboard\ncreate_wandb_plot(x_data=individuals.head(30)[\"index\"], \n y_data=individuals.head(30).individual_id, \n x_name=\"Individual ID\", y_name=\"Frequency\", \n title=\"-Top 30 IDs with most appearances-\", \n log=\"frames\", plot=\"bar\")\n\ncreate_wandb_hist(x_data=individuals[\"individual_id\"], \n x_name=\"Individual Numbers\", \n title=\"Distribution of Appearances per Individual\", \n log=\"hist1\")\n\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:55:53.679931Z","iopub.execute_input":"2022-03-01T16:55:53.680304Z","iopub.status.idle":"2022-03-01T16:56:03.978088Z","shell.execute_reply.started":"2022-03-01T16:55:53.680272Z","shell.execute_reply":"2022-03-01T16:56:03.977458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Species analysis\n\nBelow I have mapped and \"example\" for each species that appears within the dataset, in order to have a better understanding of the differences and similarities in appearance between the individuals.\n\n\n\n\n\n### ๐Ÿฌ Things to be noted:\n* There are ~70% whales and 30% unique dolphins within the dataset\n* The most common [unique] species are the `dusky dolphin` and the `humpback` and `blue whale`. These species have many **distinct inidividuals** within them.\n* Regarding the most common species overall (**not taking into account the number of times an inidividual appears**) are the `bottlenose dolphin` and the `beluga` and `humpback whale`. This is because of 2 reasons:\n * many of the top *20 individuals* with most appearances are from bottlenose dolphins and the humpback whales species\n * for beluga whale there are 800 unique individuals, but they appear on an average less than 10 times within the dataset.","metadata":{}},{"cell_type":"code","source":"# ๐Ÿ W&B Experiment\nrun = wandb.init(project='happywhale', name='SpeciesAnalysis', config=CONFIG)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:56:03.978977Z","iopub.execute_input":"2022-03-01T16:56:03.979187Z","iopub.status.idle":"2022-03-01T16:56:11.349119Z","shell.execute_reply.started":"2022-03-01T16:56:03.979161Z","shell.execute_reply":"2022-03-01T16:56:11.348057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"species_simple = train[\"species\"].value_counts().reset_index()\nspecies = train.groupby(by=[\"individual_id\", \"species\"]).count()\\\n .reset_index()[\"species\"].value_counts().reset_index()\nclasses = train.groupby(by=[\"individual_id\", \"class\"]).count()\\\n .reset_index()[\"class\"].value_counts().reset_index()\nprint(clr.S+\"Individuals that are WHALES:\"+clr.E, round(classes.iloc[0][1]/classes[\"class\"].sum()*100),\"%\")\nprint(clr.S+\"Individuals that are DOLPHINS:\"+clr.E, round(classes.iloc[1][1]/classes[\"class\"].sum()*100),\"%\")\n\n# Plots\nfig = plt.figure(figsize=(20, 15))\n\nax1= fig.add_subplot(1,2,1)\nax2= fig.add_subplot(2,2,2)\nax3= fig.add_subplot(2,2,4)\nfig.suptitle('- Species Analysis -', size = 26, color = my_colors[7], weight='bold')\n\nsns.barplot(data=classes, y=\"class\", x=\"index\", ax=ax1, palette=[my_colors[3], my_colors[4]])\nshow_values_on_bars(ax1, h_v=\"v\", space=0.4)\nax1.set_title(\"Class Frequency\",\n size = 15, color = my_colors[6], weight='bold')\nax1.set_ylabel(\"Frequency\", size = 13, color = my_colors[6], weight='bold')\nax1.set_xlabel(\"Class\", size = 13, color = my_colors[6], weight='bold')\n\nsns.barplot(data=species_simple, x=\"species\", y=\"index\", ax=ax2, palette=\"PuBuGn_r\",alpha=0.65)\nshow_values_on_bars(ax2, h_v=\"h\", space=0.4)\nax2.axhspan(-0.5, 3.5, color=my_colors[4], alpha=0.2)\nax2.set_title(\"Simple Count on Species\",\n size = 15, color = my_colors[6], weight='bold')\nax2.set_ylabel(\"Simple Species\", size = 13, color = my_colors[6], weight='bold')\nax2.set_xlabel(\"\", size = 13, color = my_colors[6], weight='bold')\nax2.set_xticks([])\nax2.yaxis.set_tick_params(labelsize=12)\n\nsns.barplot(data=species, x=\"species\", y=\"index\", ax=ax3, palette=\"PuBuGn_r\")\nshow_values_on_bars(ax3, h_v=\"h\", space=0.4)\nax3.axhspan(-0.5, 3.5, color=my_colors[4], alpha=0.2)\nax3.set_title(\"vs Unique Count on Species\",\n size = 15, color = my_colors[6], weight='bold')\nax3.set_ylabel(\"Unique Species\", size = 13, color = my_colors[6], weight='bold')\nax3.set_xlabel(\"\", size = 13, color = my_colors[6], weight='bold')\nax3.set_xticks([])\nax3.yaxis.set_tick_params(labelsize=12)\n\nsns.despine(left=True, bottom=True)\nplt.subplots_adjust(left=None, bottom=None, right=None, top=0.86, wspace=0.4, hspace=None);","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T16:56:11.352698Z","iopub.execute_input":"2022-03-01T16:56:11.353080Z","iopub.status.idle":"2022-03-01T16:56:13.320820Z","shell.execute_reply.started":"2022-03-01T16:56:11.353026Z","shell.execute_reply":"2022-03-01T16:56:13.320187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ๐Ÿ Log plots into W&B Dashboard\ncreate_wandb_plot(x_data=classes[\"index\"], \n y_data=classes[\"class\"], \n x_name=\"Class\", y_name=\"Count\", \n title=\"-Class Frequency-\", \n log=\"class\", plot=\"bar\")\n\ncreate_wandb_plot(x_data=species[\"index\"], \n y_data=species[\"species\"], \n x_name=\"Species\", y_name=\"Count\", \n title=\"-Unique Count on Species-\", \n log=\"species\", plot=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:56:13.322701Z","iopub.execute_input":"2022-03-01T16:56:13.323241Z","iopub.status.idle":"2022-03-01T16:56:14.187193Z","shell.execute_reply.started":"2022-03-01T16:56:13.323203Z","shell.execute_reply":"2022-03-01T16:56:14.186591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below you can see the 2 graphs that explain the shift between **simple count on species vs unique count on species**:","metadata":{}},{"cell_type":"code","source":"top_species = train[train[\"individual_id\"].isin(individuals.head(20)[\"index\"].tolist())][\"species\"]\\\n .value_counts().reset_index()\nbeluga = train[train[\"species\"]==\"beluga_whale\"][\"individual_id\"].value_counts().reset_index()\nbeluga_perc = round(beluga[beluga[\"individual_id\"]<=10].shape[0]/beluga.shape[0]*100)\n\n# Plots\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 7))\nfig.suptitle('- Miscellaneous -', size = 26, color = my_colors[7], weight='bold')\naxs = [ax1, ax2]\n\nsns.histplot(data=beluga, x=\"individual_id\", ax=ax1, color=my_colors[1])\nax1.set_title(\"Distribution of appearances on BELUGA Species\",\n size = 15, color = my_colors[6], weight='bold')\nax1.set_xlabel(\"Beluga Inidivual ID\", size = 13, color = my_colors[6], weight='bold')\nax1.set_ylabel(\"Count\", size = 13, color = my_colors[6], weight='bold')\nax1.arrow(x=8, y=200, dx=0, dy=-100, \n width = 0.05, head_width = 2, head_length=10, color=my_colors[6])\nax1.text(8, 210, f'{beluga_perc}% individuals have less that 10 apparitions', \n size=13, color=my_colors[6], weight='bold')\nax1.axvspan(-0.5, 10.5, color=my_colors[4], alpha=0.3)\n\nsns.barplot(data=top_species, x=\"species\", y=\"index\", ax=ax2, palette=\"Blues_r\")\nshow_values_on_bars(ax2, h_v=\"h\", space=0.4)\nax2.set_title(\"Top 10 IDs and their Species\",\n size = 15, color = my_colors[6], weight='bold')\nax2.set_ylabel(\"Individual ID\", size = 13, color = my_colors[6], weight='bold')\nax2.set_xlabel(\"Frequency\", size = 13, color = my_colors[6], weight='bold')\nax2.yaxis.set_tick_params(labelsize=13)\n\nsns.despine(left=True, bottom=True)\nplt.subplots_adjust(left=None, bottom=None, right=None, top=0.86, wspace=0.4, hspace=None);","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T16:56:14.189954Z","iopub.execute_input":"2022-03-01T16:56:14.190555Z","iopub.status.idle":"2022-03-01T16:56:15.197336Z","shell.execute_reply.started":"2022-03-01T16:56:14.190516Z","shell.execute_reply":"2022-03-01T16:56:15.196651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ๐Ÿ Log plots into W&B Dashboard\ncreate_wandb_hist(x_data=beluga[\"individual_id\"], \n x_name=\"Beliga Individual ID\", \n title=\"Distribution of Appearances on Beluga Species\", \n log=\"hist2\")\n\ncreate_wandb_plot(x_data=top_species[\"index\"], \n y_data=top_species[\"species\"], \n x_name=\"Species\", y_name=\"Count\", \n title=\"-Top 10 IDs and their Species-\", \n log=\"sp2\", plot=\"bar\")\n\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:56:15.198376Z","iopub.execute_input":"2022-03-01T16:56:15.198578Z","iopub.status.idle":"2022-03-01T16:56:28.205256Z","shell.execute_reply.started":"2022-03-01T16:56:15.198553Z","shell.execute_reply":"2022-03-01T16:56:28.204519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Images Analysis\n\n## I. Specimen Explore","metadata":{}},{"cell_type":"code","source":"# ๐Ÿ W&B Experiment\nrun = wandb.init(project='happywhale', name='ImageAnalysis', config=CONFIG)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:56:28.206897Z","iopub.execute_input":"2022-03-01T16:56:28.207526Z","iopub.status.idle":"2022-03-01T16:56:35.347111Z","shell.execute_reply.started":"2022-03-01T16:56:28.207482Z","shell.execute_reply":"2022-03-01T16:56:35.345938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image_species(species_name, sample_size):\n '''\n Shows a sample of n random images from a certain species. Logs the images to W&B as well.\n species_name: string containing the desired species name\n sample_size: number of random images to be printed on a row\n '''\n # Get image info\n data = train[train[\"species\"]==species_name].sample(sample_size, random_state=24)\n image_nr = data[\"image\"].to_list()\n image_path = data[\"path\"].to_list()\n\n # Plot\n fig, axs = plt.subplots(1, sample_size, figsize=(23, 4))\n axs = axs.flatten()\n wandb_images = []\n\n for k, path in enumerate(image_path):\n axs[k].set_title(f\"{k+1}. {species_name}-{image_nr[k]}\", \n fontsize = 13, color = my_colors[7], weight='bold')\n\n img = plt.imread(path)\n wandb_images.append(wandb.Image(img))\n axs[k].imshow(img)\n axs[k].axis(\"off\")\n\n plt.tight_layout()\n plt.show()\n\n # ๐Ÿ Log Image to W&B\n wandb.log({f\"{species_name}\": wandb_images})","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T16:56:35.349092Z","iopub.execute_input":"2022-03-01T16:56:35.352869Z","iopub.status.idle":"2022-03-01T16:56:35.861936Z","shell.execute_reply.started":"2022-03-01T16:56:35.352765Z","shell.execute_reply":"2022-03-01T16:56:35.860103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ๐Ÿฌ Things to be noted:\n* **image_size:** the images width and height is very different from one picture to another\n* **night view:** not all the pictures were made during the day - some of them were also caught during the night (see `beluga_whale` pictures 1 to 3)\n* **multiple individuals:** there are some pictures with 2 or more subjects present in it (see `cuviers_beaked_whale` picture 2 and `frasiers_dolphin` picture 2)\n* **ladscape:** in some of the images the subject is very close, however in others the landscape is much more predominant (see `minke_whale` pictures 3 and 4), which could impose some issues in identifying subtle characteristics within the individual\n* **additional noise:** there are some images that have digital marking on them that could pollute the algorithm (see `blue_whale` picture 1)","metadata":{}},{"cell_type":"code","source":"for species_name in train[\"species\"].unique().tolist():\n # Custom function to prin images & log into ๐ŸW&B\n show_image_species(species_name, sample_size=4)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T16:56:35.864048Z","iopub.execute_input":"2022-03-01T16:56:35.864399Z","iopub.status.idle":"2022-03-01T17:02:31.709644Z","shell.execute_reply.started":"2022-03-01T16:56:35.864361Z","shell.execute_reply":"2022-03-01T17:02:31.708785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## II. Same Individuals\n\nLet's also look at a few examples that contain the same individual.\n\n### ๐Ÿฌ Things to be noted:\n* **could there be duplicated images?** - in the first example there are 2 different images that look exactly the same ... at first glimpse. On a closer look, you can see some *very subtle* differences between the water waves. The pictures are taken mere moments appart.\n* **increased noice** - as we have seen above as well, there are many cases where the same individual appears onto very different backgrounds, angles (sometimes from the front, sometimes from the side) or shapes (in some cases we see a tail, in others a fin).\n* **lighting** - lighting is another pretty noisy aspect and it should be dealt with during the image augmentation phase.","metadata":{}},{"cell_type":"code","source":"def show_same_individual(example_id, sample_size):\n '''\n Shows a sample of n random images from a certain individual. Logs the images to W&B as well.\n example_id: string containing individual id to be displayed\n sample_size: number of random images to be printed on a row\n '''\n \n data = train[train[\"individual_id\"]==example_id].sample(sample_size, random_state=24)\n image_nr = data[\"image\"].to_list()\n image_path = data[\"path\"].to_list()\n\n # Plot\n fig, axs = plt.subplots(2, round(sample_size/2), figsize=(23, 6))\n fig.suptitle(f'- Individual {example_id} -', size = 20, color = my_colors[7], weight='bold')\n axs = axs.flatten()\n wandb_images = []\n\n for k, path in enumerate(image_path):\n axs[k].set_title(f\"Img. {image_nr[k]}\", \n fontsize = 13, color = my_colors[7], weight='bold')\n\n img = plt.imread(path)\n wandb_images.append(wandb.Image(img))\n axs[k].imshow(img)\n axs[k].axis(\"off\")\n\n plt.tight_layout()\n plt.show()\n\n # ๐Ÿ Log Image to W&B\n wandb.log({f\"{example_id}\": wandb_images})","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T17:02:31.711129Z","iopub.execute_input":"2022-03-01T17:02:31.711609Z","iopub.status.idle":"2022-03-01T17:02:32.198241Z","shell.execute_reply.started":"2022-03-01T17:02:31.711573Z","shell.execute_reply":"2022-03-01T17:02:32.196875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_id = all_indivs.iloc[1, :][0]\nshow_same_individual(example_id, sample_size=8)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:02:32.199922Z","iopub.execute_input":"2022-03-01T17:02:32.200260Z","iopub.status.idle":"2022-03-01T17:03:28.969002Z","shell.execute_reply.started":"2022-03-01T17:02:32.200219Z","shell.execute_reply":"2022-03-01T17:03:28.967934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_id = all_indivs.iloc[5, :][0]\nshow_same_individual(example_id, sample_size=8)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:28.970497Z","iopub.execute_input":"2022-03-01T17:03:28.971076Z","iopub.status.idle":"2022-03-01T17:04:15.531170Z","shell.execute_reply.started":"2022-03-01T17:03:28.971035Z","shell.execute_reply":"2022-03-01T17:04:15.530355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## III. Image sizes\n\n> ๐Ÿฌ **Note**: I am using `imagesize` library in order to retrieve the `width` and `height` of the images without overloading the memory of the notebook. And it's faster :)","metadata":{}},{"cell_type":"code","source":"# Save image size to a new column within the training dataset\nwidths, heights = [], []\n\nfor path in tqdm(train[\"path\"]):\n width, height = imagesize.get(path)\n widths.append(width)\n heights.append(height)\n \ntrain[\"width\"] = widths\ntrain[\"height\"] = heights\ntrain[\"dimension\"] = train[\"width\"] * train[\"height\"]","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:15.532143Z","iopub.execute_input":"2022-03-01T17:04:15.532677Z","iopub.status.idle":"2022-03-01T17:11:28.068218Z","shell.execute_reply.started":"2022-03-01T17:04:15.532631Z","shell.execute_reply":"2022-03-01T17:11:28.066689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ๐Ÿฌ Things to be noted:\n* distributions are quite alongated, with varying values for both width and height.\n* there are some images with very low values for either `width` or `height` (less than 100 pixels)","metadata":{}},{"cell_type":"code","source":"data_w = train[[\"species\", \"width\", \"class\"]]\ndata_h = train[[\"species\", \"height\", \"class\"]]\n\nprint(clr.S+\"WIDTH - Min Value:\"+clr.E, data_w[\"width\"].min(), \"pixels\")\nprint(clr.S+\"WIDTH - Max Value:\"+clr.E, data_w[\"width\"].max(), \"pixels\", \"\\n\")\nprint(clr.S+\"HEIGHT - Min Value:\"+clr.E, data_h[\"height\"].min(), \"pixels\")\nprint(clr.S+\"HEIGHT - Max Value:\"+clr.E, data_h[\"height\"].max(), \"pixels\")\n\n# Plots\nfig, (ax1, ax2) = plt.subplots(2, 1, figsize=(20, 19))\nfig.suptitle('- Image Size distribution on Species -', size = 26, color = my_colors[7], weight='bold')\naxs = [ax1, ax2]\n\nv1 = sns.violinplot(data=data_w, x=\"species\", y=\"width\", hue=\"class\", \n palette=[my_colors[1], my_colors[3]], ax=ax1)\nax1.set_title(\"Width\", y=0.97,\n size = 15, color = my_colors[6], weight='bold')\nax1.set_xlabel(\"\")\nax1.set_ylabel(\"Width\", size = 13, color = my_colors[6], weight='bold')\nax1.set_xticklabels(ax1.get_xticklabels(), rotation=45, ha='right')\n\n\nv2 = sns.violinplot(data=data_h, x=\"species\", y=\"height\", hue=\"class\", \n palette=[my_colors[6], my_colors[4]], ax=ax2)\nax2.set_title(\"Height\", y=0.9,\n size = 15, color = my_colors[6], weight='bold')\nax2.set_ylabel(\"Height\", size = 13, color = my_colors[6], weight='bold')\nax2.set_xlabel(\"\")\nax2.yaxis.set_tick_params(labelsize=13)\nax2.set_xticklabels(ax2.get_xticklabels(), rotation=45, ha='right')\n\nsns.despine(left=True, bottom=True)\nplt.subplots_adjust(left=None, bottom=None, right=None, top=0.93, wspace=None, hspace=None);","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T17:11:28.071255Z","iopub.execute_input":"2022-03-01T17:11:28.071856Z","iopub.status.idle":"2022-03-01T17:11:31.257396Z","shell.execute_reply.started":"2022-03-01T17:11:28.071789Z","shell.execute_reply":"2022-03-01T17:11:31.256429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"๐Ÿฌ So I went ahead to see which images might be with possibly concerning \"low resolution\":\n* **frasiers dolphin**: all images have the lowest dimension out of the entire dataset\n* **pygmy killer whale** and **duskin dolphin**: have low resolutions too, with the majority of images being smaller","metadata":{}},{"cell_type":"code","source":"data_d = train[[\"species\", \"dimension\", \"class\"]]\n\n# Plots\nfig, (ax1) = plt.subplots(1, 1, figsize=(20, 5))\nfig.suptitle('- Image Dimension distribution on Species -', size = 26, color = my_colors[7], weight='bold')\n\nsns.violinplot(data=data_d, x=\"species\", y=\"dimension\", hue=\"class\", \n palette=[my_colors[1], my_colors[3]], ax=ax1)\n# ax1.set_title(\"Dimension\", y=0.97,\n# size = 15, color = my_colors[6], weight='bold')\nax1.set_xlabel(\"\")\nax1.set_ylabel(\"Dimension\", size = 13, color = my_colors[6], weight='bold')\nax1.set_xticklabels(ax1.get_xticklabels(), rotation=45, ha='right')\n\nsns.despine(left=True, bottom=True)\nplt.subplots_adjust(left=None, bottom=None, right=None, top=0.93, wspace=None, hspace=None);","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T17:11:31.258728Z","iopub.execute_input":"2022-03-01T17:11:31.259547Z","iopub.status.idle":"2022-03-01T17:11:32.930033Z","shell.execute_reply.started":"2022-03-01T17:11:31.259494Z","shell.execute_reply":"2022-03-01T17:11:32.929098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## IV. Other weird image examples\n\nWhile browsing through the training/test images *(yes, I have looked at 70k+ images)*, I have found a few examples that I will share here that are ... complicated, to say the least.\n\nFound myself nervous laughing, which I guess it's better than crying. How could we preprocess these images? ๐Ÿ˜…๐Ÿคง!\n\n### ๐Ÿฌ Things to be noted:\n* **penguins!** - there are penguins in the data! I find that absolutely adorable.\n* **people** - there are also people (tourists and scientists) within the images (not as adorable as the penguins tho).\n* **afar objects** - there are many images where the subject is so far in the distance that you can barely see it.\n* **very similar photos** - there are images where you would think that they are identicap copies - in fact, they are not. But they are pictures takes moments appart, so the differences between then are extremely subtle. This could mess up the CV score.\n* **cannot see the subject** - there are a few examples where I myself cannot see the subject. At all. Maybe I need better glasses.\n* **water and ice** - initially I thought I would see only water, however, for the individuals that live mostly in arctic waters, many pictures contain ice\n* **don't put your finger over the camera** - innevitably, there are some pictures where there is a human finger blocking partially the image ๐Ÿ˜…\n* **beautiful subject** - I have also added a few images where I believe the takes are absolutely gorgeaus.\n\n... Did I already mention the penguins? ๐Ÿง๐Ÿง๐Ÿง","metadata":{}},{"cell_type":"code","source":"def plot_weird_images(ids, cols, rows, figsize, title, root_path):\n \n image_path = [root_path+i+\".jpg\" for i in ids]\n\n # Plot\n fig, axs = plt.subplots(cols, rows, figsize=figsize)\n fig.suptitle(title, size = 20, color = my_colors[7], weight='bold')\n axs = axs.flatten()\n\n for k, path in enumerate(image_path):\n axs[k].set_title(f\"Img. {ids[k]}\", \n fontsize = 13, color = my_colors[7], weight='bold')\n\n img = plt.imread(path)\n axs[k].imshow(img)\n axs[k].axis(\"off\")\n\n plt.tight_layout()\n plt.show()\n \n \n# Hand picked IDs\ntrain_ids = [\"0c42057255dbd6\", \"0c5821546292dd\", \"02f0606c99c41e\", \"2b3441bc1f27ce\", \"2c1f13d5f6d09d\",\n \"2c54be7b88181a\", \"3c15e996c183aa\", \"3cfa63a3bbebb7\", \"3dace1d4074b97\", \"3dd2c145275816\",\n \"5b850348ea63f8\", \"5c0c29e4993000\", \"5c90d285552ea2\", \"5e334d096b864e\", \"6e73f4c12d7d54\",\n \"7ad3a277f55107\", \"9a236360f50155\", \"9f94de1a3c768b\", \"12a7b25090e1b9\", \"13a25d81619913\",\n \"16d6fb560bd7bc\", \"35d677992a4f2e\", \"083a0fee112e3c\", \"090d7f9228a6bc\", \"292ceb0ffef4e8\",\n \"7940e462f75dd9\", \"24326cc55fe303\", \"ae680fc65c1ba0\", \"bb875ffcb8d064\", \"cd5fe465c60cb9\",\n \"d4d8ac80cb3a4b\", \"d5b42024509635\", \"dae4589b0f8dc5\", \"f7942e041d9963\", \"fc55d004bdc2da\"]\nplot_weird_images(ids=train_ids, cols=5, rows=7, \n figsize=(23, 14), title=\"- [Weird] Train Samples -\",\n root_path=\"../input/happy-whale-and-dolphin/train_images/\")\nprint(\"\\n\")\n\ntest_ids = [\"5bf1396d350169\", \"5e4a1ef591f291\", \"6caa20cf5526cb\", \"8c660e44867f8a\", \"9b0b44b19ba412\",\n \"43f1e346be1ddd\", \"67e5fb9a6110b0\", \"d5794a831a1b23\", \"db6e6c2b29ba40\", \"e4acbbdc2feb58\"]\nplot_weird_images(ids=test_ids, cols=2, rows=5, \n figsize=(23, 6), title=\"- [Weird] Test Samples -\",\n root_path=\"../input/happy-whale-and-dolphin/test_images/\")","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-01T17:11:32.932075Z","iopub.execute_input":"2022-03-01T17:11:32.932539Z","iopub.status.idle":"2022-03-01T17:12:12.697891Z","shell.execute_reply.started":"2022-03-01T17:11:32.932508Z","shell.execute_reply":"2022-03-01T17:12:12.696978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the end, we might be better of just erasing some of the outlier images.\n\nFor example the image below (image: `cd5fe465c60cb9.jpg`) is cathegorized as `gray_whale`, but there is no subject within it. \n\nMoreover, the subject's `individual_id` is `fc0f7c162cc0`, and if we take a look it has 72 more apparitions, so there are planty examples to choose from.\n\n","metadata":{}},{"cell_type":"code","source":"print(clr.S+\"Image characteristics:\"+clr.E)\ntrain[train[\"image\"]==\"cd5fe465c60cb9.jpg\"]","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:12:12.698846Z","iopub.execute_input":"2022-03-01T17:12:12.699053Z","iopub.status.idle":"2022-03-01T17:12:16.332957Z","shell.execute_reply.started":"2022-03-01T17:12:12.699028Z","shell.execute_reply":"2022-03-01T17:12:16.331974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(clr.S+\"Total number of apparitions:\"+clr.E, len(train[train[\"individual_id\"]==\"fc0f7c162cc0\"]))\nshow_same_individual(\"fc0f7c162cc0\", sample_size=8)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:12:16.334349Z","iopub.execute_input":"2022-03-01T17:12:16.334595Z","iopub.status.idle":"2022-03-01T17:13:04.865900Z","shell.execute_reply.started":"2022-03-01T17:12:16.334565Z","shell.execute_reply":"2022-03-01T17:13:04.864889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:13:04.867728Z","iopub.execute_input":"2022-03-01T17:13:04.868316Z","iopub.status.idle":"2022-03-01T17:13:14.282270Z","shell.execute_reply.started":"2022-03-01T17:13:04.868271Z","shell.execute_reply":"2022-03-01T17:13:14.281354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Preprocess\n\n## I. .csv preprocess\n๐Ÿฌ `target` -> column that incorporates for each observation all images that contain the same individual.","metadata":{}},{"cell_type":"code","source":"# Create a unique id column based on image name\ntrain[\"image_code\"] = train[\"image\"].apply(lambda x: x.split(\".\")[0])\n\n# Create a 'target' column\ntmp = train.groupby('individual_id')['image_code'].agg('unique').to_dict()\ntrain['target'] = train['individual_id'].map(tmp)\n\n# Map the individual id to a unique key (integer, not string)\nindividual_mapping = train[\"individual_id\"].value_counts().reset_index().drop(columns=[\"individual_id\"])\nindividual_mapping.columns = [\"individual_id\"]\nindividual_mapping[\"individual_key\"] = np.arange(start=0, stop=len(individual_mapping), step=1)\n\ntrain = pd.merge(train, individual_mapping, on=\"individual_id\")\n\n# Add Validation Fold\n### based on individual key group\nskf = StratifiedKFold(n_splits=5)\nskf_splits = skf.split(X=train.drop(columns=\"individual_key\"), y=train[\"individual_key\"])\n\nfor fold, (train_index, valid_index) in enumerate(skf_splits):\n train.loc[valid_index , \"kfold\"] = np.int(fold)\n \ntrain[\"kfold\"] = train[\"kfold\"].astype(int)\n \n# The adjusted training data\ntrain.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:13:14.283768Z","iopub.execute_input":"2022-03-01T17:13:14.285512Z","iopub.status.idle":"2022-03-01T17:13:15.662696Z","shell.execute_reply.started":"2022-03-01T17:13:14.285456Z","shell.execute_reply":"2022-03-01T17:13:15.661972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = \"../input/happy-whale-and-dolphin/test_images\"\n\ntest = pd.DataFrame({\"image\" : os.listdir(TEST_PATH)})\ntest[\"path\"] = TEST_PATH + \"/\" + test[\"image\"]\ntest[\"image_code\"] = test[\"image\"].apply(lambda x: x.split(\".\")[0])\n\n\nwidths, heights = [], []\n\nfor path in tqdm(test[\"path\"]):\n width, height = imagesize.get(path)\n widths.append(width)\n heights.append(height)\n \ntest[\"width\"] = widths\ntest[\"height\"] = heights\ntest[\"dimension\"] = test[\"width\"] * test[\"height\"]\n\ntest.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:13:15.665870Z","iopub.execute_input":"2022-03-01T17:13:15.666077Z","iopub.status.idle":"2022-03-01T17:17:42.825130Z","shell.execute_reply.started":"2022-03-01T17:13:15.666052Z","shell.execute_reply":"2022-03-01T17:17:42.824247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.to_csv(\"train.csv\", index=False)\n# test.to_csv(\"test.csv\", index=False)\n\n# ๐Ÿ Save datasets to W&B\nsave_dataset_artifact(run_name=\"TrainArtifact\", artifact_name=\"train\",\n path=\"../input/happywhale-2022/train.csv\")\n\nsave_dataset_artifact(run_name=\"TestArtifact\", artifact_name=\"test\",\n path=\"../input/happywhale-2022/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:17:42.826762Z","iopub.execute_input":"2022-03-01T17:17:42.827012Z","iopub.status.idle":"2022-03-01T17:18:08.053903Z","shell.execute_reply.started":"2022-03-01T17:17:42.826983Z","shell.execute_reply":"2022-03-01T17:18:08.052891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n

๐Ÿณ Part II. Image Similarities - Sisters or Twins ๐Ÿฌ

\n\nHow does our brain figure out so easily that the synonym of *car* could be *vehicle*? How does our brain know how to group similar words together, like *bear, mountain, hike*? We even have games such as \"spot the intruder\", where we ask kids as small as kindergarden **recognize which word or image is the intruder** from a pool of words, like *table, chair, bed, kitchen, lion*.\n\nWe can do this with faces too. There is much controvercy over some actors that look VERY similar to one another - like they are twins. However, a glimpse at a few images makes our brain imediately recognize which is which. Have you ever had twin friends? At the beginning you cannot tell them appart, but after a little you could even wonder how you could confuse them in the first place.\n\n
\n\n๐Ÿณ Can we do this with whales and dolphins too?\n\n### โฌ‡๏ธ Other Useful Libraries","metadata":{}},{"cell_type":"code","source":"# Helpful Imports\n!pip install -q efficientnet_pytorch\n\nimport albumentations\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom efficientnet_pytorch import EfficientNet\nfrom numpy import dot, sqrt\nfrom scipy import spatial\n\nfrom transformers import *\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(clr.S+'Device available now:'+clr.E, device)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-01T17:18:08.055547Z","iopub.execute_input":"2022-03-01T17:18:08.056502Z","iopub.status.idle":"2022-03-01T17:18:26.231170Z","shell.execute_reply.started":"2022-03-01T17:18:08.056464Z","shell.execute_reply":"2022-03-01T17:18:26.230229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Get Image Embeddings\n\n### ๐Ÿณ Why do we need these?\n\nSimply put, the **embeddings** are the last layer of a neural network, right before the *classification* part. Usually, we take the pixels of an image, we get them through a neural network of some kind, with multiple layers and neurons per layer. The last layer contains all the *good juices* from the image.\n\nThink of it like a zip file. You take an image, to compress all that good information into a zip file, and then if you reopen the zip file, the image is the same. We want that compressed vector, the **embedding** (to understand more about this you can read **[this amazing article by Chris Deotte - image below from there too](https://www.kaggle.com/c/shopee-product-matching/discussion/226279)**).\n\n
\n\n**How do we get these?**\n\nWe get these by training a neural network, such as a CNN. **ArcFace is a great Loss Function**, in order to force the network to [make similar class embeddings to be close and dissimilar embeddings to be far from each other](https://www.kaggle.com/c/shopee-product-matching/discussion/226279).\n\nHence, I will be using a very simple backbone such as an already trained **EffNet B7**, just to explore how the image embeddings would associate with one another.","metadata":{}},{"cell_type":"code","source":"# ---- PARAMETERS ----\nSTATE = 24\nKEYS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\nIMG_SIZE = 256\nBATCH_SIZE = 16\n# --------------------","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:18:26.233035Z","iopub.execute_input":"2022-03-01T17:18:26.233277Z","iopub.status.idle":"2022-03-01T17:18:26.238337Z","shell.execute_reply.started":"2022-03-01T17:18:26.233247Z","shell.execute_reply":"2022-03-01T17:18:26.236858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select only a sample from the training data\ndf = pd.read_csv(\"../input/happywhale-2022/train.csv\")\ndf = df[df[\"individual_key\"].isin(KEYS)].reset_index(drop=True)\ndf[\"path\"] = \"../input/happy-whale-and-dolphin/train_images/\" + df[\"image\"]\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:18:26.239570Z","iopub.execute_input":"2022-03-01T17:18:26.240401Z","iopub.status.idle":"2022-03-01T17:18:26.660221Z","shell.execute_reply.started":"2022-03-01T17:18:26.240355Z","shell.execute_reply":"2022-03-01T17:18:26.659556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### I. The Dataset","metadata":{}},{"cell_type":"code","source":"def get_transforms(img_size=256):\n '''Function to apply albumentations to the image.\n Keeping it simple for now - Just a resizing and normalization.'''\n \n return albumentations.Compose([\n albumentations.Resize(img_size, img_size),\n albumentations.Normalize()\n ])\n\nclass HappyWhaleDataset(Dataset):\n def __init__(self, csv, transforms=get_transforms(img_size=256)):\n\n self.csv = csv\n self.transform = transforms\n\n def __len__(self):\n return self.csv.shape[0]\n\n def __getitem__(self, index):\n row = self.csv.iloc[index]\n \n image = cv2.imread(row.path)\n image = image[:, :, ::-1]\n \n transformed_img = self.transform(image=image)\n transformed_img = transformed_img['image'].astype(np.float32)\n image = transformed_img.transpose(2, 0, 1)\n \n target = torch.tensor(row.individual_key)\n\n return torch.tensor(image), target\n \n\n# Get the data loader\ndataset = HappyWhaleDataset(df, transforms=get_transforms(img_size=IMG_SIZE))\nloader = DataLoader(dataset, batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:18:26.661351Z","iopub.execute_input":"2022-03-01T17:18:26.661556Z","iopub.status.idle":"2022-03-01T17:18:26.670286Z","shell.execute_reply.started":"2022-03-01T17:18:26.661532Z","shell.execute_reply":"2022-03-01T17:18:26.669820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### II. The EffNet Model","metadata":{}},{"cell_type":"code","source":"class BackboneModel(nn.Module):\n def __init__(self):\n super(BackboneModel, self).__init__()\n # Retrieve pretrained weights\n self.backbone = EfficientNet.from_pretrained('efficientnet-b7')\n \n def forward(self, img): \n img = self.backbone(img)\n return img\n \n# Initiate the model\nmodel = BackboneModel().to(device)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:18:26.671077Z","iopub.execute_input":"2022-03-01T17:18:26.671795Z","iopub.status.idle":"2022-03-01T17:18:32.663840Z","shell.execute_reply.started":"2022-03-01T17:18:26.671764Z","shell.execute_reply":"2022-03-01T17:18:32.662981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### III. Retrieving the Embeddings\n\n> ๐Ÿณ **Note**: the example below is a very simple one, where the model is NOT trained on the new whales/dolphin images. Hence, we would need to create a training pipeline to fine tune the embeddings.\n\n**Training Pipeline for Embeddings and clusterization**: my 2nd notebook [๐ŸณWhales&Dolphins: EffNet Train & RAPIDS Clusters](https://www.kaggle.com/andradaolteanu/whales-dolphins-effnet-train-rapids-clusters)","metadata":{}},{"cell_type":"code","source":"# Retrieve all embeddings for each image\nall_embeddings = []\nall_targets = []\n\nwith torch.no_grad():\n for img, target in tqdm(loader): \n img = img.to(device)\n img_embedding = model(img)\n img_embedding = img_embedding.detach().cpu().numpy()\n all_embeddings.append(img_embedding)\n all_targets.append(target.numpy())","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:18:32.665556Z","iopub.execute_input":"2022-03-01T17:18:32.665875Z","iopub.status.idle":"2022-03-01T17:42:14.028263Z","shell.execute_reply.started":"2022-03-01T17:18:32.665832Z","shell.execute_reply":"2022-03-01T17:42:14.027140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Concatenate batches together\nimage_embeddings = np.concatenate(all_embeddings)\nimage_targets = np.concatenate(all_targets)\n\nprint(clr.S+\"Shape of the embeddings:\"+clr.E, image_embeddings[0].shape)\n\n# Save embeddings and corresponding image\nnp.save('effnet_image_embeddings.npy', image_embeddings)\nnp.save('effnet_image_targets.npy', image_targets)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:42:14.030461Z","iopub.execute_input":"2022-03-01T17:42:14.031066Z","iopub.status.idle":"2022-03-01T17:42:14.049949Z","shell.execute_reply.started":"2022-03-01T17:42:14.031016Z","shell.execute_reply":"2022-03-01T17:42:14.048723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ๐Ÿ Save baseline (no train) embeddings to W&B\nsave_dataset_artifact(run_name=\"BaseEmbeddings\", artifact_name=\"effnet_embeds_notrain\",\n path=\"../input/happywhale-2022/effnet_image_embeddings.npy\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:42:14.052386Z","iopub.execute_input":"2022-03-01T17:42:14.052994Z","iopub.status.idle":"2022-03-01T17:42:26.286253Z","shell.execute_reply.started":"2022-03-01T17:42:14.052944Z","shell.execute_reply":"2022-03-01T17:42:26.285229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Cosine Distance\n\n> ๐Ÿณ **Note**: My main inspiration was this article on [face recognition with ArcFace](https://learnopencv.com/face-recognition-with-arcface/) - absolutely amazing, highly recommend.\n\n### I. The Cosine Distance Function","metadata":{}},{"cell_type":"code","source":"def get_cosine_similarity(embeddings):\n '''Compute cos distance between n embedding vector and itself.'''\n similarity_matrix = []\n \n for embed1 in embeddings:\n similarity_row = []\n for embed2 in embeddings:\n similarity_row.append(1 - spatial.distance.cosine(embed1, embed2))\n similarity_matrix.append(similarity_row)\n \n return np.array(similarity_matrix, dtype=\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:42:26.288165Z","iopub.execute_input":"2022-03-01T17:42:26.288427Z","iopub.status.idle":"2022-03-01T17:42:26.296422Z","shell.execute_reply.started":"2022-03-01T17:42:26.288396Z","shell.execute_reply":"2022-03-01T17:42:26.295377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### II. Select images from same individual","metadata":{}},{"cell_type":"code","source":"# Select few examples from the same individual\nexample_index = df[df[\"individual_key\"]==1].sample(5, random_state=24).index.tolist()\nexample_paths = df[df[\"individual_key\"]==1].sample(5, random_state=24)[\"path\"].tolist()\nexample_embeds = image_embeddings[example_index]\n\n# Compute similarity matrix\ncos_matrix = get_cosine_similarity(example_embeds)\n\nmask = np.zeros_like(cos_matrix)\nmask[np.triu_indices_from(mask)] = True","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:42:26.297836Z","iopub.execute_input":"2022-03-01T17:42:26.298326Z","iopub.status.idle":"2022-03-01T17:42:26.321294Z","shell.execute_reply.started":"2022-03-01T17:42:26.298287Z","shell.execute_reply":"2022-03-01T17:42:26.320566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### III. Compute cos Distance Matrix\n\n> ๐Ÿณ **Note**: These will improve substantially when the embeddings will be fitted on the competition dataset.\n\n[Source](https://www.geeksforgeeks.org/how-to-create-different-subplot-sizes-in-matplotlib/) of how I learned to do this chart :).\n\nBecause the embeddings are not trained, the similarity between the images is extremely low. In [my notebook here](https://www.kaggle.com/andradaolteanu/whales-dolphins-effnet-train-rapids-clusters) I train the model and then group the images by clusters, so the similarity matrix looks different (and improved :) ).","metadata":{}},{"cell_type":"code","source":"# Plots\nfig = plt.figure(figsize=(12, 12))\nax1 = plt.subplot2grid(shape=(6, 6), loc=(5, 1), colspan=1)\nax2 = plt.subplot2grid(shape=(6, 6), loc=(5, 2), colspan=1)\nax3 = plt.subplot2grid(shape=(6, 6), loc=(5, 3), colspan=1)\nax4 = plt.subplot2grid(shape=(6, 6), loc=(5, 4), colspan=1)\nax5 = plt.subplot2grid(shape=(6, 6), loc=(5, 5), colspan=1)\nh_axes = [ax1, ax2, ax3, ax4, ax5]\n\nax6 = plt.subplot2grid(shape=(6, 6), loc=(0, 0), colspan=1)\nax7 = plt.subplot2grid(shape=(6, 6), loc=(1, 0), colspan=1)\nax8 = plt.subplot2grid(shape=(6, 6), loc=(2, 0), colspan=1)\nax9 = plt.subplot2grid(shape=(6, 6), loc=(3, 0), colspan=1)\nax10 = plt.subplot2grid(shape=(6, 6), loc=(4, 0), colspan=1)\nv_axes = [ax6, ax7, ax8, ax9, ax10]\n\nax11 = plt.subplot2grid(shape=(6, 6), loc=(0, 1), colspan=5, rowspan=5)\n\nfig.suptitle('- Cosine Distance -', size = 21, color = my_colors[7], weight='bold')\nfor k, ax in enumerate(h_axes):\n ax.imshow(plt.imread(example_paths[k]))\n ax.set_axis_off()\n \nfor k, ax in enumerate(v_axes):\n ax.imshow(plt.imread(example_paths[k]))\n ax.set_axis_off()\n \nsns.heatmap(cos_matrix, ax=ax11, fmt=\".5\",\n cbar=False, annot=True, linewidths=0.5, mask=mask, square=True, cmap=\"winter_r\")\n\nplt.tight_layout()\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:42:26.323232Z","iopub.execute_input":"2022-03-01T17:42:26.324635Z","iopub.status.idle":"2022-03-01T17:42:34.929850Z","shell.execute_reply.started":"2022-03-01T17:42:26.324591Z","shell.execute_reply":"2022-03-01T17:42:34.928931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n### ๐Ÿ W&B Dashboard\n\n> My [W&B Dashboard](https://wandb.ai/andrada/happywhale?workspace=user-andrada).\n\n\n
\n\n
\n\n### My Specs\n\n* ๐Ÿ–ฅ Z8 G4 Workstation\n* ๐Ÿ’พ 2 CPUs & 96GB Memory\n* ๐ŸŽฎ NVIDIA Quadro RTX 8000\n* ๐Ÿ’ป Zbook Studio G7 on the go","metadata":{}}]}