{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nPyTorch embeddings classifier of whales and dolphins","metadata":{}},{"cell_type":"markdown","source":"## Importing libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport copy\nimport cv2\nimport random\nimport time\nfrom datetime import datetime\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import datasets, models, transforms\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import NearestNeighbors\n\nfrom PIL import Image\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport joblib\n\nimport optuna\nfrom optuna.trial import TrialState\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-16T10:07:40.996566Z","iopub.execute_input":"2022-03-16T10:07:40.997064Z","iopub.status.idle":"2022-03-16T10:07:45.272113Z","shell.execute_reply.started":"2022-03-16T10:07:40.996974Z","shell.execute_reply":"2022-03-16T10:07:45.271387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Wandb","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/rwightman/pytorch-image-models\n!pip install --upgrade wandb","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:08:08.048267Z","iopub.execute_input":"2022-03-16T10:08:08.048522Z","iopub.status.idle":"2022-03-16T10:08:33.482392Z","shell.execute_reply.started":"2022-03-16T10:08:08.048495Z","shell.execute_reply":"2022-03-16T10:08:33.481401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nimport wandb\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret('wandb_api')\n    wandb.login(key=api_key)\n    anony = None\nexcept:\n    anony = 'must'\n    print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:08:33.487001Z","iopub.execute_input":"2022-03-16T10:08:33.487285Z","iopub.status.idle":"2022-03-16T10:08:35.589546Z","shell.execute_reply.started":"2022-03-16T10:08:33.487248Z","shell.execute_reply":"2022-03-16T10:08:35.588747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Constants","metadata":{}},{"cell_type":"code","source":"CONFIG = {'seed': 2022,\n          'epochs': 5,\n          'img_size': 256,\n          'model_name': 'tf_efficientnet_b0',\n          'embedding_size': 256,\n          'train_batch_size': 32,\n          'valid_batch_size': 64,\n          'learning_rate': 1e-4,\n          'scheduler': 'CosineAnnealingLR',\n          'min_lr': 1e-6,\n          'T_max': 500,\n          'weight_decay': 1e-6,\n          'n_fold': 5,\n          'neigh': 100,\n          'margin': 0,\n          'device': torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n         }\n          \nDATA_DIR = '../input/happy-whale-and-dolphin'\nTRAIN_DIR = '../input/happy-whale-and-dolphin/train_images'\nTEST_DIR = '../input/happy-whale-and-dolphin/test_images'","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:38:41.135231Z","iopub.execute_input":"2022-03-16T10:38:41.135489Z","iopub.status.idle":"2022-03-16T10:38:41.141866Z","shell.execute_reply.started":"2022-03-16T10:38:41.135461Z","shell.execute_reply":"2022-03-16T10:38:41.140951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"## Loading Metadata","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(f'{DATA_DIR}/train.csv')\ndf_train['path'] = f'{DATA_DIR}/train_images/' + df_train['image']\n\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:37:25.818661Z","iopub.execute_input":"2022-03-16T10:37:25.818948Z","iopub.status.idle":"2022-03-16T10:37:25.888763Z","shell.execute_reply.started":"2022-03-16T10:37:25.818896Z","shell.execute_reply":"2022-03-16T10:37:25.887919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv')\ndf_test['path'] = '../input/happy-whale-and-dolphin/test_images/' + df_test['image']\n\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:35:49.461124Z","iopub.execute_input":"2022-03-16T10:35:49.461392Z","iopub.status.idle":"2022-03-16T10:35:49.523739Z","shell.execute_reply.started":"2022-03-16T10:35:49.461363Z","shell.execute_reply":"2022-03-16T10:35:49.522949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Analysis","metadata":{}},{"cell_type":"code","source":"print('Train samples count: ', len(df_train.index))\ndf_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:09:20.337245Z","iopub.execute_input":"2022-03-13T18:09:20.340497Z","iopub.status.idle":"2022-03-13T18:09:20.358388Z","shell.execute_reply.started":"2022-03-13T18:09:20.340421Z","shell.execute_reply":"2022-03-13T18:09:20.357314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Species Count: ',len(df_train['species'].value_counts().index))\ndf_train['species'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:09:20.359917Z","iopub.execute_input":"2022-03-13T18:09:20.364563Z","iopub.status.idle":"2022-03-13T18:09:20.409586Z","shell.execute_reply.started":"2022-03-13T18:09:20.364411Z","shell.execute_reply":"2022-03-13T18:09:20.408193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fixing duplicate labels\n`beluga` -> `beluga_whale`  \n`kiler_whale` -> `killer_whale`  \n`bottlenose_dolpin` -> `bottlenose_dolphin`  \n`globis` -> `globis_whale`","metadata":{}},{"cell_type":"code","source":"print('Before fixing duplicate labels : ')\nprint('Number of unique species : ', df_train['species'].nunique())\n\ndf_train['species'].replace({\n    'bottlenose_dolpin': 'bottlenose_dolphin',\n    'kiler_whale': 'killer_whale',\n    'beluga': 'beluga_whale',\n    'globis': 'globis_whale',\n},inplace =True)\n\nprint('\\nAfter fixing duplicate labels : ')\nprint('Number of unique species : ', df_train['species'].nunique())\n\n\ndf_train['class'] = df_train['species'].apply(lambda x: x.split('_')[-1])\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:37:32.907441Z","iopub.execute_input":"2022-03-16T10:37:32.908109Z","iopub.status.idle":"2022-03-16T10:37:32.965165Z","shell.execute_reply.started":"2022-03-16T10:37:32.908064Z","shell.execute_reply":"2022-03-16T10:37:32.964398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:37:37.941184Z","iopub.execute_input":"2022-03-16T10:37:37.941846Z","iopub.status.idle":"2022-03-16T10:37:37.977406Z","shell.execute_reply.started":"2022-03-16T10:37:37.941808Z","shell.execute_reply":"2022-03-16T10:37:37.976772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('../input/happy-whale-and-dolphin/train_images'))","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:09:20.566306Z","iopub.execute_input":"2022-03-13T18:09:20.566584Z","iopub.status.idle":"2022-03-13T18:09:21.529392Z","shell.execute_reply.started":"2022-03-13T18:09:20.566547Z","shell.execute_reply":"2022-03-13T18:09:21.528359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization of data","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (15,12))\nfor idx,i in enumerate(df_train.species.unique()):\n    plt.subplot(4,7,idx + 1)\n    df = df_train[df_train['species'] == i].reset_index(drop=True)\n        \n    image_path = df.loc[random.randint(0, len(df) - 1),'path']\n    img = Image.open(image_path)\n    img = img.resize((224,224))\n    plt.imshow(img)\n    plt.axis('off')\n    plt.title(i)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:09:21.53123Z","iopub.execute_input":"2022-03-13T18:09:21.53179Z","iopub.status.idle":"2022-03-13T18:09:27.465858Z","shell.execute_reply.started":"2022-03-13T18:09:21.53174Z","shell.execute_reply":"2022-03-13T18:09:27.464779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_species(df,species_name):\n    plt.figure(figsize = (12,12))\n    species_df = df[df['species'] ==species_name].reset_index(drop = True)\n    plt.suptitle(species_name)\n    for idx,i in enumerate(np.random.choice(species_df['path'],8)):\n        plt.subplot(8,8,idx+1)\n        image_path = i\n        img = Image.open(image_path)\n        img = img.resize((224,224))\n        plt.imshow(img)\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:09:27.467266Z","iopub.execute_input":"2022-03-13T18:09:27.467648Z","iopub.status.idle":"2022-03-13T18:09:27.477439Z","shell.execute_reply.started":"2022-03-13T18:09:27.467598Z","shell.execute_reply":"2022-03-13T18:09:27.476369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for species in df_train['species'].unique():\n    plot_species(df_train, species)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:09:27.479259Z","iopub.execute_input":"2022-03-13T18:09:27.479867Z","iopub.status.idle":"2022-03-13T18:10:10.018896Z","shell.execute_reply.started":"2022-03-13T18:09:27.479824Z","shell.execute_reply":"2022-03-13T18:10:10.017884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_individual(df,individual_id):\n    plt.figure(figsize = (12,12))\n    species_df = df[df['individual_id'] == individual_id].reset_index(drop = True)\n    plt.suptitle(individual_id)\n    for idx,i in enumerate(np.random.choice(species_df['path'],8)):\n        plt.subplot(8,8,idx+1)\n        image_path = i\n        img = Image.open(image_path)\n        img = img.resize((224,224))\n        plt.imshow(img)\n        plt.axis('off')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:10:10.020924Z","iopub.execute_input":"2022-03-13T18:10:10.021317Z","iopub.status.idle":"2022-03-13T18:10:10.030258Z","shell.execute_reply.started":"2022-03-13T18:10:10.021268Z","shell.execute_reply":"2022-03-13T18:10:10.028611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_5_ids = df_train.individual_id.value_counts().head(5)\nfor i in top_5_ids.index:\n    plot_individual(df_train , i)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:10:10.032497Z","iopub.execute_input":"2022-03-13T18:10:10.032945Z","iopub.status.idle":"2022-03-13T18:10:21.382981Z","shell.execute_reply.started":"2022-03-13T18:10:10.032898Z","shell.execute_reply":"2022-03-13T18:10:21.381977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Distribution Analysis","metadata":{}},{"cell_type":"markdown","source":"## Bar chart of whales/dolphins","metadata":{}},{"cell_type":"code","source":"plot = sns.countplot(x = df_train['class'], color = '#2596be')\nsns.despine()\nplot.set_title('Class Distribution\\n', font = 'serif', x = 0.1, y=1, fontsize = 16)\nplot.set_ylabel('Count', x = 0.02, font = 'serif', fontsize = 12)\nplot.set_xlabel('Species', fontsize = 12, font = 'serif')\n\nfor p in plot.patches:\n    plot.annotate(format(p.get_height(), '.0f'), (p.get_x() + p.get_width() / 2, p.get_height()), \n       ha = 'center', va = 'center', xytext = (0, -20),font = 'serif', textcoords = 'offset points', size = 15)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:12:12.593167Z","iopub.execute_input":"2022-03-13T18:12:12.5935Z","iopub.status.idle":"2022-03-13T18:12:12.863176Z","shell.execute_reply.started":"2022-03-13T18:12:12.593443Z","shell.execute_reply":"2022-03-13T18:12:12.862109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Circle diagram of whales/dolphins","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(5,5))\nclass_cnt = df_train.groupby(['class']).size().reset_index(name = 'counts')\ncolors = sns.color_palette('Paired')[0:9]\nplt.pie(class_cnt['counts'], labels=class_cnt['class'], colors=colors, autopct='%1.1f%%')\nplt.legend(loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:12:17.282427Z","iopub.execute_input":"2022-03-13T18:12:17.282732Z","iopub.status.idle":"2022-03-13T18:12:17.431307Z","shell.execute_reply.started":"2022-03-13T18:12:17.282701Z","shell.execute_reply":"2022-03-13T18:12:17.430287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bar chart by species","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nsns.countplot(data=df_train, y = 'species',  palette='crest', dodge=False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:12:22.709167Z","iopub.execute_input":"2022-03-13T18:12:22.709494Z","iopub.status.idle":"2022-03-13T18:12:23.168879Z","shell.execute_reply.started":"2022-03-13T18:12:22.709455Z","shell.execute_reply":"2022-03-13T18:12:23.168007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bar charts for most frequent whales and dolphins","metadata":{}},{"cell_type":"code","source":"fig,ax = plt.subplots(1,2,figsize=(10,5))\n\nwhales = df_train[df_train['class'] == 'whale']\ndolphins = df_train[df_train['class'] != 'whale']\n\nsns.countplot(y='species', data=whales, order=whales.iloc[0:]['species'].value_counts().index, ax=ax[0], color='#0077b6')\nax[0].set_title('Most frequent whales')\nax[0].set_ylabel(None)\n    \nsns.countplot(y='species', data=dolphins,order=dolphins.iloc[0:]['species'].value_counts().index, ax=ax[1], color='#90e0ef')\nax[1].set_title('Most frequent dolphins')\nax[1].set_ylabel(None)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:12:51.076649Z","iopub.execute_input":"2022-03-13T18:12:51.076989Z","iopub.status.idle":"2022-03-13T18:12:51.75915Z","shell.execute_reply.started":"2022-03-13T18:12:51.076943Z","shell.execute_reply":"2022-03-13T18:12:51.758098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data normalization","metadata":{}},{"cell_type":"code","source":"# %%time\n# transform = transforms.Compose([transforms.Resize(255), transforms.CenterCrop(224), transforms.ToTensor()])\n# dataset = datasets.ImageFolder(DATA_DIR, transform=transform)\n\n# kwargs = {'num_workers': 10, 'pin_memory': True, 'persistent_workers': True} if CONFIG['device'] == 'cuda' else {}\n# dataloader = torch.utils.data.DataLoader(dataset, batch_size=256, shuffle=True, **kwargs)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:10:21.702311Z","iopub.status.idle":"2022-03-13T18:10:21.703077Z","shell.execute_reply.started":"2022-03-13T18:10:21.70257Z","shell.execute_reply":"2022-03-13T18:10:21.702601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# images, labels = next(iter(dataloader))\n# plt.imshow(images[0].permute(1, 2, 0))","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:10:21.705451Z","iopub.status.idle":"2022-03-13T18:10:21.70613Z","shell.execute_reply.started":"2022-03-13T18:10:21.705749Z","shell.execute_reply":"2022-03-13T18:10:21.705781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mean_and_std(dataloader, device):\n    channels_sum, channels_squared_sum = 0, 0\n    num_batches = len(dataloader)\n    \n    for data in tqdm(dataloader):        \n        # mean over batch, height and width, but not over the channels\n        images = data[0].to(device, dtype=torch.float)\n        channels_sum += torch.mean(images, dim=[0, 2, 3])\n        channels_squared_sum += torch.mean(images ** 2, dim=[0, 2, 3])\n    \n    mean = channels_sum / num_batches\n\n    # std = sqrt(E[X^2] - (E[X])^2)\n    std = (channels_squared_sum / num_batches - mean ** 2) ** 0.5\n\n    return mean, std","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:13:01.921305Z","iopub.execute_input":"2022-03-13T18:13:01.921647Z","iopub.status.idle":"2022-03-13T18:13:01.928954Z","shell.execute_reply.started":"2022-03-13T18:13:01.921604Z","shell.execute_reply":"2022-03-13T18:13:01.927988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# # ~2h 30min\n\n# print(datetime.now())\n# mean, std = get_mean_and_std(dataloader, CONFIG['device'])\n\n# print('Mean:', mean)\n# print('Standard deviation:', std)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:10:21.711747Z","iopub.status.idle":"2022-03-13T18:10:21.712845Z","shell.execute_reply.started":"2022-03-13T18:10:21.712523Z","shell.execute_reply":"2022-03-13T18:10:21.712559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mean: `tensor([0.4286, 0.4748, 0.5269])`  \nStandard deviation: `tensor([0.2193, 0.2160, 0.2262])`","metadata":{}},{"cell_type":"code","source":"mean = torch.tensor([0.4286, 0.4748, 0.5269])\nstd = torch.tensor([0.2193, 0.2160, 0.2262])","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:29:20.650749Z","iopub.execute_input":"2022-03-16T10:29:20.653206Z","iopub.status.idle":"2022-03-16T10:29:20.664630Z","shell.execute_reply.started":"2022-03-16T10:29:20.653163Z","shell.execute_reply":"2022-03-16T10:29:20.663935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntransform = transforms.Compose([transforms.Resize(CONFIG['img_size']), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])\n\nkwargs = {'num_workers': 10, 'pin_memory': True, 'persistent_workers': True} if CONFIG['device'] == 'cuda' else {}\ndataset = datasets.ImageFolder(DATA_DIR, transform=transform)\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=256, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:13:22.23232Z","iopub.execute_input":"2022-03-13T18:13:22.233231Z","iopub.status.idle":"2022-03-13T18:14:06.42378Z","shell.execute_reply.started":"2022-03-13T18:13:22.232961Z","shell.execute_reply":"2022-03-13T18:14:06.421968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(iter(dataloader))\nplt.imshow(images[0].permute(1, 2, 0))","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:14:06.426126Z","iopub.execute_input":"2022-03-13T18:14:06.426768Z","iopub.status.idle":"2022-03-13T18:14:39.848076Z","shell.execute_reply.started":"2022-03-13T18:14:06.426721Z","shell.execute_reply":"2022-03-13T18:14:39.846981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training preparation","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:36:56.003976Z","iopub.execute_input":"2022-03-16T10:36:56.004515Z","iopub.status.idle":"2022-03-16T10:36:56.010914Z","shell.execute_reply.started":"2022-03-16T10:36:56.004475Z","shell.execute_reply":"2022-03-16T10:36:56.009965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=CONFIG['n_fold'])\n\nfor fold, ( _, val_) in enumerate(gkf.split(X=df_train, y=df_train.individual_id, groups=df_train.individual_id)):\n      df_train.loc[val_ , 'kfold'] = fold","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:38:57.767379Z","iopub.execute_input":"2022-03-16T10:38:57.767648Z","iopub.status.idle":"2022-03-16T10:38:57.906050Z","shell.execute_reply.started":"2022-03-16T10:38:57.767617Z","shell.execute_reply":"2022-03-16T10:38:57.905393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.kfold.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:38:58.880894Z","iopub.execute_input":"2022-03-16T10:38:58.881616Z","iopub.status.idle":"2022-03-16T10:38:58.891138Z","shell.execute_reply.started":"2022-03-16T10:38:58.881572Z","shell.execute_reply":"2022-03-16T10:38:58.890248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset class","metadata":{}},{"cell_type":"code","source":"class HappyWhaleDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.groups = df.groupby('individual_id').groups\n        self.keys = list(self.groups.keys())\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.groups)\n    \n    def __getitem__(self, index):         \n        # get first random image\n        image_indices_1 = self.groups[self.keys[index]]\n        image_path_1 = self.df.iloc[image_indices_1, :].sample(n=1)['path'].values[0]\n        image_1 = cv2.cvtColor(cv2.imread(image_path_1), cv2.COLOR_BGR2RGB)\n        individual_id_1 = self.df.iloc[image_indices_1, :]['individual_id'].values[0]\n        \n        # get second random image\n        image_index_2 = self.df.sample(n=1).index\n        image_path_2 = self.df.iloc[image_index_2, :]['path'].values[0]\n        image_2 = cv2.cvtColor(cv2.imread(image_path_2), cv2.COLOR_BGR2RGB)\n        individual_id_2 = self.df.iloc[image_index_2, :]['individual_id'].values[0]\n        \n        # 1 if individual ids match, -1 otherwise\n        target = 1 if individual_id_1 == individual_id_2 else -1\n        \n        # transform the dataset if transformations were specified\n        if self.transforms:\n            image_1 = self.transforms(image=image_1)['image']\n            image_2 = self.transforms(image=image_2)['image']\n        \n        return {\n            'image1': image_1,\n            'image2': image_2,\n            'target': torch.tensor(target, dtype=torch.int)\n        }","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:29:10.784018Z","iopub.execute_input":"2022-03-16T10:29:10.784728Z","iopub.status.idle":"2022-03-16T10:29:10.795982Z","shell.execute_reply.started":"2022-03-16T10:29:10.784688Z","shell.execute_reply":"2022-03-16T10:29:10.794846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    'train': A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.Normalize(\n                mean=mean, \n                std=std, \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.),\n    \n    'valid': A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.Normalize(\n                mean=mean, \n                std=std, \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:29:25.989698Z","iopub.execute_input":"2022-03-16T10:29:25.989983Z","iopub.status.idle":"2022-03-16T10:29:25.996961Z","shell.execute_reply.started":"2022-03-16T10:29:25.989955Z","shell.execute_reply":"2022-03-16T10:29:25.996208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n    train_dataset = HappyWhaleDataset(df_train, transforms=data_transforms['train'])\n    valid_dataset = HappyWhaleDataset(df_valid, transforms=data_transforms['valid'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n                              num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:15:43.806448Z","iopub.execute_input":"2022-03-13T18:15:43.80726Z","iopub.status.idle":"2022-03-13T18:15:43.820194Z","shell.execute_reply.started":"2022-03-13T18:15:43.807223Z","shell.execute_reply":"2022-03-13T18:15:43.819011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare dataloaders\ntrain_loader, valid_loader = prepare_loaders(df_train, fold=0)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:15:46.316066Z","iopub.execute_input":"2022-03-13T18:15:46.316393Z","iopub.status.idle":"2022-03-13T18:15:46.607912Z","shell.execute_reply.started":"2022-03-13T18:15:46.316361Z","shell.execute_reply":"2022-03-13T18:15:46.606718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{"execution":{"iopub.status.busy":"2022-02-20T17:00:12.566401Z","iopub.execute_input":"2022-02-20T17:00:12.566906Z","iopub.status.idle":"2022-02-20T17:00:12.570015Z","shell.execute_reply.started":"2022-02-20T17:00:12.566867Z","shell.execute_reply":"2022-02-20T17:00:12.569356Z"}}},{"cell_type":"code","source":"def criterion(outputs1, outputs2, targets):\n    return nn.CosineEmbeddingLoss(margin=CONFIG['margin'])(outputs1, outputs2, targets)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:15:46.988119Z","iopub.execute_input":"2022-03-13T18:15:46.988395Z","iopub.status.idle":"2022-03-13T18:15:46.993735Z","shell.execute_reply.started":"2022-03-13T18:15:46.988363Z","shell.execute_reply":"2022-03-13T18:15:46.992525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create model","metadata":{}},{"cell_type":"code","source":"class HappyWhaleModel(nn.Module):\n    def __init__(self, model_name, pretrained=True):\n        super(HappyWhaleModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained, num_classes=0)\n        self.fc = nn.LazyLinear(CONFIG['embedding_size'])\n        self.dropout = nn.Dropout(p=0.3)\n\n    def forward(self, images):\n        features = self.model(images)\n        features = self.dropout(features)\n        output = self.fc(features)\n        return output\n    \nmodel = HappyWhaleModel(CONFIG['model_name'])\nmodel.to(CONFIG['device'])","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:29:39.655106Z","iopub.execute_input":"2022-03-16T10:29:39.655358Z","iopub.status.idle":"2022-03-16T10:29:44.060160Z","shell.execute_reply.started":"2022-03-16T10:29:39.655331Z","shell.execute_reply":"2022-03-16T10:29:44.059352Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dummy run to initialize the layers \nimg = torch.randn(1, 3, CONFIG['img_size'], CONFIG['img_size']).to(CONFIG['device'])\nmodel(img)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:03.434769Z","iopub.execute_input":"2022-03-13T18:16:03.435117Z","iopub.status.idle":"2022-03-13T18:16:09.62887Z","shell.execute_reply.started":"2022-03-13T18:16:03.435036Z","shell.execute_reply":"2022-03-13T18:16:09.62792Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find best hyperparameters with optuna","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    N_TRAIN_EXAMPLES = CONFIG['train_batch_size'] * 12\n    N_VALID_EXAMPLES = CONFIG['valid_batch_size'] * 4\n    device = CONFIG['device']\n\n    # Generate the model.\n    model = HappyWhaleModel(CONFIG['model_name']).to(device)\n\n    # Generate the optimizers.\n    optimizer_number = trial.suggest_categorical('optimizer number (0: Adam; 1: SGD)', [0, 1])\n    lr = trial.suggest_float('learning rate', 1e-2, 1e-1, log=True)\n    \n    optimizer_map = {\n        0: 'Adam',\n        1: 'SGD',\n    }\n    \n    optimizer = getattr(optim, optimizer_map[optimizer_number])(model.parameters(), lr=lr)\n    \n    config = {\n        'optimizer (0: Adam; 1: SGD)': optimizer_number,\n        'learning rate': lr,\n    }\n    \n    run = wandb.init(project='HappyWhale',\n                     name=f'trial_{trial.number + 1}',\n                     group='optuna research',\n                     config=config,\n                     anonymous='must')\n\n    # Training of the model.\n    for epoch in range(CONFIG['epochs']):\n        model.train()\n    \n        dataset_size = 0\n        running_loss = 0.0\n        \n        for batch_idx, data in enumerate(train_loader):\n            # Limiting training data for faster epochs.\n            if batch_idx * CONFIG['train_batch_size'] > N_TRAIN_EXAMPLES:\n                break\n\n            images1 = data['image1'].to(device, dtype=torch.float)\n            images2 = data['image2'].to(device, dtype=torch.float)\n            targets = data['target'].to(device, dtype=torch.int)\n\n            batch_size = images1.size(0)\n\n            outputs1 = model(images1)\n            outputs2 = model(images2)\n            loss = criterion(outputs1, outputs2, targets)\n            loss.backward()\n            optimizer.step()\n            \n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n        # Validation of the model.\n        model.eval()\n        correct = 0\n        with torch.no_grad():\n            for batch_idx, data in enumerate(valid_loader):\n                # Limiting validation data.\n                if batch_idx * CONFIG['valid_batch_size'] > N_VALID_EXAMPLES:\n                    break\n                \n                images1 = data['image1'].to(device, dtype=torch.float)\n                images2 = data['image2'].to(device, dtype=torch.float)\n                targets = data['target'].to(device, dtype=torch.int)\n\n                batch_size = images1.size(0)\n\n                outputs1 = model(images1)\n                outputs2 = model(images2)\n                loss = criterion(outputs1, outputs2, targets)\n\n                running_loss += (loss.item() * batch_size)\n                dataset_size += batch_size\n\n        epoch_loss = running_loss / dataset_size\n        run.log({'Cosine Embedding Loss': epoch_loss})\n\n        trial.report(epoch_loss, epoch)\n\n        # handle pruning based on the intermediate value.\n        if trial.should_prune():\n            raise optuna.exceptions.TrialPruned()\n\n    run.finish()\n    \n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-03-06T19:49:52.198934Z","iopub.execute_input":"2022-03-06T19:49:52.199515Z","iopub.status.idle":"2022-03-06T19:49:52.233373Z","shell.execute_reply.started":"2022-03-06T19:49:52.199427Z","shell.execute_reply":"2022-03-06T19:49:52.232541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = optuna.create_study()\nstudy.optimize(objective, n_trials=20, timeout=3600, show_progress_bar=True)\n\nprint('Number of finished trials: {}'.format(len(study.trials)))\n\nprint('Best trial:')\ntrial = study.best_trial\n\nprint('  Value: {}'.format(trial.value))\n\nprint('  Params: ')\nfor key, value in trial.params.items():\n    print('    {}: {}'.format(key, value))\n          \n# Create the summary run.\nsummary = wandb.init(project='HappyWhale',\n                     group='optuna summary',\n                     name='summary')\n\n# Getting the study trials.\ntrials = study.trials\n\n# WandB summary.\nfor step, trial in enumerate(trials):\n    # Logging the loss.\n    summary.log({'Cosine Embedding Loss': trial.value}, step=step, commit=False)\n\n    # Logging the parameters.        \n    summary.log(trial.params, commit=True)\n    \nsummary.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-06T19:49:52.235092Z","iopub.execute_input":"2022-03-06T19:49:52.235442Z","iopub.status.idle":"2022-03-06T19:49:52.296148Z","shell.execute_reply.started":"2022-03-06T19:49:52.235414Z","shell.execute_reply":"2022-03-06T19:49:52.295304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training function","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        images1 = data['image1'].to(device, dtype=torch.float)\n        images2 = data['image2'].to(device, dtype=torch.float)\n        targets = data['target'].to(device, dtype=torch.int)\n        \n        batch_size = images1.size(0)\n\n        outputs1 = model(images1)\n        outputs2 = model(images2)\n        \n        loss = criterion(outputs1, outputs2, targets)\n        loss.backward()\n\n        optimizer.step()\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n\n        if scheduler is not None:\n            scheduler.step()\n                \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])\n    gc.collect()\n    \n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:24.076187Z","iopub.execute_input":"2022-03-13T18:16:24.076483Z","iopub.status.idle":"2022-03-13T18:16:24.087616Z","shell.execute_reply.started":"2022-03-13T18:16:24.07645Z","shell.execute_reply":"2022-03-13T18:16:24.086435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation function","metadata":{}},{"cell_type":"code","source":"@torch.inference_mode()\ndef valid_one_epoch(model, dataloader, device, epoch):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        images1 = data['image1'].to(device, dtype=torch.float)\n        images2 = data['image2'].to(device, dtype=torch.float)\n        targets = data['target'].to(device, dtype=torch.int)\n        \n        batch_size = images1.size(0)\n\n        outputs1 = model(images1)\n        outputs2 = model(images2)\n        loss = criterion(outputs1, outputs2, targets)\n        \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])   \n    \n    gc.collect()\n    \n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:28.845385Z","iopub.execute_input":"2022-03-13T18:16:28.84584Z","iopub.status.idle":"2022-03-13T18:16:28.863033Z","shell.execute_reply.started":"2022-03-13T18:16:28.845788Z","shell.execute_reply":"2022-03-13T18:16:28.862061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run training","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n    # To automatically log gradients\n    wandb.watch(model, log_freq=100)\n    \n    if torch.cuda.is_available():\n        print('[INFO] Using GPU: {}\\n'.format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_epoch_loss = np.inf\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n        train_epoch_loss = train_one_epoch(model, optimizer, scheduler, \n                                           dataloader=train_loader, \n                                           device=CONFIG['device'], epoch=epoch)\n        \n        val_epoch_loss = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n                                         epoch=epoch)\n    \n        history['Train Loss'].append(train_epoch_loss)\n        history['Valid Loss'].append(val_epoch_loss)\n        \n        # log the metrics\n        wandb.log({'Train Loss': train_epoch_loss})\n        wandb.log({'Valid Loss': val_epoch_loss})\n        \n        # deep copy the model\n        if val_epoch_loss <= best_epoch_loss:\n            print(f'{b_}Validation Loss Improved ({best_epoch_loss} ---> {val_epoch_loss})')\n            best_epoch_loss = val_epoch_loss\n            run.summary['Best Loss'] = best_epoch_loss\n            best_model_wts = copy.deepcopy(model.state_dict())\n            PATH = 'Loss{:.4f}_epoch{:.0f}.bin'.format(best_epoch_loss, epoch)\n            torch.save(model.state_dict(), PATH)\n            # Save a model file from the current directory\n            print(f'Model Saved{sr_}')\n            \n        print()\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print('Best Loss: {:.4f}'.format(best_epoch_loss))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:38.222354Z","iopub.execute_input":"2022-03-13T18:16:38.222658Z","iopub.status.idle":"2022-03-13T18:16:38.236483Z","shell.execute_reply.started":"2022-03-13T18:16:38.222623Z","shell.execute_reply":"2022-03-13T18:16:38.235396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CONFIG['scheduler'] == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n                                                   eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n                                                             eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == None:\n        return None\n        \n    return scheduler","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:39.966009Z","iopub.execute_input":"2022-03-13T18:16:39.969154Z","iopub.status.idle":"2022-03-13T18:16:39.983518Z","shell.execute_reply.started":"2022-03-13T18:16:39.969036Z","shell.execute_reply":"2022-03-13T18:16:39.982427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define optimizer and scheduler\noptimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n                       weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:43.180489Z","iopub.execute_input":"2022-03-13T18:16:43.18078Z","iopub.status.idle":"2022-03-13T18:16:43.187904Z","shell.execute_reply.started":"2022-03-13T18:16:43.180749Z","shell.execute_reply":"2022-03-13T18:16:43.186861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='HappyWhale', \n                 config=CONFIG,\n                 name='training',\n                 group='training')","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:44.910691Z","iopub.execute_input":"2022-03-13T18:16:44.911216Z","iopub.status.idle":"2022-03-13T18:16:52.181285Z","shell.execute_reply.started":"2022-03-13T18:16:44.911179Z","shell.execute_reply":"2022-03-13T18:16:52.179998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler, \n                              device=CONFIG['device'],\n                              num_epochs=CONFIG['epochs'])","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:16:57.993409Z","iopub.execute_input":"2022-03-13T18:16:57.99372Z","iopub.status.idle":"2022-03-13T18:20:12.966326Z","shell.execute_reply.started":"2022-03-13T18:16:57.993685Z","shell.execute_reply":"2022-03-13T18:20:12.965092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving model","metadata":{}},{"cell_type":"code","source":"torch.save(model.state_dict(), 'happy-whale-model.pth')\n\n# Save as artifact for version control.\nartifact = wandb.Artifact('model', type='model')\nartifact.add_file('happy-whale-model.pth')\n\nrun.log_artifact(artifact)\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:21:38.463176Z","iopub.execute_input":"2022-03-13T18:21:38.463478Z","iopub.status.idle":"2022-03-13T18:21:46.917764Z","shell.execute_reply.started":"2022-03-13T18:21:38.463447Z","shell.execute_reply":"2022-03-13T18:21:46.916027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading model\nUncomment if you would like to load your model","metadata":{}},{"cell_type":"code","source":"entity = 'artmalygin'\nmodel_number = 0","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:30:29.129663Z","iopub.execute_input":"2022-03-16T10:30:29.129939Z","iopub.status.idle":"2022-03-16T10:30:29.134087Z","shell.execute_reply.started":"2022-03-16T10:30:29.129892Z","shell.execute_reply":"2022-03-16T10:30:29.133025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run = wandb.init(project='HappyWhale',\n#                  name='loading',\n#                  group='loading')\n\n# artifact = run.use_artifact(f'{entity}/HappyWhale/model:v{model_number}', type='model')\n# artifact_dir = artifact.download()\n\n# run.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:30:31.654032Z","iopub.execute_input":"2022-03-16T10:30:31.654755Z","iopub.status.idle":"2022-03-16T10:30:45.338920Z","shell.execute_reply.started":"2022-03-16T10:30:31.654716Z","shell.execute_reply":"2022-03-16T10:30:45.338169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wandb_model = HappyWhaleModel(CONFIG['model_name']).to(CONFIG['device'])\n# wandb_model.load_state_dict(torch.load(os.path.join(artifact_dir, 'happy-whale-model.pth')))\n# wandb_model.eval()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:31:14.423823Z","iopub.execute_input":"2022-03-16T10:31:14.424546Z","iopub.status.idle":"2022-03-16T10:31:14.647408Z","shell.execute_reply.started":"2022-03-16T10:31:14.424510Z","shell.execute_reply":"2022-03-16T10:31:14.646742Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"class HappyWhaleEvaluationDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df.index)\n    \n    def __getitem__(self, index):\n        item = self.df.iloc[index]\n        image_path = item['path']\n        image = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n        \n        # transform the dataset if transformations were specified\n        if self.transforms:\n            image = self.transforms(image=image)['image']\n        \n        return {\n            'image': image,\n        }","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:35:25.174264Z","iopub.execute_input":"2022-03-16T10:35:25.175006Z","iopub.status.idle":"2022-03-16T10:35:25.181183Z","shell.execute_reply.started":"2022-03-16T10:35:25.174968Z","shell.execute_reply":"2022-03-16T10:35:25.180455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fold = 0\ndf_train_eval = df_train[df_train.kfold != train_fold].reset_index(drop=True)\ndf_valid_eval = df_train[df_train.kfold == train_fold].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:39:13.410794Z","iopub.execute_input":"2022-03-16T10:39:13.411571Z","iopub.status.idle":"2022-03-16T10:39:13.432557Z","shell.execute_reply.started":"2022-03-16T10:39:13.411532Z","shell.execute_reply":"2022-03-16T10:39:13.431857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders_eval(df_train, df_valid, df_test):\n    train_dataset = HappyWhaleEvaluationDataset(df_train, transforms=data_transforms['train'])\n    valid_dataset = HappyWhaleEvaluationDataset(df_valid, transforms=data_transforms['valid'])\n    test_dataset = HappyWhaleEvaluationDataset(df_test, transforms=data_transforms['valid'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n                              num_workers=2, shuffle=False, pin_memory=True)\n    test_loader = DataLoader(test_dataset, batch_size=CONFIG['valid_batch_size'],\n                             num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader, test_loader","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:39:15.920665Z","iopub.execute_input":"2022-03-16T10:39:15.920949Z","iopub.status.idle":"2022-03-16T10:39:15.927615Z","shell.execute_reply.started":"2022-03-16T10:39:15.920913Z","shell.execute_reply":"2022-03-16T10:39:15.926947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader_eval, valid_loader_eval, test_loader_eval = prepare_loaders_eval(df_train_eval, df_valid_eval, df_test)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:39:19.367661Z","iopub.execute_input":"2022-03-16T10:39:19.367936Z","iopub.status.idle":"2022-03-16T10:39:19.371795Z","shell.execute_reply.started":"2022-03-16T10:39:19.367886Z","shell.execute_reply":"2022-03-16T10:39:19.371139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.inference_mode()\ndef inference(model, dataloader, device):\n    model.eval()  \n    embedding = torch.randn(1, CONFIG['embedding_size'])\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        images = data['image'].to(device, dtype=torch.float)\n        \n        outputs = model(images).cpu()\n        \n        embedding = torch.cat((embedding, outputs), 0)\n    \n    return embedding","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:53:44.423997Z","iopub.execute_input":"2022-03-16T10:53:44.424812Z","iopub.status.idle":"2022-03-16T10:53:44.431452Z","shell.execute_reply.started":"2022-03-16T10:53:44.424758Z","shell.execute_reply":"2022-03-16T10:53:44.430503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_grid(train_predictions, valid_predictions, train_labels, valid_labels, new_individual_thres):\n    neigh = NearestNeighbors(n_neighbors=CONFIG['neigh'], metric='cosine')\n    neigh.fit(train_predictions)\n    \n    distances, idxs = neigh.kneighbors(valid_predictions, return_distance=True)\n    conf = 1 - distances\n    preds = []\n\n    for j in range(len(idxs)):\n        preds.append(list(train_labels[idxs[j]]))\n\n    all_top_5_preds = []\n    valid_labels_list = []\n    \n    print(len(preds))\n    \n    for i in range(len(preds)):\n        valid_labels_list.append((valid_labels[i]))\n\n        predict_top = preds[i][:5]\n        top_5_conf = conf[i][:5]\n\n        if top_5_conf[0] < new_individual_thres:\n           \n            temp_list = ['new_individual', predict_top[0], predict_top[1], predict_top[2], predict_top[3]]\n            all_top_5_preds.append(temp_list)   \n           \n        elif top_5_conf[1] < new_individual_thres:\n   \n            temp_list=[predict_top[0], 'new_individual', predict_top[1], predict_top[2], predict_top[3]]\n            all_top_5_preds.append(temp_list)    \n           \n        elif top_5_conf[2] < new_individual_thres:\n\n            temp_list=[predict_top[0], predict_top[1], 'new_individual', predict_top[2], predict_top[3]]\n            all_top_5_preds.append(temp_list)    \n           \n        elif top_5_conf[3] < new_individual_thres:\n           \n            temp_list=[predict_top[0], predict_top[1], predict_top[2], 'new_individual', predict_top[3]]\n            all_top_5_preds.append(temp_list)  \n           \n        elif top_5_conf[4] < new_individual_thres:\n\n            temp_list=[predict_top[0], predict_top[1], predict_top[2], predict_top[3], 'new_individual']\n            all_top_5_preds.append(temp_list)        \n           \n        else:\n            all_top_5_preds.append(predict_top)\n\n        if (('new_individual' in all_top_5_preds[-1]) and (valid_labels_list[i] not in train_labels)):\n            all_top_5_preds[-1] = [valid_labels_list[i] if x == 'new_individual' else x for x in all_top_5_preds[-1]]\n\n    score = map_per_set(valid_labels_list, all_top_5_preds)\n\n    return score","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:42:38.194310Z","iopub.execute_input":"2022-03-16T10:42:38.195050Z","iopub.status.idle":"2022-03-16T10:42:38.208748Z","shell.execute_reply.started":"2022-03-16T10:42:38.194997Z","shell.execute_reply":"2022-03-16T10:42:38.207987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_predictions = np.array(inference(model, train_loader_eval, CONFIG['device']))[1:]\nvalid_predictions = np.array(inference(model, valid_loader_eval, CONFIG['device']))[1:]\n\ntrain_labels = np.array(df_train_eval['individual_id'].values)\nvalid_labels = np.array(df_valid_eval['individual_id'].values)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:48:24.696632Z","iopub.execute_input":"2022-03-16T10:48:24.696887Z","iopub.status.idle":"2022-03-16T10:49:05.392772Z","shell.execute_reply.started":"2022-03-16T10:48:24.696859Z","shell.execute_reply":"2022-03-16T10:49:05.391786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\ndef map_per_set(labels, predictions):\n    \"\"\"Computes the average over multiple images.\n\n    Parameters\n    ----------\n    labels : list\n             A list of the true labels. (Only one true label per images allowed!)\n    predictions : list of list\n             A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"\n    return np.mean([map_per_image(l, p) for l,p in zip(labels, predictions)])","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:47:41.043752Z","iopub.execute_input":"2022-03-16T10:47:41.044295Z","iopub.status.idle":"2022-03-16T10:47:41.050530Z","shell.execute_reply.started":"2022-03-16T10:47:41.044254Z","shell.execute_reply":"2022-03-16T10:47:41.049731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iteration = 0\nbest_score = 0\nbest_thres = 0\n\nfor thres in np.arange(0.1, 0.9, 0.1):\n    print('iteration', iteration, 'of', len(np.arange(0.3, 0.9, 0.1)))        \n    iteration += 1\n    score = predict_grid(train_predictions, valid_predictions, train_labels, valid_labels, new_individual_thres=thres)\n    \n    if score > best_score:\n        best_score = score\n        best_thres = thres\n    \n    print('thres:', thres, ', score:', score)\n    \nprint('Best score is:', best_score)\nprint('Best thres is:', best_thres)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:49:41.324773Z","iopub.execute_input":"2022-03-16T10:49:41.325275Z","iopub.status.idle":"2022-03-16T10:49:41.476581Z","shell.execute_reply.started":"2022-03-16T10:49:41.325236Z","shell.execute_reply":"2022-03-16T10:49:41.475856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_submission(train_data, valid_data, train_labels, neighbors=100, metric='cosine', new_individual_thres=0.6):\n    neigh = NearestNeighbors(n_neighbors=neighbors, metric=metric)\n    neigh.fit(train_data)\n    \n    distances, idxs = neigh.kneighbors(valid_data, return_distance=True)\n    conf = 1-distances\n    preds = []\n    \n    df = pd.read_csv(\"../input/happy-whale-and-dolphin/sample_submission.csv\")\n    for i in range(len(idxs)):\n        preds.append(train_labels[idxs[i]])\n        predict_top_decoded = {}\n        \n    for i in range(len(distances)):\n        \n        predict_top= list(preds[i][:5])\n        topValues = conf[i][:5]\n\n        if topValues[0] < new_individual_thres:\n            \n            temp_list = ['new_individual',predict_top[0],predict_top[1],predict_top[2],predict_top[3]]\n            predict_top_decoded[df.iloc[i]['image']] = temp_list\n            \n        elif topValues[1] < new_individual_thres:\n    \n            temp_list = [predict_top[0], 'new_individual', predict_top[1], predict_top[2], predict_top[3]]\n            predict_top_decoded[df.iloc[i]['image']] = temp_list\n            \n        elif topValues[2] < new_individual_thres:\n\n            temp_list = [predict_top[0], predict_top[1], 'new_individual', predict_top[2], predict_top[3]]\n            predict_top_decoded[df.iloc[i]['image']] = temp_list\n            \n        elif topValues[3] < new_individual_thres:\n            \n            temp_list = [predict_top[0], predict_top[1], predict_top[2], 'new_individual', predict_top[3]]\n            predict_top_decoded[df.iloc[i]['image']] = temp_list\n            \n        elif topValues[4] < new_individual_thres:\n\n            temp_list = [predict_top[0], predict_top[1], predict_top[2], predict_top[3], 'new_individual']\n            predict_top_decoded[df.iloc[i]['image']] = temp_list\n            \n        else:\n            predict_top_decoded[df.iloc[i]['image']] = predict_top\n             \n    for x in tqdm(predict_top_decoded):\n        predict_top_decoded[x] = ' '.join(predict_top_decoded[x])\n    \n    predictions = pd.Series(predict_top_decoded).reset_index()\n    predictions.columns = ['image', 'predictions']\n    predictions.to_csv('happy-whale-submission.csv', index=False)\n    predictions.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:50:00.145967Z","iopub.execute_input":"2022-03-16T10:50:00.146613Z","iopub.status.idle":"2022-03-16T10:50:00.160932Z","shell.execute_reply.started":"2022-03-16T10:50:00.146577Z","shell.execute_reply":"2022-03-16T10:50:00.160204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = np.array(inference(model, test_loader_eval, CONFIG['device']))[1:]\nall_train_data = np.concatenate((train_predictions, valid_predictions))\nall_training_labels = np.concatenate((train_labels, valid_labels))\nget_submission(all_train_data, test_predictions, all_training_labels, neighbors=CONFIG['neigh'], metric='cosine', new_individual_thres=best_thres)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T11:40:44.068675Z","iopub.execute_input":"2022-03-16T11:40:44.068963Z","iopub.status.idle":"2022-03-16T12:06:51.021859Z","shell.execute_reply.started":"2022-03-16T11:40:44.068928Z","shell.execute_reply":"2022-03-16T12:06:51.021103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving submission to artifacts\nrun = wandb.init(project='HappyWhale', \n                 config=CONFIG,\n                 name='submission',\n                 group='submission')\n\nartifact = wandb.Artifact('submission', type='submission')\nartifact.add_file('happy-whale-submission.csv')\n\nrun.log_artifact(artifact)\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2022-03-16T12:06:51.023432Z","iopub.execute_input":"2022-03-16T12:06:51.023815Z","iopub.status.idle":"2022-03-16T12:07:05.158486Z","shell.execute_reply.started":"2022-03-16T12:06:51.023781Z","shell.execute_reply":"2022-03-16T12:07:05.157772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion\n[View the Complete Dashboard Here](https://wandb.ai/artmalygin/HappyWhale)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}