{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Happywhale Whale and Dolphin Identification Using Wide ResNet50 Transfer Learning","metadata":{}},{"cell_type":"markdown","source":"# Imports and installation","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os # for os stuff\nfrom sklearn import preprocessing  # for preprocessing\nfrom tqdm import tqdm # for progress bar\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# pytorch imports\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.models as models\nfrom torchvision.io import read_image\n\n# for visualizing\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\nprint(dir(models))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-18T21:14:17.635976Z","iopub.execute_input":"2022-03-18T21:14:17.636310Z","iopub.status.idle":"2022-03-18T21:14:20.123739Z","shell.execute_reply.started":"2022-03-18T21:14:17.636228Z","shell.execute_reply":"2022-03-18T21:14:20.122964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.125330Z","iopub.execute_input":"2022-03-18T21:14:20.126006Z","iopub.status.idle":"2022-03-18T21:14:20.131819Z","shell.execute_reply.started":"2022-03-18T21:14:20.125967Z","shell.execute_reply":"2022-03-18T21:14:20.130962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformers\nFor the training set, resize down to 256x256, take a random crop of 224, then applies random horizontal flip and normalization.\n\nFor test set, same thing as training set except the crop is centered and no flip.","metadata":{}},{"cell_type":"code","source":"train_transformer = nn.Sequential(\n    transforms.ConvertImageDtype(torch.float),\n    transforms.Resize(256),\n    transforms.RandomCrop(224),\n    transforms.RandomHorizontalFlip(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n)\n\ntest_transformer = nn.Sequential(\n    transforms.ConvertImageDtype(torch.float),\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.133521Z","iopub.execute_input":"2022-03-18T21:14:20.134369Z","iopub.status.idle":"2022-03-18T21:14:20.142521Z","shell.execute_reply.started":"2022-03-18T21:14:20.134331Z","shell.execute_reply":"2022-03-18T21:14:20.141628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing the data\n","metadata":{}},{"cell_type":"markdown","source":"Putting the metadata into a dataframe and correcting duplicate species labels. Adding a path column so it's easier to find the images.","metadata":{}},{"cell_type":"code","source":"train_dataframe = pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\ntrain_dataframe['species'].replace({\n    'bottlenose_dolpin': 'bottlenose_dolphin',\n    'kiler_whale': 'killer_whale'\n}, inplace=True)\ntrain_dataframe['path'] = '../input/happy-whale-and-dolphin/train_images/'+ train_dataframe['image']\n\ntest_images = os.listdir('../input/happy-whale-and-dolphin/test_images')\ntest_dataframe = pd.DataFrame(data=test_images, columns=['image'])\ntest_dataframe['path'] = '../input/happy-whale-and-dolphin/test_images/'+ test_dataframe['image']\n\nnum_individuals = train_dataframe['individual_id'].nunique()\nprint(\"num individuals:\", num_individuals)\nprint(\"num species:\", train_dataframe['species'].nunique())\nprint(train_dataframe.species.value_counts())\nprint(train_dataframe)\nprint(test_dataframe)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.144570Z","iopub.execute_input":"2022-03-18T21:14:20.145265Z","iopub.status.idle":"2022-03-18T21:14:20.780248Z","shell.execute_reply.started":"2022-03-18T21:14:20.145179Z","shell.execute_reply":"2022-03-18T21:14:20.779454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we also convert the individual ids to integers so we can work with tensors for the labels.","metadata":{}},{"cell_type":"code","source":"label_encoder = preprocessing.LabelEncoder()\ntrain_dataframe['individual_id'] = label_encoder.fit_transform(train_dataframe['individual_id'])\nprint(train_dataframe)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.782395Z","iopub.execute_input":"2022-03-18T21:14:20.782931Z","iopub.status.idle":"2022-03-18T21:14:20.842829Z","shell.execute_reply.started":"2022-03-18T21:14:20.782882Z","shell.execute_reply":"2022-03-18T21:14:20.841923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset class.\nThe image preprocessing is done on the GPU.","metadata":{}},{"cell_type":"code","source":"class HappyDataSet(torch.utils.data.Dataset):\n    def __init__(self, dataframe, transformer, test=False):\n        self.dataframe = dataframe\n        self.transformer = transformer\n        self.test = test\n        \n    def __len__(self):\n        return self.dataframe.shape[0]\n    \n    def __getitem__(self, idx):\n        path = self.dataframe.loc[idx, \"path\"]\n        image = read_image(path, torchvision.io.ImageReadMode.RGB)\n        image = image.to(device)\n        if self.transformer is not None:\n            image = self.transformer(image)\n        label = self.dataframe.loc[idx, 'image'] if self.test else self.dataframe.loc[idx, 'individual_id']\n        return image, label\n        ","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.844330Z","iopub.execute_input":"2022-03-18T21:14:20.844578Z","iopub.status.idle":"2022-03-18T21:14:20.851924Z","shell.execute_reply.started":"2022-03-18T21:14:20.844544Z","shell.execute_reply":"2022-03-18T21:14:20.850889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is an example of one of the grayscale images that need to be converted to RGB.","metadata":{}},{"cell_type":"code","source":"Image.open(train_dataframe.loc[38, \"path\"]).convert('RGB')","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.853351Z","iopub.execute_input":"2022-03-18T21:14:20.854358Z","iopub.status.idle":"2022-03-18T21:14:20.986297Z","shell.execute_reply.started":"2022-03-18T21:14:20.854317Z","shell.execute_reply":"2022-03-18T21:14:20.985611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make the dataloaders","metadata":{}},{"cell_type":"code","source":"train_set = HappyDataSet(train_dataframe, transformer=train_transformer, test=False)\ntest_set = HappyDataSet(test_dataframe, transformer=test_transformer, test=True)\n\ntrain_loader = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.987254Z","iopub.execute_input":"2022-03-18T21:14:20.987482Z","iopub.status.idle":"2022-03-18T21:14:20.994506Z","shell.execute_reply.started":"2022-03-18T21:14:20.987450Z","shell.execute_reply":"2022-03-18T21:14:20.993695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Check if dataloader breaks\n# dataiter = iter(train_loader)\n# images, labels = dataiter.next()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:20.995958Z","iopub.execute_input":"2022-03-18T21:14:20.996510Z","iopub.status.idle":"2022-03-18T21:14:21.007319Z","shell.execute_reply.started":"2022-03-18T21:14:20.996471Z","shell.execute_reply":"2022-03-18T21:14:21.006450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function","metadata":{}},{"cell_type":"code","source":"def train(model, optimizer, criterion, train_loader, start_epoch=0, epochs=30, lr_schedule={}, state=None, checkpoint_schedule=[]):\n    model.to(device)\n    model.train()\n    \n    if state:\n        # load provided state\n        model.load_state_dict(state['model'])\n        optimizer.load_state_dict(state['optimizer'])\n        start_epoch = state['epoch']\n        \n    for epoch in range(start_epoch):\n        # Go through learning rate schedule up to the start epoch\n        if epoch in lr_schedule:\n            # Update learning rate at scheduled epoch\n            for group in optimizer.param_groups:\n                group['lr'] = lr_schedule[epoch]\n    \n    for epoch in range(epochs):\n        if epoch in lr_schedule:\n            # Update learning rate at scheduled epoch\n            for group in optimizer.param_groups:\n                group['lr'] = lr_schedule[epoch]\n\n        epoch_loss = 0.\n        for batch in tqdm(train_loader):\n            images, labels = batch[0].to(device), batch[1].to(device)\n        \n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            epoch_loss += loss.item()\n            loss.backward()\n            optimizer.step()\n\n        print(f'loss: {epoch_loss / len(train_loader)}, e={epoch}')\n        if epoch in checkpoint_schedule:\n            state = {'epoch': epoch+1, 'model': model.state_dict(), 'optimizer': optimizer.state_dict()}\n            torch.save(state, f'./checkpoint_{epoch+1}.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:21.010364Z","iopub.execute_input":"2022-03-18T21:14:21.010879Z","iopub.status.idle":"2022-03-18T21:14:21.023015Z","shell.execute_reply.started":"2022-03-18T21:14:21.010840Z","shell.execute_reply":"2022-03-18T21:14:21.022070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get the model and train it","metadata":{}},{"cell_type":"markdown","source":"This is using Adam as the optimizer, with cross entropy loss.","metadata":{}},{"cell_type":"code","source":"# resnet = models.wide_resnet50_2(pretrained=True)\n# # change the final layer to num_individuals predictions\n# resnet.fc = nn.Linear(2048, num_individuals)\n\n# criterion = nn.CrossEntropyLoss()\n# optimizer = optim.Adam(resnet.parameters())\n\n# train(resnet, optimizer, criterion, train_loader, epochs=2, state=torch.load('../input/happywhale-checkpoints/checkpoint_1.pkl'), lr_schedule={0:.1, 1:.01, 2:.001}, checkpoint_schedule=[0,1,2])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:21.024514Z","iopub.execute_input":"2022-03-18T21:14:21.024992Z","iopub.status.idle":"2022-03-18T21:14:21.033209Z","shell.execute_reply.started":"2022-03-18T21:14:21.024956Z","shell.execute_reply":"2022-03-18T21:14:21.032364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make predictions","metadata":{}},{"cell_type":"code","source":"# evaluate from a checkpoint\nresnet = models.wide_resnet50_2()\nresnet.fc = nn.Linear(2048, num_individuals)\nstate = torch.load('../input/happywhale-checkpoints/checkpoint_1.pkl')\nresnet.load_state_dict(state['model'])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:21.035452Z","iopub.execute_input":"2022-03-18T21:14:21.036051Z","iopub.status.idle":"2022-03-18T21:14:34.461966Z","shell.execute_reply.started":"2022-03-18T21:14:21.036013Z","shell.execute_reply":"2022-03-18T21:14:34.461129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For testing out the prediction\nresnet.to(device)\nresnet.eval()\nwith torch.no_grad():\n    dataiter = iter(test_loader)\n    image, label = dataiter.next()\n    label, = label\n    print(label)\n    image = image.to(device)\n    output = resnet(image)\n    flattened_output = torch.flatten(output)\n    probs = F.softmax(flattened_output, dim=0)\n    top_5_values, top_5_predicted = torch.topk(probs, k=5)\n    print(top_5_values, top_5_predicted)\n    top_5_ids = label_encoder.inverse_transform(top_5_predicted.detach().cpu().numpy())\n    print(top_5_values, top_5_ids)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:34.463412Z","iopub.execute_input":"2022-03-18T21:14:34.463668Z","iopub.status.idle":"2022-03-18T21:14:40.526159Z","shell.execute_reply.started":"2022-03-18T21:14:34.463632Z","shell.execute_reply":"2022-03-18T21:14:40.524608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# outfile = open('submission.csv', 'w')\n# outfile.write('image,predictions\\n')\n# resnet.to(device)\n# resnet.eval()\n# new_individual_threshold = 100/num_individuals\n\n# with torch.no_grad():\n#     for image, label in tqdm(test_loader):\n#         image = image.to(device)\n#         label, = label\n#         line = f'{label},' # the line we will write to file\n#         output = resnet(image)\n#         flattened_output = torch.flatten(output)\n#         output_probs = F.softmax(flattened_output, dim=0)\n        \n#         top_5_values, top_5_predicted = torch.topk(output_probs, k=5)\n#         top_5_ids = label_encoder.inverse_transform(top_5_predicted.detach().cpu().numpy())\n        \n#         # put in the top 4 scorers first\n#         for i in range(4):\n#             line += top_5_ids[i] + \" \"\n#         if torch.min(top_5_values) < new_individual_threshold:\n#             # predict new individual\n#             line += \"new_individual\"\n#         else:\n#             # predict the last top 5 scorer\n#             line += top_5_ids[4]\n#         line += \"\\n\"\n#         outfile.write(line)\n        \n# outfile.close()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:40.527621Z","iopub.execute_input":"2022-03-18T21:14:40.527868Z","iopub.status.idle":"2022-03-18T21:14:40.534704Z","shell.execute_reply.started":"2022-03-18T21:14:40.527834Z","shell.execute_reply":"2022-03-18T21:14:40.533877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The code for generating a random submission","metadata":{}},{"cell_type":"code","source":"# # Random guessing\n# outfile = open('submission.csv', 'w')\n# outfile.write('image,predictions\\n')\n\n# with torch.no_grad():\n#     for image, label in tqdm(test_loader):\n#         image = image.to(device)\n#         label, = label\n#         line = f'{label},' # the line we will write to file\n#         top_5_predicted = torch.randperm(num_individuals+1)\n#         if num_individuals in top_5_predicted:\n#             top_5_predicted = top_5_predicted[top_5_predicted != num_individuals]\n#         top_5_ids = label_encoder.inverse_transform(top_5_predicted.detach().cpu().numpy())\n        \n#         # put in the top 4 scorers first\n#         for i in range(4):\n#             line += top_5_ids[i] + \" \"\n#         if len(top_5_ids) == 4:\n#             # predict new individual\n#             line += \"new_individual\"\n#         else:\n#             # predict the last top 5 scorer\n#             line += top_5_ids[4]\n#         line += \"\\n\"\n#         outfile.write(line)\n        \n# outfile.close()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T21:14:40.535888Z","iopub.execute_input":"2022-03-18T21:14:40.536664Z","iopub.status.idle":"2022-03-18T21:49:20.587618Z","shell.execute_reply.started":"2022-03-18T21:14:40.536617Z","shell.execute_reply":"2022-03-18T21:49:20.586860Z"},"trusted":true},"execution_count":null,"outputs":[]}]}