{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport copy\nimport os\nDATA_PATH = \"../input/humpback-whale-identification\"\nprint(os.listdir(DATA_PATH))\n\nfrom tqdm import tnrange, tqdm_notebook as tqdm\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"603d932eb2b8777e7fec931c4d57e8efd0702be5","trusted":false},"cell_type":"code","source":"TRAIN_PATH = DATA_PATH+\"/train/\"\ntrain_files = list(os.listdir(TRAIN_PATH))[100:]\nf = TRAIN_PATH+train_files[1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dee8362587bd2c3c795a121134c0ee6f03941563","trusted":false},"cell_type":"code","source":"im = mpimg.imread(f); im.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a10c7a230ffe5d649a01c96c3fc173f888227fd4","trusted":false},"cell_type":"code","source":"plt.imshow(im)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7841d16e1072fa2f827f6716cdfaae4cf4ce032b","trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import ShuffleSplit","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb31e803d1d9a3d5034890023357322ca5db698a","trusted":false},"cell_type":"code","source":"trainalldf = pd.read_csv(DATA_PATH+\"/train.csv\", nrows=64)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d0dc843183ef3ee997bba5d02b5eff531c97542","trusted":false},"cell_type":"code","source":"trainalldf.count()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"92be9088b64854f9f321b7170f1a275b104112d5","trusted":false},"cell_type":"code","source":"whaleids = sorted(list(trainalldf['Id'].drop_duplicates()))\nprint(whaleids[:5]); print(len(whaleids))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1207db1ddf0a693cd979041153a15e9c9e61083e","trusted":false},"cell_type":"code","source":"whaleids_dict = dict((k,v) for v,k in enumerate(whaleids))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"963bc825cc59193b0621d330269e78fbbedacc4d"},"cell_type":"markdown","source":"## Cut resnet into new model"},{"metadata":{"_uuid":"147769f418eb1707078c5ccfc795807ca838e6e2","trusted":false},"cell_type":"code","source":"BS = 32\nimage_input_size = 224","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"533b97d3ee9500512f4227b68aed25d1d4af1d89","trusted":false},"cell_type":"code","source":"norm = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\ninv_normalize = transforms.Normalize(\n    mean=[-0.485/0.229, -0.456/0.224, -0.406/0.255],\n    std=[1/0.229, 1/0.224, 1/0.255]\n)\n\ntransforms_dict = {\n    'train': transforms.Compose([transforms.RandomResizedCrop(image_input_size),\n                                 transforms.RandomHorizontalFlip(),\n                                 transforms.ToTensor(),\n                                 norm]),\n    'val': transforms.Compose([transforms.Resize(image_input_size),\n                                 transforms.CenterCrop(image_input_size),\n                                 transforms.ToTensor(),\n                                 norm])\n}\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fdbe11abe9676467846a58a810f48b3a2ae56b0","trusted":false},"cell_type":"code","source":"class WhaleImageDataset(torchvision.datasets.folder.ImageFolder):\n    def __init__(self, ROOT_PATH, tfm, images, targets=None):\n        self.ROOT_PATH = ROOT_PATH\n        self.images = images\n        self.targets = targets\n        self.trans = tfm\n        self.loader = torchvision.datasets.folder.default_loader\n    \n    def __getitem__(self, index):\n        f = self.ROOT_PATH + self.images[index]\n        im = self.loader(f)\n        if self.targets is None: # Test mode has no targets\n            return self.trans(im)\n        return self.trans(im), self.targets[index]\n    \n    def __len__(self):\n        return len(self.images)\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fbf2004e9d4803937789ee235973532d0e99ae72"},"cell_type":"markdown","source":"## Split data into train/val"},{"metadata":{"_uuid":"f1b77df2586d10f33b8aeef60813003c3e69e6ea","trusted":false},"cell_type":"code","source":"def split_into_train_val(trainalldf, whaleids_dict, test_size=None, train_size=None, batch_size=BS):\n    trainallimages = trainalldf['Image'].values\n    trainallids = trainalldf['Id'].values\n    trainallclasses = np.array([whaleids_dict[id] for id in trainallids])\n    \n    splitter = ShuffleSplit(n_splits=1, test_size=test_size, train_size=train_size)\n    (train_idxs, val_idxs) = next(splitter.split(trainallimages, trainallclasses))\n    idxs = {'train': train_idxs, 'val': val_idxs}\n    \n    images_dict = {phase: trainallimages[idxs[phase]] for phase in ['train', 'val']}\n    classes_dict = {phase: trainallclasses[idxs[phase]] for phase in ['train', 'val']}\n    \n    datasets_dict = {phase: WhaleImageDataset(TRAIN_PATH, transforms_dict[phase], images_dict[phase], classes_dict[phase]) for phase in ['train','val']}\n    \n    dataloaders_dict = {phase: torch.utils.data.DataLoader(datasets_dict[phase], batch_size=batch_size, shuffle=True, num_workers=1, pin_memory=True) \n                    for phase in ['train', 'val']}\n    \n    return dataloaders_dict","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8e03f7cf0b5ef900ed209359f027f02914ce20c","trusted":false},"cell_type":"code","source":"# Try to overfit just one batch\ndataloaders_dict = split_into_train_val(trainalldf, whaleids_dict, test_size=32, train_size=32)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc28c9cce5e6c86921b8eb16a6685c9b39eea975","scrolled":true,"trusted":false},"cell_type":"code","source":"#im, c = datasets_dict['train'][1]\n#print(im.shape)\n#im = im.permute(1,2,0)\n#im2 = inv_normalize(im)\n#print(im2.shape)\n#plt.imshow(im2)\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f093b06d6454686e2acb5f18514f35b44d938f6"},"cell_type":"markdown","source":"## Set up optimiser"},{"metadata":{"_uuid":"e2a3599871a66848d621d2ff753f8eeb80d77dba","trusted":false},"cell_type":"code","source":"def avprec_cutoff(inds, targets, N=5, m=1):\n    rels = (inds.numpy() == targets.numpy()).astype('int')\n    pks = []\n    for ki in range(1,N+1):\n        pk = rels[:,0:ki].sum(axis=1).reshape(-1,1)/ki\n        pks.append(pk/m)\n\n    return (np.concatenate(pks, axis=1) * rels).sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"905cecbd589155fbb225c19f0d1c57d10c524874","trusted":false},"cell_type":"code","source":"def train_model(model, opt, crit, NUM_EPOCHS, dataloaders_dict, choose_best_acc=False, freeze_bn=False):\n    val_acc_history = []\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(NUM_EPOCHS):\n        print('Epoch {}/{}'.format(epoch, NUM_EPOCHS - 1))\n        print('-' * 10)\n\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n                if freeze_bn:\n                    def set_bn_eval(m):\n                        classname = m.__class__.__name__\n                        if classname.find('BatchNorm') != -1:\n                          m.eval()\n\n                    model.apply(set_bn_eval)\n                \n            else:\n                model.eval()\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            for X_batch, y_batch in dataloaders_dict[phase]:\n                X_batch = X_batch.to('cuda')\n                y_batch = y_batch.to('cuda')\n\n                opt.zero_grad()\n\n                outputs = model(X_batch)\n\n                loss = crit(outputs, y_batch)\n\n                _, preds = torch.max(outputs, 1)\n\n                if phase == 'train':\n                    loss.backward()\n                    opt.step()\n\n                running_loss += loss.item() * X_batch.size(0)\n                running_corrects += torch.sum(preds == y_batch.data)\n\n            epoch_loss = running_loss / len(dataloaders_dict[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders_dict[phase].dataset)\n\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))     \n\n        print('\\n')\n\n    print('Best acc: {:.4f}'.format(best_acc))\n    if choose_best_acc:\n        print('Loading best weights')\n        model.load_state_dict(best_model_wts)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d3cc5e78fca9669769e48bfee2e265c948bc61c"},"cell_type":"markdown","source":"Best training loss is 0.74 - we need more parameters.\n### Resnet50"},{"metadata":{"_uuid":"2b5f4332227986da12c39716edc55f86994ac044","trusted":false},"cell_type":"code","source":"resnet50 = torchvision.models.resnet50(pretrained=True)\nfor p in resnet50.parameters():\n    p.requires_grad = False # Freeze all existing layers","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49d5bfe92160116d95cb67cf4f5010fc941cf19b","trusted":false},"cell_type":"code","source":"resnet50.fc = nn.Linear(2048, len(whaleids))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37a9810027394086403919f43e8c7f7203f936f1","trusted":false},"cell_type":"code","source":"resnet50.to('cuda')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ec5c5b97e6c0a87ac45dbce5049797c50b6aaed","trusted":false},"cell_type":"code","source":"opt = torch.optim.Adam(resnet50.fc.parameters(), lr=3e-4)\ncrit = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c2b0a1f7e965b7c848f9e53b93bc0948cae430b","trusted":false},"cell_type":"code","source":"train_model(resnet50, opt, crit, 50, dataloaders_dict, freeze_bn=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d4da46b0bbf97578ad135a49f516616d0aef55db","trusted":false},"cell_type":"code","source":"for p in resnet50.parameters():\n    p.requires_grad = True # Unfreeze all layers","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6077eb9743bc872432623d6abcbe933c4efac9f6","trusted":false},"cell_type":"code","source":"opt = torch.optim.Adam(resnet50.fc.parameters(), lr=3e-5)\ncrit = nn.CrossEntropyLoss()\ntrain_model(resnet50, opt, crit, 50, dataloaders_dict, freeze_bn=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21c7d8ddc02af56dfac2188d4cf1b2cde9115b17"},"cell_type":"markdown","source":"What does the data look like?"},{"metadata":{"_uuid":"bbc4c76e03b26e1d7f4e6ed797fe3502a41d7d7e","trusted":false},"cell_type":"code","source":"x_batch, y_batch = next(iter(dataloaders_dict['train']))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c48518d69024183ccf451b13769ecc0f9b6b4c23","trusted":false},"cell_type":"code","source":"y_batch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8666644fd9a6297c4f4623e89eef267264bcaa0","trusted":false},"cell_type":"code","source":"preds = resnet50(x_batch.to('cuda')); preds.max(dim=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d06605290aa7ca3458d5a66014c00e9745b4873"},"cell_type":"markdown","source":"Training can't seem to shake learning class 0 as a clear preference - the non-zeros are all accurate.\n\nCan we weight against the class."},{"metadata":{"_uuid":"54a919607c624e0e6432a870fda1aceab402fbb3","trusted":false},"cell_type":"code","source":"freqs = trainalldf['Id'].value_counts() ; freqs[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27198d3e69175ea3790700a88d0190ec7342a8d8","trusted":false},"cell_type":"code","source":"wts = [1/freqs[w] for w in whaleids] ; wts[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b4e42f5d1e4e1a0ac49d3316e5c3381e6d63646","trusted":false},"cell_type":"code","source":"opt = torch.optim.Adam(resnet50.fc.parameters(), lr=3e-4)\ncrit = nn.CrossEntropyLoss(weight=torch.Tensor(wts).to('cuda'))\ntrain_model(resnet50, opt, crit, 50, dataloaders_dict, freeze_bn=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"046d40b98429178c07ab6bc8901a8fd1cae8b0e2"},"cell_type":"markdown","source":"Accuracy gets as high as 1.0, so the model is hopefully big enough and training strategy OK"},{"metadata":{"trusted":true,"_uuid":"480c09b058d5e109097e6712a0f51244da031213"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}