{"cells":[{"metadata":{"_uuid":"07f75b75ce5be8111a3049038f2a0d6f2e25b0b6"},"cell_type":"markdown","source":"## V6\n- Trained with augmented data.\n\n## Dealing with the imbalanced dataset\n- new_whale: Calculate softmax of the output scores. If the maximum probability is less than a certain threshold, then we classify it as new_whale. This 'threshold' would be a hyperparameter.\n    * We'll remove new_whale images from the training set. \n        - Put new_whale as the first prediction for all images and others made by the model after it.\n- For all other classes perform some transformations to increase size of the dataset.\n    - Don't perform horizontal flip initially.\n    \n### Transformation to use\n- RandomAffine\n- ColorJitter\n\n## Other ideas\n- Increase batch size."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import torch\nimport numpy as np\nimport pandas as pd\nimport os\n\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nimport torchvision.utils as utils\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport pickle\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"TRAIN_DIR = '../input/humpback-whale-identification/train'\nTEST_DIR = '../input/humpback-whale-identification/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d9c8930aaedf68cffe304e1124979011d206f97"},"cell_type":"code","source":"os.listdir('../input/data-preprocessing-and-serialization')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e621129a8deb6499a771e2121c5a7d27cf535bec"},"cell_type":"code","source":"train_file = '../input/data-preprocessing-and-serialization/processed_data'\ntest_file = '../input/data-preprocessing-and-serialization/test_data'\nval_file = '../input/validation-data/val_data'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f35f4bf3dcc28f81f77abca0f191c7eb81edf1a8","trusted":true},"cell_type":"code","source":"class W_dataset(Dataset):\n    def __init__(self, data_file, transform=None):\n        # data_file: handle to the preprocessed data file\n        self.data_file = data_file\n        self.transform = transform\n        self.train_dict = pickle.load(self.data_file)\n        \n    def __len__(self):\n        return len(self.train_dict['labels'])\n    def __getitem__(self, idx):\n        img_np = self.train_dict['data'][idx]\n        label = self.train_dict['labels'][idx]\n        x = self.transform(img_np)\n        return (x, label)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e4f2cda32793430c15c988ffc001f30d3e52d5e","trusted":true},"cell_type":"code","source":"transform = T.Compose([\n            T.ToTensor(),\n            T.Normalize((0.51401635, 0.55264414, 0.59649817), (0.26610398, 0.2555096,  0.25559797))\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91c191c6edfd79c879817e240fb4f143548d5eb2","trusted":true},"cell_type":"code","source":"f = open(train_file, 'rb')\ntrain_dset = W_dataset(f, transform=transform)\ntrain_loader = DataLoader(train_dset, batch_size=64, shuffle=True)\nf.close()\nprint(len(train_dset))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d52588be7da78ca7542f299132526e225375dd7"},"cell_type":"code","source":"max(train_dset.train_dict['labels'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d4123e1a97809ea38627492f37c4f51177b6af7c","trusted":true},"cell_type":"code","source":"for t, (x,y) in enumerate(train_loader):\n    print(t, x.size(), y.size())\n    if t>2:\n        break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ae0233022040772a904a04edcd849f739c0b80f","trusted":true},"cell_type":"code","source":"def display_batch(batch):\n    grid = utils.make_grid(batch, nrow=4)\n    plt.figure(figsize=(10,10))\n    plt.imshow(grid.numpy().transpose(1,2,0))\nfor x, label in train_loader:\n    display_batch(x)\n    break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9751aa19883a2a82670b1eca591381c50f79aa6","trusted":true},"cell_type":"code","source":"USE_GPU = True\n\ndtype = torch.float32 # we will be using float throughout this tutorial\n\nif USE_GPU and torch.cuda.is_available():\n    device = torch.device('cuda')\nelse:\n    device = torch.device('cpu')\n\n# Constant to control how frequently we print train loss\nprint_every = 100\n\nprint('using device:', device)\n\ndef flatten(x):\n    N = x.shape[0] # read in N, C, H, W\n    return x.view(N, -1)  # \"flatten\" the C * H * W values into a single vector per image\n\ndef test_flatten():\n    x = torch.arange(12).view(2, 1, 3, 2)\n    print('Before flattening: ', x)\n    print('After flattening: ', flatten(x))\n\ntest_flatten()\n\n# We need to wrap `flatten` function in a module in order to stack it\n# in nn.Sequential\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return flatten(x)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a8d85b1c771b4ac675dbd12f5b8fc00e058c691","trusted":true},"cell_type":"code","source":"def check_accuracy_part34(loader, model):\n    print(\"Checking accuracy on training set: \")  \n    num_correct = 0\n    num_samples = 0\n    model.eval()  # set model to evaluation mode\n    with torch.no_grad():\n        for x, y in loader:\n            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n            y = y.to(device=device, dtype=torch.long)\n            scores = model(x)\n            _, preds = scores.max(1)\n            num_correct += (preds == y).sum()\n            num_samples += preds.size(0)\n        acc = float(num_correct) / num_samples\n        print('Got %d / %d correct (%.2f)' % (num_correct, num_samples, 100 * acc))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5a52aea0fc279920ca4c2d6d2c9d1a8a3e65265","trusted":true},"cell_type":"code","source":"def train_part34(model, optimizer, loader, epochs=1):\n    \"\"\"\n    Inputs:\n    - model: A PyTorch Module giving the model to train.\n    - optimizer: An Optimizer object we will use to train the model\n    - epochs: (Optional) A Python integer giving the number of epochs to train for\n    \n    Returns: Nothing, but prints model accuracies during training.\n    \"\"\"\n    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.80, last_epoch=-1)\n    \n    model = model.to(device=device)  # move the model parameters to CPU/GPU\n    for e in range(1, epochs+1):\n        loss_history = []\n        t_history = []\n        print('Start of epoch: ', e)\n        for t, (x, y) in enumerate(loader):\n            model.train()  # put model to training mode\n            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n            y = y.to(device=device, dtype=torch.long)\n\n            scores = model(x)\n            loss = F.cross_entropy(scores, y)\n\n            # Zero out all of the gradients for the variables which the optimizer\n            # will update.\n            optimizer.zero_grad()\n\n            # This is the backwards pass: compute the gradient of the loss with\n            # respect to each  parameter of the model.\n            loss.backward()\n            if t%50 == 0:\n                loss_history.append(float(loss))\n                t_history.append(t)\n            \n            # Actually update the parameters of the model using the gradients\n            # computed by the backwards pass.\n            optimizer.step()\n\n            if t % print_every == 0:\n                print('Iteration %d, loss = %.4f' % (t, loss.item()))\n                print()\n                #print(loss_history)\n                #print(t_history)\n        if e==epochs:\n            check_accuracy_part34(loader, model)\n        plt.plot(t_history, loss_history, 'o-')\n        plt.show()\n        for param_group in optimizer.param_groups:\n             print(param_group['lr'])\n             break\n        scheduler.step()\n        for param_group in optimizer.param_groups:\n             print(param_group['lr'])\n             break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"428f1ff186dda396304f6902b2eda67486fbff4c","trusted":true},"cell_type":"code","source":"model = None\noptimizer = None\nchannel_1 = 16\nchannel_2 = 16\nchannel_3 = 128\nnum_units_1 = 8192\nnum_units_2 = 128\nbias = False\nnum_classes = 5005\ndropout_prob = 0.0\n\nmodel = nn.Sequential(\n        # Conv_1\n            nn.Conv2d(3, channel_1, (5,5), padding=2, bias=bias),\n            nn.ReLU(),\n            nn.BatchNorm2d(num_features= channel_1),\n            nn.MaxPool2d(kernel_size=(2,2), stride=2),\n            nn.Dropout2d(p=dropout_prob),\n        # Conv_2\n            nn.Conv2d(channel_1, channel_2, (3,3), padding=1, bias=bias),\n            nn.ReLU(),\n            nn.BatchNorm2d(num_features= channel_2),\n            nn.MaxPool2d(kernel_size=(2,2), stride=2),\n            nn.Dropout2d(p=dropout_prob),\n            Flatten(),\n        # Linear_1\n            nn.Linear(channel_2*16*32, num_units_1, bias=bias),\n            nn.ReLU(),\n            nn.BatchNorm1d(num_features= num_units_1),\n        # output\n            nn.Linear(num_units_1, num_classes, bias=True)\n)\n\noptimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=0)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6778d4c211cb83658362298c41e31388aaed8193","trusted":true},"cell_type":"code","source":"train_part34(model, optimizer, train_loader, epochs=6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b7e896b5973c25919ad40fbdc6d2a4f34fb7b1f"},"cell_type":"code","source":"# find out threshold for new_whale class\n'''def select_threshold(loader, model, thresh):\n    print(\"Finding the best threshold value: \")  \n    num_correct = 0\n    num_samples = 0\n    model.eval()  # set model to evaluation mode\n    with torch.no_grad():\n        for x, y in loader:\n            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n            y = y.to(device=device, dtype=torch.long)\n            scores = model(x)\n            probs = F.softmax(scores, dim=1)\n            max_value, preds = probs.max(1)\n            preds[max_value <= thresh] = 0\n            num_correct += (preds == y).sum()\n            num_samples += preds.size(0)\n        acc = float(num_correct) / num_samples\n        print('Got %d / %d correct (%.2f)' % (num_correct, num_samples, 100 * acc))\n    return acc\nthresh_values = np.linspace(0.2, 0.8)\nbest_acc = -1\nfor thresh in thresh_values:\n    acc = select_threshold(val_loader, model, thresh)\n    print('acc = ', acc)\n    print('thresh = ', thresh)\n    print()\n    if acc > best_acc:\n        best_acc = acc\n        best_thresh = thresh\nprint('Best threshold value found to be: ', best_thresh)'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c593fd1af35ddcd39804cbe08ae54e705fa319ad"},"cell_type":"markdown","source":"## Threshold value = 0.3836"},{"metadata":{"trusted":true,"_uuid":"ab2f57d9d08fc223f397dc85d9cebf26622cfd9b"},"cell_type":"code","source":"# save the model\ntorch.save(model.state_dict(), 'model_state_dict')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd232186a82e81bc3af9afdd2f133d63faed6de2"},"cell_type":"code","source":"class Test_dataset(Dataset):\n    def __init__(self, data_file, transform=None):\n        # data_file: handle to the preprocessed data file\n        self.data_file = data_file\n        self.transform = transform\n        self.test_dict = pickle.load(self.data_file)\n        \n    def __len__(self):\n        return len(self.test_dict['data'])\n    def __getitem__(self, idx):\n        img_np = self.test_dict['data'][idx]\n        img_name = self.test_dict['img_names'][idx]\n        x = self.transform(img_np)\n        return (x, img_name)\n    \nf = open(test_file, 'rb')\ntest_dset = Test_dataset(f, transform=transform)\ntest_loader = DataLoader(test_dset, batch_size=64, shuffle=False)\nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1943008d7e3b3657961adf2010134b136cf0230"},"cell_type":"code","source":"for t, (x,y) in enumerate(test_loader):\n    print(t, x.size(), len(y))\n    if t>2:\n        break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63ac416d8a75141f67635b617303f0e9008b9d26","trusted":true},"cell_type":"code","source":"def gen_test_csv(model, loader):\n    thresh = 1.0\n    columns = ['Image', 'Id']\n    test_df = pd.DataFrame(columns=columns)\n    # map whale ids to labels\n    train_df = pd.read_csv('../input/humpback-whale-identification/train.csv')\n    IDs = list(train_df.Id.unique())\n    IDs.sort() \n    print('Number of whale ids = ', len(IDs))\n    model = model.to(device=device)  # move the model parameters to CPU/GPU\n    model.eval()\n    with torch.no_grad():\n        for t, (x, img_name) in enumerate(loader):\n            x = x.to(device=device, dtype=dtype)\n            scores = model(x)\n            #scores = scores.cpu().numpy()\n            # argsort sorts in ascending order. So, we take the last five elements of each row.\n            # Calculate softmax\n            probs = F.softmax(scores, dim=1)\n            probs, indices = torch.topk(probs, 5, dim=1)  # top 5 probabilities\n            #whale_classes = np.argsort(scores, axis=1)[:,-5:]\n            whale_Ids = []\n            for i in range(scores.shape[0]):\n                whale_Ids.append([])\n                new_whale = False    # has new whale been put in\n                for j in range(5):\n                    #if j==0:\n                        #whale_Ids[i].append('new_whale')\n                    if probs[i, j] < thresh and not new_whale:\n                        label = 0\n                        new_whale = True\n                    else:\n                        label = indices[i, j]\n                    Id = IDs[label]\n                    whale_Ids[i].append(Id)\n            for i in range(len(whale_Ids)):\n                whale_Ids[i] = '\\n'.join(whale_Ids[i])\n            whale_imgs = list(img_name)\n            pred = {'Image':whale_imgs, 'Id':whale_Ids}\n            test_df = test_df.append(pd.DataFrame(pred), ignore_index=True, sort=False)\n    test_df.to_csv('submission.csv', index=False)\n    return test_df\n\ntest_df = gen_test_csv(model, test_loader)\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66cf9a3452070ed8f94037e799d3788c949e8c6c"},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}