{"cells":[{"metadata":{"trusted":true,"_uuid":"eb48fe7eb908e18fe7adc2e80c5f7b11190052aa","scrolled":false},"cell_type":"code","source":"!ls -1 ./target/valid | wc -l","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mplimg\nfrom matplotlib.pyplot import imshow\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time \nimport tqdm\nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport warnings\nwarnings.simplefilter(\"ignore\", category=DeprecationWarning)\n\nfrom collections import OrderedDict\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"full_train_df = pd.read_csv(\"../input/humpback-whale-identification/train.csv\")\nfull_train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cc9c2621821449d058f0413ad60a5bb84289df3"},"cell_type":"code","source":"print(f\"There are {len(os.listdir('../input/humpback-whale-identification/train'))} images in train dataset with {full_train_df.Id.nunique()} unique classes.\")\nprint(f\"There are {len(os.listdir('../input/humpback-whale-identification/test'))} images in test dataset.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8b47c1184c55ed9971cf80b606acf7528f7abcb"},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 4))\ntrain_imgs = os.listdir(\"../input/humpback-whale-identification/train\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(\"../input/humpback-whale-identification/train/\" + img)\n    plt.imshow(im)\n    lab = full_train_df.loc[full_train_df.Image == img, 'Id'].values[0]\n    ax.set_title(f'Label: {lab}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7abc7491b70c87afe7fc206f90e5d9b77e3f4e07"},"cell_type":"code","source":"full_train_df.Id.value_counts().sort_values(ascending=False).head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbe6c01800b59971028266e7a0a33d06d9d153f6"},"cell_type":"markdown","source":"# remove new whale\nOnly new_whale has more pictures than other classes. It causes new_whale detection frequentry. That's why I remove new_whale training set.\n- https://www.kaggle.com/suicaokhoailang/removing-class-new-whale-is-a-good-idea"},{"metadata":{"trusted":true,"_uuid":"9195c2b82ef3ee8b965f26d38f77a9b2e85853ab"},"cell_type":"code","source":"import copy\nnew_whale_df = full_train_df.query(\"Id == 'new_whale'\")\ntrain_df = full_train_df.query(\"Id != 'new_whale'\")\nprint(new_whale_df.shape)\nprint(train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8395bfc7341e16b2569900e997c57d4a55bf86d0"},"cell_type":"code","source":"if not os.path.exists('./target'):\n    os.system(\"mkdir ./target\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"170e6b933fd15f757555b46b4c9404f28598aab7"},"cell_type":"code","source":"if not os.path.exists('./target/train'):\n    os.system(\"mkdir ./target/train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f279e551a6f4a1e765cbcbea952d4661c9b75d4"},"cell_type":"code","source":"for image_name in train_df.Image.values:\n    src_path = os.path.join('../input/humpback-whale-identification/train', image_name)\n    dist_path = os.path.join('./target/train')\n    os.system(\"cp \" + src_path + \" \" + dist_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8df5690e4446a8cd48b4589745424fe1309d701"},"cell_type":"code","source":"for i in range(1, 4):\n    print(f'There are {train_df.Id.value_counts()[train_df.Id.value_counts().values==i].shape[0]} classes with {i} samples in train data.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8940cbfc7a9fc272193d74de187b378c1e95b911"},"cell_type":"code","source":"plt.title('Distribution of classes excluding new_whale');\ntrain_df.Id.value_counts()[1:].plot(kind='hist');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94b16fe99403b056d847b7a0b0560b5621a9eac1"},"cell_type":"code","source":"np.array(im).shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9cea3a7c47549f8b4a199cec3b1a55b5a1edc95"},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true,"_uuid":"317b278f0582cbb43724864dc2f8166c1873a21f"},"cell_type":"code","source":"def prepare_labels(y):\n    # From here: https://www.kaggle.com/pestipeti/keras-cnn-starter\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n\n    y = onehot_encoded\n    return y, label_encoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"869082384b8d8acd60eacc90be65766acb04e7b8"},"cell_type":"code","source":"class WhaleDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', df=None, transform = transforms.Compose([transforms.ToTensor()]), y=None):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.y = y\n        if self.datatype == 'train':\n            self.df = df.values\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_files_list)\n    \n    def __getitem__(self, idx):\n        if self.datatype == 'train':\n            img_name = os.path.join(self.datafolder, self.df[idx][0])\n            label = self.y[idx]\n            \n        elif self.datatype == 'test':\n            img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n            label = np.zeros((5004,))\n            \n        image = Image.open(img_name).convert('RGB')\n        image = self.transform(image)\n        if self.datatype == 'train':\n            return image, label\n        elif self.datatype == 'test':\n            # so that the images will be in a correct order\n            return image, label, self.image_files_list[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5506fc8e7b06d4d9824ad384f56a618baa301f1"},"cell_type":"code","source":"y, le_full = prepare_labels(train_df['Id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85103d4084b165e1d3d42d28105d8e7f2f82b3e8"},"cell_type":"code","source":"data_transforms = transforms.Compose([\n    transforms.Resize((100, 100)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225])\n    ])\ntrain_dataset = WhaleDataset(datafolder='./target/train/', datatype='train', df=train_df, transform=data_transforms, y=y)\n\nbatch_size = 32\nnum_workers = 2\n\ntrain_sampler = SubsetRandomSampler(list(range(len(os.listdir('./target/train')))))\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e500c3144ec743b1e8847d890ee9e2799532f505"},"cell_type":"markdown","source":"# Basic CNN without New Whale"},{"metadata":{"trusted":true,"_uuid":"b980131c117fef167f2691e4873677da9218ab4a"},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(3, 32, 7, padding=1)\n        self.conv2_bn = nn.BatchNorm2d(32)\n        self.pool = nn.MaxPool2d(2, 2)\n        \n        self.conv2 = nn.Conv2d(32, 64, 3, padding=1)        \n        self.pool2 = nn.AvgPool2d(3, 3)\n        \n        self.fc1 = nn.Linear(64 * 4 * 4 * 16, 1024)\n        self.fc2 = nn.Linear(1024, 5004)\n\n        self.dropout = nn.Dropout(0.5)        \n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv2_bn(self.conv1(x))))\n        x = self.pool2(F.relu(self.conv2(x)))\n        x = x.view(-1, 64 * 4 * 4 * 16)\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        x = self.fc2(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd56689a949f7f2b4b5cd3125f2a4671e8519f99"},"cell_type":"code","source":"model_conv = Net()\ncriterion = nn.BCEWithLogitsLoss()\n\noptimizer = optim.Adam(model_conv.parameters(), lr=0.01)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a3a01d309adaa953e8ea07581ed26799c03862b"},"cell_type":"code","source":"model_conv.cuda()\nn_epochs = 7\nloss_list = []\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n\n    train_loss = []\n    exp_lr_scheduler.step()\n\n    for batch_i, (data, target) in  enumerate(train_loader):\n        #print(batch_i)\n        data, target = data.cuda(), target.cuda()\n        optimizer.zero_grad()\n        output = model_conv(data)\n        loss = criterion(output, target.float())\n        train_loss.append(loss.item())\n\n        loss.backward()\n        optimizer.step()\n        \n    loss_list.append(np.mean(train_loss))\n    print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20f37193cb5d1dec07717f439d90bd3cfad4e799"},"cell_type":"code","source":"plt.plot(range(n_epochs), loss_list, 'r-', label='train_loss')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"445aa15810c5ecb55d255901d1b1d4858faef7a4"},"cell_type":"code","source":"torch.save(model_conv, './without_newwhale_model')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5e4d3927e87f6f1bdd1e13d58ddebba198b8c3f"},"cell_type":"markdown","source":"# Find fixed threshold of new whale\n- https://www.kaggle.com/suicaokhoailang/removing-class-new-whale-is-a-good-idea\n- At first, I couldn't understand what he does in the kernel. I got he found the fixed detection rate of new whale."},{"metadata":{"trusted":true,"_uuid":"4ab069b9f3e5e7826d74e1d6ccaa0d1f1ceb70a3"},"cell_type":"code","source":"len_valid_nw = new_whale_df.shape[0]\nvalid_new_whale_df = new_whale_df.head(int(len_valid_nw * 0.2))\nvalid_df = pd.concat([train_df, valid_new_whale_df]).reset_index(drop=True)\nvalid_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edf39fbbe56cccc299e0157a3ee4c6f6a9e0575d"},"cell_type":"code","source":"del train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f244a8e50b96cd248e64318ee7ecca43e5342ea3"},"cell_type":"code","source":"if not os.path.exists('./target/valid'):\n    os.system(\"mkdir ./target/valid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77730f1f02b1d8630a4bbf6c21fca6df99772b4d"},"cell_type":"code","source":"!rm -f ./target/valid/*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5aea4ce2dbf7d025df8beca7649861d8c605ee89"},"cell_type":"code","source":"!rm -f ./target/train/*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7de9a69bfff6b92e8613ccdacd6341b6d5ca9443"},"cell_type":"code","source":"valid_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51648986fb7fac8030001b6185ee5200f472b10b"},"cell_type":"code","source":"for image_name in valid_df.Image.values:\n    src_path = os.path.join('../input/humpback-whale-identification/train', image_name)\n    dist_path = os.path.join('./target/valid')\n    os.system(\"cp \" + src_path + \" \" + dist_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6959436844521fe244a8a9ab80b8538e0d0dc4e"},"cell_type":"code","source":"!ls -1 ./target/valid | wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b591b5809d83a271406f8ef21568b85e9bc73c38"},"cell_type":"code","source":"y, le_full = prepare_labels(valid_df['Id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cef8f512e318138127fc3a9f8625ce129ee7415"},"cell_type":"code","source":"data_transforms_valid = transforms.Compose([\n    transforms.Resize((100, 100)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225])\n])\n\nvalid_set = WhaleDataset(datafolder='./target/valid', datatype='test', transform=data_transforms_valid)\n\nbatch_size = 32\nnum_workers = 2\n\nvalid_loader = torch.utils.data.DataLoader(valid_set, batch_size=32, num_workers=num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ce67539c537c46b93a0c6d705bf0d95af2fb73d"},"cell_type":"code","source":"model_conv = torch.load('./without_newwhale_model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e5b3bdb5b035d9fa04b3f58e342b6bd2a5f31e0"},"cell_type":"code","source":"model_conv.eval()\nfor (data, target, name) in valid_loader:\n    data = data.cuda()\n    output = model_conv(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        e_str = [str(s) for s in e]\n        valid_df.loc[valid_df['Image'] == n, 'Predict'] = ' '.join(e_str)\n\nvalid_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bcbd71dffd84e772d8c1bf7841863117ef56915"},"cell_type":"code","source":"preds = valid_df['Predict']\nids = valid_df['Id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e5cfe54feab00b5e869a99e617a7ec715fc484d"},"cell_type":"code","source":"def map5(X, y, th):\n    score = 0\n    for i in range(X.shape[0]):\n        str_X = X[i].split(' ')\n        result = [float(s) for s in str_X]\n        result.insert(0, float(th))\n        result = np.array(result)\n        pred = le_full.inverse_transform(result.argsort()[-5:][::-1])\n        for j in range(pred.shape[0]):\n            if pred[j] == y[i]:\n                score += (5 - j)/5\n                break\n    return float(score/X.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"522a47a99d68eb0a28b8eaf3dc938d507647e661"},"cell_type":"code","source":"best_th = 0\nbest_score = 0\nfor th in np.arange(-10, 0, 1):\n    score = map5(preds, ids, th)\n    if score > float(best_score):\n        best_score = score\n        best_th = th\n    print(\"Threshold = {:.3f}, MAP5 = {:.3f}\".format(th,score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04d73fa522848dd2513ee70a7274bc16d65997ee"},"cell_type":"code","source":"print(\"Best Threshold = {:.3f}, Best MAP5 = {:.3f}\".format(best_th,best_score))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50233bbd910a8a9ed2e371351c65bda097fb50aa"},"cell_type":"markdown","source":"# Predict"},{"metadata":{"trusted":true,"_uuid":"be15bf660a584f324f6b09500659c1a105ff4cf4"},"cell_type":"code","source":"data_transforms_test = transforms.Compose([\n    transforms.Resize((100, 100)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225])\n])\n\ntest_set = WhaleDataset(datafolder='../input/humpback-whale-identification/test/', datatype='test', transform=data_transforms_test)\n\nbatch_size = 32\nnum_workers = 2\n\ntest_sampler = SubsetRandomSampler(list(range(len(os.listdir('../input/humpback-whale-identification/test')))))\n# less size for test loader.\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=32, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d694b5a250a0812e1b3a6550ea59be3052a401c9"},"cell_type":"code","source":"sub = pd.read_csv('../input/humpback-whale-identification/sample_submission.csv')\n\nmodel_conv.eval()\nfor index, (data, target, name) in enumerate(test_loader):\n    data = data.cuda()\n    output = model_conv(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        e = np.insert(e, 0, float(best_th))\n        sub.loc[sub['Image'] == n, 'Id'] = ' '.join(le_full.inverse_transform(e.argsort()[-5:][::-1]))\n        \nsub.to_csv('submission_without_new_whale.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0d2dbc38808ccf27f578a9110a2ae466bea9c11"},"cell_type":"code","source":"!rm -rf ./target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"202ccccec670b5ab0b8566ed71365c11934cb36a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}