{"cells":[{"metadata":{"trusted":true,"_uuid":"65655386f03bf88bb08a5086a45aac0e2a528b3d"},"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport math\nimport copy\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torchvision\n\n# import skimage\nfrom skimage.io import imread\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea585d58a85d22857cc8dc5911a23fd3be78ffb3"},"cell_type":"markdown","source":"**Configurations**"},{"metadata":{"trusted":true,"_uuid":"875595f5b6e66679d8d765216ab75ed952176c67"},"cell_type":"code","source":"EPOCHS = 5\nUSE_GPU = True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39c14c5d34d1ac13c7a518dec23d2a61bdf609f7"},"cell_type":"markdown","source":"**Load labels**"},{"metadata":{"trusted":true,"_uuid":"09fea8658ef1ff21c10b746baf42bc63f01c5b13"},"cell_type":"code","source":"labels_df = pd.read_csv(\"../input/train_labels.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"109ec345268d504edfa3238195a324995b066663"},"cell_type":"code","source":"labels_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd9a1343d32e56c29fee7a887fb9bfd8cf8dcbc5"},"cell_type":"markdown","source":"**Make sure we are not having the imbalanced classification problem**"},{"metadata":{"trusted":true,"_uuid":"53d98e0f0b23ed253810c4e253720273ff9cf90f"},"cell_type":"code","source":"labels_df[\"label\"].value_counts().plot(kind=\"pie\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4ce9243ba78c8383e60ac1e510b2ee50f27a090"},"cell_type":"markdown","source":"**Train test split for model selection**"},{"metadata":{"trusted":true,"_uuid":"2b6a7879cbf70f542c75941d164ba4fef9958b0e"},"cell_type":"code","source":"train_indices, test_indices = train_test_split(labels_df.index, test_size=0.25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4136f257e4a6f8ddd9531924ff3f5bb92049cd47"},"cell_type":"code","source":"train_indices.shape, test_indices.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d079deb1192c827bd301e3a33565f9df08f6f0b4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d400510a381dd032a986bb552fa3e3d3bbab4672"},"cell_type":"code","source":"class HistopathologicCancerDataset(torch.utils.data.Dataset):\n    \"\"\"\n    This is our custom dataset class which will load the images, perform transforms on them,\n    and load their corresponding labels.\n    \"\"\"\n    \n    def __init__(self, img_dir, labels_csv_file=None, transform=None):\n        self.img_dir = img_dir\n        \n        if labels_csv_file:\n            self.labels_df = pd.read_csv(labels_csv_file)\n        else:\n            self.images = [os.path.join(img_dir, f) for f in os.listdir(img_dir) if f.endswith(\".tif\")]\n            \n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        try:\n            img_path = os.path.join(\n                self.img_dir,\n                \"{}.tif\".format(self.labels_df.iloc[idx, 0])\n            )\n        except AttributeError:\n            img_path = self.images[idx]\n\n#         print(\"img_path:\", img_path)\n        img = imread(img_path)\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        sample = {\n            \"image\": img,\n        }\n        try:\n            sample[\"label\"] = self.labels_df.loc[idx, \"label\"]\n            sample[\"id\"] = self.labels_df.loc[idx, \"id\"]\n        except AttributeError:\n            sample[\"id\"] = os.path.basename(self.images[idx]).replace(\".tif\", \"\")\n        \n        return sample\n    \n    def __len__(self):\n        try:\n            return self.labels_df.shape[0]\n        except AttributeError:\n            return len(self.images)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"241df72febc76df14d21be298435bac5df1a546f"},"cell_type":"markdown","source":"**Image tranformation pipeline**"},{"metadata":{"trusted":true,"_uuid":"c51370550ec0a7a92fd8fc0ebedd0a02412cfa22"},"cell_type":"code","source":"transform_pipe = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(), # Convert np array to PILImage\n    \n    # Resize image to 224 x 224 as required by most vision models\n    torchvision.transforms.Resize(\n        size=(224, 224)\n    ),\n    \n    # Convert PIL image to tensor with image values in [0, 1]\n    torchvision.transforms.ToTensor(),\n    \n    torchvision.transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69c25803dccad729a4bfb95bdf032514e7b92064"},"cell_type":"code","source":"train_data = HistopathologicCancerDataset(\n    img_dir=\"../input/train/\",\n    labels_csv_file=\"../input/train_labels.csv\",\n    transform=transform_pipe\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"219bf456dfd642daea9cc82a77ff159071debe49"},"cell_type":"markdown","source":"**The training dataset loader will randomly sample from the train samples**"},{"metadata":{"trusted":true,"_uuid":"0b26dc3d91fa365d8f23a8667a48647c2d27fbfc"},"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(\n    train_data,\n    batch_size=64,\n    sampler=torch.utils.data.SubsetRandomSampler(\n        train_indices\n    )\n#     shuffle=True,\n#     num_workers=8\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48375a7572ee065377f8fa3a18ade4553c17edcb"},"cell_type":"markdown","source":"**The training dataset loader will randomly sample from the test samples**"},{"metadata":{"trusted":true,"_uuid":"24d1263c31ff663b5c2ff168d3b81ec0fa86f72b"},"cell_type":"code","source":"test_loader = torch.utils.data.DataLoader(\n    train_data,\n    batch_size=64,\n    sampler=torch.utils.data.SubsetRandomSampler(\n        test_indices\n    )\n#     shuffle=True,\n#     num_workers=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94cf65bc49ab5b1881d9d571cf5f29ed287d2432"},"cell_type":"code","source":"dataloaders = {\n    \"train\": train_loader,\n    \"test\": test_loader\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5094af5c399ef22267b9145a540c7f6407bcbfd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4664185fcbb0be997a64f04e49bdabb926f5b5a5"},"cell_type":"code","source":"class Flatten(torch.nn.Module):\n    \"\"\"\n    Custom flatten module like what is available in Keras.\n    \"\"\"\n    \n    def forward(self, input):\n        return input.view(input.size(0), -1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76628ba5dd13103c32f2c1ad3361185676b121d2"},"cell_type":"markdown","source":"**Model definition**"},{"metadata":{"trusted":true,"_uuid":"48205e7aa4bc2b12c0d07d3c767f37eee146af22"},"cell_type":"code","source":"# model = torch.nn.Sequential(\n#     torch.nn.Conv2d(\n#         in_channels=3,\n#         out_channels=8,\n#         kernel_size=3,\n#     ),\n#     torch.nn.MaxPool2d(\n#         kernel_size=2\n#     ),\n#     torch.nn.ReLU(),\n    \n#     torch.nn.Conv2d(\n#         in_channels=8,\n#         out_channels=16,\n#         kernel_size=3\n#     ),\n#     torch.nn.MaxPool2d(\n#         kernel_size=2\n#     ),\n#     torch.nn.ReLU(),\n    \n#     torch.nn.Conv2d(\n#         in_channels=16,\n#         out_channels=32,\n#         kernel_size=3\n#     ),\n#     torch.nn.MaxPool2d(\n#         kernel_size=2\n#     ),\n#     torch.nn.ReLU(),\n    \n#     torch.nn.Conv2d(\n#         in_channels=32,\n#         out_channels=64,\n#         kernel_size=3\n#     ),\n#     torch.nn.MaxPool2d(\n#         kernel_size=2\n#     ),\n#     torch.nn.ReLU(),\n    \n#     Flatten(),\n    \n#     torch.nn.Linear(\n#         in_features=1024,\n#         out_features=1\n#     ),\n#     torch.nn.Sigmoid()\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5be2f990b6f87bd67d288a42940020c92380e3cf"},"cell_type":"code","source":"# model = torchvision.models.resnet50(pretrained=True)\nmodel = torchvision.models.resnet50()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79a19634b183fa2355038cdec379078bbb894d2e"},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6d19b75eb04fe2d3f837fa8d8bac0c487cf1d52"},"cell_type":"markdown","source":"**Replace the final fully connected layer to suite the problem**"},{"metadata":{"trusted":true,"_uuid":"8f91bc25091c8c78c9fba5f282ff844d23dfa524"},"cell_type":"code","source":"model.fc = torch.nn.Sequential(\n    torch.nn.Linear(\n        in_features=2048,\n        out_features=1\n    ),\n    torch.nn.Sigmoid()\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"764fe1ca0e6848a1c7b27e8977b72c0b5bc13f78"},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67ae74875924d72ccafb15ca5d46498117e875da"},"cell_type":"code","source":"# out = model(train_data[0][\"image\"].view(1, 3, 224, 224))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3e5a45f190b1546a0fac7bc39ab0bafffa0e630"},"cell_type":"code","source":"# out.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bac38d2858c52d8e44966c53b1b49d59211e48b"},"cell_type":"code","source":"# Some utils functions.\n# Seems like PyTorch does not auto-infer tensor shapes in a sequential model, so we need to figure the shapes ourself.\n\ndef compute_conv2d_output_dimensions(Hin, Win, kernel_size, padding=(0, 0), dilation=(1, 1), stride=(1, 1)):\n    Hout = math.floor(((Hin + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) / stride[0]) + 1)\n    Wout = math.floor(((Win + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) / stride[1]) + 1)\n    return Hout, Wout\n\n\ndef compute_maxpooling2d_output_dimensions(Hin, Win, kernel_size, stride=None, padding=(0, 0), dilation=(1, 1)):\n    if stride is None:\n        stride = kernel_size\n    \n    Hout = math.floor(((Hin + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) / stride[0]) + 1)\n    Wout = math.floor(((Win + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) / stride[1]) + 1)\n    return Hout, Wout","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc24450fe47fd3d2b15002385ffc78e07b1bd781"},"cell_type":"code","source":"# compute_conv2d_output_dimensions(96, 96, (3, 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ea6b3aa71cf8fb4e654f6e38247b6de31eba60b"},"cell_type":"code","source":"# compute_maxpooling2d_output_dimensions(94, 94, kernel_size=(2, 2))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f29b180dd8ad7e3a23ad495fadaa6c41587777d"},"cell_type":"markdown","source":"**Model training**"},{"metadata":{"trusted":true,"_uuid":"d59db9eaf79310f397694fb80235d444c0e87ad2"},"cell_type":"code","source":"if USE_GPU:\n    model = model.cuda() # Should be called before instantiating optimizer according to docs: https://pytorch.org/docs/stable/nn.html\n\noptimizer = torch.optim.Adam(model.parameters())\ncriterion = torch.nn.BCELoss()\n\nbest_model_wts = copy.deepcopy(model.state_dict())\nbest_acc = 0.0\n\nfor i in range(EPOCHS):\n    for phase in [\"train\", \"test\"]:\n        if phase == \"train\":\n            model.train()\n        else:\n            model.eval()\n        \n        samples = 0\n        loss_sum = 0\n        correct_sum = 0\n        for j, batch in enumerate(dataloaders[phase]):\n            X = batch[\"image\"]\n            labels = batch[\"label\"]\n            if USE_GPU:\n                X = X.cuda()\n                labels = labels.cuda()\n\n            optimizer.zero_grad()\n\n            with torch.set_grad_enabled(phase == 'train'):\n                y = model(X)\n                loss = criterion(\n                    y, \n                    labels.view(-1, 1).float()\n                )\n\n                if phase == \"train\":\n                    loss.backward()\n                    optimizer.step()\n                    \n                loss_sum += loss.item() * X.shape[0] # We need to multiple by batch size as loss is the mean loss of the samples in the batch\n                samples += X.shape[0]\n                num_corrects = torch.sum((y >= 0.5).float() == labels.view(-1, 1).float())\n                correct_sum += num_corrects\n                \n                # Print batch statistics every 50 batches\n                if j % 50 == 49 and phase == \"train\":\n                    print(\"{}:{} - loss: {}, acc: {}\".format(\n                        i + 1, \n                        j + 1, \n                        float(loss_sum) / float(samples), \n                        float(correct_sum) / float(samples)\n                    ))\n                \n        # Print epoch statistics\n        epoch_acc = float(correct_sum) / float(samples)\n        epoch_loss = float(loss_sum) / float(samples)\n        print(\"epoch: {} - {} loss: {}, {} acc: {}\".format(i + 1, phase, epoch_loss, phase, epoch_acc))\n        \n        # Deep copy the model\n        if phase == \"test\" and epoch_acc > best_acc:\n            best_acc = epoch_acc\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(best_model_wts, \"resnet50.pth\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"868d7222941e9f12ccb81d2236b5f6fc5343b67c"},"cell_type":"markdown","source":"**Persist latest model**"},{"metadata":{"trusted":false,"_uuid":"b088d1389ec07e2062ad8ee021d090206adade37"},"cell_type":"code","source":"# torch.save(best_model_wts, \"resnet50.pth\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a71c3ae63bb75481a42d72076a318f87f7c1f781"},"cell_type":"markdown","source":"**Reconstruct model from saved weights**"},{"metadata":{"trusted":false,"_uuid":"f47a9938a9696fa2d51e1087efb2d0a5bcd7689c"},"cell_type":"code","source":"model1 = torchvision.models.resnet50()\nmodel1.fc = torch.nn.Sequential(\n    torch.nn.Linear(\n        in_features=2048,\n        out_features=1\n    ),\n    torch.nn.Sigmoid()\n)\nmodel1.load_state_dict(torch.load(\"resnet50.pth\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb5f948c07f5a13ea174f0c90d86d95f12efc7a6"},"cell_type":"markdown","source":"**Make predictions**"},{"metadata":{"trusted":false,"_uuid":"b27cb59293b4645d4421fae368195f59c792c5b0"},"cell_type":"code","source":"test_data = HistopathologicCancerDataset(\n    img_dir=\"../input/test/\",\n    transform=transform_pipe\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"69726b65aa54a73effdc571cafa1880b24c83603"},"cell_type":"code","source":"test_loader1 = torch.utils.data.DataLoader(\n    test_data,\n    batch_size=64,\n#     shuffle=True,\n#     num_workers=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"89bdaad8df07738b0af1c8d536f62ef7207165c6"},"cell_type":"code","source":"model1.eval()\nif USE_GPU:\n    model1 = model1.cuda()\n\nids_all = []\npredictions = []\n\nfor j, batch in enumerate(test_loader1):\n    X = batch[\"image\"]\n    ids = batch[\"id\"]\n    if USE_GPU:\n        X = X.cuda()\n    \n    for _id in ids:\n        ids_all.append(_id)\n\n    with torch.set_grad_enabled(False):\n        y_pred = model1(X)\n        predictions.append((y_pred >= 0.5).float().cpu().numpy())\n        \nprint(\"Done making predictions!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b4872db3ea679655a43d7c24aa79b0583a2016ba"},"cell_type":"code","source":"submissions = pd.DataFrame({\n    \"id\": ids_all,\n    \"label\": np.concatenate(predictions).reshape(-1,).astype(\"int\")\n}).set_index(\"id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4bfe438e6e6a8efb18fe12c59c56aed3bd4150b3"},"cell_type":"code","source":"submissions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3c726f83f64226d321af3a15d8149960679aab20"},"cell_type":"code","source":"submissions.to_csv(\"submissions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c958435d7f7fc5298fc4d5c288d9ec871a8ba9e5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}