{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import Libraries and Setup ","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport copy\nimport random\nimport time\nfrom PIL import Image\nfrom PIL import Image, ImageDraw\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import models\nimport torchvision.transforms as transforms\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset, random_split","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:19.714632Z","iopub.execute_input":"2021-10-12T08:21:19.715600Z","iopub.status.idle":"2021-10-12T08:21:25.284574Z","shell.execute_reply.started":"2021-10-12T08:21:19.715503Z","shell.execute_reply":"2021-10-12T08:21:25.283825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Device configuration\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using {} device\".format(device))","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:25.286345Z","iopub.execute_input":"2021-10-12T08:21:25.286607Z","iopub.status.idle":"2021-10-12T08:21:25.331609Z","shell.execute_reply.started":"2021-10-12T08:21:25.286573Z","shell.execute_reply":"2021-10-12T08:21:25.330648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:25.333557Z","iopub.execute_input":"2021-10-12T08:21:25.334022Z","iopub.status.idle":"2021-10-12T08:21:33.825241Z","shell.execute_reply.started":"2021-10-12T08:21:25.333984Z","shell.execute_reply":"2021-10-12T08:21:33.824483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GLobal Variables","metadata":{}},{"cell_type":"code","source":"N_EPOCHS = 20\nBATCH_SIZE = 128\n# IMG_HEIGHT = 96\n# IMG_WIDTH = 96\nSEED = 1234\n\n## For normalization required by pretrained models in torchvision.models\nMEAN = [0.485, 0.456, 0.406]\nSTD = [0.229, 0.224, 0.225]","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:33.828763Z","iopub.execute_input":"2021-10-12T08:21:33.828990Z","iopub.status.idle":"2021-10-12T08:21:33.833012Z","shell.execute_reply.started":"2021-10-12T08:21:33.828964Z","shell.execute_reply":"2021-10-12T08:21:33.832338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configure for Reproducibility","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:33.833901Z","iopub.execute_input":"2021-10-12T08:21:33.834162Z","iopub.status.idle":"2021-10-12T08:21:33.852511Z","shell.execute_reply.started":"2021-10-12T08:21:33.834118Z","shell.execute_reply":"2021-10-12T08:21:33.851746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"train_dir = '../input/histopathologic-cancer-detection/train'\ntest_dir = '../input/histopathologic-cancer-detection/test'\ntrain_labels = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\nsubmission = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:33.855637Z","iopub.execute_input":"2021-10-12T08:21:33.855854Z","iopub.status.idle":"2021-10-12T08:21:34.500868Z","shell.execute_reply.started":"2021-10-12T08:21:33.855831Z","shell.execute_reply":"2021-10-12T08:21:34.500108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = len(os.listdir(train_dir))\ntest_size = len(os.listdir(test_dir))\n\nprint(train_size,test_size)\nprint(train_labels.shape)\nprint(submission.shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:34.502121Z","iopub.execute_input":"2021-10-12T08:21:34.502369Z","iopub.status.idle":"2021-10-12T08:21:38.827034Z","shell.execute_reply.started":"2021-10-12T08:21:34.502337Z","shell.execute_reply":"2021-10-12T08:21:38.826060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_labels.head())\ndisplay(submission.head())","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:38.828586Z","iopub.execute_input":"2021-10-12T08:21:38.828878Z","iopub.status.idle":"2021-10-12T08:21:38.858734Z","shell.execute_reply.started":"2021-10-12T08:21:38.828843Z","shell.execute_reply":"2021-10-12T08:21:38.858095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:38.859744Z","iopub.execute_input":"2021-10-12T08:21:38.860009Z","iopub.status.idle":"2021-10-12T08:21:38.873162Z","shell.execute_reply.started":"2021-10-12T08:21:38.859973Z","shell.execute_reply":"2021-10-12T08:21:38.872348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(13, 6))\ntrain_labels['label'].value_counts().plot(kind='bar')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:38.876741Z","iopub.execute_input":"2021-10-12T08:21:38.876958Z","iopub.status.idle":"2021-10-12T08:21:39.094585Z","shell.execute_reply.started":"2021-10-12T08:21:38.876936Z","shell.execute_reply":"2021-10-12T08:21:39.093838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the Data","metadata":{}},{"cell_type":"code","source":"# get the ids of cancer cases\ncancer = train_labels.loc[train_labels['label']==1]['id'].to_numpy() \n\n# get the ids of the normal cases\nnormal = train_labels.loc[train_labels['label']==0]['id'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:39.095849Z","iopub.execute_input":"2021-10-12T08:21:39.096084Z","iopub.status.idle":"2021-10-12T08:21:39.132211Z","shell.execute_reply.started":"2021-10-12T08:21:39.096053Z","shell.execute_reply":"2021-10-12T08:21:39.131552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize'] = (15, 6)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:39.133310Z","iopub.execute_input":"2021-10-12T08:21:39.133746Z","iopub.status.idle":"2021-10-12T08:21:39.138450Z","shell.execute_reply.started":"2021-10-12T08:21:39.133712Z","shell.execute_reply":"2021-10-12T08:21:39.137522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize cancer cases\n\nnrows, ncols= 6, 15\nplt.subplots_adjust(wspace=0, hspace=0) \n\nfor i, image_id in enumerate(cancer[: nrows * ncols]):\n    img_path = os.path.join(train_dir , image_id +'.tif')\n    img = Image.open(img_path)\n    idcol = ImageDraw.Draw(img)\n    idcol.rectangle(((0,0),(95,95)),outline='red')\n    plt.subplot(nrows, ncols, i+1) \n    plt.imshow(np.array(img))\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:39.139589Z","iopub.execute_input":"2021-10-12T08:21:39.140293Z","iopub.status.idle":"2021-10-12T08:21:43.401759Z","shell.execute_reply.started":"2021-10-12T08:21:39.140257Z","shell.execute_reply":"2021-10-12T08:21:43.401048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize Normal Cases\n\nnrows, ncols= 6, 15\nplt.subplots_adjust(wspace=0, hspace=0) \n\nfor i, image_id in enumerate(normal[: nrows * ncols]):\n    img_path = os.path.join(train_dir , image_id +'.tif')\n    img = Image.open(img_path)\n    idcol = ImageDraw.Draw(img)\n    idcol.rectangle(((0,0),(95,95)),outline='green')\n    plt.subplot(nrows, ncols, i+1) \n    plt.imshow(np.array(img))\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:43.402944Z","iopub.execute_input":"2021-10-12T08:21:43.403232Z","iopub.status.idle":"2021-10-12T08:21:47.974752Z","shell.execute_reply.started":"2021-10-12T08:21:43.403196Z","shell.execute_reply":"2021-10-12T08:21:47.973738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Custom Dataset","metadata":{}},{"cell_type":"code","source":"# class CustomDataset(Dataset):\n    \n#     def __init__(self, labels, img_dir, transform=None, target_transform=None):\n#         self.img_labels = labels\n#         self.img_dir = img_dir\n#         self.transform = transform\n#         self.target_transform = target_transform\n\n#     def __len__(self):\n#         return len(self.img_labels)\n\n#     def __getitem__(self, idx):\n#         img_name, label = self.img_labels.iloc[idx]\n#         img_path = os.path.join(self.img_dir, img_name + '.tif')\n#         image =  Image.open(img_path)\n\n#         if self.transform is not None:\n#             image = self.transform(image)\n#         if self.target_transform is not None:\n#             label = self.target_transform(label)\n            \n#         return image, label","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:47.976331Z","iopub.execute_input":"2021-10-12T08:21:47.976691Z","iopub.status.idle":"2021-10-12T08:21:47.981600Z","shell.execute_reply.started":"2021-10-12T08:21:47.976652Z","shell.execute_reply":"2021-10-12T08:21:47.980943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    \n    def __init__(self, labels, img_dir, dataset_type = 'train', transform=None, target_transform=None):\n        self.img_labels = labels\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n        self.dataset_type = dataset_type\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_id = self.img_labels.loc[idx, 'id']\n        img_path = os.path.join(self.img_dir, img_id + '.tif')\n        image =  Image.open(img_path)\n\n        if self.transform is not None:\n            image = self.transform(image)\n            \n        # For Test Dataset, we don't have class label.\n        # So for Test Dataset, we will return the image 'id' as label\n        label = 0\n        \n        if self.dataset_type == 'train':\n            label = self.img_labels.loc[idx, 'label']\n            \n            if self.target_transform is not None:\n                label = self.target_transform(label)\n        \n        elif self.dataset_type == 'test':\n            label = img_id\n            \n        return image, label","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:47.983095Z","iopub.execute_input":"2021-10-12T08:21:47.983677Z","iopub.status.idle":"2021-10-12T08:21:47.995752Z","shell.execute_reply.started":"2021-10-12T08:21:47.983638Z","shell.execute_reply":"2021-10-12T08:21:47.994966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define transformation that converts a PIL image into PyTorch tensors.\nimport torchvision.transforms as transforms\ndata_transformer = transforms.Compose([transforms.ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:47.997226Z","iopub.execute_input":"2021-10-12T08:21:47.997689Z","iopub.status.idle":"2021-10-12T08:21:48.008082Z","shell.execute_reply.started":"2021-10-12T08:21:47.997638Z","shell.execute_reply":"2021-10-12T08:21:48.006950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define an instance of the custom dataset for the train folder.\ndataset = CustomDataset(train_labels, train_dir, transform = data_transformer)\n\nimg, label = dataset[24]\nprint(img.shape, torch.min(img), torch.max(img))\nprint(label)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.009899Z","iopub.execute_input":"2021-10-12T08:21:48.010177Z","iopub.status.idle":"2021-10-12T08:21:48.096724Z","shell.execute_reply.started":"2021-10-12T08:21:48.010143Z","shell.execute_reply":"2021-10-12T08:21:48.096069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Splitting Dataset into Training and Validation set","metadata":{}},{"cell_type":"code","source":"dataset_size = len(dataset)\ntrain_size = int(0.8 * dataset_size)\nvalid_size = dataset_size - train_size\n\n# Split Pytorch tensor\ntrain_dataset, valid_dataset = random_split(dataset, [train_size, valid_size]) # random split 80/20\n\nprint(\"train dataset size:\", len(train_dataset))\nprint(\"validation dataset size:\", len(valid_dataset))\n","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.097819Z","iopub.execute_input":"2021-10-12T08:21:48.098054Z","iopub.status.idle":"2021-10-12T08:21:48.125203Z","shell.execute_reply.started":"2021-10-12T08:21:48.098023Z","shell.execute_reply":"2021-10-12T08:21:48.124335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply Transformations to Train and Validation Set","metadata":{}},{"cell_type":"code","source":"# Define the following transformations for the training dataset\ntrain_transformer = transforms.Compose([\n    transforms.RandomHorizontalFlip(p=0.5), \n    transforms.RandomVerticalFlip(p=0.5),  \n    transforms.RandomRotation(20),         \n#     transforms.RandomResizedCrop(96, scale=(0.8,1.0),ratio=(1.0,1.0)),\n    transforms.ToTensor()]\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.126484Z","iopub.execute_input":"2021-10-12T08:21:48.126838Z","iopub.status.idle":"2021-10-12T08:21:48.132300Z","shell.execute_reply.started":"2021-10-12T08:21:48.126777Z","shell.execute_reply":"2021-10-12T08:21:48.131358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For the validation dataset, we don't need any augmentation; simply convert images into tensors\nvalid_transformer = transforms.Compose([\n    transforms.ToTensor()]\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.133618Z","iopub.execute_input":"2021-10-12T08:21:48.134027Z","iopub.status.idle":"2021-10-12T08:21:48.142717Z","shell.execute_reply.started":"2021-10-12T08:21:48.133991Z","shell.execute_reply":"2021-10-12T08:21:48.142026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After defining the transformations, overwrite the transform functions of train_ts, val_ts\ntrain_dataset.transform = train_transformer\nvalid_dataset.transform = valid_transformer","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.144121Z","iopub.execute_input":"2021-10-12T08:21:48.144461Z","iopub.status.idle":"2021-10-12T08:21:48.151562Z","shell.execute_reply.started":"2021-10-12T08:21:48.144425Z","shell.execute_reply":"2021-10-12T08:21:48.150798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define DataLoader","metadata":{}},{"cell_type":"code","source":"# Define two dataloaders for the datasets\n\ntrain_dataloader = DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle=True)\nvalid_dataloader = DataLoader(valid_dataset, batch_size = BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.153054Z","iopub.execute_input":"2021-10-12T08:21:48.153418Z","iopub.status.idle":"2021-10-12T08:21:48.159813Z","shell.execute_reply.started":"2021-10-12T08:21:48.153383Z","shell.execute_reply":"2021-10-12T08:21:48.159075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch, (X, y) in enumerate(train_dataloader):\n    print(\"Batch : {}\".format(batch))\n    print(X.shape)\n    print(y.shape)\n    \n    if batch == 4:\n        break","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:48.161291Z","iopub.execute_input":"2021-10-12T08:21:48.161600Z","iopub.status.idle":"2021-10-12T08:21:52.633821Z","shell.execute_reply.started":"2021-10-12T08:21:48.161567Z","shell.execute_reply":"2021-10-12T08:21:52.632418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define a Model","metadata":{}},{"cell_type":"markdown","source":"### DenseNet169\n\nAll pre-trained models expect input images normalized in the same way, i.e. mini-batches of 3-channel RGB images of shape (3 x H x W), where H and W are expected to be at least 224. The images have to be loaded in to a range of [0, 1] and then normalized using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]","metadata":{}},{"cell_type":"code","source":"# model = models.densenet169(pretrained=True)\n# model","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:52.634976Z","iopub.execute_input":"2021-10-12T08:21:52.635677Z","iopub.status.idle":"2021-10-12T08:21:52.640885Z","shell.execute_reply.started":"2021-10-12T08:21:52.635629Z","shell.execute_reply":"2021-10-12T08:21:52.639923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Basic CNN","metadata":{}},{"cell_type":"code","source":"class BasicCNN(nn.Module):\n    \n    def __init__(self):\n        super(BasicCNN, self).__init__()\n        \n        self.layer1 = nn.Sequential(\n            nn.Conv2d(3, 32, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2, stride = 2)\n        )\n        \n        self.layer2 = nn.Sequential(\n            nn.Conv2d(32, 64, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2, stride = 2)\n        )\n        \n        self.layer3 = nn.Sequential(\n            nn.Conv2d(64, 128, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2, stride = 2)\n        )\n        \n        self.layer4 = nn.Sequential(\n            nn.Conv2d(128, 256, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2, stride = 2)\n        )\n        \n        self.layer5 = nn.Sequential(\n            nn.Conv2d(256, 512, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2, stride = 2)\n        )\n        \n        self.avg_pool = nn.AvgPool2d(kernel_size = 3)\n        self.dropout1 = nn.Dropout(0.7)\n        self.fc1 = nn.Linear(1*1*512, 64)\n        self.relu = nn.ReLU()\n        self.dropout2 = nn.Dropout(0.5)\n        self.fc2 = nn.Linear(64, 2)\n        \n        \n    def forward(self,x):\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        x = self.layer5(x)\n#         print(x.shape)\n        x = self.avg_pool(x)\n#         print(x.shape)\n        x = x.view(-1, 1 * 1 * 512)\n#         print(x.shape)\n        x = self.dropout1(x)\n        x = self.relu(self.fc1(x))\n        x = self.dropout2(x)\n        x = self.fc2(x)\n#         x = F.softmax(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:52.642454Z","iopub.execute_input":"2021-10-12T08:21:52.642794Z","iopub.status.idle":"2021-10-12T08:21:52.659116Z","shell.execute_reply.started":"2021-10-12T08:21:52.642748Z","shell.execute_reply":"2021-10-12T08:21:52.658332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If we need to move a model to GPU via .cuda(), \n# we must do it before constructing optimizers for it. \n# Parameters of a model after .cuda() will be different objects with those before the call.\n\n# In general, we should make sure that optimized parameters live in consistent locations \n# when optimizers are constructed and used.\n\nmodel = BasicCNN().to(device)\nmodel","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:52.660413Z","iopub.execute_input":"2021-10-12T08:21:52.660754Z","iopub.status.idle":"2021-10-12T08:21:58.119796Z","shell.execute_reply.started":"2021-10-12T08:21:52.660720Z","shell.execute_reply":"2021-10-12T08:21:58.119105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\nsummary(model, input_size=(3,96,96))","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:58.126164Z","iopub.execute_input":"2021-10-12T08:21:58.129208Z","iopub.status.idle":"2021-10-12T08:21:58.254990Z","shell.execute_reply.started":"2021-10-12T08:21:58.129150Z","shell.execute_reply":"2021-10-12T08:21:58.254330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Optimizer and Loss Function\n\nWe are using nn.CrossEntropyLoss as criterion and torch.optim.Adam as optimizer. \n\nnn.CrossEntropyLoss combines nn.LogSoftmax and nn.NLLLoss. We pass our model’s output logits to nn.CrossEntropyLoss, which will normalize the logits and compute the prediction error.","metadata":{}},{"cell_type":"code","source":"LEARNING_RATE = 0.001\noptimizer = torch.optim.Adam(params = model.parameters(),lr=LEARNING_RATE)\ncriterion = nn.CrossEntropyLoss(reduction='mean')","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:58.258829Z","iopub.execute_input":"2021-10-12T08:21:58.260734Z","iopub.status.idle":"2021-10-12T08:21:58.268127Z","shell.execute_reply.started":"2021-10-12T08:21:58.260679Z","shell.execute_reply":"2021-10-12T08:21:58.267332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Optimizing the Model Parameters","metadata":{}},{"cell_type":"code","source":"def train(dataloader, model, loss_fn, optimizer):\n    ds_size = len(dataloader.dataset)\n    num_batch = len(dataloader)\n    \n    total_accurate_preds = 0.0\n    total_loss = 0.0\n    \n    model.train()\n    for batch_idx, (X, y) in enumerate(dataloader):\n        X, y = X.to(device), y.to(device)\n\n        # Compute prediction error\n        logits = model(X)\n        loss = loss_fn(logits, y) # average loss per batch \n        preds = logits.argmax(dim=1)\n        num_accurate_preds = (preds == y).float().sum() # number of correct predictions for a single batch\n        \n        total_loss += loss.item()\n        total_accurate_preds += num_accurate_preds.item()\n    \n        # Backpropagation\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        if batch_idx % 100 == 0:\n            current = batch_idx * len(X)\n            print(f\"loss: {loss.item():>7f}  [{current}/{ds_size}]\")\n\n    accuracy = total_accurate_preds/ ds_size\n    avg_loss = total_loss/ num_batch\n\n    return  accuracy, avg_loss\n\n# train(train_dataloader, model, criterion, optimizer)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:58.273276Z","iopub.execute_input":"2021-10-12T08:21:58.275591Z","iopub.status.idle":"2021-10-12T08:21:58.287695Z","shell.execute_reply.started":"2021-10-12T08:21:58.275555Z","shell.execute_reply":"2021-10-12T08:21:58.286984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.eval() will notify all your layers that we are in eval mode, that way, \n# batchnorm or dropout layers will work in eval mode instead of training mode.\n\n# torch.no_grad() impacts the autograd engine and deactivate it. \n# It will reduce memory usage and speed up computations \n# but we won’t be able to backprop (which we don’t want in an eval mode).\n\ndef evaluate(dataloader, model, loss_fn):\n    ds_size = len(dataloader.dataset)\n    num_batch = len(dataloader)\n    \n    total_accurate_preds = 0\n    total_loss = 0.0\n    \n    model.eval()\n    with torch.no_grad():\n        for X, y in dataloader:\n            X, y = X.to(device), y.to(device)\n            \n            # Compute prediction error\n            logits = model(X)\n            loss = loss_fn(logits, y) # average loss per batch \n            preds = logits.argmax(dim=1)\n            num_accurate_preds = (preds == y).float().sum() # number of correct predictions for a single batch\n    \n            total_loss += loss.item()\n            total_accurate_preds += num_accurate_preds.item()\n    \n    accuracy = total_accurate_preds/ ds_size\n    avg_loss = total_loss/ num_batch\n\n    #     print(f\"Validation Error: \\n Accuracy: {(100*accuracy):>0.1f}%, Avg loss: {avg_loss:>8f} \\n\")\n    \n    return  accuracy, avg_loss\n\n# evaluate(valid_dataloader, model, criterion)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:58.292862Z","iopub.execute_input":"2021-10-12T08:21:58.294927Z","iopub.status.idle":"2021-10-12T08:21:58.306083Z","shell.execute_reply.started":"2021-10-12T08:21:58.294890Z","shell.execute_reply":"2021-10-12T08:21:58.305248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history = {\"loss\": [],\"accuracy\": []} # history of accuracy and loss for train set in each epoch\nvalidation_history = {\"loss\": [],\"accuracy\": []} # history of accuracy and loss for validation set in each epoch\nbest_model_weights = copy.deepcopy(model.state_dict()) # a deep copy of weights for the best performing model\nbest_loss = float('inf') # initialize best loss to a large value\n\nfor epoch in range(N_EPOCHS):\n    print(f\"Epoch {epoch+1}/{N_EPOCHS}\\n-------------------------------\")\n    \n    start = time.time()\n    train_accuracy, train_loss = train(train_dataloader, model, criterion, optimizer)\n    train_history['loss'].append(train_loss)\n    train_history['accuracy'].append(train_accuracy)\n    print('\\nTrain accuracy : {}, Train loss : {}'.format(train_accuracy, train_loss))\n#     print(train_history)\n    \n    validation_accuracy, validation_loss = evaluate(valid_dataloader, model, criterion)\n    validation_history['loss'].append(validation_loss)\n    validation_history['accuracy'].append(validation_accuracy)\n    print('Validation accuracy : {}, Validation loss : {}'.format(validation_accuracy,validation_loss))\n#     print(validation_history)\n    \n    print(\"Execution time for an Epoch : {}\\n\".format(time.time() - start))\n    \n    # store best model\n    if validation_loss < best_loss:\n        best_loss = validation_loss\n        best_model_weights = copy.deepcopy(model.state_dict())\n\n        ## store weights into a local file\n        # torch.save(model.state_dict(), weight_path)\n    \nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:21:58.311324Z","iopub.execute_input":"2021-10-12T08:21:58.313990Z","iopub.status.idle":"2021-10-12T08:22:58.831996Z","shell.execute_reply.started":"2021-10-12T08:21:58.313954Z","shell.execute_reply":"2021-10-12T08:22:58.830605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot Accuracy and Losses","metadata":{}},{"cell_type":"code","source":"print(train_history)\nprint(validation_history)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:22:58.833060Z","iopub.status.idle":"2021-10-12T08:22:58.833480Z","shell.execute_reply.started":"2021-10-12T08:22:58.833250Z","shell.execute_reply":"2021-10-12T08:22:58.833272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = list(range(N_EPOCHS)) \n\nplt.clf()\nplt.plot(epoch_range,train_history['loss'] , label=\"Train\")\nplt.plot(epoch_range, validation_history['loss'], label=\"Validation\")\nplt.xlabel(\"#epoch\")\nplt.ylabel('loss')\nplt.title('Training Loss and Validation Loss')\nplt.legend()\nplt.show()\n\nplt.clf()\nplt.plot(epoch_range, train_history['accuracy'], label=\"Train\")\nplt.plot(epoch_range, validation_history['accuracy'], label=\"Validation\")\nplt.xlabel(\"#epoch\")\nplt.ylabel('Accuracy')\nplt.title('Training Accuracy and Validation Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:22:58.835528Z","iopub.status.idle":"2021-10-12T08:22:58.835963Z","shell.execute_reply.started":"2021-10-12T08:22:58.835715Z","shell.execute_reply":"2021-10-12T08:22:58.835737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving model\n\nA common PyTorch convention is to save models using either a .pt or .pth file extension.\n\nIt is however not [recommended to use .pth extension](https://stackoverflow.com/questions/59095824/what-is-the-difference-between-pt-pth-and-pwf-extentions-in-pytorch) when checkpointing models because it collides with Python path (.pth) configuration files.","metadata":{}},{"cell_type":"code","source":"weight_save_path = './model.pt'\ntorch.save(best_model_weights, weight_save_path)\nprint(\"Saved PyTorch Best Model State to model.pt\")","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:17:50.809429Z","iopub.execute_input":"2021-10-12T08:17:50.810358Z","iopub.status.idle":"2021-10-12T08:17:50.840916Z","shell.execute_reply.started":"2021-10-12T08:17:50.810314Z","shell.execute_reply":"2021-10-12T08:17:50.839812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Predictions for Test Images","metadata":{}},{"cell_type":"code","source":"# Define transformation that converts a PIL image into PyTorch tensors.\ntest_transformer = transforms.Compose([transforms.ToTensor()])\n\n# Define an instance of the custom dataset for the train folder.\ntest_dataset = CustomDataset(submission, test_dir, dataset_type = 'test', transform = test_transformer)\n\nimg, img_id = test_dataset[24]\nprint(img.shape, torch.min(img), torch.max(img))\nprint(img_id)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:17:50.842392Z","iopub.execute_input":"2021-10-12T08:17:50.842933Z","iopub.status.idle":"2021-10-12T08:17:50.864694Z","shell.execute_reply.started":"2021-10-12T08:17:50.842892Z","shell.execute_reply":"2021-10-12T08:17:50.863567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a Dataloader for test dataset\ntest_dataloader = DataLoader(test_dataset, batch_size = BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:18:09.373281Z","iopub.execute_input":"2021-10-12T08:18:09.373606Z","iopub.status.idle":"2021-10-12T08:18:09.379730Z","shell.execute_reply.started":"2021-10-12T08:18:09.373579Z","shell.execute_reply":"2021-10-12T08:18:09.378051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configure model with best model weights \n\n# model = BasicCNN().to(device)\n# model.load_state_dict(torch.load(\"model.pth\", map_location=device))\n\nmodel.load_state_dict(best_model_weights)\n# print(model.state_dict())","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:18:14.835460Z","iopub.execute_input":"2021-10-12T08:18:14.835756Z","iopub.status.idle":"2021-10-12T08:18:14.848221Z","shell.execute_reply.started":"2021-10-12T08:18:14.835724Z","shell.execute_reply":"2021-10-12T08:18:14.846885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds_size = len(test_dataloader.dataset)\ncancer_probs = []\ntest_image_ids = []\n\nmodel.eval()\nwith torch.no_grad():\n    for batch_idx, (images, image_ids) in enumerate(test_dataloader):\n        images = images.to(device)\n        logits = model(images)\n        preds = F.softmax(logits, dim=1)[:, 1]\n        preds = torch.flatten(preds).detach().cpu().numpy()\n        cancer_probs += list(preds)\n        test_image_ids += list(image_ids)\n        \n        if batch_idx % 100 == 0:\n            current = batch_idx * len(images)\n            print(f\"Prediction Done:  [{current}/{test_ds_size}]\")    ","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:18:20.529322Z","iopub.execute_input":"2021-10-12T08:18:20.530040Z","iopub.status.idle":"2021-10-12T08:18:21.573653Z","shell.execute_reply.started":"2021-10-12T08:18:20.529999Z","shell.execute_reply":"2021-10-12T08:18:21.572423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'id':test_image_ids, 'label':cancer_probs})\n\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-12T08:18:24.676196Z","iopub.execute_input":"2021-10-12T08:18:24.677052Z","iopub.status.idle":"2021-10-12T08:18:24.690646Z","shell.execute_reply.started":"2021-10-12T08:18:24.677020Z","shell.execute_reply":"2021-10-12T08:18:24.689641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}