{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom PIL import Image\nimport cv2 as cv\nimport matplotlib.pyplot as plt\nimport os\nimport time\nimport math\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nfrom torchvision.datasets import ImageFolder\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets\nfrom torchvision.transforms import ToTensor, Lambda\nimport torch.optim as optim\n\nimport sklearn\nfrom torch.utils.data import Dataset\n\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport gc\nimport random\n\ndevice = torch.device(\"cuda\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-11T04:47:55.831168Z","iopub.execute_input":"2022-09-11T04:47:55.831890Z","iopub.status.idle":"2022-09-11T04:47:55.841042Z","shell.execute_reply.started":"2022-09-11T04:47:55.831847Z","shell.execute_reply":"2022-09-11T04:47:55.839911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\nseed_everything(1001)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:47:59.164487Z","iopub.execute_input":"2022-09-11T04:47:59.165420Z","iopub.status.idle":"2022-09-11T04:47:59.175941Z","shell.execute_reply.started":"2022-09-11T04:47:59.165371Z","shell.execute_reply":"2022-09-11T04:47:59.174699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dir = \"../input/brest-cancer/Breast Cancer DataSet/Train\"\n#test_dir =  \"../input/brest-cancer/Breast Cancer DataSet/Test\"\n#valid_dir = \"../input/brest-cancer/Breast Cancer DataSet/valid\"\n# benig = os.path.join(root_dir, \"Benign\")\n# malig = os.path.join(root_dir, \"Malignant\")","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:20.096378Z","iopub.execute_input":"2022-09-10T04:20:20.097037Z","iopub.status.idle":"2022-09-10T04:20:20.103154Z","shell.execute_reply.started":"2022-09-10T04:20:20.097002Z","shell.execute_reply":"2022-09-10T04:20:20.102299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir='../input/histopathologic-cancer-detection/train/'\ntest_dir='../input/histopathologic-cancer-detection/test/'\n#valid_dir=''","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:03.984534Z","iopub.execute_input":"2022-09-11T04:48:03.985568Z","iopub.status.idle":"2022-09-11T04:48:03.990597Z","shell.execute_reply.started":"2022-09-11T04:48:03.985524Z","shell.execute_reply":"2022-09-11T04:48:03.989291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_df = pd.read_csv(\"../input/histopathologic-cancer-detection/train_labels.csv\")\nfull_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:07.168237Z","iopub.execute_input":"2022-09-11T04:48:07.169194Z","iopub.status.idle":"2022-09-11T04:48:07.555594Z","shell.execute_reply.started":"2022-09-11T04:48:07.169140Z","shell.execute_reply":"2022-09-11T04:48:07.554650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SIZE =20000\ndf_negatives = full_train_df[full_train_df['label'] == 0].sample(SAMPLE_SIZE, random_state=42)\ndf_positives = full_train_df[full_train_df['label'] == 1].sample(SAMPLE_SIZE, random_state=42)\n\n# Concatenate the two dfs and shuffle them up\ntrain_df = sklearn.utils.shuffle(pd.concat([df_positives, df_negatives], axis=0).reset_index(drop=True))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:10.908377Z","iopub.execute_input":"2022-09-11T04:48:10.909041Z","iopub.status.idle":"2022-09-11T04:48:10.952869Z","shell.execute_reply.started":"2022-09-11T04:48:10.909000Z","shell.execute_reply":"2022-09-11T04:48:10.951807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:16:43.965980Z","iopub.execute_input":"2022-09-11T04:16:43.966399Z","iopub.status.idle":"2022-09-11T04:16:43.973489Z","shell.execute_reply.started":"2022-09-11T04:16:43.966334Z","shell.execute_reply":"2022-09-11T04:16:43.972255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn import model_selection","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:20.774238Z","iopub.execute_input":"2022-09-10T04:20:20.774852Z","iopub.status.idle":"2022-09-10T04:20:20.782339Z","shell.execute_reply.started":"2022-09-10T04:20:20.774814Z","shell.execute_reply":"2022-09-10T04:20:20.780800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df, valid_df = model_selection.train_test_split(\n #   df, test_size=0.1, random_state=42, stratify=df.label.values\n#)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:20.784001Z","iopub.execute_input":"2022-09-10T04:20:20.784396Z","iopub.status.idle":"2022-09-10T04:20:20.792007Z","shell.execute_reply.started":"2022-09-10T04:20:20.784360Z","shell.execute_reply":"2022-09-10T04:20:20.791052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# benig_list = os.listdir(benig)\n# malig_list = os.listdir(malig)\n\n# benig_path = [os.path.join(benig, x) for x in benig_list]\n# malig_path = [os.path.join(malig, x) for x in malig_list]","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:20.793597Z","iopub.execute_input":"2022-09-10T04:20:20.794285Z","iopub.status.idle":"2022-09-10T04:20:20.800440Z","shell.execute_reply.started":"2022-09-10T04:20:20.794227Z","shell.execute_reply":"2022-09-10T04:20:20.799455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass CreateDataset(Dataset):\n    def __init__(self, df_data, data_dir = './', transform=None):\n        super().__init__()\n        self.df = df_data.values\n        self.data_dir = data_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_name,label = self.df[index]\n        img_path = os.path.join(self.data_dir, img_name+'.tif')\n        image = cv.imread(img_path)\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:16.924400Z","iopub.execute_input":"2022-09-11T04:48:16.925176Z","iopub.status.idle":"2022-09-11T04:48:16.933573Z","shell.execute_reply.started":"2022-09-11T04:48:16.925135Z","shell.execute_reply":"2022-09-11T04:48:16.932289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n                       transforms.ToPILImage(),\n                        #transforms.Resize(size = (224, 224)),\n                        transforms.RandomHorizontalFlip(p=0.3),\n                        transforms.RandomVerticalFlip(p=0.3),\n                        transforms.RandomResizedCrop(96),    \n                        transforms.ToTensor(),                        \n                        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n                        transforms.ConvertImageDtype(dtype= torch.float32)\n                   ])\n\n\ntest_transforms = transform=transforms.Compose([\n                       transforms.ToPILImage(),\n                        #transforms.Resize(size = (224, 224)),\n                       transforms.ToTensor(),\n                        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n                        transforms.ConvertImageDtype(dtype= torch.float32)\n                   ])\ntransforms_valid = transforms.Compose(\n    [\n        transforms.ToPILImage(),\n        #transforms.Resize((224,224)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)\n# Set Batch Size\nbatch_size = 128\n\n# Percentage of training set to use as validation\nvalid_size = 0.25\n\ntrain_data=CreateDataset(df_data=train_df, data_dir=train_dir, transform=train_transforms)\nvalid_data=CreateDataset(df_data=train_df, data_dir=train_dir, transform=transforms_valid)\n\n\n\n\n# obtain training indices that will be used for validation\nnum_train = len(train_data)\nindices = list(range(num_train))\n#np.random.shuffle(indices)\nsplit = int(np.floor(valid_size * num_train))\ntrain_idx, valid_idx = indices[split:], indices[:split]\n\n# Create Samplers\ntrain_sampler = SubsetRandomSampler(train_idx)\nvalid_sampler = SubsetRandomSampler(valid_idx)\n\n\n\n\n\ntrain_dataloader = DataLoader(train_data, batch_size=batch_size,sampler=train_sampler)\nvalid_dataloader = DataLoader(train_data, batch_size=batch_size,sampler=valid_sampler)\n\nsample_sub = pd.read_csv(\"../input/histopathologic-cancer-detection/sample_submission.csv\")\ntest_data = CreateDataset(df_data=sample_sub, data_dir=test_dir, transform=test_transforms)\n\ntest_dataloader = DataLoader(test_data, batch_size=batch_size,shuffle=False)\n\n\n# valid_data = ImageFolder(valid_dir)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:20.610962Z","iopub.execute_input":"2022-09-11T04:48:20.611709Z","iopub.status.idle":"2022-09-11T04:48:20.754586Z","shell.execute_reply.started":"2022-09-11T04:48:20.611665Z","shell.execute_reply":"2022-09-11T04:48:20.753501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = next(iter(train_dataloader))\nsample[0][0]\nplt.imshow(sample[0][0].permute(2, 1, 0))\n#img = cv.imread('../input/histopathologic-cancer-detection/train/00001b2b5609af42ab0ab276dd4cd41c3e7745b5.tif')\n#img.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:25.309275Z","iopub.execute_input":"2022-09-11T04:48:25.310431Z","iopub.status.idle":"2022-09-11T04:48:26.813321Z","shell.execute_reply.started":"2022-09-11T04:48:25.310368Z","shell.execute_reply":"2022-09-11T04:48:26.812298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0][0]","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:17:03.656965Z","iopub.execute_input":"2022-09-11T04:17:03.658157Z","iopub.status.idle":"2022-09-11T04:17:03.668924Z","shell.execute_reply.started":"2022-09-11T04:17:03.658098Z","shell.execute_reply":"2022-09-11T04:17:03.667541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchEmbed(nn.Module):\n    \"\"\" Image to Patch Embedding\n    \"\"\"\n    def __init__(self, img_dims, patch_size, in_chans, embed_dim):\n        super().__init__()\n        patch_size = (patch_size, patch_size)\n        num_patches = (img_dims[1] // patch_size[1]) * (img_dims[0] // patch_size[0])\n        self.img_dims = img_dims\n        self.patch_size = patch_size\n        self.num_patches = num_patches\n\n        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)\n\n    def forward(self, x):\n        B, C, H, W = x.shape\n        assert H == self.img_dims[0] and W == self.img_dims[1], \\\n            f\"Input image size ({H}*{W}) doesn't match model ({self.img_dims[0]}*{self.img_dims[1]}).\"\n        x = self.proj(x).flatten(2).transpose(1, 2)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:32.328251Z","iopub.execute_input":"2022-09-11T04:48:32.328933Z","iopub.status.idle":"2022-09-11T04:48:32.337260Z","shell.execute_reply.started":"2022-09-11T04:48:32.328891Z","shell.execute_reply":"2022-09-11T04:48:32.335840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mlp(nn.Module):\n    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):\n        super().__init__()\n        out_features = out_features or in_features\n        hidden_features = hidden_features or in_features\n        self.fc1 = nn.Linear(in_features, hidden_features)\n        self.act = act_layer()\n        self.fc2 = nn.Linear(hidden_features, out_features)\n        self.drop = nn.Dropout(drop)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.act(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:35.804306Z","iopub.execute_input":"2022-09-11T04:48:35.805303Z","iopub.status.idle":"2022-09-11T04:48:35.814707Z","shell.execute_reply.started":"2022-09-11T04:48:35.805263Z","shell.execute_reply":"2022-09-11T04:48:35.813428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, dim, num_heads=12, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):\n        super().__init__()\n        self.num_heads = num_heads\n        head_dim = dim // num_heads\n        # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights\n        self.scale = qk_scale or head_dim ** -0.5\n\n        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)\n        self.attn_drop = nn.Dropout(attn_drop)\n        self.proj = nn.Linear(dim, dim)\n        self.proj_drop = nn.Dropout(proj_drop)\n\n    def forward(self, x):\n        B, N, C = x.shape\n        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)\n\n        attn = (q @ k.transpose(-2, -1)) * self.scale\n        attn = attn.softmax(dim=-1)\n        attn = self.attn_drop(attn)\n\n        x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:38.305009Z","iopub.execute_input":"2022-09-11T04:48:38.306172Z","iopub.status.idle":"2022-09-11T04:48:38.317141Z","shell.execute_reply.started":"2022-09-11T04:48:38.306118Z","shell.execute_reply":"2022-09-11T04:48:38.315830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/69175642/droppath-in-timm-seems-like-a-dropout\n\nclass Block(nn.Module):\n\n    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,\n                 drop_rate=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):\n        super().__init__()\n        self.norm1 = norm_layer(dim)\n        self.attn = Attention(\n            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)\n        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here\n#         self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n        self.drop = nn.Dropout(p=drop_rate)\n        self.norm2 = norm_layer(dim)\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n    def forward(self, x):\n        x = x + self.drop(self.attn(self.norm1(x)))\n        x = x + self.drop(self.mlp(self.norm2(x)))\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:41.377724Z","iopub.execute_input":"2022-09-11T04:48:41.378132Z","iopub.status.idle":"2022-09-11T04:48:41.387126Z","shell.execute_reply.started":"2022-09-11T04:48:41.378089Z","shell.execute_reply":"2022-09-11T04:48:41.385898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VisionTransformer(nn.Module):\n    \"\"\" Vision Transformer with support for patch or hybrid CNN input stage\n    \"\"\"\n    def __init__(self, img_dims = (96,96), patch_size=16, in_chans=3, num_classes=2, embed_dim=768, depth=12,\n                 num_heads=8, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,\n                 drop_h_rate=0.4, norm_layer=nn.LayerNorm):\n        super().__init__()\n        self.num_classes = num_classes\n        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models\n        self.patch_embed = PatchEmbed(\n            img_dims = img_dims, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)\n        num_patches = self.patch_embed.num_patches\n        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))\n        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))\n        self.pos_drop = nn.Dropout(p=drop_rate)\n        dpr = [x.item() for x in torch.linspace(0, drop_h_rate, depth)]  # stochastic depth decay rule\n        self.blocks = nn.ModuleList([\n            Block(\n                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,\n                drop=drop_rate, attn_drop=attn_drop_rate, drop_rate=dpr[i], norm_layer=norm_layer)\n            for i in range(depth)])\n        self.norm = norm_layer(embed_dim)\n\n        # Classifier head\n        self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()\n\n    def forward_features(self, x):\n        B = x.shape[0]\n        x = self.patch_embed(x)\n\n        cls_tokens = self.cls_token.expand(B, -1, -1)  \n        x = torch.cat((cls_tokens, x), dim=1)\n        x = x + self.pos_embed\n        x = self.pos_drop(x)\n\n        for blk in self.blocks:\n            x = blk(x)\n\n        x = self.norm(x)\n        return x[:, 0]\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        x = self.head(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:43.927733Z","iopub.execute_input":"2022-09-11T04:48:43.928160Z","iopub.status.idle":"2022-09-11T04:48:43.941565Z","shell.execute_reply.started":"2022-09-11T04:48:43.928125Z","shell.execute_reply":"2022-09-11T04:48:43.940008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-11T03:45:17.860199Z","iopub.execute_input":"2022-09-11T03:45:17.861292Z","iopub.status.idle":"2022-09-11T03:45:17.866861Z","shell.execute_reply.started":"2022-09-11T03:45:17.861253Z","shell.execute_reply":"2022-09-11T03:45:17.865698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = VisionTransformer()\nmodel = model.to(device)\n# img = img.to(device)\n# out = model(img)\n# out\n","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:48:47.947214Z","iopub.execute_input":"2022-09-11T04:48:47.947995Z","iopub.status.idle":"2022-09-11T04:48:52.167927Z","shell.execute_reply.started":"2022-09-11T04:48:47.947953Z","shell.execute_reply":"2022-09-11T04:48:52.166850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(train_loader, criterion, optimizer):\n        # keep track of training loss\n        epoch_loss = 0.0\n        epoch_accuracy = 0.0\n\n        ###################\n        # train the model #\n        ###################\n        model.train()\n        for i, (data, target) in enumerate(train_loader):\n            data, target = data.cuda(), target.cuda()\n            # clear the gradients of all optimized variables\n            optimizer.zero_grad()\n            # forward pass: compute predicted outputs by passing inputs to the model\n            output = model(data)\n            # calculate the batch loss\n            loss = criterion(output, target)\n            # backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n            optimizer.step()\n            # Calculate Accuracy\n            accuracy = (output.argmax(dim=1) == target).float().mean()\n            # update training loss and accuracy\n            \n            epoch_loss += loss\n            epoch_accuracy += accuracy\n            if i % 100 == 0:\n                    print(f\"\\tBATCH {i+1}/{len(train_loader)} - LOSS: {loss}\")\n        return epoch_loss / len(train_loader), epoch_accuracy / len(train_loader)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:49:14.751546Z","iopub.execute_input":"2022-09-11T04:49:14.752222Z","iopub.status.idle":"2022-09-11T04:49:14.760826Z","shell.execute_reply.started":"2022-09-11T04:49:14.752182Z","shell.execute_reply":"2022-09-11T04:49:14.759813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_one_epoch(valid_loader, criterion):\n        # keep track of validation loss\n        valid_loss = 0.0\n        valid_accuracy = 0.0\n\n        ######################\n        # validate the model #\n        ######################\n        model.eval()\n        for data, target in valid_loader:\n            data, target = data.cuda(), target.cuda()\n            \n            with torch.no_grad():\n                # forward pass: compute predicted outputs by passing inputs to the model\n                output = model(data)\n                # calculate the batch loss\n                loss = criterion(output, target)\n                # Calculate Accuracy\n                accuracy = (output.argmax(dim=1) == target).float().mean()\n                # update average validation loss and accuracy\n                valid_loss += loss\n                valid_accuracy += accuracy\n        return valid_loss / len(valid_loader), valid_accuracy / len(valid_loader)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:49:18.528287Z","iopub.execute_input":"2022-09-11T04:49:18.529122Z","iopub.status.idle":"2022-09-11T04:49:18.536842Z","shell.execute_reply.started":"2022-09-11T04:49:18.529080Z","shell.execute_reply":"2022-09-11T04:49:18.535280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_loss_min = np.Inf  # track change in validation loss\noptimizer = optim.Adam(model.parameters(), lr=2e-05)\ncriterion=nn.CrossEntropyLoss()\n\n    # keeping track of losses as it happen\ntrain_losses = []\nvalid_losses = []\ntrain_accs = []\nvalid_accs = []\nepochs=20\n\nfor epoch in range(1, epochs + 1):\n        gc.collect()\n        print(f\"{'='*50}\")\n        print(f\"EPOCH {epoch} - TRAINING...\")\n        train_loss, train_acc = train_one_epoch(train_dataloader,criterion, optimizer)\n        print(\n            f\"\\n\\t[TRAIN] EPOCH {epoch} - LOSS: {train_loss}, ACCURACY: {train_acc}\\n\"\n        )\n        train_losses.append(train_loss)\n        train_accs.append(train_acc)\n        gc.collect()\n        if valid_dataloader is not None:\n            gc.collect()\n            print(f\"EPOCH {epoch} - VALIDATING...\")\n            valid_loss, valid_acc = validate_one_epoch(valid_dataloader, criterion)\n            print(f\"\\t[VALID] LOSS: {valid_loss}, ACCURACY: {valid_acc}\\n\")\n            valid_losses.append(valid_loss)\n            valid_accs.append(valid_acc)\n            gc.collect()\n            #trying to plot loss and accuracy at each epoch\n            #plot_loss(train_losses,valid_losses)\n            #plot_acc(train_accs,valid_accs)     \n            # save model if validation loss has decreased\n            if valid_loss <= valid_loss_min and epoch != 1:\n                print(\n                    \"Validation loss decreased ({:.4f} --> {:.4f}).  Saving model ...\".format(\n                        valid_loss_min, valid_loss\n                    )\n                )\n                torch.save(model.state_dict(), 'best_model.pt')\n            valid_loss_min = valid_loss\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2022-09-11T04:49:21.213993Z","iopub.execute_input":"2022-09-11T04:49:21.215203Z","iopub.status.idle":"2022-09-11T06:39:08.807606Z","shell.execute_reply.started":"2022-09-11T04:49:21.215150Z","shell.execute_reply":"2022-09-11T06:39:08.806154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\ntrain_accs=torch.tensor(train_accs,device='cpu')\nvalid_accs=torch.tensor(valid_accs,device='cpu')\ntl= np.array(train_accs)\nvl=np.array(valid_accs)\n\nplt.plot(tl, label='Training accuracy')\nplt.plot(vl, label='Validation accuracy')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:39:18.758652Z","iopub.execute_input":"2022-09-11T06:39:18.759265Z","iopub.status.idle":"2022-09-11T06:39:19.105748Z","shell.execute_reply.started":"2022-09-11T06:39:18.759226Z","shell.execute_reply":"2022-09-11T06:39:19.104780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\ntrain_losses=torch.tensor(train_losses,device='cpu')\nvalid_losses=torch.tensor(valid_losses,device='cpu')\ntl= np.array(train_losses)\nvl=np.array(valid_losses)\n\nplt.plot(tl, label='Training loss')\nplt.plot(vl, label='Validation loss')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"loss\")\nplt.legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:39:26.869350Z","iopub.execute_input":"2022-09-11T06:39:26.870141Z","iopub.status.idle":"2022-09-11T06:39:28.662306Z","shell.execute_reply.started":"2022-09-11T06:39:26.870100Z","shell.execute_reply":"2022-09-11T06:39:28.661314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(torch.load('best_model.pt'))","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:40:29.899682Z","iopub.execute_input":"2022-09-11T06:40:29.900635Z","iopub.status.idle":"2022-09-11T06:40:30.091622Z","shell.execute_reply.started":"2022-09-11T06:40:29.900596Z","shell.execute_reply":"2022-09-11T06:40:30.090432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Turn off gradients\nmodel.eval()\n\npreds = []\nfor batch_i, (data, target) in enumerate(test_dataloader):\n    data, target = data.cuda(), target.cuda()\n    output= model(data)\n\n    pr = output.detach().cpu().numpy()\n    for i in pr:\n        preds.append(i)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:40:34.429382Z","iopub.execute_input":"2022-09-11T06:40:34.429763Z","iopub.status.idle":"2022-09-11T06:51:08.845384Z","shell.execute_reply.started":"2022-09-11T06:40:34.429730Z","shell.execute_reply":"2022-09-11T06:51:08.843990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Submission file        \nsample_sub['label'] = preds","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:51:40.001458Z","iopub.execute_input":"2022-09-11T06:51:40.001853Z","iopub.status.idle":"2022-09-11T06:51:40.020346Z","shell.execute_reply.started":"2022-09-11T06:51:40.001818Z","shell.execute_reply":"2022-09-11T06:51:40.019401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(sample_sub)):\n    sample_sub.label[i] = np.array(sample_sub.label[i],float) ","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:51:50.691504Z","iopub.execute_input":"2022-09-11T06:51:50.692644Z","iopub.status.idle":"2022-09-11T06:52:23.788000Z","shell.execute_reply.started":"2022-09-11T06:51:50.692597Z","shell.execute_reply":"2022-09-11T06:52:23.786918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv('submission.csv', index=False)\nsample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:52:48.141332Z","iopub.execute_input":"2022-09-11T06:52:48.142508Z","iopub.status.idle":"2022-09-11T06:52:52.545200Z","shell.execute_reply.started":"2022-09-11T06:52:48.142467Z","shell.execute_reply":"2022-09-11T06:52:52.544025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imshow(img):\n    '''Helper function to un-normalize and display an image'''\n    # unnormalize\n    img = img / 2 + 0.5\n    # convert from Tensor image and display\n    plt.imshow(np.transpose(img, (1, 2, 0)))","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:53:18.789998Z","iopub.execute_input":"2022-09-11T06:53:18.790408Z","iopub.status.idle":"2022-09-11T06:53:18.797036Z","shell.execute_reply.started":"2022-09-11T06:53:18.790364Z","shell.execute_reply":"2022-09-11T06:53:18.795826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# obtain one batch of training images\ndataiter = iter(test_dataloader)\nimages, labels = dataiter.next()\nimages = images.numpy() # convert images to numpy for display\n\n# plot the images in the batch, along with the corresponding labels\nfig = plt.figure(figsize=(25, 4))\n# display 20 images\nfor idx in np.arange(20):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    imshow(images[idx])\n    prob = \"Cancer\" if(abs(sample_sub.label[idx][1]) <= 0.5) else \"Normal\" \n    ax.set_title('{}'.format(prob))","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:53:22.772825Z","iopub.execute_input":"2022-09-11T06:53:22.774025Z","iopub.status.idle":"2022-09-11T06:53:24.840674Z","shell.execute_reply.started":"2022-09-11T06:53:22.773974Z","shell.execute_reply":"2022-09-11T06:53:24.839407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}