{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":52279,"databundleVersionId":5822112,"sourceType":"competition"},{"sourceId":5756795,"sourceType":"datasetVersion","datasetId":3308073}],"dockerImageVersionId":30498,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom glob import glob\nimport json\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchmetrics import AveragePrecision\nimport torch\nimport torchvision\nfrom torchvision.transforms import transforms\nimport albumentations as A \nfrom albumentations.pytorch.transforms import ToTensorV2\nimport os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport torch.nn.functional as F\nimport time\nimport base64\nimport typing as t\nimport zlib","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-04T23:20:31.481722Z","iopub.execute_input":"2023-06-04T23:20:31.482666Z","iopub.status.idle":"2023-06-04T23:20:31.489563Z","shell.execute_reply.started":"2023-06-04T23:20:31.482622Z","shell.execute_reply":"2023-06-04T23:20:31.488585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!mkdir /kaggle/working/packages\n!cp -r /kaggle/input/pycocotools/* /kaggle/working/packages\nos.chdir(\"/kaggle/working/packages/pycocotools-2.0.6/\")\n!python setup.py install\n!pip install . --no-index --find-links /kaggle/working/packages/\nos.chdir(\"/kaggle/working\")\nfrom pycocotools import _mask as coco_mask","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:27:51.326793Z","iopub.execute_input":"2023-06-04T22:27:51.327146Z","iopub.status.idle":"2023-06-04T22:28:44.828931Z","shell.execute_reply.started":"2023-06-04T22:27:51.327114Z","shell.execute_reply":"2023-06-04T22:28:44.823239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    \n    batch_size= 64\n    n_epochs = 10\n    learning_rate = 0.001\n    \n    \n    \nconfig = Config()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:44.834805Z","iopub.execute_input":"2023-06-04T22:28:44.837591Z","iopub.status.idle":"2023-06-04T22:28:44.848396Z","shell.execute_reply.started":"2023-06-04T22:28:44.837533Z","shell.execute_reply":"2023-06-04T22:28:44.847457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Acquisition:\n    \n    def get_datframe(self,path):\n        return pd.read_csv(path)\n    \n    def get_json_dataframe(self, json_file):\n        data = []\n        with open(json_file, 'r') as file:\n            for line in file:\n                item = json.loads(line)\n                data.append(item)\n        \n        json_df = pd.DataFrame(data)\n        return json_df\n    \n        \n        \nacq = Acquisition()      ","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:44.852029Z","iopub.execute_input":"2023-06-04T22:28:44.85624Z","iopub.status.idle":"2023-06-04T22:28:44.868554Z","shell.execute_reply.started":"2023-06-04T22:28:44.856193Z","shell.execute_reply":"2023-06-04T22:28:44.867623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title=acq.get_datframe(path='/kaggle/input/hubmap-hacking-the-human-vasculature/tile_meta.csv')\ntitle.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:44.870025Z","iopub.execute_input":"2023-06-04T22:28:44.870768Z","iopub.status.idle":"2023-06-04T22:28:44.941288Z","shell.execute_reply.started":"2023-06-04T22:28:44.870731Z","shell.execute_reply":"2023-06-04T22:28:44.940176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wsi = acq.get_datframe(path='/kaggle/input/hubmap-hacking-the-human-vasculature/wsi_meta.csv')\nwsi.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:44.943104Z","iopub.execute_input":"2023-06-04T22:28:44.943835Z","iopub.status.idle":"2023-06-04T22:28:44.984514Z","shell.execute_reply.started":"2023-06-04T22:28:44.943782Z","shell.execute_reply":"2023-06-04T22:28:44.983422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:44.986462Z","iopub.execute_input":"2023-06-04T22:28:44.987198Z","iopub.status.idle":"2023-06-04T22:28:45.175419Z","shell.execute_reply.started":"2023-06-04T22:28:44.987162Z","shell.execute_reply":"2023-06-04T22:28:45.173204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"polygons_df = acq.get_json_dataframe('/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl')\npolygons_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:45.176848Z","iopub.execute_input":"2023-06-04T22:28:45.186239Z","iopub.status.idle":"2023-06-04T22:28:49.919961Z","shell.execute_reply.started":"2023-06-04T22:28:45.186204Z","shell.execute_reply":"2023-06-04T22:28:49.918748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageHuBMAPDataset(Dataset):\n    \n    def __init__(self, image_dir, labels_file, transform=None):\n        \n        with open(labels_file, 'r') as json_file:\n            self.json_labels = [json.loads(line) for line in json_file]\n            \n        self.image_dir = image_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.json_labels)\n\n    def __getitem__(self, idx):\n        image_path = os.path.join(self.image_dir, f\"{self.json_labels[idx]['id']}.tif\")\n        image = Image.open(image_path)\n\n        mask = np.zeros((512, 512), dtype=np.float32)\n\n        for annot in self.json_labels[idx]['annotations']:\n            cords = annot['coordinates']\n            if annot['type'] == \"blood_vessel\":\n                for cd in cords:\n                    rr, cc = np.array([i[1] for i in cd]), np.asarray([i[0] for i in cd])\n                    mask[rr, cc] = 1\n\n        image = torch.tensor(np.array(image), dtype=torch.float32,requires_grad=True).permute(2, 0, 1)  \n        mask = torch.tensor(mask, dtype=torch.float32)\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, mask","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:49.921779Z","iopub.execute_input":"2023-06-04T22:28:49.922218Z","iopub.status.idle":"2023-06-04T22:28:49.933152Z","shell.execute_reply.started":"2023-06-04T22:28:49.922169Z","shell.execute_reply":"2023-06-04T22:28:49.93202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class get_images_labels:\n    def __init__(self, root_path):\n        self.root_path = root_path\n    \n    def set_path(self,filename):\n        return os.path.join(self.root_path,filename)\n    \n    def get_test_path(self,filename):\n        return os.path.join(self.root_path,filename)\n        \n\n        \nimg_lble = get_images_labels(root_path='/kaggle/input/hubmap-hacking-the-human-vasculature')      \n\nimage_folder = img_lble.set_path(filename='train')\nlabels_file = img_lble.set_path(filename='polygons.jsonl')\ntest_image_folder =img_lble.get_test_path(filename='test')","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:49.938061Z","iopub.execute_input":"2023-06-04T22:28:49.93874Z","iopub.status.idle":"2023-06-04T22:28:49.947504Z","shell.execute_reply.started":"2023-06-04T22:28:49.938681Z","shell.execute_reply":"2023-06-04T22:28:49.946475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_mask():\n    plt.figure(figsize=(12, 8))\n    dataset = ImageHuBMAPDataset(image_dir=image_folder, labels_file=labels_file)\n    num_samples = 8\n    num_rows = (num_samples + 3) // 4  \n    num_cols = min(num_samples, 4)\n    for i in range(num_samples):\n        image, mask = dataset[i]\n        image = image.permute(1, 2, 0).detach().numpy() / 255\n        subplot_index = i + 1\n        plt.subplot(num_rows, 2 * num_cols, 2 * subplot_index - 1)\n        plt.imshow(image)\n        plt.axis('off')\n        plt.title('Train_Image')\n\n    mask_subplot_index = (subplot_index - 1) % num_samples + 1\n    plt.subplot(num_rows, 2 * num_cols, 2 * subplot_index)\n    plt.imshow(mask, cmap='gray')\n    plt.axis('off')\n    plt.title('Train_Mask')\n    plt.tight_layout(pad=0.2)\n    plt.show()\n        \n        \ndef plot_losses(history):\n    train_losses = [x.get('Train_loss') for x in history]\n    plt.plot(train_losses)\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend('Training')\n    plt.title('Loss vs. No. of epochs');","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:49.949248Z","iopub.execute_input":"2023-06-04T22:28:49.949713Z","iopub.status.idle":"2023-06-04T22:28:49.962657Z","shell.execute_reply.started":"2023-06-04T22:28:49.949603Z","shell.execute_reply":"2023-06-04T22:28:49.9615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_mask()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:49.964216Z","iopub.execute_input":"2023-06-04T22:28:49.964744Z","iopub.status.idle":"2023-06-04T22:28:56.054109Z","shell.execute_reply.started":"2023-06-04T22:28:49.964711Z","shell.execute_reply":"2023-06-04T22:28:56.053077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HuBMAPClassificationNN(torch.nn.Module):\n    def __init__(self):\n        super(HuBMAPClassificationNN, self).__init__()\n        \n        self.network_1 = torch.nn.Sequential(\n            torch.nn.Conv2d(3, 32, kernel_size=3, padding=1),\n            torch.nn.ReLU(),\n            torch.nn.Conv2d(32, 32, kernel_size=3, padding=1),\n            torch.nn.ReLU(),\n            torch.nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n\n        self.network_2 = torch.nn.Sequential(\n            torch.nn.Conv2d(32, 32, kernel_size=3, padding=1),\n            torch.nn.ReLU(),\n            torch.nn.Conv2d(32, 32, kernel_size=3, padding=1),\n            torch.nn.ReLU(),\n            torch.nn.ConvTranspose2d(32, 1, kernel_size=2, stride=2)\n        )\n\n    def forward(self, x):\n        x = self.network_1(x)\n        x = self.network_2(x)\n        x = torch.sigmoid(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:56.055152Z","iopub.execute_input":"2023-06-04T22:28:56.055524Z","iopub.status.idle":"2023-06-04T22:28:56.072408Z","shell.execute_reply.started":"2023-06-04T22:28:56.055492Z","shell.execute_reply":"2023-06-04T22:28:56.070743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = HuBMAPClassificationNN().to(device)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:28:56.074114Z","iopub.execute_input":"2023-06-04T22:28:56.074609Z","iopub.status.idle":"2023-06-04T22:29:02.679007Z","shell.execute_reply.started":"2023-06-04T22:28:56.074575Z","shell.execute_reply":"2023-06-04T22:29:02.677959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    \n    def train_dataloader(self,image_folder,labels_file):\n        dataset = ImageHuBMAPDataset(image_folder,labels_file)\n        return DataLoader(dataset, batch_size=config.batch_size, shuffle=True)\n        \n  \n    def fit(self,epochs, lr, model, train_loader,opt_func):\n        history =[]\n        result = {}\n        \n        criterion = torch.nn.BCELoss() \n        optimizer = opt_func(model.parameters(), lr=lr)\n        \n        for epoch in range(epochs):\n            model.train() \n            running_loss = 0.0\n            accuracy = 0.0\n            start_time = time.time()\n            \n            for images, masks in train_loader:\n                images = images.to(device)\n                masks = masks.to(device)\n                optimizer.zero_grad()\n                outputs = model(images)\n                masks = masks.unsqueeze(1) \n                loss = criterion(outputs, masks)\n                loss.backward()\n                optimizer.step()\n                running_loss += loss.item()\n\n            epoch_loss = running_loss / len(train_loader)\n            epoch_time = time.time() - start_time\n            print(f\"Epoch {epoch+1}/{epochs}, Loss: {epoch_loss:.4f}, Time: {epoch_time:.2f} seconds\")\n            result['Train_loss'] = epoch_loss\n            history.append(result)\n        return history \n        \n    \ntrainer = Trainer()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:29:02.680474Z","iopub.execute_input":"2023-06-04T22:29:02.681149Z","iopub.status.idle":"2023-06-04T22:29:02.692211Z","shell.execute_reply.started":"2023-06-04T22:29:02.681112Z","shell.execute_reply":"2023-06-04T22:29:02.69111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=trainer.fit(config.n_epochs, config.learning_rate, model, \n                    trainer.train_dataloader(image_folder,labels_file),\n                    torch.optim.Adam)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:29:02.693967Z","iopub.execute_input":"2023-06-04T22:29:02.69469Z","iopub.status.idle":"2023-06-04T22:37:16.123119Z","shell.execute_reply.started":"2023-06-04T22:29:02.694655Z","shell.execute_reply":"2023-06-04T22:37:16.122149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_losses(history)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T22:37:16.124618Z","iopub.execute_input":"2023-06-04T22:37:16.12742Z","iopub.status.idle":"2023-06-04T22:37:16.436617Z","shell.execute_reply.started":"2023-06-04T22:37:16.127392Z","shell.execute_reply":"2023-06-04T22:37:16.435719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageHuBMAPDatasetTest(Dataset):\n    \n    def __init__(self, testFiles, testFolder,transforms):\n        self.testFiles = testFiles \n        self.testFolder = testFolder \n        self.transforms = transforms \n        \n    def __getitem__(self, idx):\n        \n        testId = self.testFiles[idx]\n        inputPath = os.path.join(self.testFolder, testId)\n        im  = Image.open(inputPath)\n        if self.transforms is not None: \n            transformed = self.transforms(image = np.array(im))\n            return testId.split(\".\")[0], transformed['image']\n        \n    def __len__(self):\n        return len(self.testFiles)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T23:20:42.720045Z","iopub.execute_input":"2023-06-04T23:20:42.720402Z","iopub.status.idle":"2023-06-04T23:20:42.72875Z","shell.execute_reply.started":"2023-06-04T23:20:42.720373Z","shell.execute_reply":"2023-06-04T23:20:42.727539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_path='/kaggle/input/hubmap-hacking-the-human-vasculature'","metadata":{"execution":{"iopub.status.busy":"2023-06-04T23:20:43.758553Z","iopub.execute_input":"2023-06-04T23:20:43.758956Z","iopub.status.idle":"2023-06-04T23:20:43.763365Z","shell.execute_reply.started":"2023-06-04T23:20:43.758923Z","shell.execute_reply":"2023-06-04T23:20:43.76234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Test:\n    \n    def encode_binary_mask(self,mask):\n        if mask.dtype != bool:\n            raise ValueError(\n                \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n                mask.dtype)\n\n        mask = np.squeeze(mask)\n        if len(mask.shape) != 2:\n            raise ValueError(\n                \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n                mask.shape)\n\n        mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n        mask_to_encode = mask_to_encode.astype(np.uint8)\n        mask_to_encode = np.asfortranarray(mask_to_encode)\n        encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n        binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n        base64_str = base64.b64encode(binary_str)\n        return base64_str\n    \n    \n    def encode_output(self,outputs,idx):\n        blood_vessel = torch.argmax(outputs, 1) \n        blood_vessel = blood_vessel == 1\n        blood_vessel = blood_vessel * 1\n    \n        blood_vessel = blood_vessel.cpu().numpy()\n        all_encode = {} \n        for i in range(blood_vessel.shape[0]):\n            list_encode = []\n            sliceImage = blood_vessel[i,:,:]\n            binarized = sliceImage > 0\n            coded_len = self.encode_binary_mask(binarized)\n            list_encode.append(coded_len)\n            all_encode[idx[i]] =list_encode\n        return all_encode\n\n    \n   \n    def get_test_transforms(self):\n        return A.Compose([A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),ToTensorV2()])\n    \n    def test_dataloader(self,image_folder):\n        dataset = ImageHuBMAPDatasetTest(os.listdir(test_image_folder),\n                                            test_image_folder,self.get_test_transforms())\n        return DataLoader(dataset, batch_size=config.batch_size)\n    \n    \n\n    def evaluate(self,model):\n        model.eval()\n        predictions = []\n        outputdict ={}\n        outputsoftmax = torch.nn.Softmax2d() \n        with torch.no_grad():\n            for idx,images in tqdm(self.test_dataloader(test_image_folder)):\n                images = images.to(device)\n                outputs = model(images)\n                outputs = outputsoftmax(outputs)\n                encoded = self.encode_output(outputs, idx)\n                for key in encoded: \n                    outputdict[key] = \" \".join([f\"0 1.0 {x.decode('utf-8')}\" for x in  encoded[key]])\n                print(outputdict)    \n        return outputdict\n        \n    \n    \n        \ntest = Test()\noutputdict=test.evaluate(model)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T23:23:21.064707Z","iopub.execute_input":"2023-06-04T23:23:21.065167Z","iopub.status.idle":"2023-06-04T23:23:21.124872Z","shell.execute_reply.started":"2023-06-04T23:23:21.065135Z","shell.execute_reply":"2023-06-04T23:23:21.123827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Submission:\n    \n    def submit_results(self,outputdict):\n        submission = pd.DataFrame(outputdict.items(), columns= [\"id\", \"prediction_string\"]) \n        submission[\"height\"] = 512\n        submission[\"width\"] = 512 \n        submission= submission[[\"id\",\"height\",\"width\",\"prediction_string\"]]\n        submission.to_csv(\"submission.csv\", index= False)\n        print(\"Submission Completed!!!\")\n        \n        \nsubmit = Submission()\nsubmit.submit_results(outputdict)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-04T23:24:22.736585Z","iopub.execute_input":"2023-06-04T23:24:22.737094Z","iopub.status.idle":"2023-06-04T23:24:22.751121Z","shell.execute_reply.started":"2023-06-04T23:24:22.73704Z","shell.execute_reply":"2023-06-04T23:24:22.749648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}