{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Goals of this Kernel\nThis kernel will provide you with a starter template to load all images and masks into memory and gets them ready for pytorch lightning.  \n  \nWe will also use Splitter to create a 10 fold Crossvalidation dataset which can easily be extended with more data.\n\n## Why is memory an issue?\nEach image is annotated with every single cell of interest. Our goal is to segment them individually.  \nBecause they can overlap, we cannot simply store a number for each pixel coressponding to the cell, but need a mask for each single neuron.  \nThis array would be very big and not fit in memory (I tried).  \nBut because it almost only contains zeros, we can use [sparse matrices](https://sparse.pydata.org/en/stable/) and only store the positive pixels.  \nWe can easily convert this back into a dense representation at runtime.\n  \n## Where can I follow you?  \nI am glad you asked: https://twitter.com/PSodmann","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/p-sodmann/splitter -q\n!pip install sparse -q","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:11:58.764107Z","iopub.execute_input":"2021-10-17T15:11:58.764419Z","iopub.status.idle":"2021-10-17T15:12:17.472660Z","shell.execute_reply.started":"2021-10-17T15:11:58.764360Z","shell.execute_reply":"2021-10-17T15:12:17.471602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom splitter.splitter import Splitter\nfrom tqdm.auto import tqdm\nimport sparse\nfrom torch.utils.data import DataLoader, Dataset\nimport pandas as pd\nimport imageio\nimport matplotlib.pyplot as plt\nimport torch","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:29:46.202343Z","iopub.execute_input":"2021-10-17T15:29:46.203493Z","iopub.status.idle":"2021-10-17T15:29:46.214338Z","shell.execute_reply.started":"2021-10-17T15:29:46.203448Z","shell.execute_reply":"2021-10-17T15:29:46.213283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref: https://www.kaggle.com/inversion/run-length-decoding-quick-start\ndef rle2mask(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [\n        np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])\n    ]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.bool)\n\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    \n    return img.reshape(shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:12:17.484922Z","iopub.execute_input":"2021-10-17T15:12:17.485417Z","iopub.status.idle":"2021-10-17T15:12:17.495013Z","shell.execute_reply.started":"2021-10-17T15:12:17.485335Z","shell.execute_reply":"2021-10-17T15:12:17.494338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pads and truncates the mask to max_size in z direction (number of possible annotated cells in one image)\ndef pad(array, max_size=128):\n    if array.shape[0] <= max_size:\n        padded = np.zeros((max_size, array.shape[1], array.shape[2]))\n        padded[:array.shape[0]] = array\n    else:\n        padded = array[:max_size]\n    \n    return padded\n\nclass CellDataset(Dataset):\n    def __init__(self, data):\n        self.data = data\n        \n        # tile size\n        self.size = 256\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        img = self.data[idx][\"image\"]\n        \n        # get a random crop\n        x = np.random.randint(img.shape[0] - self.size)\n        y = np.random.randint(img.shape[1] - self.size)\n        \n        # make mask dense\n        dense_mask = self.data[idx][\"sparse_mask\"].todense()\n        dense_mask = dense_mask[:, x:x+self.size,y:y+self.size]\n        \n        # get only masks in the image, which contain positive pixels (neurons)\n        filled_mask = dense_mask[np.where(np.sum(dense_mask, axis=(1,2)) > 0)]\n        \n        # pad in z direction\n        padded_mask = pad(filled_mask)\n        \n        # crop image and return image and mask\n        return np.array(np.expand_dims(img[x:x+self.size,y:y+self.size], 0)), padded_mask","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:12:17.496657Z","iopub.execute_input":"2021-10-17T15:12:17.496870Z","iopub.status.idle":"2021-10-17T15:12:17.514185Z","shell.execute_reply.started":"2021-10-17T15:12:17.496844Z","shell.execute_reply":"2021-10-17T15:12:17.512917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pytorch_lightning as pl\n\nclass CellDataModule(pl.LightningDataModule):\n    def __init__(self, dataframe, batch_size: int = 32):\n        super().__init__()\n        \n        self.dataframe = dataframe\n        self.batch_size = batch_size\n        \n        # make 10 split cross validation.\n        self.splitter = Splitter(10, seed=21188)\n\n    def setup(self, stage=None):\n        self.cell_ids = self.dataframe[\"id\"].unique()\n        \n        # add data to cross-validation.\n        # we can add more semi supervised data later without changing the splits\n        # https://medium.com/analytics-vidhya/splitting-your-data-growing-beyond-train-test-split-dc0eb83d7dac\n        for cell_id in self.cell_ids:\n            self.splitter.add(cell_id)\n        \n        # 8 folds for training, 1 for validation, 1 for testing\n        self.train_ids, self.valid_ids, self.tests_ids = self.splitter.get_split([[0,1,2,3,4,5,6,7], [8], [9]])\n    \n    def load_data(self, item_ids):\n        items = []\n        for item_id in tqdm(item_ids):\n            image = imageio.imread(f'../input/sartorius-cell-instance-segmentation/train/{item_id}.png')\n            \n            mask = []\n            \n            # get all annotations for one image\n            cells = self.dataframe.loc[self.dataframe[\"id\"] == item_id]\n            \n            # get all masks for a particular image\n            for index, cell in cells.iterrows():\n                mask.append(rle2mask(cell[\"annotation\"], image.shape))\n                \n            # make it sparse, so it fits into memory\n            mask = sparse.COO(np.array(mask))\n            \n            items.append({\"image\":image, \"sparse_mask\":mask})\n        \n        return items\n        \n    def train_dataloader(self):\n        self.train_data = CellDataset(self.load_data(self.train_ids))\n        return DataLoader(self.train_data, batch_size=self.batch_size)\n\n    def val_dataloader(self):\n        self.valid_data = CellDataset(self.load_data(self.valid_ids))\n        return DataLoader(self.valid_data, batch_size=self.batch_size)\n\n    def test_dataloader(self):\n        self.tests_data = CellDataset(self.load_data(self.tests_ids))\n        return DataLoader(self.tests_data, batch_size=self.batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:12:17.515715Z","iopub.execute_input":"2021-10-17T15:12:17.516491Z","iopub.status.idle":"2021-10-17T15:12:17.534022Z","shell.execute_reply.started":"2021-10-17T15:12:17.516444Z","shell.execute_reply":"2021-10-17T15:12:17.532450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotation_df = pd.read_csv(\"../input/sartorius-cell-instance-segmentation/train.csv\")\n\ncdm = CellDataModule(annotation_df)\ncdm.setup()\n\ntdl = cdm.train_dataloader()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:12:17.535748Z","iopub.execute_input":"2021-10-17T15:12:17.536166Z","iopub.status.idle":"2021-10-17T15:13:37.005981Z","shell.execute_reply.started":"2021-10-17T15:12:17.536118Z","shell.execute_reply":"2021-10-17T15:13:37.004713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We managed to load all data into memory, this only works, because we saved the masks in a sparse format.  \nBefore using them in a neural network, we need to convert them back into a dense representation, this happens in the dataset.","metadata":{}},{"cell_type":"code","source":"image_number = 50\ncell_number = 1\n\ndata = cdm.train_data[image_number]","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:36:08.648781Z","iopub.execute_input":"2021-10-17T15:36:08.649268Z","iopub.status.idle":"2021-10-17T15:36:08.670323Z","shell.execute_reply.started":"2021-10-17T15:36:08.649217Z","shell.execute_reply":"2021-10-17T15:36:08.669655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_number = 3\n\nplt.imshow(data[1][cell_number,:,:])\nplt.show()\n\nplt.imshow(data[0][0])\nplt.imshow(data[1][cell_number,:,:], alpha=0.3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:36:08.993607Z","iopub.execute_input":"2021-10-17T15:36:08.994340Z","iopub.status.idle":"2021-10-17T15:36:09.432828Z","shell.execute_reply.started":"2021-10-17T15:36:08.994293Z","shell.execute_reply":"2021-10-17T15:36:09.432035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overlay of all cells in the image\n\nplt.imshow(data[0][0])\n\nall_masks = np.zeros([256, 256])\nfor mask in data[1]:\n    all_masks += mask\n    \nplt.imshow(all_masks, alpha=0.3)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T15:36:09.527680Z","iopub.execute_input":"2021-10-17T15:36:09.527963Z","iopub.status.idle":"2021-10-17T15:36:09.788602Z","shell.execute_reply.started":"2021-10-17T15:36:09.527921Z","shell.execute_reply":"2021-10-17T15:36:09.787472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Awesome, we managed to load the data and fit it into memory and get our mask back.  \nHave fun building a model and competing in this challenge!\n  \n🐱 Phil","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}