{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook's content\n\nWe have divided this notebook into following parts:\n1. Getting your environment ready (a.k.a. loading data and packages)\n2. Dataset&dataloader for the original competition data (`siim-covid19-detection`)\n3. Basic data augmentation\n4. Dataset&dataloader for the preprocessed competition data (`siimcovid19-512-img-png-600-study-png/study`)\n\n# Getting ready","metadata":{}},{"cell_type":"code","source":"%%bash\nconda install -c conda-forge gdcm -y","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:18:51.267103Z","iopub.execute_input":"2021-06-15T23:18:51.267514Z","iopub.status.idle":"2021-06-15T23:19:24.644392Z","shell.execute_reply.started":"2021-06-15T23:18:51.267424Z","shell.execute_reply":"2021-06-15T23:19:24.643171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport io\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nfrom PIL import Image\nimport pydicom\nimport seaborn as sns\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:24.646874Z","iopub.execute_input":"2021-06-15T23:19:24.647218Z","iopub.status.idle":"2021-06-15T23:19:25.111744Z","shell.execute_reply.started":"2021-06-15T23:19:24.647163Z","shell.execute_reply":"2021-06-15T23:19:25.110923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = '../input/siim-covid19-detection/train'\nTEST_DIR = '../input/siim-covid19-detection/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.113489Z","iopub.execute_input":"2021-06-15T23:19:25.114109Z","iopub.status.idle":"2021-06-15T23:19:25.118081Z","shell.execute_reply.started":"2021-06-15T23:19:25.114071Z","shell.execute_reply":"2021-06-15T23:19:25.117324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrain_study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.11949Z","iopub.execute_input":"2021-06-15T23:19:25.120267Z","iopub.status.idle":"2021-06-15T23:19:25.185384Z","shell.execute_reply.started":"2021-06-15T23:19:25.120227Z","shell.execute_reply":"2021-06-15T23:19:25.184527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study['id'] = train_study['id'].apply(lambda i: i.split('_')[0])\ntrain_study.rename(columns={'Negative for Pneumonia': '0','Typical Appearance': '1',\"Indeterminate Appearance\": '2',\n                   \"Atypical Appearance\": \"3\"}, inplace=True)\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.186838Z","iopub.execute_input":"2021-06-15T23:19:25.187157Z","iopub.status.idle":"2021-06-15T23:19:25.206352Z","shell.execute_reply.started":"2021-06-15T23:19:25.187124Z","shell.execute_reply":"2021-06-15T23:19:25.205253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\ndef get_label(row):\n    for c in train_study.columns:\n        if row[c] == 1:\n            labels.append(int(c))\n            \ntrain_study.apply(get_label, axis=1)\ntrain_study.drop(columns=['0', '1','2', '3'], inplace=True)\ntrain_study['label'] = labels\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.207658Z","iopub.execute_input":"2021-06-15T23:19:25.208039Z","iopub.status.idle":"2021-06-15T23:19:25.453678Z","shell.execute_reply.started":"2021-06-15T23:19:25.208005Z","shell.execute_reply":"2021-06-15T23:19:25.452719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Do I need a dataloader?\n\nIf you are planning to use `pytorch` package, then answer is simply **yes**, you do! Dataloader is a simple data iterator, increasing clarity of your code and providing an easy way to split dataset into batches. Dataloader wraps around a Dataset object - a way to tell your computer, how to find your data. You can find more details in this official pytorch tutorial -> [link](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html)\n\n### Dataloader for original competition data\n\nSpoiler alert! in the following code we:\n1. extract image path from previously loaded dataframe\n2. extract image data from the dcm object\n3. resize images\n4. do data augmentations (explenations in the next part of this notebook)\n5. return a image-label pair","metadata":{}},{"cell_type":"code","source":"from os import listdir, walk\nfrom skimage import exposure\nimport torch\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.455026Z","iopub.execute_input":"2021-06-15T23:19:25.455368Z","iopub.status.idle":"2021-06-15T23:19:25.94811Z","shell.execute_reply.started":"2021-06-15T23:19:25.455336Z","shell.execute_reply":"2021-06-15T23:19:25.947317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CovLungDataset(Dataset):\n    def __init__(self, dir_path, labels_data, transforms=None, new_size=(512, 512)):\n        self.dir_path = dir_path\n        self.labels_data = labels_data\n        self.new_size = new_size\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.labels_data)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        image_dir = self.labels_data.iloc[idx]['id']\n        if self.labels_data is not None:\n            img_label = self.labels_data.iloc[idx]['label']\n        else:\n            img_label = -1\n        \n        path_to_img = os.path.join(self.dir_path, image_dir)\n        # get first image path only\n        path_to_img = os.path.join(path_to_img, listdir(path_to_img)[0])\n        path_to_img = os.path.join(path_to_img, next(walk(path_to_img))[2][0])\n        \n        # read image\n        data = pydicom.dcmread(path_to_img)\n        image = data.pixel_array\n        image = exposure.equalize_hist(image)\n        \n        good_height, good_width = self.new_size\n        image = cv2.resize(image, (good_width, good_height), interpolation=Image.LANCZOS)\n        \n        # data augmentation\n        if self.transforms:\n            # doesn't work on floats\n            image = (image * 255).astype(np.uint8)\n            image = self.transforms(image=image)['image']\n        \n        sample = {'image': image, 'label': img_label}\n        return sample","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.950473Z","iopub.execute_input":"2021-06-15T23:19:25.951017Z","iopub.status.idle":"2021-06-15T23:19:25.962404Z","shell.execute_reply.started":"2021-06-15T23:19:25.950959Z","shell.execute_reply":"2021-06-15T23:19:25.96128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Should I do data augmentation?\n\nProbably **yes**, unless you're not afraid of overfitting.\n\nAccording to [Wikipedia](https://en.wikipedia.org/wiki/Data_augmentation):\n> Data augmentation in data analysis are techniques used to increase the amount of data by adding slightly modified copies of already existing data or newly created synthetic data from existing data. It acts as a regularizer and helps reduce overfitting when training a machine learning model.\n\nIn this tutorial, we will experiment with brightness and rotation of the images, but be aware that you can do much more.","metadata":{}},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ntransform = A.Compose([A.RandomBrightnessContrast(brightness_limit=[-0.2, 0.2], \n                                                  contrast_limit=[-0.2, 0.2], \n                                                  p=0.5),\n                       A.ShiftScaleRotate(scale_limit=[-0.1, 0.3], \n                                          shift_limit=0.1, \n                                          rotate_limit=20, \n                                          border_mode=cv2.BORDER_CONSTANT,\n                                          p=0.5),\n                       # reshape image of size (k, n, 1) into (1, k, n)\n                       ToTensorV2(p=1.0)\n                      ])","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:25.964309Z","iopub.execute_input":"2021-06-15T23:19:25.964758Z","iopub.status.idle":"2021-06-15T23:19:26.328392Z","shell.execute_reply.started":"2021-06-15T23:19:25.964712Z","shell.execute_reply":"2021-06-15T23:19:26.327322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's combine that with our dataloader and check the results.","metadata":{}},{"cell_type":"code","source":"transformed_dataset = CovLungDataset(dir_path=TRAIN_DIR,\n                                     labels_data=train_study[['id', 'label']],\n                                     transforms=transform,\n                                     new_size=(512, 512))\n\n\ntrain_dataloader = DataLoader(transformed_dataset, batch_size=8, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:26.329707Z","iopub.execute_input":"2021-06-15T23:19:26.330027Z","iopub.status.idle":"2021-06-15T23:19:26.336736Z","shell.execute_reply.started":"2021-06-15T23:19:26.329994Z","shell.execute_reply":"2021-06-15T23:19:26.336012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport numpy as np\n\n\ndataiter = iter(train_dataloader)\nsample = dataiter.next()\nimages = sample['image'] \nlabels = sample['label']\n\nfig = plt.figure(figsize=(15., 25.))\ngrid = ImageGrid(fig, 111, \n                 nrows_ncols=(2, 4),\n                 axes_pad=.4,\n                 )\n\nlabels_dict = {0: 'Negative for Pneumonia',  1: 'Typical Appearance',  2: 'Indeterminate Appearance',  3: 'Atypical Appearance'}\n\nj = 0\nfor ax, im in zip(grid, images):\n    im = im.numpy()\n    im = np.transpose(im, (1,2,0))\n    ax.imshow(im, cmap='gray')\n    ax.set_title(labels_dict[labels[j].item()], fontsize=12)\n    j += 1\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:19:26.337836Z","iopub.execute_input":"2021-06-15T23:19:26.338289Z","iopub.status.idle":"2021-06-15T23:19:34.622786Z","shell.execute_reply.started":"2021-06-15T23:19:26.33824Z","shell.execute_reply":"2021-06-15T23:19:34.620399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataloader for the preprocessed image data","metadata":{}},{"cell_type":"code","source":"class CovLungDataset(Dataset):\n    def __init__(self, dir_path, labels_data, transforms=None):\n        self.dir_path = dir_path\n        self.labels_data = labels_data\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.labels_data)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        image_dir = self.labels_data.iloc[idx]['id']\n        if self.labels_data is not None:\n            img_label = self.labels_data.iloc[idx]['label']\n        else:\n            img_label = -1\n        \n        path_to_img = os.path.join(self.dir_path, f'{image_dir}.png')\n        image = cv2.imread(path_to_img)\n        \n        if self.transforms:\n            image = self.transforms(image=image)['image']\n        \n        return {'image': image, 'label': img_label}","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:20:12.985161Z","iopub.execute_input":"2021-06-15T23:20:12.985541Z","iopub.status.idle":"2021-06-15T23:20:12.994075Z","shell.execute_reply.started":"2021-06-15T23:20:12.985509Z","shell.execute_reply":"2021-06-15T23:20:12.993262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformed_dataset = CovLungDataset(dir_path='../input/siimcovid19-512-img-png-600-study-png/study',\n                                     labels_data=train_study,\n                                     transforms=transform,\n                                     )\n\n\ntrain_dataloader = DataLoader(transformed_dataset, batch_size=8, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T23:20:13.311662Z","iopub.execute_input":"2021-06-15T23:20:13.312173Z","iopub.status.idle":"2021-06-15T23:20:13.31679Z","shell.execute_reply.started":"2021-06-15T23:20:13.312141Z","shell.execute_reply":"2021-06-15T23:20:13.316086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}