{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nimport PIL\nimport glob\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport albumentations as alb\nfrom pydicom import dcmread","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:29.076414Z","iopub.execute_input":"2021-07-17T14:21:29.077068Z","iopub.status.idle":"2021-07-17T14:21:29.083665Z","shell.execute_reply.started":"2021-07-17T14:21:29.077012Z","shell.execute_reply":"2021-07-17T14:21:29.082833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper","metadata":{}},{"cell_type":"code","source":"def plot(img):\n    plt.figure(figsize=(10, 10))\n    plt.title(\"Raw visualization\")\n    plt.imshow(img, 'gray')\n\n\ndef get_labels(df):\n    for i, row in df.iterrows():\n        if(row[labels[0]] == 1):\n            df.loc[i, 'label']=labels[0]\n        elif(row[labels[1]] == 1):\n            df.loc[i, 'label']=labels[1]\n        elif(row[labels[2]] == 1):\n            df.loc[i, 'label']=labels[2]\n        elif(row[labels[3]] == 1):\n            df.loc[i, 'label']=labels[3]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_paths(df):\n    list_, labels=[], []\n\n    for i, path_ in tqdm(enumerate(df.path), total=len(df)):\n        for folder in glob.glob(path_+'*'):\n            list_.extend(glob.glob(folder+'/*.dcm'))\n            labels.extend([df.iloc[i, 6] for _ in glob.glob(folder+'/*.dcm')])\n            if(len(list_)!=len(labels)):\n                print(len(list_),len(labels))\n    return list_, labels\n\n\ndef convert_dicom(path):\n    dicom = dcmread(path)\n    return dicom.pixel_array\n\ndef augmentations(path):\n    img = convert_dicom(path)\n    \n    aug = alb.Compose([\n#         alb.RandomBrightnessContrast(p=1),\n        alb.Emboss(p=1),\n#         alb.Downscale(scale_min=0.1, scale_max=0.5,p=1),\n#         alb.Equalize(p=1),\n        alb.RandomGamma(p=1),\n    ])\n    \n    return aug(image=img)['image']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data processing","metadata":{}},{"cell_type":"code","source":"study_level = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nstudy_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:29.085870Z","iopub.execute_input":"2021-07-17T14:21:29.086597Z","iopub.status.idle":"2021-07-17T14:21:29.118031Z","shell.execute_reply.started":"2021-07-17T14:21:29.086520Z","shell.execute_reply":"2021-07-17T14:21:29.116912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_level['id'] = study_level.apply(lambda row: row.id.split('_')[0], axis=1)\nstudy_level['path'] = study_level.apply(lambda row: \"../input/siim-covid19-detection/train/\"+row.id+'/', axis=1)\n\nstudy_level.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:29.120175Z","iopub.execute_input":"2021-07-17T14:21:29.120824Z","iopub.status.idle":"2021-07-17T14:21:29.350512Z","shell.execute_reply.started":"2021-07-17T14:21:29.120771Z","shell.execute_reply":"2021-07-17T14:21:29.349349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']\nstudy_level['label']=''\n\nget_labels(study_level)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:29.352167Z","iopub.execute_input":"2021-07-17T14:21:29.352682Z","iopub.status.idle":"2021-07-17T14:21:32.070149Z","shell.execute_reply.started":"2021-07-17T14:21:29.352629Z","shell.execute_reply":"2021-07-17T14:21:32.068969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_level","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:32.071988Z","iopub.execute_input":"2021-07-17T14:21:32.072470Z","iopub.status.idle":"2021-07-17T14:21:32.096044Z","shell.execute_reply.started":"2021-07-17T14:21:32.072420Z","shell.execute_reply":"2021-07-17T14:21:32.094964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image and label segregation","metadata":{}},{"cell_type":"code","source":"train_images_path, train_images_labels = get_paths(study_level)\nlen(train_images_path), len(train_images_labels)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:32.107132Z","iopub.execute_input":"2021-07-17T14:21:32.107629Z","iopub.status.idle":"2021-07-17T14:21:41.814994Z","shell.execute_reply.started":"2021-07-17T14:21:32.107597Z","shell.execute_reply":"2021-07-17T14:21:41.814038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Distribution","metadata":{}},{"cell_type":"code","source":"sns.countplot(study_level['Negative for Pneumonia'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:41.816808Z","iopub.execute_input":"2021-07-17T14:21:41.817102Z","iopub.status.idle":"2021-07-17T14:21:41.958507Z","shell.execute_reply.started":"2021-07-17T14:21:41.817073Z","shell.execute_reply":"2021-07-17T14:21:41.957126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(study_level['Typical Appearance'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:41.960587Z","iopub.execute_input":"2021-07-17T14:21:41.961077Z","iopub.status.idle":"2021-07-17T14:21:42.106299Z","shell.execute_reply.started":"2021-07-17T14:21:41.961029Z","shell.execute_reply":"2021-07-17T14:21:42.105156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(study_level['Indeterminate Appearance'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:42.107968Z","iopub.execute_input":"2021-07-17T14:21:42.108402Z","iopub.status.idle":"2021-07-17T14:21:42.247608Z","shell.execute_reply.started":"2021-07-17T14:21:42.108353Z","shell.execute_reply":"2021-07-17T14:21:42.246661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(study_level['Atypical Appearance'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:42.248999Z","iopub.execute_input":"2021-07-17T14:21:42.249292Z","iopub.status.idle":"2021-07-17T14:21:42.386184Z","shell.execute_reply.started":"2021-07-17T14:21:42.249263Z","shell.execute_reply":"2021-07-17T14:21:42.385039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(study_level.label)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:21:42.388146Z","iopub.execute_input":"2021-07-17T14:21:42.388602Z","iopub.status.idle":"2021-07-17T14:21:42.560790Z","shell.execute_reply.started":"2021-07-17T14:21:42.388552Z","shell.execute_reply":"2021-07-17T14:21:42.559645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Visualization","metadata":{}},{"cell_type":"code","source":"dicom = dcmread(train_images_path[10])\nimg = dicom.pixel_array\nplot(img)\n\nprint(train_images_labels[10])","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:39:48.198406Z","iopub.execute_input":"2021-07-17T14:39:48.198889Z","iopub.status.idle":"2021-07-17T14:39:50.744021Z","shell.execute_reply.started":"2021-07-17T14:39:48.198851Z","shell.execute_reply":"2021-07-17T14:39:50.743036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Extraction","metadata":{}},{"cell_type":"markdown","source":"## Albumentations","metadata":{}},{"cell_type":"code","source":"img = augmentations(train_images_path[10])\nplot(np.array(img))","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:39:58.743616Z","iopub.execute_input":"2021-07-17T14:39:58.744040Z","iopub.status.idle":"2021-07-17T14:40:01.154065Z","shell.execute_reply.started":"2021-07-17T14:39:58.744002Z","shell.execute_reply":"2021-07-17T14:40:01.152891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OpenCV Image Threshold","metadata":{}},{"cell_type":"code","source":"img = convert_dicom(train_images_path[5])\nretval, threshold = cv2.threshold(img, 2500, 255, cv2.THRESH_TOZERO_INV)\nplt.imshow(threshold)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:40:49.262621Z","iopub.execute_input":"2021-07-17T14:40:49.263130Z","iopub.status.idle":"2021-07-17T14:40:50.958103Z","shell.execute_reply.started":"2021-07-17T14:40:49.263092Z","shell.execute_reply":"2021-07-17T14:40:50.956858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = convert_dicom(train_images_path[5])\nretval, threshold = cv2.threshold(img, 2500, 255, cv2.THRESH_TOZERO)\nplt.imshow(threshold)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:42:04.462799Z","iopub.execute_input":"2021-07-17T14:42:04.463233Z","iopub.status.idle":"2021-07-17T14:42:06.140913Z","shell.execute_reply.started":"2021-07-17T14:42:04.463199Z","shell.execute_reply":"2021-07-17T14:42:06.139578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = convert_dicom(train_images_path[5])\nretval, threshold = cv2.threshold(img, 2500, 255, cv2.THRESH_TRUNC)\nplt.imshow(threshold)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:41:59.050086Z","iopub.execute_input":"2021-07-17T14:41:59.050553Z","iopub.status.idle":"2021-07-17T14:42:00.726283Z","shell.execute_reply.started":"2021-07-17T14:41:59.050502Z","shell.execute_reply":"2021-07-17T14:42:00.724930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = convert_dicom(train_images_path[5])\nretval, threshold = cv2.threshold(img, 2500, 255, cv2.THRESH_BINARY_INV)\nplt.imshow(threshold)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:42:08.466440Z","iopub.execute_input":"2021-07-17T14:42:08.466855Z","iopub.status.idle":"2021-07-17T14:42:10.129122Z","shell.execute_reply.started":"2021-07-17T14:42:08.466821Z","shell.execute_reply":"2021-07-17T14:42:10.128027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = convert_dicom(train_images_path[5])\nretval, threshold = cv2.threshold(img, 2500, 255, cv2.THRESH_BINARY)\nplt.imshow(threshold)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:42:10.130824Z","iopub.execute_input":"2021-07-17T14:42:10.131168Z","iopub.status.idle":"2021-07-17T14:42:11.780378Z","shell.execute_reply.started":"2021-07-17T14:42:10.131136Z","shell.execute_reply":"2021-07-17T14:42:11.779187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normal Deep 2D Convolution","metadata":{}},{"cell_type":"code","source":"class FeatureExtractor(nn.Module):\n    def __init__(self, fil1, fil2, fil3):\n        super(FeatureExtractor, self).__init__()\n        self.conv1 = nn.Conv2d(1, 1, fil1)\n        self.conv2 = nn.Conv2d(1, 1, fil2)\n        self.conv3 = nn.Conv2d(1, 1, fil3)\n        \n    def forward(self, img):\n        img = self.conv1(img)\n        img = self.conv2(img)\n        img = self.conv3(img)\n        return img\n    \nmodel = FeatureExtractor(32, 32, 64)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:48:03.393312Z","iopub.execute_input":"2021-07-17T14:48:03.393744Z","iopub.status.idle":"2021-07-17T14:48:03.403586Z","shell.execute_reply.started":"2021-07-17T14:48:03.393696Z","shell.execute_reply":"2021-07-17T14:48:03.402772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = convert_dicom(train_images_path[10])\nimg = torch.Tensor(img.astype('float32')).unsqueeze(0).unsqueeze(0)\nplt.imshow(model(img).squeeze(0).squeeze(0).squeeze(0).detach().numpy())","metadata":{"execution":{"iopub.status.busy":"2021-07-17T14:48:05.486487Z","iopub.execute_input":"2021-07-17T14:48:05.487075Z","iopub.status.idle":"2021-07-17T14:48:26.527722Z","shell.execute_reply.started":"2021-07-17T14:48:05.487031Z","shell.execute_reply":"2021-07-17T14:48:26.526323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}