{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\nimport glob\nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.pylab as pylab\nimport seaborn as sns\nimport pprint\nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport albumentations as A\nimport cv2\nimport wandb\n\nfrom PIL import Image\nfrom colorama import Fore, Back, Style\n# colored output\ny_ = Fore.YELLOW\nr_ = Fore.RED\ng_ = Fore.GREEN\nb_ = Fore.BLUE\nm_ = Fore.MAGENTA\n\nsns.set(font=\"Serif\",style =\"white\")\n!conda install -c conda-forge gdcm -y\n\nimport gdcm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-26T12:37:33.063109Z","iopub.execute_input":"2021-10-26T12:37:33.063591Z","iopub.status.idle":"2021-10-26T12:38:32.130675Z","shell.execute_reply.started":"2021-10-26T12:37:33.063495Z","shell.execute_reply":"2021-10-26T12:38:32.129713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_study_level = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:40:32.492844Z","iopub.execute_input":"2021-10-26T12:40:32.493260Z","iopub.status.idle":"2021-10-26T12:40:32.579672Z","shell.execute_reply.started":"2021-10-26T12:40:32.493220Z","shell.execute_reply":"2021-10-26T12:40:32.578557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:40:35.818005Z","iopub.execute_input":"2021-10-26T12:40:35.818297Z","iopub.status.idle":"2021-10-26T12:40:35.839098Z","shell.execute_reply.started":"2021-10-26T12:40:35.818266Z","shell.execute_reply":"2021-10-26T12:40:35.838148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:40:37.019431Z","iopub.execute_input":"2021-10-26T12:40:37.020033Z","iopub.status.idle":"2021-10-26T12:40:37.031723Z","shell.execute_reply.started":"2021-10-26T12:40:37.019977Z","shell.execute_reply":"2021-10-26T12:40:37.030837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_directory = \"../input/siim-covid19-detection/train/\"\ntest_directory = \"../input/siim-covid19-detection/test/\"\n\ntrain_study_level['StudyInstanceUID'] = train_study_level['id'].apply(lambda x: x.replace('_study', ''))\ndel train_study_level['id']\ntrain_df = train_image_level.merge(train_study_level, on='StudyInstanceUID')","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:40:39.406953Z","iopub.execute_input":"2021-10-26T12:40:39.407271Z","iopub.status.idle":"2021-10-26T12:40:39.444596Z","shell.execute_reply.started":"2021-10-26T12:40:39.407235Z","shell.execute_reply":"2021-10-26T12:40:39.443293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:40:40.361408Z","iopub.execute_input":"2021-10-26T12:40:40.362311Z","iopub.status.idle":"2021-10-26T12:40:40.380587Z","shell.execute_reply.started":"2021-10-26T12:40:40.362257Z","shell.execute_reply":"2021-10-26T12:40:40.378743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_paths = []\n\nfor sid in tqdm(train_df['StudyInstanceUID']):\n    training_paths.append(glob.glob(os.path.join(train_directory, sid +\"/*/*\"))[0])\n\ntrain_df['path'] = training_paths","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:40:45.602956Z","iopub.execute_input":"2021-10-26T12:40:45.604122Z","iopub.status.idle":"2021-10-26T12:41:13.952056Z","shell.execute_reply.started":"2021-10-26T12:40:45.604053Z","shell.execute_reply":"2021-10-26T12:41:13.951145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:41:30.192622Z","iopub.execute_input":"2021-10-26T12:41:30.193583Z","iopub.status.idle":"2021-10-26T12:41:30.209635Z","shell.execute_reply.started":"2021-10-26T12:41:30.193531Z","shell.execute_reply":"2021-10-26T12:41:30.208468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'legend.fontsize': 'x-large',\n          'figure.figsize': (20, 32),\n         'axes.labelsize': 'x-large',\n         'axes.titlesize':'x-large',\n         'xtick.labelsize':'x-large',\n         'ytick.labelsize':'x-large'}\npylab.rcParams.update(params)\n\nfig, ax = plt.subplots(4,2)\nsns.kdeplot(train_df[\"Negative for Pneumonia\"], shade=True,ax=ax[0,0],color=\"#ffb4a2\")\nax[0,0].set_title(\"Negative for Pneumonia Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\nsns.countplot(x = train_df[\"Negative for Pneumonia\"], ax=ax[0,1],color=\"#ffb4a2\")\nax[0,1].set_title(\"Negative for Pneumonia Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nsns.kdeplot(train_df[\"Typical Appearance\"], shade=True,ax=ax[1,0],color=\"#e5989b\")\nax[1,0].set_title(\"Typical Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\nsns.countplot(x = train_df[\"Typical Appearance\"], ax=ax[1,1],color=\"#e5989b\")\nax[1,1].set_title(\"Typical Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nsns.kdeplot(train_df[\"Indeterminate Appearance\"], shade=True,ax=ax[2,0],color=\"#b5838d\")\nax[2,0].set_title(\"Indeterminate Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\nsns.countplot(x = train_df[\"Indeterminate Appearance\"], ax=ax[2,1],color=\"#b5838d\")\nax[2,1].set_title(\"Indeterminate Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nsns.kdeplot(train_df[\"Atypical Appearance\"], shade=True,ax=ax[3,0],color=\"#6d6875\")\nax[3,0].set_title(\"Atypical Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\nsns.countplot(x = train_df[\"Atypical Appearance\"], ax=ax[3,1],color=\"#6d6875\")\nax[3,1].set_title(\"Atypical Appearance Distribution\",font=\"Serif\", fontsize=20,weight=\"bold\")\n\nfig.subplots_adjust(wspace=0.2, hspace=0.4, top=0.93)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:41:34.343713Z","iopub.execute_input":"2021-10-26T12:41:34.344114Z","iopub.status.idle":"2021-10-26T12:41:36.028007Z","shell.execute_reply.started":"2021-10-26T12:41:34.344072Z","shell.execute_reply":"2021-10-26T12:41:36.026956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voi_lut=True\nfix_monochrome=True\n\ndef dicom_dataset_to_dict(filename,func):\n    \n    dicom_header = dicom.dcmread(filename) \n    \n    #====== DICOM FILE DATA ======\n    dicom_dict = {}\n    repr(dicom_header)\n    for dicom_value in dicom_header.values():\n        if dicom_value.tag == (0x7fe0, 0x0010):\n            #discard pixel data\n            continue\n        if type(dicom_value.value) == dicom.dataset.Dataset:\n            dicom_dict[dicom_value.name] = dicom_dataset_to_dict(dicom_value.value)\n        else:\n            v = _convert_value(dicom_value.value)\n            dicom_dict[dicom_value.name] = v\n      \n    del dicom_dict['Pixel Representation']\n    \n    if func!='metadata_df':\n        #====== DICOM IMAGE DATA ======\n        # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n        if voi_lut:\n            data = apply_voi_lut(dicom_header.pixel_array, dicom_header)\n        else:\n            data = dicom_header.pixel_array\n        # depending on this value, X-ray may look inverted - fix that:\n        if fix_monochrome and dicom_header.PhotometricInterpretation == \"MONOCHROME1\":\n            data = np.amax(data) - data\n        data = data - np.min(data)\n        data = data / np.max(data)\n        modified_image_data = (data * 255).astype(np.uint8)\n    \n        return dicom_dict, modified_image_data\n    \n    else:\n        return dicom_dict\n\ndef _sanitise_unicode(s):\n    return s.replace(u\"\\u0000\", \"\").strip()\n\ndef _convert_value(v):\n    t = type(v)\n    if t in (list, int, float):\n        cv = v\n    elif t == str:\n        cv = _sanitise_unicode(v)\n    elif t == bytes:\n        s = v.decode('ascii', 'replace')\n        cv = _sanitise_unicode(s)\n    elif t == dicom.valuerep.DSfloat:\n        cv = float(v)\n    elif t == dicom.valuerep.IS:\n        cv = int(v)\n    else:\n        cv = repr(v)\n    return cv\n\nfor filename in train_df.path[5:15]:\n    df, img_array = dicom_dataset_to_dict(filename, 'fetch_both_values')\n    \n    fig, ax = plt.subplots(1, 2, figsize=[15, 8])\n    ax[0].imshow(img_array, cmap=plt.cm.gray)\n    ax[1].imshow(img_array, cmap=plt.cm.plasma)    \n    plt.show()\n    \n    pprint.pprint(df)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:42:28.255060Z","iopub.execute_input":"2021-10-26T12:42:28.255719Z","iopub.status.idle":"2021-10-26T12:42:56.477894Z","shell.execute_reply.started":"2021-10-26T12:42:28.255667Z","shell.execute_reply":"2021-10-26T12:42:56.475506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:44:35.165350Z","iopub.execute_input":"2021-10-26T12:44:35.166396Z","iopub.status.idle":"2021-10-26T12:44:35.174658Z","shell.execute_reply.started":"2021-10-26T12:44:35.166319Z","shell.execute_reply":"2021-10-26T12:44:35.173602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:44:36.147451Z","iopub.execute_input":"2021-10-26T12:44:36.147748Z","iopub.status.idle":"2021-10-26T12:44:36.156512Z","shell.execute_reply.started":"2021-10-26T12:44:36.147718Z","shell.execute_reply":"2021-10-26T12:44:36.155626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:44:37.706946Z","iopub.execute_input":"2021-10-26T12:44:37.707770Z","iopub.status.idle":"2021-10-26T12:44:37.714436Z","shell.execute_reply.started":"2021-10-26T12:44:37.707726Z","shell.execute_reply":"2021-10-26T12:44:37.713346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:44:38.523832Z","iopub.execute_input":"2021-10-26T12:44:38.524164Z","iopub.status.idle":"2021-10-26T12:44:38.563633Z","shell.execute_reply.started":"2021-10-26T12:44:38.524129Z","shell.execute_reply":"2021-10-26T12:44:38.562547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/siim-covid19-detection/train/ae3e63d94c13/288554eb6182/e00f9fe0cce5.dcm'\ndicom = pydicom.read_file(path)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:44:40.890202Z","iopub.execute_input":"2021-10-26T12:44:40.890502Z","iopub.status.idle":"2021-10-26T12:44:41.222175Z","shell.execute_reply.started":"2021-10-26T12:44:40.890473Z","shell.execute_reply":"2021-10-26T12:44:41.220950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_jpg_directory = '../input/siim-covid19-resized-to-256px-jpg/train'\ntest_jpg_directory = '../input/siim-covid19-resized-to-256px-jpg/test'\n\ndef getImagePaths(path):\n    image_names = []\n    for dirname, _, filenames in os.walk(path):\n        for filename in filenames:\n            fullpath = os.path.join(dirname, filename)\n            image_names.append(fullpath)\n    return image_names\n\ntrain_images_path = getImagePaths(train_jpg_directory)\ntest_images_path = getImagePaths(test_jpg_directory)\n\nprint(f\"{y_}Number of train images: {g_} {len(train_images_path)}\\n\")\nprint(f\"{y_}Number of test images: {g_} {len(test_images_path)}\\n\")\n\ndef getShape(data, images_paths):\n    shape = cv2.imread(images_paths[0]).shape\n    for image_path in images_paths:\n        image_shape=cv2.imread(image_path).shape\n        if (image_shape!=shape):\n            return data +\" - Different image shape\"\n        else:\n            return data +\" - Same image shape \" + str(shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:55:10.296487Z","iopub.execute_input":"2021-10-26T12:55:10.296940Z","iopub.status.idle":"2021-10-26T12:55:18.869094Z","shell.execute_reply.started":"2021-10-26T12:55:10.296893Z","shell.execute_reply":"2021-10-26T12:55:18.867625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_augmentations(images, titles, sup_title):\n    fig, axes = plt.subplots(figsize=(20, 16), nrows=3, ncols=4, squeeze=False)\n    \n    for indx, (img, title) in enumerate(zip(images, titles)):\n        axes[indx // 4][indx % 4].imshow(img)\n        axes[indx // 4][indx % 4].set_title(title, fontsize=15)\n        \n    plt.tight_layout()\n    fig.suptitle(sup_title, fontsize = 20)\n    fig.subplots_adjust(wspace=0.2, hspace=0.2, top=0.93)\n    plt.show()\n    \ndef augment(paths, data):\n    \n    # list of albumentations\n    albumentations = [A.RandomSunFlare(p=1), A.RandomFog(p=1), A.RandomBrightness(p=1),\n                              A.RandomCrop(p=1,height = 128, width = 128), A.Rotate(p=1, limit=90),\n                              A.RGBShift(p=1), A.RandomSnow(p=1),\n                              A.HorizontalFlip(p=1), A.VerticalFlip(p=1), A.RandomContrast(limit = 0.5,p = 1),\n                              A.HueSaturationValue(p=1,hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=50)]\n    \n    # image titles\n    titles = [\"RandomSunFlare\",\"RandomFog\",\"RandomBrightness\",\n                       \"RandomCrop\",\"Rotate\", \"RGBShift\", \"RandomSnow\",\"HorizontalFlip\", \"VerticalFlip\", \"RandomContrast\",\"HSV\"]\n    \n    for i in paths:\n        image_path = i\n        \n        # getting image name from path\n        image_name = image_path.split(\"/\")[4].split(\".\")[0]\n        \n        # reading image\n        image = cv2.imread(image_path)\n\n        # list of images\n        images = []\n        \n        # creating image augmentations\n        for augmentation_type in albumentations:\n            augmented_img = augmentation_type(image = image)['image']\n            images.append(augmented_img)\n\n        # original image\n        titles.insert(0, \"Original\")\n        images.insert(0,image)  \n        \n        sup_title = \"Image Augmentation for \" + data + \" - \" + image_name\n        plot_augmentations(images, titles, sup_title)\n        \n        titles.remove(\"Original\")","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:55:42.498630Z","iopub.execute_input":"2021-10-26T12:55:42.499278Z","iopub.status.idle":"2021-10-26T12:55:42.514852Z","shell.execute_reply.started":"2021-10-26T12:55:42.499236Z","shell.execute_reply":"2021-10-26T12:55:42.514111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"getShape('train',train_images_path)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:55:43.413290Z","iopub.execute_input":"2021-10-26T12:55:43.413604Z","iopub.status.idle":"2021-10-26T12:55:43.433551Z","shell.execute_reply.started":"2021-10-26T12:55:43.413571Z","shell.execute_reply":"2021-10-26T12:55:43.432768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"getShape('test',test_images_path)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:55:44.308972Z","iopub.execute_input":"2021-10-26T12:55:44.309278Z","iopub.status.idle":"2021-10-26T12:55:44.327713Z","shell.execute_reply.started":"2021-10-26T12:55:44.309242Z","shell.execute_reply":"2021-10-26T12:55:44.326966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augment(train_images_path[5:15],'train')","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:56:58.724364Z","iopub.execute_input":"2021-10-26T12:56:58.724718Z","iopub.status.idle":"2021-10-26T12:57:27.626347Z","shell.execute_reply.started":"2021-10-26T12:56:58.724683Z","shell.execute_reply":"2021-10-26T12:57:27.625543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augment(test_images_path[5:15],'test')","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:57:38.735909Z","iopub.execute_input":"2021-10-26T12:57:38.736667Z","iopub.status.idle":"2021-10-26T12:58:10.543480Z","shell.execute_reply.started":"2021-10-26T12:57:38.736609Z","shell.execute_reply":"2021-10-26T12:58:10.541985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}