{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pydicom as dicom\nimport albumentations as A\nimport glob\nimport os\nfrom tqdm import tqdm\nimport cv2\nimport pprint","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-20T11:19:10.349866Z","iopub.execute_input":"2022-02-20T11:19:10.350282Z","iopub.status.idle":"2022-02-20T11:19:13.288964Z","shell.execute_reply.started":"2022-02-20T11:19:10.350194Z","shell.execute_reply":"2022-02-20T11:19:13.287244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imagelvl=pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_studylvl=pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:19:13.290844Z","iopub.execute_input":"2022-02-20T11:19:13.29118Z","iopub.status.idle":"2022-02-20T11:19:13.364068Z","shell.execute_reply.started":"2022-02-20T11:19:13.29115Z","shell.execute_reply":"2022-02-20T11:19:13.362732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imagelvl.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:19:13.36625Z","iopub.execute_input":"2022-02-20T11:19:13.366619Z","iopub.status.idle":"2022-02-20T11:19:13.391181Z","shell.execute_reply.started":"2022-02-20T11:19:13.366585Z","shell.execute_reply":"2022-02-20T11:19:13.390279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_studylvl.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:19:13.392849Z","iopub.execute_input":"2022-02-20T11:19:13.393238Z","iopub.status.idle":"2022-02-20T11:19:13.40569Z","shell.execute_reply.started":"2022-02-20T11:19:13.393203Z","shell.execute_reply":"2022-02-20T11:19:13.404913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"../input/siim-covid19-detection/train\"\ntest_dir = \"../input/siim-covid19-detection/test\"\ntrain_studylvl['StudyInstanceUID'] = train_studylvl['id'].apply(lambda x: x.replace('_study', ''))\ndel train_studylvl['id']\ntrain_df = train_imagelvl.merge(train_studylvl, on='StudyInstanceUID')","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:19:13.407111Z","iopub.execute_input":"2022-02-20T11:19:13.407649Z","iopub.status.idle":"2022-02-20T11:19:13.435995Z","shell.execute_reply.started":"2022-02-20T11:19:13.407612Z","shell.execute_reply":"2022-02-20T11:19:13.434899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:19:13.437081Z","iopub.execute_input":"2022-02-20T11:19:13.437508Z","iopub.status.idle":"2022-02-20T11:19:13.452763Z","shell.execute_reply.started":"2022-02-20T11:19:13.437478Z","shell.execute_reply":"2022-02-20T11:19:13.451708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nfrom bokeh.plotting import figure,show\nfrom bokeh.models import HoverTool\nimport matplotlib.pyplot as plt\nsns.set()\ngraph=sns.scatterplot(x=\"boxes\",y=\"label\",data=train_df)\ngraph","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:19:13.453905Z","iopub.execute_input":"2022-02-20T11:19:13.454436Z","iopub.status.idle":"2022-02-20T11:32:37.179299Z","shell.execute_reply.started":"2022-02-20T11:19:13.454405Z","shell.execute_reply":"2022-02-20T11:32:37.178475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from colorama import Fore as F, Back as B, Style as S # colored output\n_y = F.YELLOW\n_r = F.RED\n_g = F.GREEN\n_b = F.BLUE\n_m = F.MAGENTA","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:32:37.181622Z","iopub.execute_input":"2022-02-20T11:32:37.182043Z","iopub.status.idle":"2022-02-20T11:32:37.187385Z","shell.execute_reply.started":"2022-02-20T11:32:37.182013Z","shell.execute_reply":"2022-02-20T11:32:37.186279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_paths = []\n\nfor sid in tqdm(train_df['StudyInstanceUID']):\n    training_paths.append(glob.glob(os.path.join(train_dir, sid +\"/*/*\"))[0])\n\ntrain_df['path'] = training_paths","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:32:37.189145Z","iopub.execute_input":"2022-02-20T11:32:37.189532Z","iopub.status.idle":"2022-02-20T11:33:25.279311Z","shell.execute_reply.started":"2022-02-20T11:32:37.18949Z","shell.execute_reply":"2022-02-20T11:33:25.277953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:33:25.281432Z","iopub.execute_input":"2022-02-20T11:33:25.281809Z","iopub.status.idle":"2022-02-20T11:33:25.29971Z","shell.execute_reply.started":"2022-02-20T11:33:25.281779Z","shell.execute_reply":"2022-02-20T11:33:25.298807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(2,2,figsize=(14,10))\nsns.kdeplot(train_df[\"Negative for Pneumonia\"], shade=True,ax=ax[0,0],color=\"#ffb4a2\")\nax[0,0].set_title(\"Negative for Pneumonia Distribution\",font=\"Serif\", fontsize=8)\nsns.kdeplot(train_df[\"Typical Appearance\"], shade=True,ax=ax[0,1],color=\"#e5989b\")\nax[0,1].set_title(\"Typical Appearance Distribution\",font=\"Serif\", fontsize=8)\nsns.kdeplot(train_df[\"Indeterminate Appearance\"], shade=True,ax=ax[1,0],color=\"#e5989b\")\nax[1,0].set_title(\"Indeterminate Appearance Distribution\",font=\"Serif\", fontsize=8)\nsns.kdeplot(train_df[\"Atypical Appearance\"], shade=True,ax=ax[1,1],color=\"#ffb4a2\")\nax[1,1].set_title(\"Atypical Appearance Distribution\",font=\"Serif\", fontsize=8)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:33:25.301172Z","iopub.execute_input":"2022-02-20T11:33:25.301593Z","iopub.status.idle":"2022-02-20T11:33:26.463912Z","shell.execute_reply.started":"2022-02-20T11:33:25.301551Z","shell.execute_reply":"2022-02-20T11:33:26.462485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \"\"\"Credit: https://github.com/pydicom/pydicom/issues/319\"\"\"\ndef dicom_dataset_to_dict(filename):\n    dicom_header = dicom.dcmread(filename) \n    dicom_dict = {}\n    repr(dicom_header)\n    for dicom_value in dicom_header.values():\n        if dicom_value.tag == (0x7fe0, 0x0010):\n            continue\n        if type(dicom_value.value) == dicom.dataset.Dataset:\n            dicom_dict[dicom_value.name] = dicom_dataset_to_dict(dicom_value.value)\n        else:\n            v = _convert_value(dicom_value.value)\n            dicom_dict[dicom_value.name] = v\n    fig, ax = plt.subplots(1, 2, figsize=[10, 8])\n    ax[0].imshow(dicom_header.pixel_array, cmap=plt.cm.gray)\n    ax[1].imshow(dicom_header.pixel_array, cmap=plt.cm.plasma)\n    plt.show()\n    \n    return dicom_dict\n\n\ndef _sanitise_unicode(s):\n    return s.replace(u\"\\u0000\", \"\").strip()\n\ndef _convert_value(v):\n    t = type(v)\n    if t in (list, int, float):\n        cv = v\n    elif t == str:\n        cv = _sanitise_unicode(v)\n    elif t == bytes:\n        s = v.decode('ascii', 'replace')\n        cv = _sanitise_unicode(s)\n    elif t == dicom.valuerep.DSfloat:\n        cv = float(v)\n    elif t == dicom.valuerep.IS:\n        cv = int(v)\n    else:\n        cv = repr(v)\n    return cv\n\nfor filename in train_df.path[0:5]:\n    df = dicom_dataset_to_dict(filename)\n    pprint.pprint(df)","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:33:26.465854Z","iopub.execute_input":"2022-02-20T11:33:26.466195Z","iopub.status.idle":"2022-02-20T11:33:39.061834Z","shell.execute_reply.started":"2022-02-20T11:33:26.466159Z","shell.execute_reply":"2022-02-20T11:33:39.056682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing import image\nfrom keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:33:39.06431Z","iopub.execute_input":"2022-02-20T11:33:39.064655Z","iopub.status.idle":"2022-02-20T11:33:47.096218Z","shell.execute_reply.started":"2022-02-20T11:33:39.064628Z","shell.execute_reply":"2022-02-20T11:33:47.094974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen=ImageDataGenerator(\nrotation_range=40,\nwidth_shift_range=0.2,\nheight_shift_range=0.2,\nshear_range=0.2,\nzoom_range=0.2,\nhorizontal_flip=True,\nfill_mode='nearest')","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:37:29.748805Z","iopub.execute_input":"2022-02-20T11:37:29.749144Z","iopub.status.idle":"2022-02-20T11:37:29.754533Z","shell.execute_reply.started":"2022-02-20T11:37:29.749114Z","shell.execute_reply":"2022-02-20T11:37:29.753379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(6053):\n    test_img=train_images_path[i]\n    img=train_imagelvl.img_to_array(test_img)\n    img=img.reshape((1,) + img.shape)\n    \ni=0\nfor batch in datagen.flow(img,save_prefix='test',save_format='jpeg'):\n    plt.figure(i)\n    plot=plt.imshow(image.img_to_array(batch[0]))\n    i+=1\n    if i>4:\n        break\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:37:33.128798Z","iopub.execute_input":"2022-02-20T11:37:33.129455Z","iopub.status.idle":"2022-02-20T11:37:33.167002Z","shell.execute_reply.started":"2022-02-20T11:37:33.12942Z","shell.execute_reply":"2022-02-20T11:37:33.165439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Working on these errors:(","metadata":{}},{"cell_type":"markdown","source":"# ****Its time to create a Convolutional Neural Network to pass these images****","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers,models\ntrain_images,test_images= train_images/255.0,test_images/255.0\nclass_names=['Pneumonia','Atypical']","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:39:11.127575Z","iopub.execute_input":"2022-02-20T11:39:11.127948Z","iopub.status.idle":"2022-02-20T11:39:11.160113Z","shell.execute_reply.started":"2022-02-20T11:39:11.127915Z","shell.execute_reply":"2022-02-20T11:39:11.158353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=models.Sequential()\nmodel.add(layers.Conv2D(32,(3,3),activation='relu',input_shape=(256,256,3)))\nmodel.add(layers.MaxPooling2D((2,2)))\nmodel.add(layers.Conv2D(64,(3,3),activation=\"relu\"))\nmodel.add(layers.MaxPooling2D((2,2)))\nmodel.add(layers.Conv2D(64,(3,3),activation='relu'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:37:46.68423Z","iopub.execute_input":"2022-02-20T11:37:46.684539Z","iopub.status.idle":"2022-02-20T11:37:46.717192Z","shell.execute_reply.started":"2022-02-20T11:37:46.684513Z","shell.execute_reply":"2022-02-20T11:37:46.714972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.add(layers.Flatten())\nmodel.add(layers.Dense(64,activation='relu'))\nmodel.add(layers.Dense(4))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n             metrics=['accuracy'])\nhistory=model.fit(train_images,train_labels,epochs=40000,validation_data=(test_images,test_labels))","metadata":{"execution":{"iopub.status.busy":"2022-02-20T11:37:53.757678Z","iopub.execute_input":"2022-02-20T11:37:53.75801Z","iopub.status.idle":"2022-02-20T11:37:53.790602Z","shell.execute_reply.started":"2022-02-20T11:37:53.757975Z","shell.execute_reply":"2022-02-20T11:37:53.789008Z"},"trusted":true},"execution_count":null,"outputs":[]}]}