{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#Importing packages\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom glob import glob\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detail_class_info=pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\ntrain_label=pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\nsamp_submission=pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Viewing the dataset on a high level-detailed class info file\ndetail_class_info.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Viewing the dataset on a high level-train_labels info file\ntrain_label.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Viewing the dataset on a high level-sample submission file\nsamp_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*Understanding the dataset *\n\nIn reading the dataset detail_class_info file, it shows the list of patient ID's and under which class they are classified, i.e., if they are \"No Lung Opacity / Not Normal\", \"Normal\" or \"Lung Opacity\"\n\nThe dataset train_labels, it shows the patient id loaded and contains image details for the cases confirmed with 1"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check the number of records loaded and the number of unique patients in the Class upload file and Train label file\nprint(\"Details under Class Upload file\")\nprint(detail_class_info.shape[0],'Number of Patient ID Loaded')\nprint(detail_class_info['patientId'].value_counts().shape[0],'Number of Patient cases available')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check the number of records loaded and the number of unique patients in the Train label file\nprint(\"Details under Train Label file\")\nprint(train_label.shape[0],'Number of Patient ID Loaded')\nprint(train_label['patientId'].value_counts().shape[0],'Number of Patient cases available')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Find the list of patient ID loaded under various class\ndetail_class_info.groupby('class').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Find the total list of patient ID who are identified with Cases confirmed or not\ntrain_label.groupby('Target').patientId.count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check for missing values, if any\nprint(\"Number of missing values in Details class file: \",pd.isnull(detail_class_info).sum().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Visualize the split in class based on Detail Class Information\ndetail_class_info.groupby('class').size().plot.bar(rot=0,color=['red','green','blue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Visualize the number of cases confirmed or not from the training table\ntrain_label.groupby('Target').size().plot.bar(rot=0,color=['green','red'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Insight on DCM files provided in Train and Test Folders\n\n\nMedical images are stored in a special format known as DICOM files (*.dcm). They contain a combination of header metadata as well as a raw image arrays for pixel data. We will use a library called pydicom to read the files. This file can be read against a particular given patientId and matching for the respective file in the stage_2_train_images and stage_2_test_images folders, and then use the pydicom.read_file() method to load the data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We will read a patient detail from the Test Image Folder\npatID=train_label['patientId'][4]\nimg_train_folder='../input/rsna-pneumonia-detection-challenge/stage_2_train_images/%s.dcm' % patID\ndcm_data=pydicom.read_file(img_train_folder)\n\nprint(dcm_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We will access the above primary patient details with pixel data in array form\nim=dcm_data.pixel_array\nprint(type(im))\nprint(im.dtype)\nprint(im.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pylab\npylab.imshow(im,cmap=pylab.cm.gist_gray)\npylab.axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Exploring the Data and Labels\n\nAny given patient may potentially have many boxes if there are several different suspicious areas of pneumonia. To collapse the current CSV file dataframe into a dictionary with unique entries, we will consider the following method:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_data(df):\n    \"\"\"\n    Method to read a CSV file (Pandas dataframe) and parse the \n    data into the following nested dictionary:\n\n      parsed = {\n        \n        'patientId-00': {\n            'dicom': path/to/dicom/file,\n            'label': either 0 or 1 for normal or pnuemonia, \n            'boxes': list of box(es)\n        },\n        'patientId-01': {\n            'dicom': path/to/dicom/file,\n            'label': either 0 or 1 for normal or pnuemonia, \n            'boxes': list of box(es)\n        }, ...\n\n      }\n\n    \"\"\"\n    # --- Define lambda to extract coords in list [y, x, height, width]\n    extract_box = lambda row: [row['y'], row['x'], row['height'], row['width']]\n\n    parsed = {}\n    for n, row in df.iterrows():\n        # --- Initialize patient entry into parsed \n        pid = row['patientId']\n        if pid not in parsed:\n            parsed[pid] = {\n                'dicom': '../input/rsna-pneumonia-detection-challenge/stage_2_train_images/%s.dcm' % pid,\n                'label': row['Target'],\n                'boxes': []}\n\n        # --- Add box if opacity is present\n        if parsed[pid]['label'] == 1:\n            parsed[pid]['boxes'].append(extract_box(row))\n\n    return parsed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parsed = parse_data(train_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(parsed['00436515-870c-4b36-a041-de91049b9ab4'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw(data):\n    \"\"\"\n    Method to draw single patient with bounding box(es) if present \n\n    \"\"\"\n    # --- Open DICOM file\n    d = pydicom.read_file(data['dicom'])\n    im = d.pixel_array\n\n    # --- Convert from single-channel grayscale to 3-channel RGB\n    im = np.stack([im] * 3, axis=2)\n\n    # --- Add boxes with random color if present\n    for box in data['boxes']:\n        rgb = np.floor(np.random.rand(3) * 256).astype('int')\n        im = overlay_box(im=im, box=box, rgb=rgb, stroke=6)\n\n    pylab.imshow(im, cmap=pylab.cm.gist_gray)\n    pylab.axis('off')\n\ndef overlay_box(im, box, rgb, stroke=1):\n    \"\"\"\n    Method to overlay single box on image\n\n    \"\"\"\n    # --- Convert coordinates to integers\n    box = [int(b) for b in box]\n    \n    # --- Extract coordinates\n    y1, x1, height, width = box\n    y2 = y1 + height\n    x2 = x1 + width\n\n    im[y1:y1 + stroke, x1:x2] = rgb\n    im[y2:y2 + stroke, x1:x2] = rgb\n    im[y1:y2, x1:x1 + stroke] = rgb\n    im[y1:y2, x2:x2 + stroke] = rgb\n\n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(parsed['00436515-870c-4b36-a041-de91049b9ab4'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"def formatMetadataString(val):\n    return str(val).split(':')[1].replace('\\'', '')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\n\nimage_data = []\n\ndataDir='../input/rsna-pneumonia-detection-challenge'\ntrainimg_folder='../input/rsna-pneumonia-detection-challenge/stage_2_train_images'\n\ntrainfiles = glob.glob(os.path.join(dataDir,trainimg_folder, \"*.dcm\"))\n\nfor f in trainfiles:    \n    \n    patientId = formatMetadataString(ds['PatientID'])\n    age = formatMetadataString(ds['PatientAge'])\n    gender = formatMetadataString(ds['PatientSex'])\n    viewPos = formatMetadataString(ds['ViewPosition'])    \n    image_data.append([patientId, int(age), gender, viewPos])\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}