{"cells":[{"metadata":{"trusted":true,"_uuid":"83c74cdf44b27e6b206556b8979bf54eda588e30"},"cell_type":"code","source":"# basic imports\nimport os, random\nfrom tqdm import tqdm\nimport pydicom\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcab09c98e7e02740e00ac5bd2e533658316484d"},"cell_type":"code","source":"# what do we have here?\nprint (os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12173594123e32289906a8d18507a069337e1631"},"cell_type":"code","source":"# global variables\nTRAIN_DIR=\"../input/rsna-pneumonia-detection-challenge/stage_1_train_images\"\n# list of training images\nTRAIN_LIST=sorted(os.listdir(TRAIN_DIR))\n\n# dictionary to map string class identifiers to numerical\nCLASSES_DICT={'Normal': 0, 'Lung Opacity' : 1, 'No Lung Opacity / Not Normal' : 2}\n\n# we might need this later for saving into hdf5 files\nCLASSES=np.array([['Normal'.encode(\"utf-8\")],\n                  ['Lung Opacity'.encode(\"utf-8\")],\n                  ['No Lung Opacity / Not Normal'.encode(\"utf-8\")]])\nNUMCLASSES=CLASSES.shape[0]\n\nTEST_DIR=\"../input/rsna-pneumonia-detection-challenge/stage_1_test_images\"\n# list of test images\nTEST_LIST=sorted(os.listdir(TEST_DIR))\n\nIMAGE_SIZE=[1024,1024]\n\n# dictionaries to map string sex and viewposition to numerical\nPATIENTSEX_DICT={'M': 0, 'F' : 1}\nVIEWPOSITION_DICT={'AP': 0, 'PA' : 1}\n\n# datasets from RSNA Preprocessed Non-Image Inputs\nTRAIN_PROCESSED_CSV_FILE=\"../input/rsna-preprocessed-nonimage-inputs/stage_1_train_processed.csv\"\nTRAIN_PROCESSED_CSV_COLUMN_NAMES=['patientId', 'label', 'class', 'sex', 'age', 'viewPosition']\n\nTRAIN_BOUNDINGBOX_CSV_FILE=\"../input/rsna-preprocessed-nonimage-inputs/stage_1_train_boundingboxes.csv\"\nTRAIN_BOUNDINGBOX_CSV_COLUMN_NAMES=['patientId', 'boundingbox']\n\nTEST_PROCESSED_CSV_FILE=\"../input/rsna-preprocessed-nonimage-inputs/stage_1_test_processed.csv\"\nTEST_PROCESSED_CSV_COLUMN_NAMES=['patientId', 'sex', 'age', 'viewPosition']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d87cb5f3006407858fede5a4269684083be6097c"},"cell_type":"code","source":"# how many unique records does TRAIN_BOUNDINGBOX_CSV_FILE contain?\n!printf \"Number of bounding box data records (not unique) stored: \"; grep -v \"patientId,boundingbox\" ../input/rsna-preprocessed-nonimage-inputs/stage_1_train_boundingboxes.csv | sort | uniq| wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7725cbc34c5548cd730bdf0283dccd31a1eebfc"},"cell_type":"code","source":"# what does the stage_1_train_boundingboxes.csv file look like?\n!printf \"First 10 rows, including header, in stage_1_train_boundingboxes.csv:\\n\\n\"; \\\n         head -10 ../input/rsna-preprocessed-nonimage-inputs/stage_1_train_boundingboxes.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f428106119d9fd942db061ac418828e91498243"},"cell_type":"code","source":"# how many unique records does TRAIN_PROCESSED_CSV_FILE contain?\n!printf \"Number of unique train processed data records stored: \"; grep -v \"patientId,label,class,sex,age,viewPosition\" ../input/rsna-preprocessed-nonimage-inputs/stage_1_train_processed.csv | sort | uniq| wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b728de80699e1f576c11ef1996ceff256c9ea7ac"},"cell_type":"code","source":"# what does the stage_1_train_processed.csv file look like?\n!printf \"First 10 rows, including header, in stage_1_train_processed.csv:\\n\\n\"; \\\n         head -10 ../input/rsna-preprocessed-nonimage-inputs/stage_1_train_processed.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34818a3c92ddb301a2a2dc5a514740524f5b9b0a"},"cell_type":"code","source":"# how many unique records does TEST_PROCESSED_CSV_FILE contain?\n!printf \"Number of unique train processed data records stored: \"; grep -v \"patientId,sex,age,viewPosition\" ../input/rsna-preprocessed-nonimage-inputs/stage_1_test_processed.csv | sort | uniq| wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"353df679575368bc48efd622b6f8585448d05679"},"cell_type":"code","source":"# what does the stage_1_test_processed.csv file look like?\n!printf \"First 10 rows, including header, in stage_1_test_processed.csv:\\n\\n\"; \\\n         head -10 ../input/rsna-preprocessed-nonimage-inputs/stage_1_test_processed.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc2f9239fcbf28cfbe77717d15bb611a31f9a6cd"},"cell_type":"code","source":"# read TRAIN_PROCESSED_CSV_FILE into a pandas dataframe\ntraindf = pd.read_csv(TRAIN_PROCESSED_CSV_FILE,\n                      names=TRAIN_PROCESSED_CSV_COLUMN_NAMES,\n                      # skip the header line\n                      header=0,\n                      # index the dataframe on patientId\n                      index_col='patientId')\n\nprint (traindf.shape)\nprint (traindf.head(n=10))\n\n# read TRAIN_BOUNDINGBOX_CSV_FILE into a pandas dataframe\nboundingboxesdf = pd.read_csv(TRAIN_BOUNDINGBOX_CSV_FILE,\n                              names=TRAIN_BOUNDINGBOX_CSV_COLUMN_NAMES,\n                              # skip the header line\n                              header=0,\n                              # index the dataframe on patientId\n                              index_col='patientId')\n\nprint (boundingboxesdf.shape)\nprint (boundingboxesdf.head(n=10))\n\n# read TEST_PROCESSED_CSV_FILE into a pandas dataframe\ntestdf = pd.read_csv(TEST_PROCESSED_CSV_FILE,\n                     names=TEST_PROCESSED_CSV_COLUMN_NAMES,\n                     # skip the header line\n                     header=0,\n                     # index the dataframe on patientId\n                     index_col='patientId')\n\nprint (testdf.shape)\nprint (testdf.head(n=10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dba1921160d0bacb805d4e1cf1efd5a0136676f5"},"cell_type":"code","source":"# combine bounding boxes by unique patientId (multiple bounding boxes put in a list)\nbboxes=boundingboxesdf.copy().groupby(['patientId'])['boundingbox'].apply(list)\nprint (bboxes.head(n=10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"cb1ecf84990ec77b2a1ecced7cf678664814927a"},"cell_type":"code","source":"# save keys (unique patientIds) for 'Normal' 'Lung Opacity,' and 'No Lung Opacity / Not Normal' examples\nnormalkeys=traindf.index[traindf['class']==0].tolist()\nlungopacitykeys=traindf.index[traindf['class']==1].tolist()\notherabnormalkeys=traindf.index[traindf['class']==2].tolist()\n\nprint (\"Out of a total of {} unique X-rays:\".format(len(normalkeys+lungopacitykeys+otherabnormalkeys)))\nprint (\">{} X-rays are labeled as '{}'\".format(len(normalkeys), np.squeeze(CLASSES)[0].decode(\"utf-8\")))\nprint (\">{} X-rays are labeled as having '{}'\".format(len(lungopacitykeys), np.squeeze(CLASSES)[1].decode(\"utf-8\")))\nprint (\">{} X-rays are labeled as having '{}'\".format(len(otherabnormalkeys), np.squeeze(CLASSES)[2].decode(\"utf-8\")))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d641b4a60107bae696eb80cdf4d7c3d2bdc2da5"},"cell_type":"code","source":"# extract test keys while we are at it\ntestkeys=testdf.index.tolist()\nprint (\"There are a total of {} test X-rays\".format(len(testkeys)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24f0f7afe3fa4c912b0a7f20f5d085677d8a87f1"},"cell_type":"code","source":"# utility to extract bounding boxes for a given patientId\ndef getBoundingBoxes (bboxes, key):\n    bboxlist=list(bboxes[key])\n    return bboxlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b2b7b33cf182a6b12a11922888e3ff9772f696a"},"cell_type":"code","source":"# utility to extract x, y, width, and height for a single bounding box\ndef getBoundingBoxParameters (bbox):\n    for i in range(len(bbox)):\n        bboxparsed=bbox.split()\n        x=bboxparsed[0]\n        y=bboxparsed[1]\n        width=bboxparsed[2]\n        height=bboxparsed[3]  \n        return x, y, width, height","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d77faa887d443b4c7d4aed175f9e091ec779cb3"},"cell_type":"code","source":"# make sure we can extract bounding box details from lungopacitykeys\nfor i, key in enumerate(lungopacitykeys):\n    print (\"Count: {}, Key={}\".format(i+1, key))\n    bboxlist=getBoundingBoxes(bboxes, key)\n    # print (bboxlist)\n    for i in range(len(bboxlist)):\n        print(\"  Bounding Box {}:: \".format(i+1), end=\"\")\n        x, y, width, height = getBoundingBoxParameters(bboxlist[i])\n        print(\"x= {}, y= {}, width={}, height={}\".format(x, y, width, height))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93ab36c99009ca59fb4b002e28299e9b2522b686"},"cell_type":"code","source":"# extract and check counts for boundingboxes\nzeroboundingboxcount=0\noneboundingboxcount=0\ntwoboundingboxcount=0\nthreeboundingboxcount=0\nfor key in lungopacitykeys: \n    bboxlist=getBoundingBoxes(bboxes, key)\n    #print(len(bboxes))\n    if (len(bboxlist)==1):\n        zeroboundingboxcount +=1\n    if (len(bboxlist))==2:\n        oneboundingboxcount +=1\n    elif (len(bboxlist))==3:\n        twoboundingboxcount +=1\n    elif (len(bboxlist))==4:\n        threeboundingboxcount +=1\nprint (\"Out of a total of {} X-rays labeled as having '{}':\".format(len(lungopacitykeys), np.squeeze(CLASSES)[1].decode(\"utf-8\")))\nprint (\">{}/{} have no bounding boxes\".format(zeroboundingboxcount,len(lungopacitykeys)))\nprint (\">{}/{} have bounding boxes\".format(oneboundingboxcount+twoboundingboxcount+threeboundingboxcount, len(lungopacitykeys)))\nprint (\">>{} have 1 bounding box\".format(oneboundingboxcount))\nprint (\">>{} have 2 bounding boxes\".format(twoboundingboxcount))\nprint (\">>{} have 3 bounding boxes\".format(threeboundingboxcount))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7178ef492fac41d37302a452e143d613adcf5143"},"cell_type":"code","source":"# analyze train and test data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be19d5ed33500a55b4566e676cd7f5305900e188"},"cell_type":"code","source":"# double-check train and test samples\nprint (\"Total train examples are: {}\".format(traindf.shape[0]))\nprint (\"Total test examples are: {}\".format(testdf.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c388a5520e8bdc44c2ed207e696f6d1d0b8b1646"},"cell_type":"code","source":"# what does the gender mix look like for train data?\ntraindf.groupby(['sex']).size().reset_index(name='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0e1ee536bc07f7eaec8b169bfab30ff32f855b2"},"cell_type":"code","source":"# visually\ntraindf.groupby(['sex']).size().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f04aa315ff69ba03467e8c75e88cd9665513f0b3"},"cell_type":"code","source":"# what does the gender mix look like for test data?\ntestdf.groupby(['sex']).size().reset_index(name='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a7d6954046634600f25d38d90446fe39434261a"},"cell_type":"code","source":"# visually\ntestdf.groupby(['sex']).size().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ab74d983752f8f0f9850fc204f966837120a2a1"},"cell_type":"code","source":"# 56.1% Male in train data and 57.1% in test data is pretty similar mix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b4488f1a4e9b373a8ee6132ed82d40312313f28"},"cell_type":"code","source":"# what does the viewPosition mix look like for train data?\ntraindf.groupby(['viewPosition']).size().reset_index(name='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35072fcbdc4d4bc1d416f587066ff54156c48873"},"cell_type":"code","source":"# what does the viewPosition mix look like for test data?\ntestdf.groupby(['viewPosition']).size().reset_index(name='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5d03bab36551a83ee4ebd08630b80ef254ae483"},"cell_type":"code","source":"# 45.6% AP in train data and 46.8% in test data is similar mix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"125b534b8f295ba15ba96c9319e41878cadaf077"},"cell_type":"code","source":"# what does the viewPosition mix look like by sex for train data?\ntraindf.groupby(['sex','viewPosition']).size().reset_index(name='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4888fdd3193c9688c79e93c113b8c7ca19ac0a21"},"cell_type":"code","source":"# visually\ntraindf.groupby(['sex','viewPosition']).size().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"227ac7fe4688715fca67e5b7a30f20fea4b5ebcd"},"cell_type":"code","source":"# what does the viewPosition mix look like by sex for test data?\ntestdf.groupby(['sex','viewPosition']).size().reset_index(name='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5cfb63a6e3f0e1a12e328c57cdfb3ae8612568e"},"cell_type":"code","source":"# visually\ntestdf.groupby(['sex','viewPosition']).size().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65f09027bd2e3a0519f27d2093dcc2821393fd22"},"cell_type":"code","source":"# not quite similar, especially for female examples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61f20eb0a61e2f5e3c08588e18e600c703d65159"},"cell_type":"code","source":"# what does the age distribution look like for train data?\ntraindf.groupby(['age']).size().plot.bar(figsize=(10,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"136e0c066a93e319df1aea95aad300ae1f971708"},"cell_type":"code","source":"# by the numbers\ntraindf['age'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b077e46de9720469b3a7d54662ca0168c4e2536c"},"cell_type":"code","source":"testdf.groupby(['age']).size().plot.bar(figsize=(10,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"134832bc5c1e3a53d9f4f78dc293291efb3b6298"},"cell_type":"code","source":"# by the numbers\ntestdf['age'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"762e9b184bdcce64fce66b8221d9ea1bb0c691eb"},"cell_type":"code","source":"# that is pretty darn close, except for minimum and maximum age examples\n# SUMMARY: the datasets provided appear to be pretty balanced, but be careful when using smaller samples\n# SUMMARY: as has been noted by others, there does not appear to be any meaningful classification information in the meta-data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d382bb46237cd8d05caf9a0b7698adebbefd708f"},"cell_type":"code","source":"# review images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c9aa76001c132a1a66debdd7fb591c1e62a49bf"},"cell_type":"code","source":"# utility to load a dicom image and/or key attributes\ndef loadImage (directory, filename, mode=\"metadata\"):\n    imagearray=np.zeros(IMAGE_SIZE)\n    patientid= filename.split(\".\")[0]\n    \n    if mode==\"metadata\":\n        # load patient meta-data only from file\n        patientdata = pydicom.dcmread(os.path.join(directory, filename), stop_before_pixels=True)\n    elif mode==\"image\":\n        # load patient meta-data and image from file\n        patientdata = pydicom.dcmread(os.path.join(directory, filename))\n        imagearray=patientdata.pixel_array\n    patientid=patientdata.PatientID\n    attributes=\"{} {} {}\".format(patientdata.PatientSex,\n                                 patientdata.PatientAge,\n                                 patientdata.ViewPosition)\n    \n    return patientid, attributes, imagearray","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"761cd1ed924770629180597050fcbb059e9891f8"},"cell_type":"code","source":"# review training images (rerun to see different images)\n# number of images per class we want to see (configurable)\nnumimages=3\n\nfigure, axes = plt.subplots(NUMCLASSES, numimages, figsize=(20,20))\ntotalaxes=axes.flatten()\n# print (totalaxes.shape)\n\n# display normal images in the first row\nnormalsample=random.sample(normalkeys, numimages)\nfor i, key in enumerate(normalsample):\n    filename=\"{}.dcm\".format(key)\n    # print (filename)\n    patientid, attributes, imagearray=loadImage(TRAIN_DIR, filename, mode=\"image\")\n    #print (patientid)\n    attributes=list(attributes.split())\n    #print (\"{}-{}-{}\".format(attributes[0], attributes[1], attributes[2]))\n    totalaxes[0*numimages+i].imshow(imagearray, cmap='bone')\n    title=\"{}\\nPatient ID: {}\\nView Position: {}\".format(np.squeeze(CLASSES)[0].decode(\"utf-8\"), patientid, attributes[2])\n    totalaxes[0*numimages+i].set_title(title)\n# display images with lung opacity in the second row\nlungopacitysample=random.sample(lungopacitykeys, numimages)\nfor i, key in enumerate(lungopacitysample):\n    filename=\"{}.dcm\".format(key)\n    #print (filename)\n    patientid, attributes, imagearray=loadImage(TRAIN_DIR, filename, mode=\"image\")\n    #print (patientid)\n    attributes=list(attributes.split())\n    #print (\"{}-{}-{}\".format(attributes[0], attributes[1], attributes[2]))\n    totalaxes[1*numimages+i].imshow(imagearray, cmap='bone')\n    title=\"{}\\nPatient ID: {}\\nView Position: {}\".format(np.squeeze(CLASSES)[1].decode(\"utf-8\"), patientid, attributes[2])\n    totalaxes[1*numimages+i].set_title(title)\n# display images with other abnormalities in the third row\notherabnormalsample=random.sample(otherabnormalkeys, numimages)\nfor i, key in enumerate(otherabnormalsample):\n    filename=\"{}.dcm\".format(key)\n    #print (filename)\n    patientid, attributes, imagearray=loadImage(TRAIN_DIR, filename, mode=\"image\")\n    #print (patientid)\n    attributes=list(attributes.split())\n    #print (\"{}-{}-{}\".format(attributes[0], attributes[1], attributes[2]))\n    totalaxes[2*numimages+i].imshow(imagearray, cmap='bone')\n    title=\"{}\\nPatient ID: {}\\nView Position: {}\".format(np.squeeze(CLASSES)[2].decode(\"utf-8\"), patientid, attributes[2])\n    totalaxes[2*numimages+i].set_title(title)\nplt.show()\nplt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"220726c99d13551f46653ecff016add912116f2e"},"cell_type":"code","source":"# review test images (rerun to see different images)\n# we will take numimages from above\n\nfigure, axes = plt.subplots(1, numimages, figsize=(20,20))\ntotalaxes=axes.flatten()\n# print (totalaxes.shape)\n\ntestsample=random.sample(testkeys, numimages)\nfor i, key in enumerate(testsample):\n    filename=\"{}.dcm\".format(key)\n    # print (filename)\n    patientid, attributes, imagearray=loadImage(TEST_DIR, filename, mode=\"image\")\n    #print (patientid)\n    attributes=list(attributes.split())\n    #print (\"{}-{}-{}\".format(attributes[0], attributes[1], attributes[2]))\n    totalaxes[0*numimages+i].imshow(imagearray, cmap='bone')\n    title=\"Patient ID: {}\\nView Position: {}\".format(patientid, attributes[2])\n    totalaxes[0*numimages+i].set_title(title)\nplt.show()\nplt.close()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}