{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nfrom matplotlib.patches import Rectangle\nimport seaborn as sns\nimport pydicom as dcm\n%matplotlib inline \nIS_LOCAL = False\nimport os\nif(IS_LOCAL):\n    PATH=\"../input/rsna-pneumonia-detection-challenge\"\nelse:\n    PATH=\"../input/rsna-pneumonia-detection-challenge/\"\nprint(os.listdir(PATH))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class_info_df = pd.read_csv(PATH+'/stage_2_detailed_class_info.csv')\ntrain_labels_df = pd.read_csv(PATH+'/stage_2_train_labels.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Detailed class info -  rows: {class_info_df.shape[0]}, columns: {class_info_df.shape[1]}\")\nprint(f\"Train labels -  rows: {train_labels_df.shape[0]}, columns: {train_labels_df.shape[1]}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_info_df.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels_df.sample(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"MISSING DATA\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def missing_data(data):\n    total = data.isnull().sum().sort_values(ascending = False)\n    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)\n    return np.transpose(pd.concat([total, percent], axis=1, keys=['Total', 'Percent']))\nmissing_data(train_labels_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_data(class_info_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(1,1, figsize=(6,4))\ntotal = float(len(class_info_df))\nsns.countplot(class_info_df['class'],order = class_info_df['class'].value_counts().index, palette='Set3')\nfor p in ax.patches:\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 3,\n            '{:1.2f}%'.format(100*height/total),\n            ha=\"center\") \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_feature_distribution(data, feature):\n    # Get the count for each label\n    label_counts = data[feature].value_counts()\n\n    # Get total number of samples\n    total_samples = len(data)\n\n    # Count the number of items in each class\n    print(\"Feature: {}\".format(feature))\n    for i in range(len(label_counts)):\n        label = label_counts.index[i]\n        count = label_counts.values[i]\n        percent = int((count / total_samples) * 10000) / 100\n        print(\"{:<30s}:   {} or {}%\".format(label, count, percent))\n\nget_feature_distribution(class_info_df, 'class')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_class_df = train_labels_df.merge(class_info_df, left_on='patientId', right_on='patientId', how='inner')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_class_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Target and class\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1,figsize=(12,6))\ntmp = train_class_df.groupby('Target')['class'].value_counts()\ndf = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\nsns.barplot(ax=ax,x = 'Target', y='Exams',hue='class',data=df, palette='Set3')\nplt.title(\"Chest exams class and Target\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Detected Lung Opacity window\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"target1 = train_class_df[train_class_df['Target']==1]\nsns.set_style('whitegrid')\nplt.figure()\nfig, ax = plt.subplots(2,2,figsize=(12,12))\nsns.distplot(target1['x'],kde=True,bins=50, color=\"red\", ax=ax[0,0])\nsns.distplot(target1['y'],kde=True,bins=50, color=\"blue\", ax=ax[0,1])\nsns.distplot(target1['width'],kde=True,bins=50, color=\"green\", ax=ax[1,0])\nsns.distplot(target1['height'],kde=True,bins=50, color=\"magenta\", ax=ax[1,1])\nlocs, labels = plt.xticks()\nplt.tick_params(axis='both', which='major', labelsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(7,7))\ntarget_sample = target1.sample(2000)\ntarget_sample['xc'] = target_sample['x'] + target_sample['width'] / 2\ntarget_sample['yc'] = target_sample['y'] + target_sample['height'] / 2\nplt.title(\"Centers of Lung Opacity rectangles (brown) over rectangles (yellow)\\nSample size: 2000\")\ntarget_sample.plot.scatter(x='xc', y='yc', xlim=(0,1024), ylim=(0,1024), ax=ax, alpha=0.8, marker=\".\", color=\"brown\")\nfor i, crt_sample in target_sample.iterrows():\n    ax.add_patch(Rectangle(xy=(crt_sample['x'], crt_sample['y']),\n                width=crt_sample['width'],height=crt_sample['height'],alpha=3.5e-3, color=\"yellow\"))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Explore DICOM data¶\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_sample_path = os.listdir(PATH+'/stage_2_train_images')[:5]\nprint(image_sample_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_train_path = os.listdir(PATH+'/stage_2_train_images')\nimage_test_path = os.listdir(PATH+'/stage_2_test_images')\nprint(\"Number of images in train set:\", len(image_train_path),\"\\nNumber of images in test set:\", len(image_test_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Unique patientId in  train_class_df: \", train_class_df['patientId'].nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = train_class_df.groupby(['patientId','Target', 'class'])['patientId'].count()\ndf = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\ntmp = df.groupby(['Exams','Target','class']).count()\ndf2 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()\ndf2.columns = ['Exams', 'Target','Class', 'Entries']\ndf2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1,figsize=(12,6))\nsns.barplot(ax=ax,x = 'Target', y='Entries', hue='Exams',data=df2, palette='Set2')\nplt.title(\"Chest exams class and Target\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"DICOM meta data\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"samplePatientID = list(train_class_df[:3].T.to_dict().values())[0]['patientId']\nsamplePatientID = samplePatientID+'.dcm'\ndicom_file_path = os.path.join(PATH,\"stage_2_train_images/\",samplePatientID)\ndicom_file_dataset = dcm.read_file(dicom_file_path)\ndicom_file_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Plot DICOM images with Target = 1¶\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dicom_images(data):\n    img_data = list(data.T.to_dict().values())\n    f, ax = plt.subplots(3,3, figsize=(16,18))\n    for i,data_row in enumerate(img_data):\n        patientImage = data_row['patientId']+'.dcm'\n        imagePath = os.path.join(PATH,\"stage_2_train_images/\",patientImage)\n        data_row_img_data = dcm.read_file(imagePath)\n        modality = data_row_img_data.Modality\n        age = data_row_img_data.PatientAge\n        sex = data_row_img_data.PatientSex\n        data_row_img = dcm.dcmread(imagePath)\n        ax[i//3, i%3].imshow(data_row_img.pixel_array, cmap=plt.cm.bone) \n        ax[i//3, i%3].axis('off')\n        ax[i//3, i%3].set_title('ID: {}\\nModality: {} Age: {} Sex: {} Target: {}\\nClass: {}\\nWindow: {}:{}:{}:{}'.format(\n                data_row['patientId'],\n                modality, age, sex, data_row['Target'], data_row['class'], \n                data_row['x'],data_row['y'],data_row['width'],data_row['height']))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_dicom_images(train_class_df[train_class_df['Target']==1].sample(9))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dicom_images_with_boxes(data):\n    img_data = list(data.T.to_dict().values())\n    f, ax = plt.subplots(3,3, figsize=(16,18))\n    for i,data_row in enumerate(img_data):\n        patientImage = data_row['patientId']+'.dcm'\n        imagePath = os.path.join(PATH,\"stage_2_train_images/\",patientImage)\n        data_row_img_data = dcm.read_file(imagePath)\n        modality = data_row_img_data.Modality\n        age = data_row_img_data.PatientAge\n        sex = data_row_img_data.PatientSex\n        data_row_img = dcm.dcmread(imagePath)\n        ax[i//3, i%3].imshow(data_row_img.pixel_array, cmap=plt.cm.bone) \n        ax[i//3, i%3].axis('off')\n        ax[i//3, i%3].set_title('ID: {}\\nModality: {} Age: {} Sex: {} Target: {}\\nClass: {}'.format(\n                data_row['patientId'],modality, age, sex, data_row['Target'], data_row['class']))\n        rows = train_class_df[train_class_df['patientId']==data_row['patientId']]\n        box_data = list(rows.T.to_dict().values())\n        for j, row in enumerate(box_data):\n            ax[i//3, i%3].add_patch(Rectangle(xy=(row['x'], row['y']),\n                        width=row['width'],height=row['height'], \n                        color=\"yellow\",alpha = 0.1))   \n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_dicom_images_with_boxes(train_class_df[train_class_df['Target']==1].sample(9))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_dicom_images(train_class_df[train_class_df['Target']==0].sample(9))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vars = ['Modality', 'PatientAge', 'PatientSex', 'BodyPartExamined', 'ViewPosition', 'ConversionType', 'Rows', 'Columns', 'PixelSpacing']\n\ndef process_dicom_data(data_df, data_path):\n    for var in vars:\n        data_df[var] = None\n    image_names = os.listdir(PATH+data_path)\n    for i, img_name in tqdm_notebook(enumerate(image_names)):\n        imagePath = os.path.join(PATH,data_path,img_name)\n        data_row_img_data = dcm.read_file(imagePath)\n        idx = (data_df['patientId']==data_row_img_data.PatientID)\n        data_df.loc[idx,'Modality'] = data_row_img_data.Modality\n        data_df.loc[idx,'PatientAge'] = pd.to_numeric(data_row_img_data.PatientAge)\n        data_df.loc[idx,'PatientSex'] = data_row_img_data.PatientSex\n        data_df.loc[idx,'BodyPartExamined'] = data_row_img_data.BodyPartExamined\n        data_df.loc[idx,'ViewPosition'] = data_row_img_data.ViewPosition\n        data_df.loc[idx,'ConversionType'] = data_row_img_data.ConversionType\n        data_df.loc[idx,'Rows'] = data_row_img_data.Rows\n        data_df.loc[idx,'Columns'] = data_row_img_data.Columns  \n        data_df.loc[idx,'PixelSpacing'] = str.format(\"{:4.3f}\",data_row_img_data.PixelSpacing[0]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"process_dicom_data(train_class_df,'stage_2_train_images/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_class_df = pd.read_csv(PATH+'/stage_2_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_class_df = test_class_df.drop('PredictionString',1)\nprocess_dicom_data(test_class_df,'stage_2_test_images/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_feature_distribution(train_class_df,'ViewPosition')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_window(data,color_point, color_window,text):\n    fig, ax = plt.subplots(1,1,figsize=(7,7))\n    plt.title(\"Centers of Lung Opacity rectangles over rectangles\\n{}\".format(text))\n    data.plot.scatter(x='xc', y='yc', xlim=(0,1024), ylim=(0,1024), ax=ax, alpha=0.8, marker=\".\", color=color_point)\n    for i, crt_sample in data.iterrows():\n        ax.add_patch(Rectangle(xy=(crt_sample['x'], crt_sample['y']),\n            width=crt_sample['width'],height=crt_sample['height'],alpha=3.5e-3, color=color_window))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target1 = train_class_df[train_class_df['Target']==1]\n\ntarget_sample = target1.sample(2000)\ntarget_sample['xc'] = target_sample['x'] + target_sample['width'] / 2\ntarget_sample['yc'] = target_sample['y'] + target_sample['height'] / 2\n\ntarget_ap = target_sample[target_sample['ViewPosition']=='AP']\ntarget_pa = target_sample[target_sample['ViewPosition']=='PA']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_ap,'green', 'yellow', 'Patient View Position: AP')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_pa,'blue', 'red', 'Patient View Position: PA')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_feature_distribution(test_class_df,'ViewPosition')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Rows: train:\",train_class_df['Rows'].unique(), \"test:\", test_class_df['Rows'].unique())\nprint(\"Columns: train:\",train_class_df['Columns'].unique(), \"test:\", test_class_df['Columns'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = train_class_df.groupby(['Target', 'PatientAge'])['patientId'].count()\ndf = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\ntmp = df.groupby(['Exams','Target', 'PatientAge']).count()\ndf2 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = train_class_df.groupby(['class', 'PatientAge'])['patientId'].count()\ndf1 = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\ntmp = df1.groupby(['Exams','class', 'PatientAge']).count()\ndf3 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(16,6))\nsns.barplot(ax=ax, x = 'PatientAge', y='Exams', hue='Target',data=df2)\nplt.title(\"Train set: Chest exams Age and Target\")\nplt.xticks(rotation=90)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(16,6))\nsns.barplot(ax=ax, x = 'PatientAge', y='Exams', hue='class',data=df3)\nplt.title(\"Train set: Chest exams Age and class\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_age1 = target_sample[target_sample['PatientAge'] < 20]\ntarget_age2 = target_sample[(target_sample['PatientAge'] >=20) & (target_sample['PatientAge'] < 35)]\ntarget_age3 = target_sample[(target_sample['PatientAge'] >=35) & (target_sample['PatientAge'] < 50)]\ntarget_age4 = target_sample[(target_sample['PatientAge'] >=50) & (target_sample['PatientAge'] < 65)]\ntarget_age5 = target_sample[target_sample['PatientAge'] >= 65]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_age1,'blue', 'red', 'Patient Age: 1-19 years')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_age2,'blue', 'red', 'Patient Age: 20-34 years')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_age3,'blue', 'red', 'Patient Age: 20-34 years')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_age4,'blue', 'red', 'Patient Age: 20-34 years')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_age5,'blue', 'red', 'Patient Age: 20-34 years')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(16,6))\nsns.countplot(test_class_df['PatientAge'], ax=ax)\nplt.title(\"Test set: Patient Age\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = train_class_df.groupby(['Target', 'PatientSex'])['patientId'].count()\ndf = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\ntmp = df.groupby(['Exams','Target', 'PatientSex']).count()\ndf2 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()\nfig, ax = plt.subplots(nrows=1,figsize=(6,6))\nsns.barplot(ax=ax, x = 'PatientSex', y='Exams', hue='Target',data=df2)\nplt.title(\"Train set: Patient Sex and Target\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = train_class_df.groupby(['class', 'PatientSex'])['patientId'].count()\ndf1 = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\ntmp = df1.groupby(['Exams','class', 'PatientSex']).count()\ndf3 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()\nfig, (ax) = plt.subplots(nrows=1,figsize=(6,6))\nsns.barplot(ax=ax, x = 'PatientSex', y='Exams', hue='class',data=df3)\nplt.title(\"Train set: Patient Sex and class\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_female = target_sample[target_sample['PatientSex']=='F']\ntarget_male = target_sample[target_sample['PatientSex']=='M']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_female,\"red\", \"magenta\",\"Patients Sex: Female\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_window(target_male,\"darkblue\", \"blue\", \"Patients Sex: Male\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(test_class_df['PatientSex'])\nplt.title(\"Test set: Patient Sex\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}