{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport pydicom\nimport cv2\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport seaborn as sns\n\nfrom tqdm import tqdm_notebook\nfrom matplotlib.patches import Rectangle","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:17.138690Z","iopub.execute_input":"2022-09-07T23:30:17.139169Z","iopub.status.idle":"2022-09-07T23:30:18.822642Z","shell.execute_reply.started":"2022-09-07T23:30:17.139062Z","shell.execute_reply":"2022-09-07T23:30:18.820357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 1: Import the data. ","metadata":{}},{"cell_type":"code","source":"classInfo = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv')\ntrainlabels = pd.read_csv(\"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv\")\ntrainImagesPath = Path(\"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images\")\ntestImagesPath = Path(\"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_test_images\")\nsampleSubPath = Path(\"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_sample_submission.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:18.825109Z","iopub.execute_input":"2022-09-07T23:30:18.826709Z","iopub.status.idle":"2022-09-07T23:30:19.007274Z","shell.execute_reply.started":"2022-09-07T23:30:18.826659Z","shell.execute_reply":"2022-09-07T23:30:19.006051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font size=4 color='blue' > EDA </font>","metadata":{}},{"cell_type":"code","source":"classInfo.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.012854Z","iopub.execute_input":"2022-09-07T23:30:19.014838Z","iopub.status.idle":"2022-09-07T23:30:19.046777Z","shell.execute_reply.started":"2022-09-07T23:30:19.014782Z","shell.execute_reply":"2022-09-07T23:30:19.045617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classInfo.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.049925Z","iopub.execute_input":"2022-09-07T23:30:19.050376Z","iopub.status.idle":"2022-09-07T23:30:19.078435Z","shell.execute_reply.started":"2022-09-07T23:30:19.050330Z","shell.execute_reply":"2022-09-07T23:30:19.077044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are two features 1. Patient ID 2. Class</font>","metadata":{}},{"cell_type":"code","source":"classInfo.patientId.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.079979Z","iopub.execute_input":"2022-09-07T23:30:19.080621Z","iopub.status.idle":"2022-09-07T23:30:19.104484Z","shell.execute_reply.started":"2022-09-07T23:30:19.080580Z","shell.execute_reply":"2022-09-07T23:30:19.102584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are 26K unique patients info available. Total numbers of records are 30K , but unqiue patien ID's are 26K . Looks there is some duplicated records for patient Id lets see.  </font>","metadata":{}},{"cell_type":"code","source":"sns.countplot(x='class',data=classInfo);","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.106234Z","iopub.execute_input":"2022-09-07T23:30:19.106838Z","iopub.status.idle":"2022-09-07T23:30:19.385665Z","shell.execute_reply.started":"2022-09-07T23:30:19.106799Z","shell.execute_reply":"2022-09-07T23:30:19.384376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > images from different classes \n1.Normal;\n2.No Lung Opacity / Not Normal; \n3.Lung Opacity </font>","metadata":{}},{"cell_type":"code","source":"def get_feature_distribution(data, feature):\n    # Get the count for each label\n    label_counts = data[feature].value_counts()\n\n    # Get total number of samples\n    total_samples = len(data)\n\n    # Count the number of items in each class\n    print(\"Feature: {}\".format(feature))\n    for i in range(len(label_counts)):\n        label = label_counts.index[i]\n        count = label_counts.values[i]\n        percent = int((count / total_samples) * 10000) / 100\n        print(\"{:<30s}:   {} or {}%\".format(label, count, percent))\n\nget_feature_distribution(classInfo, 'class')","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.387120Z","iopub.execute_input":"2022-09-07T23:30:19.387614Z","iopub.status.idle":"2022-09-07T23:30:19.402611Z","shell.execute_reply.started":"2022-09-07T23:30:19.387564Z","shell.execute_reply":"2022-09-07T23:30:19.400914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classInfo[classInfo.duplicated()]","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.404828Z","iopub.execute_input":"2022-09-07T23:30:19.405858Z","iopub.status.idle":"2022-09-07T23:30:19.435098Z","shell.execute_reply.started":"2022-09-07T23:30:19.405810Z","shell.execute_reply":"2022-09-07T23:30:19.433927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > As we suspected there are  3543 duplicated records . </font>","metadata":{}},{"cell_type":"code","source":"classInfo[classInfo.duplicated()].shape","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.436626Z","iopub.execute_input":"2022-09-07T23:30:19.437615Z","iopub.status.idle":"2022-09-07T23:30:19.460438Z","shell.execute_reply.started":"2022-09-07T23:30:19.437553Z","shell.execute_reply":"2022-09-07T23:30:19.459318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classInfo[classInfo.patientId=='c1f7889a-9ea9-4acb-b64c-b737c929599a']","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.466435Z","iopub.execute_input":"2022-09-07T23:30:19.466901Z","iopub.status.idle":"2022-09-07T23:30:19.483210Z","shell.execute_reply.started":"2022-09-07T23:30:19.466852Z","shell.execute_reply":"2022-09-07T23:30:19.482129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check for missing values ","metadata":{}},{"cell_type":"code","source":"def missing_check(df):\n    total = df.isnull().sum().sort_values(ascending=False)  # total number of null values\n    percent = (df.isnull().sum() / df.isnull().count()).sort_values(\n        ascending=False)  # percentage of values that are null\n    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])  # putting the above two together\n    return missing_data  # return the dataframe","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.486838Z","iopub.execute_input":"2022-09-07T23:30:19.487515Z","iopub.status.idle":"2022-09-07T23:30:19.495153Z","shell.execute_reply.started":"2022-09-07T23:30:19.487475Z","shell.execute_reply":"2022-09-07T23:30:19.493545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_check(classInfo)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.497136Z","iopub.execute_input":"2022-09-07T23:30:19.497616Z","iopub.status.idle":"2022-09-07T23:30:19.533969Z","shell.execute_reply.started":"2022-09-07T23:30:19.497571Z","shell.execute_reply":"2022-09-07T23:30:19.532722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are no missing values. </font>","metadata":{}},{"cell_type":"code","source":"trainlabels.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.536717Z","iopub.execute_input":"2022-09-07T23:30:19.537505Z","iopub.status.idle":"2022-09-07T23:30:19.555032Z","shell.execute_reply.started":"2022-09-07T23:30:19.537458Z","shell.execute_reply":"2022-09-07T23:30:19.554072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Train labels also contains 30K records same as meta data . </font>","metadata":{}},{"cell_type":"code","source":"trainlabels.patientId.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.556615Z","iopub.execute_input":"2022-09-07T23:30:19.557362Z","iopub.status.idle":"2022-09-07T23:30:19.572522Z","shell.execute_reply.started":"2022-09-07T23:30:19.557315Z","shell.execute_reply":"2022-09-07T23:30:19.571141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Unique patient Id count also matched with class info . </font>","metadata":{}},{"cell_type":"markdown","source":"<font size=4 color='blue' > Lets check for same patient which we verified the class Info.</font>","metadata":{}},{"cell_type":"code","source":"trainlabels[trainlabels.patientId=='c1f7889a-9ea9-4acb-b64c-b737c929599a']","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.576066Z","iopub.execute_input":"2022-09-07T23:30:19.576629Z","iopub.status.idle":"2022-09-07T23:30:19.597269Z","shell.execute_reply.started":"2022-09-07T23:30:19.576594Z","shell.execute_reply":"2022-09-07T23:30:19.596062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are two records exists for same patient  but with different dimension. This may be due to in the X-ray opacity has been deteched at multiple locations. </font>","metadata":{}},{"cell_type":"code","source":"trainlabels[trainlabels.duplicated()]","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.598724Z","iopub.execute_input":"2022-09-07T23:30:19.599222Z","iopub.status.idle":"2022-09-07T23:30:19.624759Z","shell.execute_reply.started":"2022-09-07T23:30:19.599179Z","shell.execute_reply":"2022-09-07T23:30:19.623490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are no duplicate records for train labels. </font>","metadata":{}},{"cell_type":"code","source":"sns.countplot(x='Target',data=trainlabels);","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.626520Z","iopub.execute_input":"2022-09-07T23:30:19.627134Z","iopub.status.idle":"2022-09-07T23:30:19.854249Z","shell.execute_reply.started":"2022-09-07T23:30:19.627096Z","shell.execute_reply":"2022-09-07T23:30:19.853035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are 2 values in Target column , those are either 1 or 0.  But  in classInfo we could see there are 3 different classes. Lets proceed further to check how Target and Class columns are related. </font>","metadata":{}},{"cell_type":"code","source":"missing_check(trainlabels)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.855740Z","iopub.execute_input":"2022-09-07T23:30:19.856825Z","iopub.status.idle":"2022-09-07T23:30:19.880159Z","shell.execute_reply.started":"2022-09-07T23:30:19.856780Z","shell.execute_reply":"2022-09-07T23:30:19.878875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainlabels[trainlabels.x.isna()]","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.882227Z","iopub.execute_input":"2022-09-07T23:30:19.882955Z","iopub.status.idle":"2022-09-07T23:30:19.902752Z","shell.execute_reply.started":"2022-09-07T23:30:19.882904Z","shell.execute_reply":"2022-09-07T23:30:19.901546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > There are X,Y values aremissing  for few records . This can be due to the fact that for a normal patient these values could be not applicable.  </font>","metadata":{}},{"cell_type":"code","source":"trainlabels[trainlabels.x.isna()]['Target'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.906299Z","iopub.execute_input":"2022-09-07T23:30:19.906955Z","iopub.status.idle":"2022-09-07T23:30:19.917879Z","shell.execute_reply.started":"2022-09-07T23:30:19.906908Z","shell.execute_reply":"2022-09-07T23:30:19.917059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Its proved that only for normal patients dimensions are not available. </font>","metadata":{}},{"cell_type":"code","source":"trainlabels.describe()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.919449Z","iopub.execute_input":"2022-09-07T23:30:19.920126Z","iopub.status.idle":"2022-09-07T23:30:19.963294Z","shell.execute_reply.started":"2022-09-07T23:30:19.920082Z","shell.execute_reply":"2022-09-07T23:30:19.961822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Looks , 75% of the data represents target value 1. Count plot also shows the same.  So looks its an imbalanced data set .  </font>","metadata":{}},{"cell_type":"markdown","source":"<font size=4 color='blue' > Lets concatenate classInfo and trainlabels . Before concatinating them lets remove duplicate records from class info. </font>","metadata":{}},{"cell_type":"code","source":"classInfo.drop_duplicates(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.964904Z","iopub.execute_input":"2022-09-07T23:30:19.965557Z","iopub.status.idle":"2022-09-07T23:30:19.987114Z","shell.execute_reply.started":"2022-09-07T23:30:19.965510Z","shell.execute_reply":"2022-09-07T23:30:19.985917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 2: Map training and testing images to its classes. ","metadata":{}},{"cell_type":"code","source":"#traindf = pd.concat([classInfo,trainlabels]);\ntraindf = trainlabels.merge(classInfo, left_on='patientId', right_on='patientId', how='inner')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:19.988428Z","iopub.execute_input":"2022-09-07T23:30:19.988877Z","iopub.status.idle":"2022-09-07T23:30:20.028170Z","shell.execute_reply.started":"2022-09-07T23:30:19.988836Z","shell.execute_reply":"2022-09-07T23:30:20.027161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.029626Z","iopub.execute_input":"2022-09-07T23:30:20.030431Z","iopub.status.idle":"2022-09-07T23:30:20.051357Z","shell.execute_reply.started":"2022-09-07T23:30:20.030357Z","shell.execute_reply":"2022-09-07T23:30:20.050188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Lets check for specific  patient how data has been concatinated. </font>","metadata":{}},{"cell_type":"code","source":"traindf[traindf.patientId=='c1f7889a-9ea9-4acb-b64c-b737c929599a']","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.052771Z","iopub.execute_input":"2022-09-07T23:30:20.053794Z","iopub.status.idle":"2022-09-07T23:30:20.075869Z","shell.execute_reply.started":"2022-09-07T23:30:20.053746Z","shell.execute_reply":"2022-09-07T23:30:20.074521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Good, same patient has different dimension values with same target value.  </font>\n    \n   Now lets see <font size=4 color='orange' >  how class and target are related to each other?? </font>","metadata":{}},{"cell_type":"code","source":"traindf.Target.unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.077539Z","iopub.execute_input":"2022-09-07T23:30:20.078378Z","iopub.status.idle":"2022-09-07T23:30:20.087035Z","shell.execute_reply.started":"2022-09-07T23:30:20.078335Z","shell.execute_reply":"2022-09-07T23:30:20.085846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.groupby(['class', 'Target']).size().reset_index(name='Patient Count')","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.089059Z","iopub.execute_input":"2022-09-07T23:30:20.089868Z","iopub.status.idle":"2022-09-07T23:30:20.137384Z","shell.execute_reply.started":"2022-09-07T23:30:20.089819Z","shell.execute_reply":"2022-09-07T23:30:20.135893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1,figsize=(12,6))\ntmp = traindf.groupby('Target')['class'].value_counts()\ndf = pd.DataFrame(data={'test': tmp.values}, index=tmp.index).reset_index()\nsns.barplot(ax=ax,x = 'Target', y='test',hue='class',data=df, palette='Set3')\nplt.title(\" class and Target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.146329Z","iopub.execute_input":"2022-09-07T23:30:20.147290Z","iopub.status.idle":"2022-09-07T23:30:20.413172Z","shell.execute_reply.started":"2022-09-07T23:30:20.147241Z","shell.execute_reply":"2022-09-07T23:30:20.411882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue' > Here is the answer , class with Normal and No Lung Opacity / Not Normal has been classified into single target  value that is 'O'. So we can summarize that the prediction which we have to do is like Patient has Lung Opacity ir not . Because Normal and not normal patients are combines in same Target .  </font>","metadata":{}},{"cell_type":"markdown","source":"<font size=4 color='blue'> Prediction: Binary classification  i.e. Patient has Lung Opacity or not ?   </font>","metadata":{}},{"cell_type":"code","source":"sns.boxplot(data=traindf)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.414612Z","iopub.execute_input":"2022-09-07T23:30:20.415445Z","iopub.status.idle":"2022-09-07T23:30:20.702327Z","shell.execute_reply.started":"2022-09-07T23:30:20.415406Z","shell.execute_reply":"2022-09-07T23:30:20.700771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.corr()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.703942Z","iopub.execute_input":"2022-09-07T23:30:20.704545Z","iopub.status.idle":"2022-09-07T23:30:20.727487Z","shell.execute_reply.started":"2022-09-07T23:30:20.704497Z","shell.execute_reply":"2022-09-07T23:30:20.726167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = traindf[traindf['Target']==1]\nsns.set_style('whitegrid')\nplt.figure()\nfig, ax = plt.subplots(2,2,figsize=(12,12))\nsns.distplot(target1['x'],kde=True,bins=50, color=\"red\", ax=ax[0,0]);\nsns.distplot(target1['y'],kde=True,bins=50, color=\"blue\", ax=ax[0,1]);\nsns.distplot(target1['width'],kde=True,bins=50, color=\"green\", ax=ax[1,0]);\nsns.distplot(target1['height'],kde=True,bins=50, color=\"magenta\", ax=ax[1,1]);\nlocs, labels = plt.xticks()\nplt.tick_params(axis='both', which='major', labelsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:20.729233Z","iopub.execute_input":"2022-09-07T23:30:20.729574Z","iopub.status.idle":"2022-09-07T23:30:22.028654Z","shell.execute_reply.started":"2022-09-07T23:30:20.729541Z","shell.execute_reply":"2022-09-07T23:30:22.027412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os;\nimage_train_path = os.listdir(trainImagesPath)\nimage_test_path = os.listdir(testImagesPath)\n\nprint(\"Number of images in train set:\", len(image_train_path),\"\\nNumber of images in test set:\", len(image_test_path))","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:22.030168Z","iopub.execute_input":"2022-09-07T23:30:22.031305Z","iopub.status.idle":"2022-09-07T23:30:23.517199Z","shell.execute_reply.started":"2022-09-07T23:30:22.031242Z","shell.execute_reply":"2022-09-07T23:30:23.515895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue'> Train images length matched with unique patient id's in traindf. </font>","metadata":{}},{"cell_type":"markdown","source":"## Step 3: Map training and testing images to its annotations. ","metadata":{}},{"cell_type":"code","source":"samplePatientID = list(traindf[:3].T.to_dict().values())[0]['patientId']\ndcm_path = trainImagesPath/samplePatientID\ndcm_path = dcm_path.with_suffix(\".dcm\")\ndcm = pydicom.read_file(dcm_path)\ndcm","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:31:23.609184Z","iopub.execute_input":"2022-09-08T00:31:23.610446Z","iopub.status.idle":"2022-09-08T00:31:23.636667Z","shell.execute_reply.started":"2022-09-08T00:31:23.610384Z","shell.execute_reply":"2022-09-08T00:31:23.635319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue'> We can observe that we do have available some useful information in the DICOM metadata with predictive value, for example: <br> </font>\n<font size=4 color='blue'>\nPatient sex;<br>\nPatient age;<br>\nModality;<br>\nBody part examined;<br>\nView position;<br>\nRows & Columns;<br>\nPixel Spacing. </font>","metadata":{}},{"cell_type":"code","source":"def show_dicom_images(data):\n    img_data = list(data.T.to_dict().values())\n    f, ax = plt.subplots(3,3, figsize=(16,18))\n    for i,data_row in enumerate(img_data):\n        dcm_path = trainImagesPath/data_row['patientId']\n        dcm_path = dcm_path.with_suffix(\".dcm\")\n        data_row_img_data = pydicom.read_file(dcm_path)\n        modality = data_row_img_data.Modality\n        age = data_row_img_data.PatientAge\n        sex = data_row_img_data.PatientSex\n        ax[i//3, i%3].imshow(data_row_img_data.pixel_array, cmap=plt.cm.bone) \n        ax[i//3, i%3].axis('off')\n        ax[i//3, i%3].set_title('ID: {}\\nModality: {} Age: {} Sex: {} Target: {}\\nClass: {}\\nWindow: {}:{}:{}:{}'.format(\n                data_row['patientId'],\n                modality, age, sex, data_row['Target'], data_row['class'], \n                data_row['x'],data_row['y'],data_row['width'],data_row['height']))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:23.545623Z","iopub.execute_input":"2022-09-07T23:30:23.546799Z","iopub.status.idle":"2022-09-07T23:30:23.557740Z","shell.execute_reply.started":"2022-09-07T23:30:23.546753Z","shell.execute_reply":"2022-09-07T23:30:23.556714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_dicom_images(traindf[traindf['Target']==1].sample(9))","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:23.561136Z","iopub.execute_input":"2022-09-07T23:30:23.562041Z","iopub.status.idle":"2022-09-07T23:30:26.144781Z","shell.execute_reply.started":"2022-09-07T23:30:23.561978Z","shell.execute_reply":"2022-09-07T23:30:26.143459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font size=4 color='blue'>  We would like to represent the images with the overlay boxes superposed. For this, we will need first to parse the whole dataset with Target = 1 and gather all coordinates of the windows showing a Lung Opacity on the same image. </font>","metadata":{}},{"cell_type":"code","source":"def show_dicom_images_with_boxes(data):\n    img_data = list(data.T.to_dict().values())\n    f, ax = plt.subplots(3,3, figsize=(16,18))\n    for i,data_row in enumerate(img_data):\n        dcm_path = trainImagesPath/data_row['patientId']\n        dcm_path = dcm_path.with_suffix(\".dcm\")\n        data_row_img_data = pydicom.read_file(dcm_path)\n        modality = data_row_img_data.Modality\n        age = data_row_img_data.PatientAge\n        sex = data_row_img_data.PatientSex\n        ax[i//3, i%3].imshow(data_row_img_data.pixel_array, cmap=plt.cm.bone) \n        ax[i//3, i%3].axis('off')\n        ax[i//3, i%3].set_title('ID: {}\\nModality: {} Age: {} Sex: {} Target: {}\\nClass: {}'.format(\n                data_row['patientId'],modality, age, sex, data_row['Target'], data_row['class']))\n        rows = traindf[traindf['patientId']==data_row['patientId']]\n        box_data = list(rows.T.to_dict().values())\n        for j, row in enumerate(box_data):\n            ax[i//3, i%3].add_patch(Rectangle(xy=(row['x'], row['y']),\n                        width=row['width'],height=row['height'], \n                       linewidth=2, edgecolor='r', facecolor='none'))   \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:26.146437Z","iopub.execute_input":"2022-09-07T23:30:26.146891Z","iopub.status.idle":"2022-09-07T23:30:26.160419Z","shell.execute_reply.started":"2022-09-07T23:30:26.146851Z","shell.execute_reply":"2022-09-07T23:30:26.158962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_dicom_images_with_boxes(traindf[traindf['Target']==1].sample(9))","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:29:31.036492Z","iopub.execute_input":"2022-09-08T00:29:31.036911Z","iopub.status.idle":"2022-09-08T00:29:32.577924Z","shell.execute_reply.started":"2022-09-08T00:29:31.036876Z","shell.execute_reply":"2022-09-08T00:29:32.576913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_dicom_images_with_boxes(traindf[traindf['Target']==0].sample(9))","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:28.628046Z","iopub.execute_input":"2022-09-07T23:30:28.628920Z","iopub.status.idle":"2022-09-07T23:30:31.085309Z","shell.execute_reply.started":"2022-09-07T23:30:28.628880Z","shell.execute_reply":"2022-09-07T23:30:31.084104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:31.086509Z","iopub.execute_input":"2022-09-07T23:30:31.086830Z","iopub.status.idle":"2022-09-07T23:30:31.094196Z","shell.execute_reply.started":"2022-09-07T23:30:31.086798Z","shell.execute_reply":"2022-09-07T23:30:31.092824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vars = ['Modality', 'PatientAge', 'PatientSex', 'BodyPartExamined', 'ViewPosition', 'ConversionType', 'Rows', 'Columns', 'PixelSpacing']\n\ndef process_dicom_data(data_df,imagesPath):\n    for var in vars:\n        data_df[var] = None\n    image_names = os.listdir(imagesPath)\n    for i, img_name in tqdm_notebook(enumerate(image_names)):\n        \n        dcm_path = imagesPath/img_name\n        dcm_path = dcm_path.with_suffix(\".dcm\")\n        data_row_img_data = pydicom.read_file(dcm_path)\n        idx = (data_df['patientId']==data_row_img_data.PatientID)\n        data_df.loc[idx,'Modality'] = data_row_img_data.Modality\n        data_df.loc[idx,'PatientAge'] = pd.to_numeric(data_row_img_data.PatientAge)\n        data_df.loc[idx,'PatientSex'] = data_row_img_data.PatientSex\n        data_df.loc[idx,'BodyPartExamined'] = data_row_img_data.BodyPartExamined\n        data_df.loc[idx,'ViewPosition'] = data_row_img_data.ViewPosition\n        data_df.loc[idx,'ConversionType'] = data_row_img_data.ConversionType\n        data_df.loc[idx,'Rows'] = data_row_img_data.Rows\n        data_df.loc[idx,'Columns'] = data_row_img_data.Columns  \n        data_df.loc[idx,'PixelSpacing'] = str.format(\"{:4.3f}\",data_row_img_data.PixelSpacing[0]) ","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:30:31.095910Z","iopub.execute_input":"2022-09-07T23:30:31.096544Z","iopub.status.idle":"2022-09-07T23:30:31.107407Z","shell.execute_reply.started":"2022-09-07T23:30:31.096505Z","shell.execute_reply":"2022-09-07T23:30:31.106361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_dicom_data(traindf,trainImagesPath)","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:34:28.131616Z","iopub.execute_input":"2022-09-08T00:34:28.132174Z","iopub.status.idle":"2022-09-08T00:43:23.740598Z","shell.execute_reply.started":"2022-09-08T00:34:28.132121Z","shell.execute_reply":"2022-09-08T00:43:23.739264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:41:57.103762Z","iopub.execute_input":"2022-09-07T23:41:57.104477Z","iopub.status.idle":"2022-09-07T23:41:57.130806Z","shell.execute_reply.started":"2022-09-07T23:41:57.104420Z","shell.execute_reply":"2022-09-07T23:41:57.129834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"testdf = pd.read_csv(sampleSubPath)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:43:23.646371Z","iopub.execute_input":"2022-09-07T23:43:23.646760Z","iopub.status.idle":"2022-09-07T23:43:23.669753Z","shell.execute_reply.started":"2022-09-07T23:43:23.646727Z","shell.execute_reply":"2022-09-07T23:43:23.668735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf = testdf.drop('PredictionString',1)\nprocess_dicom_data(testdf,testImagesPath)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:43:25.832145Z","iopub.execute_input":"2022-09-07T23:43:25.832559Z","iopub.status.idle":"2022-09-07T23:44:14.833825Z","shell.execute_reply.started":"2022-09-07T23:43:25.832520Z","shell.execute_reply":"2022-09-07T23:44:14.832898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T23:44:24.653838Z","iopub.execute_input":"2022-09-07T23:44:24.654629Z","iopub.status.idle":"2022-09-07T23:44:24.672844Z","shell.execute_reply.started":"2022-09-07T23:44:24.654582Z","shell.execute_reply":"2022-09-07T23:44:24.671678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Modality","metadata":{}},{"cell_type":"code","source":"traindf.Modality.unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:02:22.144363Z","iopub.execute_input":"2022-09-08T00:02:22.144755Z","iopub.status.idle":"2022-09-08T00:02:22.153890Z","shell.execute_reply.started":"2022-09-08T00:02:22.144720Z","shell.execute_reply":"2022-09-08T00:02:22.153044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf.Modality.unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:03:24.863113Z","iopub.execute_input":"2022-09-08T00:03:24.863510Z","iopub.status.idle":"2022-09-08T00:03:24.872689Z","shell.execute_reply.started":"2022-09-08T00:03:24.863477Z","shell.execute_reply":"2022-09-08T00:03:24.871423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The meaning of this modality is CR - Computer Radiography","metadata":{}},{"cell_type":"markdown","source":"#### Patient Age","metadata":{}},{"cell_type":"code","source":"traindf['PatientAge'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:15:29.324543Z","iopub.execute_input":"2022-09-08T00:15:29.324944Z","iopub.status.idle":"2022-09-08T00:15:29.337034Z","shell.execute_reply.started":"2022-09-08T00:15:29.324909Z","shell.execute_reply":"2022-09-08T00:15:29.336076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf['PatientAge'].max()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:15:46.096821Z","iopub.execute_input":"2022-09-08T00:15:46.097247Z","iopub.status.idle":"2022-09-08T00:15:46.144580Z","shell.execute_reply.started":"2022-09-08T00:15:46.097210Z","shell.execute_reply":"2022-09-08T00:15:46.143423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf['PatientAge'].min()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:15:55.110005Z","iopub.execute_input":"2022-09-08T00:15:55.110346Z","iopub.status.idle":"2022-09-08T00:15:55.155949Z","shell.execute_reply.started":"2022-09-08T00:15:55.110315Z","shell.execute_reply":"2022-09-08T00:15:55.154766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(16,6))\nsns.countplot(ax=ax, x = 'PatientAge',hue='class',data=traindf, order = traindf['PatientAge'].value_counts().index)\nplt.title(\"Train set: Age and Class\")\nplt.xticks(rotation=90)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:14:41.079003Z","iopub.execute_input":"2022-09-08T00:14:41.079421Z","iopub.status.idle":"2022-09-08T00:14:43.777074Z","shell.execute_reply.started":"2022-09-08T00:14:41.079384Z","shell.execute_reply":"2022-09-08T00:14:43.775784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(16,6))\nsns.countplot(ax=ax, x = 'PatientAge',hue='Target',data=traindf, order = traindf['PatientAge'].value_counts().index)\nplt.title(\"Train set: Age and Target\")\nplt.xticks(rotation=90)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:12:56.784232Z","iopub.execute_input":"2022-09-08T00:12:56.785783Z","iopub.status.idle":"2022-09-08T00:12:59.583751Z","shell.execute_reply.started":"2022-09-08T00:12:56.785717Z","shell.execute_reply":"2022-09-08T00:12:59.582459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the data has been captured for age group between 40 to 50. There is an outlier with age 151. There are very few data points for age group between 1 to 5 and 80 to 90.","metadata":{}},{"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(16,6))\nsns.countplot(ax=ax, x = 'PatientAge',data=testdf, order = testdf['PatientAge'].value_counts().index)\nplt.title(\"Test set: Age and Target\")\nplt.xticks(rotation=90)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:56:15.652681Z","iopub.execute_input":"2022-09-08T00:56:15.653117Z","iopub.status.idle":"2022-09-08T00:56:17.544313Z","shell.execute_reply.started":"2022-09-08T00:56:15.653081Z","shell.execute_reply":"2022-09-08T00:56:17.543061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In test set also similar kind of behaviour observed in data among different age groups. Outlier with Age 412 . ","metadata":{}},{"cell_type":"markdown","source":"#### PatientSex","metadata":{}},{"cell_type":"code","source":"traindf['PatientSex'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:52:57.248072Z","iopub.execute_input":"2022-09-08T00:52:57.248486Z","iopub.status.idle":"2022-09-08T00:52:57.259275Z","shell.execute_reply.started":"2022-09-08T00:52:57.248452Z","shell.execute_reply":"2022-09-08T00:52:57.258070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ","metadata":{}},{"cell_type":"code","source":"testdf['PatientSex'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:53:09.955379Z","iopub.execute_input":"2022-09-08T00:53:09.955805Z","iopub.status.idle":"2022-09-08T00:53:09.965787Z","shell.execute_reply.started":"2022-09-08T00:53:09.955768Z","shell.execute_reply":"2022-09-08T00:53:09.964617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the data points for Male gender both in traing and testing set .","metadata":{}},{"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(10,5))\nsns.countplot(ax=ax, x = 'PatientSex',hue='Target',data=traindf)\nplt.title(\"Train set: Gender and Target\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:55:15.177461Z","iopub.execute_input":"2022-09-08T00:55:15.177856Z","iopub.status.idle":"2022-09-08T00:55:15.413192Z","shell.execute_reply.started":"2022-09-08T00:55:15.177824Z","shell.execute_reply":"2022-09-08T00:55:15.411977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax) = plt.subplots(nrows=1,figsize=(10,5))\nsns.countplot(ax=ax, x = 'PatientSex',data=testdf)\nplt.title(\"Test set: Gender and Target\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:56:45.669779Z","iopub.execute_input":"2022-09-08T00:56:45.670208Z","iopub.status.idle":"2022-09-08T00:56:45.819477Z","shell.execute_reply.started":"2022-09-08T00:56:45.670171Z","shell.execute_reply":"2022-09-08T00:56:45.818159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BodyPartExamined","metadata":{}},{"cell_type":"code","source":"traindf['BodyPartExamined'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:57:55.518733Z","iopub.execute_input":"2022-09-08T00:57:55.519149Z","iopub.status.idle":"2022-09-08T00:57:55.536430Z","shell.execute_reply.started":"2022-09-08T00:57:55.519114Z","shell.execute_reply":"2022-09-08T00:57:55.535087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf['BodyPartExamined'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:58:05.051229Z","iopub.execute_input":"2022-09-08T00:58:05.051759Z","iopub.status.idle":"2022-09-08T00:58:05.064039Z","shell.execute_reply.started":"2022-09-08T00:58:05.051711Z","shell.execute_reply":"2022-09-08T00:58:05.063038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unique values found for this column","metadata":{}},{"cell_type":"markdown","source":"#### ConversionType\t","metadata":{}},{"cell_type":"code","source":"traindf['ConversionType'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:59:11.659814Z","iopub.execute_input":"2022-09-08T00:59:11.661883Z","iopub.status.idle":"2022-09-08T00:59:11.681038Z","shell.execute_reply.started":"2022-09-08T00:59:11.661839Z","shell.execute_reply":"2022-09-08T00:59:11.679903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf['ConversionType'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T00:59:28.123444Z","iopub.execute_input":"2022-09-08T00:59:28.123853Z","iopub.status.idle":"2022-09-08T00:59:28.139034Z","shell.execute_reply.started":"2022-09-08T00:59:28.123817Z","shell.execute_reply":"2022-09-08T00:59:28.137330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unique values found for this column","metadata":{}},{"cell_type":"markdown","source":"#### Rows\tColumns","metadata":{}},{"cell_type":"code","source":"traindf['Rows'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:00:10.110543Z","iopub.execute_input":"2022-09-08T01:00:10.110953Z","iopub.status.idle":"2022-09-08T01:00:10.124937Z","shell.execute_reply.started":"2022-09-08T01:00:10.110918Z","shell.execute_reply":"2022-09-08T01:00:10.123904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf['Columns'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:00:23.901859Z","iopub.execute_input":"2022-09-08T01:00:23.902297Z","iopub.status.idle":"2022-09-08T01:00:23.914907Z","shell.execute_reply.started":"2022-09-08T01:00:23.902259Z","shell.execute_reply":"2022-09-08T01:00:23.914070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf['Rows'].value_counts(),testdf['Columns'].value_counts(),","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:00:50.874326Z","iopub.execute_input":"2022-09-08T01:00:50.874745Z","iopub.status.idle":"2022-09-08T01:00:50.885039Z","shell.execute_reply.started":"2022-09-08T01:00:50.874709Z","shell.execute_reply":"2022-09-08T01:00:50.884043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unique values found.","metadata":{}},{"cell_type":"markdown","source":"#### ViewPosition","metadata":{}},{"cell_type":"code","source":"traindf['ViewPosition'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:01:30.921272Z","iopub.execute_input":"2022-09-08T01:01:30.921784Z","iopub.status.idle":"2022-09-08T01:01:30.941540Z","shell.execute_reply.started":"2022-09-08T01:01:30.921736Z","shell.execute_reply":"2022-09-08T01:01:30.940367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='ViewPosition',hue='Target',data=traindf)","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:02:32.309423Z","iopub.execute_input":"2022-09-08T01:02:32.309827Z","iopub.status.idle":"2022-09-08T01:02:32.583133Z","shell.execute_reply.started":"2022-09-08T01:02:32.309794Z","shell.execute_reply":"2022-09-08T01:02:32.581839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='ViewPosition',hue='class',data=traindf)","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:03:01.828960Z","iopub.execute_input":"2022-09-08T01:03:01.829388Z","iopub.status.idle":"2022-09-08T01:03:02.160691Z","shell.execute_reply.started":"2022-09-08T01:03:01.829354Z","shell.execute_reply":"2022-09-08T01:03:02.159413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\n  \n# label_encoder object knows how to understand word labels.\nlabel_encoder = preprocessing.LabelEncoder()\n  \ntraindf['ViewPosition']= label_encoder.fit_transform(traindf['ViewPosition'])\nprint(label_encoder.classes_)\ntraindf['ViewPosition'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:05:26.528918Z","iopub.execute_input":"2022-09-08T01:05:26.529796Z","iopub.status.idle":"2022-09-08T01:05:26.541687Z","shell.execute_reply.started":"2022-09-08T01:05:26.529742Z","shell.execute_reply":"2022-09-08T01:05:26.540558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='ViewPosition',hue='class',data=traindf)","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:05:46.249472Z","iopub.execute_input":"2022-09-08T01:05:46.249940Z","iopub.status.idle":"2022-09-08T01:05:46.532345Z","shell.execute_reply.started":"2022-09-08T01:05:46.249892Z","shell.execute_reply":"2022-09-08T01:05:46.531080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"AP-AnteriorPosterior=0\nPA-PostereiorAterior=1","metadata":{}},{"cell_type":"code","source":"testdf['ViewPosition'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:15:49.103645Z","iopub.execute_input":"2022-09-08T01:15:49.104099Z","iopub.status.idle":"2022-09-08T01:15:49.113922Z","shell.execute_reply.started":"2022-09-08T01:15:49.104059Z","shell.execute_reply":"2022-09-08T01:15:49.113048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label_encoder object knows how to understand word labels.\nlabel_encoder = preprocessing.LabelEncoder()\n  \ntestdf['ViewPosition']= label_encoder.fit_transform(testdf['ViewPosition'])\nprint(label_encoder.classes_)\ntestdf['ViewPosition'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:16:21.688550Z","iopub.execute_input":"2022-09-08T01:16:21.688936Z","iopub.status.idle":"2022-09-08T01:16:21.700552Z","shell.execute_reply.started":"2022-09-08T01:16:21.688904Z","shell.execute_reply":"2022-09-08T01:16:21.699260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### PixelSpacing","metadata":{}},{"cell_type":"code","source":"traindf['PixelSpacing'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:08:49.932007Z","iopub.execute_input":"2022-09-08T01:08:49.932436Z","iopub.status.idle":"2022-09-08T01:08:49.950103Z","shell.execute_reply.started":"2022-09-08T01:08:49.932401Z","shell.execute_reply":"2022-09-08T01:08:49.948770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='PixelSpacing',hue='class',data=traindf)","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:09:18.601140Z","iopub.execute_input":"2022-09-08T01:09:18.602159Z","iopub.status.idle":"2022-09-08T01:09:19.006927Z","shell.execute_reply.started":"2022-09-08T01:09:18.602114Z","shell.execute_reply":"2022-09-08T01:09:19.005796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='PixelSpacing',hue='Target',data=traindf)","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:09:49.390702Z","iopub.execute_input":"2022-09-08T01:09:49.391215Z","iopub.status.idle":"2022-09-08T01:09:49.750320Z","shell.execute_reply.started":"2022-09-08T01:09:49.391171Z","shell.execute_reply":"2022-09-08T01:09:49.748995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"0.115 Pixel spacing for agane between 2-11 years. and 0.199 is from 20-35 agae group. we have very less data points for this.","metadata":{}},{"cell_type":"code","source":"traindf[traindf['PixelSpacing']=='0.199']","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:14:54.877288Z","iopub.execute_input":"2022-09-08T01:14:54.877735Z","iopub.status.idle":"2022-09-08T01:14:54.910457Z","shell.execute_reply.started":"2022-09-08T01:14:54.877698Z","shell.execute_reply":"2022-09-08T01:14:54.909242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf[traindf['PixelSpacing']=='0.115']","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:13:47.236513Z","iopub.execute_input":"2022-09-08T01:13:47.236948Z","iopub.status.idle":"2022-09-08T01:13:47.269488Z","shell.execute_reply.started":"2022-09-08T01:13:47.236910Z","shell.execute_reply":"2022-09-08T01:13:47.268469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf=traindf.drop(['Modality','ConversionType','BodyPartExamined','Rows','Columns'],1);","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:18:06.286854Z","iopub.execute_input":"2022-09-08T01:18:06.288101Z","iopub.status.idle":"2022-09-08T01:18:06.298583Z","shell.execute_reply.started":"2022-09-08T01:18:06.288037Z","shell.execute_reply":"2022-09-08T01:18:06.297339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf=testdf.drop(['Modality','ConversionType','BodyPartExamined','Rows','Columns'],1);","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:18:34.883926Z","iopub.execute_input":"2022-09-08T01:18:34.884336Z","iopub.status.idle":"2022-09-08T01:18:34.894176Z","shell.execute_reply.started":"2022-09-08T01:18:34.884303Z","shell.execute_reply":"2022-09-08T01:18:34.893110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:18:46.426851Z","iopub.execute_input":"2022-09-08T01:18:46.427279Z","iopub.status.idle":"2022-09-08T01:18:46.455581Z","shell.execute_reply.started":"2022-09-08T01:18:46.427243Z","shell.execute_reply":"2022-09-08T01:18:46.454348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf[traindf['PatientAge'].isna()]","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:19:48.484755Z","iopub.execute_input":"2022-09-08T01:19:48.485230Z","iopub.status.idle":"2022-09-08T01:19:48.502419Z","shell.execute_reply.started":"2022-09-08T01:19:48.485190Z","shell.execute_reply":"2022-09-08T01:19:48.501451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.corr()","metadata":{"execution":{"iopub.status.busy":"2022-09-08T01:07:00.885461Z","iopub.execute_input":"2022-09-08T01:07:00.885917Z","iopub.status.idle":"2022-09-08T01:07:00.908416Z","shell.execute_reply.started":"2022-09-08T01:07:00.885882Z","shell.execute_reply":"2022-09-08T01:07:00.907260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ","metadata":{}},{"cell_type":"markdown","source":"## Step 4: Preprocessing and Visualisation of different classes ","metadata":{}},{"cell_type":"markdown","source":"## Step 5: Display images with bounding box.","metadata":{}},{"cell_type":"markdown","source":"## Step 6: Design, train and test basic CNN models for classification.","metadata":{}}]}