{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Importing the necessary libraries","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import os\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\n\n#pydicom\nimport pydicom\n\n# Suppress warnings \nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Settings for pretty nice plots\nplt.style.use('fivethirtyeight')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reading the Image datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# List files available\nprint(os.listdir(\"../input/siim-isic-melanoma-classification\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defining data path\nIMAGE_PATH = \"../input/siim-isic-melanoma-classification/\"\n\ntrain_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_df = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\n\n\n#Training data\nprint('Training data shape: ', train_df.shape)\ntrain_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Test data\nprint('Test data shape: ', test_df.shape)\ntest_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Exploration\n\n### Missing Values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Null values and Data types\nprint('Train Set')\nprint(train_df.info())\nprint('-------------')\nprint('Test Set')\nprint(test_df.info())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are some missing values in some of the columns. We shall deal with them later.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Total Number of images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Total number of images in the dataset(train+test)\nprint(\"Total images in Train set: \",train_df['image_name'].count())\nprint(\"Total images in Test set: \",test_df['image_name'].count())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Unique IDs ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The total patient ids are {train_df['patient_id'].count()}, from those the unique ids are {train_df['patient_id'].value_counts().shape[0]} \")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The number of unique patients is less than the total number of patients. This means that, patients have multiple records.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = train_df.keys()\ncolumns = list(columns)\nprint(columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploring the Target column","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df['target'].value_counts(normalize=True).plot(kind='bar', title='Distribution of the Target column in the training set')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Gender wise distribution\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['sex'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_df['sex'].value_counts(normalize=True).plot(kind='bar',  title='Gender Distribution in the training set')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Gender vs Target","execution_count":null},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"z=train_df.groupby(['target','sex'])['benign_malignant'].count().to_frame().reset_index()\nz.style.background_gradient(cmap='Reds')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"sns.catplot(x='target',y='benign_malignant', hue='sex',data=z,kind='bar')\nplt.ylabel('Count')\nplt.xlabel('benign:0 vs malignant:1')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Location of imaged site","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['anatom_site_general_challenge'].value_counts(normalize=True).sort_values()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df['anatom_site_general_challenge'].value_counts(normalize=True).sort_values().plot(kind='barh',\n                                                      title='Distribution of the imaged site in the training set')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Location of imaged site w.r.t gender","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"\nz1=train_df.groupby(['sex','anatom_site_general_challenge'])['benign_malignant'].count().to_frame().reset_index()\nz1.style.background_gradient(cmap='Reds')\nsns.catplot(x='anatom_site_general_challenge',y='benign_malignant', hue='sex',data=z1,kind='bar')\nplt.gcf().set_size_inches(10,8)\nplt.xlabel('location of imaged site')\nplt.xticks(rotation=45,fontsize='10', horizontalalignment='right')\nplt.ylabel('count of melanoma cases')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Age Distribution of patients","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['age_approx'].plot(kind='hist',bins=20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualising Age KDEs\n\n### Distribution of Ages w.r.t Target","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# KDE plot of age that were diagnosed as benign\nsns.kdeplot(train_df.loc[train_df['target'] == 0, 'age_approx'], label = 'Benign',shade=True)\n\n# KDE plot of age that were diagnosed as malignant\nsns.kdeplot(train_df.loc[train_df['target'] == 1, 'age_approx'], label = 'Malignant',shade=True)\n\n# Labeling of plot\nplt.xlabel('Age (years)'); plt.ylabel('Density'); plt.title('Distribution of Ages');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n### Distribution of Ages w.r.t gender","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# KDE plot of age that were diagnosed as benign\nsns.kdeplot(train_df.loc[train_df['sex'] == 'male', 'age_approx'], label = 'Male',shade=True)\n\n# KDE plot of age that were diagnosed as malignant\nsns.kdeplot(train_df.loc[train_df['sex'] == 'female', 'age_approx'], label = 'Female',shade=True)\n\n# Labeling of plot\nplt.xlabel('Age (years)'); plt.ylabel('Density'); plt.title('Distribution of Ages');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distribution of Diagnosis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].value_counts(normalize=True).sort_values().plot(kind='barh',\n                                                      title='Distribution in the training set')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Patient Overlap \nWe need to check that the the same patient lesion images shouldn't appear in both training and test set.","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Extract patient id's for the training set\nids_train = train_df.patient_id.values\n# Extract patient id's for the validation set\nids_test = test_df.patient_id.values\n\n# Create a \"set\" datastructure of the training set id's to identify unique id's\nids_train_set = set(ids_train)\nprint(f'There are {len(ids_train_set)} unique Patient IDs in the training set')\n# Create a \"set\" datastructure of the validation set id's to identify unique id's\nids_test_set = set(ids_test)\nprint(f'There are {len(ids_test_set)} unique Patient IDs in the test set')\n\n# Identify patient overlap by looking at the intersection between the sets\npatient_overlap = list(ids_train_set.intersection(ids_test_set))\nn_overlap = len(patient_overlap)\nprint(f'There are {n_overlap} Patient IDs in both the training and test sets')\nprint('')\nprint(f'These patients are in both the training and test datasets:')\nprint(f'{patient_overlap}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# 4. Visualising Images : JPEG\n\n## Visualizing a random selection of images","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"images = train_df['image_name'].values\n\n# Extract 9 random images from it\nrandom_images = [np.random.choice(images+'.jpg') for i in range(9)]\n\n# Location of the image dir\nimg_dir = IMAGE_PATH+'/jpeg/train'\n\nprint('Display Random Images')\n\n# Adjust the size of your images\nplt.figure(figsize=(10,8))\n\n# Iterate and plot random images\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    img = plt.imread(os.path.join(img_dir, random_images[i]))\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    \n# Adjust subplot parameters to give specified padding\nplt.tight_layout()   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We do see that the JPEG format images vary in sizes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Visualizing Images with benign lesions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"benign = train_df[train_df['benign_malignant']=='benign']\nmalignant = train_df[train_df['benign_malignant']=='malignant']","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"images = benign['image_name'].values\n\n# Extract 9 random images from it\nrandom_images = [np.random.choice(images+'.jpg') for i in range(9)]\n\n# Location of the image dir\nimg_dir = IMAGE_PATH+'/jpeg/train'\n\nprint('Display benign Images')\n\n# Adjust the size of your images\nplt.figure(figsize=(10,8))\n\n# Iterate and plot random images\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    img = plt.imread(os.path.join(img_dir, random_images[i]))\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    \n# Adjust subplot parameters to give specified padding\nplt.tight_layout()   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing Images with Malignant lesions","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"images = malignant['image_name'].values\n\n# Extract 9 random images from it\nrandom_images = [np.random.choice(images+'.jpg') for i in range(9)]\n\n# Location of the image dir\nimg_dir = IMAGE_PATH+'/jpeg/train'\n\nprint('Display malignant Images')\n\n# Adjust the size of your images\nplt.figure(figsize=(10,8))\n\n# Iterate and plot random images\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    img = plt.imread(os.path.join(img_dir, random_images[i]))\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    \n# Adjust subplot parameters to give specified padding\nplt.tight_layout()   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Histograms\n\nHistograms are a graphical representation showing how frequently various color values occur in the image i.e frequency of pixels intensity values. In a RGB color space, pixel values range from 0 to 255 where 0 stands for black and 255 stands for white. Analysis of a histogram can help us understand thee brightness, contrast and intensity distribution of an image. Now let's look at the histogram of a random selected sample from each category.\n\n### Benign category","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(16,8))\nf.add_subplot(1,2, 1)\n\nsample_img = benign['image_name'][0]+'.jpg'\nraw_image = plt.imread(os.path.join(img_dir, sample_img))\nplt.imshow(raw_image, cmap='gray')\nplt.colorbar()\nplt.title('Benign Image')\nprint(f\"Image dimensions:  {raw_image.shape[0],raw_image.shape[1]}\")\nprint(f\"Maximum pixel value : {raw_image.max():.1f} ; Minimum pixel value:{raw_image.min():.1f}\")\nprint(f\"Mean value of the pixels : {raw_image.mean():.1f} ; Standard deviation : {raw_image.std():.1f}\")\n\nf.add_subplot(1,2, 2)\n\n#_ = plt.hist(raw_image.ravel(),bins = 256, color = 'orange',)\n_ = plt.hist(raw_image[:, :, 0].ravel(), bins = 256, color = 'red', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 1].ravel(), bins = 256, color = 'Green', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 2].ravel(), bins = 256, color = 'Blue', alpha = 0.5)\n_ = plt.xlabel('Intensity Value')\n_ = plt.ylabel('Count')\n_ = plt.legend(['Red_Channel', 'Green_Channel', 'Blue_Channel'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Malignant category","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(16,8))\nf.add_subplot(1,2, 1)\n\nsample_img = malignant['image_name'][235]+'.jpg'\nraw_image = plt.imread(os.path.join(img_dir, sample_img))\nplt.imshow(raw_image, cmap='gray')\nplt.colorbar()\nplt.title('Malignant Image')\nprint(f\"Image dimensions:  {raw_image.shape[0],raw_image.shape[1]}\")\nprint(f\"Maximum pixel value : {raw_image.max():.1f} ; Minimum pixel value:{raw_image.min():.1f}\")\nprint(f\"Mean value of the pixels : {raw_image.mean():.1f} ; Standard deviation : {raw_image.std():.1f}\")\n\nf.add_subplot(1,2, 2)\n\n#_ = plt.hist(raw_image.ravel(),bins = 256, color = 'orange',)\n_ = plt.hist(raw_image[:, :, 0].ravel(), bins = 256, color = 'red', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 1].ravel(), bins = 256, color = 'Green', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 2].ravel(), bins = 256, color = 'Blue', alpha = 0.5)\n_ = plt.xlabel('Intensity Value')\n_ = plt.ylabel('Count')\n_ = plt.legend(['Red_Channel', 'Green_Channel', 'Blue_Channel'])\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}