{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.groupby(['patient_id']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = data['target']\nx = data.drop(['image_name','patient_id','target','benign_malignant'],axis=1,inplace=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.plot(figsize=(5,5))\nsns.countplot(y)\nB,M = data['target'].value_counts()\nprint('Bening: ',B)\nprint('Malignant: ',M)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"F,M = data['sex'].value_counts()\nprint('Female: ',F)\nprint('Male: ',M)\ndata['anatom_site_general_challenge'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(3,1,figsize=(10,30))\nax[0].set_xticklabels(x['anatom_site_general_challenge'],rotation=45)\nsns.countplot(x['anatom_site_general_challenge'],ax=ax[0])\n\nax[1].set_xticklabels(x['sex'],rotation=45)\nsns.countplot(x['sex'],ax=ax[1])\n\nax[2].set_xticklabels(x['age_approx'],rotation=45)\nsns.countplot(x['age_approx'],ax=ax[2],hue=y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.groupby(['age_approx', 'target']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.groupby(['sex', 'target']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.groupby(['anatom_site_general_challenge', 'target']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Sex\n\n- Total NaN: 65\n- Benign : 65\n- Malignant: 0\n\nThese all NaN values belong to only two patients IP_5205991 and IP_9835712. <br>\nFor all NaN values site location are lower extremity, upper extremity, torso or head/neck.<br> We will assign the majority sex according to site location for these NaN values.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(2,2,figsize=(14,15))\nna_sex = data[data['sex'].isna()]\nrest_sex = data[data['sex'].isna()==False]\n\nax[0][0].set_xticklabels(na_sex['anatom_site_general_challenge'],rotation=45)\nax[0][0].set_title('Site (Sex NaN) Patient Id: IP_5205991')\nsns.countplot(na_sex['anatom_site_general_challenge'][na_sex['patient_id']=='IP_5205991'],ax=ax[0][0])\n\nax[0][1].set_xticklabels(na_sex['anatom_site_general_challenge'],rotation=45)\nax[0][1].set_title('Site (Sex NaN) Patient Id: IP_9835712')\nsns.countplot(na_sex['anatom_site_general_challenge'][na_sex['patient_id']=='IP_9835712'],ax=ax[0][1])\n\nax[1][0].set_xticklabels(rest_sex['anatom_site_general_challenge'],rotation=45)\nax[1][0].set_title('Site (Sex Rest)')\nsns.countplot(rest_sex['anatom_site_general_challenge'],hue=rest_sex['sex'],ax=ax[1][0])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['sex'] =np.where(data['patient_id']=='IP_5205991','female', data['sex']) \ndata['sex'] =np.where(data['patient_id']=='IP_9835712','male', data['sex']) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Age\n\n- Total NaN: 68\n- Benign : 68\n- Malignant: 0\n\nThese all NaN values belong to only three patients IP_5205991, IP_9835712 and IP_0550106. <br>\nFor all NaN values site location are lower extremity, upper extremity, torso or head/neck.<br> Since age looks like a normal distribution. we will assign the mean age according to the site location for these NaN values.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(2,2,figsize=(15,15))\nna_age = data[data['age_approx'].isna()]\nrest_age = data[data['age_approx'].isna()==False]\n\nax[0][0].set_xticklabels(na_age['anatom_site_general_challenge'],rotation=10)\nax[0][0].set_title('Site (Age NaN) Patient Id: IP_5205991')\nsns.countplot(na_age['anatom_site_general_challenge'][na_age['patient_id']=='IP_5205991'],ax=ax[0][0])\n\nax[0][1].set_xticklabels(na_age['anatom_site_general_challenge'],rotation=10)\nax[0][1].set_title('Site (Age NaN) Patient Id: IP_9835712')\nsns.countplot(na_age['anatom_site_general_challenge'][na_age['patient_id']=='IP_9835712'],ax=ax[0][1])\n\nax[1][0].set_xticklabels(na_age['anatom_site_general_challenge'],rotation=45)\nax[1][0].set_title('Site (Age NaN) Patient Id: IP_0550106')\nsns.countplot(na_age['anatom_site_general_challenge'][na_age['patient_id']=='IP_0550106'],ax=ax[1][0])\n\n\nax[1][1].set_title('Site (Age Rest)')\nIntToSite = {0:'head/neck',1:'lower extremity',2:'oral/genital',\n             3:'palms/soles',4:'torso',5:'upper extremity'}\nlabels = [0]*6\nfor i in range(6):\n    sns.distplot(rest_age['age_approx'][rest_age['anatom_site_general_challenge']==IntToSite[i]],ax=ax[1][1],\n                 hist=False,rug=True)\n    labels[i]=IntToSite[i]\n    \n\nax[1][1].legend(labels=labels)\n\nfor i in range(6):\n    print(f\"Mean age for site {IntToSite[i]}: {rest_age['age_approx'][rest_age['anatom_site_general_challenge']==IntToSite[i]].mean()}\")\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['age_approx'] =np.where(data['patient_id']=='IP_5205991','50.0', data['age_approx']) \ndata['age_approx'] =np.where(data['patient_id']=='IP_9835712','50.0', data['age_approx']) \ndata['age_approx'] =np.where(data['patient_id']=='IP_0550106','50.0', data['age_approx'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Anatom Site\n\nFor anatom site we are assigning  'unknown' to NaN values and when we will conert anto site to one hot vector, we will give equal probability of each site to represent 'unknown' ([1/6,1/6,1/6,1/6,1/6,1/6])","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data['anatom_site_general_challenge']=data['anatom_site_general_challenge'].fillna('unknown')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['anatom_site_general_challenge'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.to_csv('modified_train.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\ntest.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['anatom_site_general_challenge']=test['anatom_site_general_challenge'].fillna('unknown')\ntest.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('modified_test.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}