{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Problem Statement\nSkin cancer is the most prevalent type of cancer. Melanoma, specifically, is responsible for 75% of skin cancer deaths, despite being the least common skin cancer.\n\nIn this competition, you’ll identify melanoma in images of skin lesions. In particular, you’ll use images within the same patient and determine which are likely to represent a melanoma. Using patient-level contextual information may help the development of image analysis tools, which could better support clinical dermatologists.\n\n![https://lh3.googleusercontent.com/proxy/bzAdtb-5DYXDHghD2eoHJlpA5QEt8q-kBDyqxCeOoOS6sRYgcWtxj2LxHzxuJ81JvBSOn1kHIZEgeSgr2yEDmyrMCA_6eTeN7vxZqejZR90_TfB_4qDLYg](https://lh3.googleusercontent.com/proxy/bzAdtb-5DYXDHghD2eoHJlpA5QEt8q-kBDyqxCeOoOS6sRYgcWtxj2LxHzxuJ81JvBSOn1kHIZEgeSgr2yEDmyrMCA_6eTeN7vxZqejZR90_TfB_4qDLYg)\n\nI think this gives an idea of what we are searching for","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Evaluation\nSubmissions are evaluated on area under the ROC curve between the predicted probability and the observed target.\n[https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc](http://https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc)\n![https://miro.medium.com/max/2010/1*cgq1HxSmqCU2rFCLUArFDg.png](https://miro.medium.com/max/2010/1*cgq1HxSmqCU2rFCLUArFDg.png)\n![https://scikit-learn.org/stable/_images/sphx_glr_plot_roc_001.png](https://scikit-learn.org/stable/_images/sphx_glr_plot_roc_001.png)\n\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split # to split the data into two parts\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\nfrom sklearn.preprocessing import MinMaxScaler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading Datasets","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/test.csv\")\nTRAIN_IMAGES_DIR = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\"\nTEST_IMAGES_DIR = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/test/\"\nsubmission_file = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Do an upvote if you think this was helpful **😬","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.shape,test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.isna().sum()/train_data.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data[test_data.patient_id.isin([train_data.patient_id.unique])].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))\nsns.distplot(train_data.groupby(\"patient_id\")[\"image_name\"].nunique(),kde = False,ax=ax1)\nsns.distplot(test_data.groupby(\"patient_id\")[\"image_name\"].nunique(),kde = False,ax=ax2)\nax1.set_title(\"Train data\")\nax2.set_title(\"Test data\")\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\npd.value_counts(train_data['sex']).plot(kind = 'pie', ax=ax1,autopct='%1.1f%%')\npd.value_counts(test_data['sex']).plot(kind ='pie', ax=ax2,autopct='%1.1f%%')\nax1.set_title(\"Train Data\")\nax2.set_title(\"Test Data\")\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))\nsns.kdeplot(train_data[train_data.sex==\"male\"].age_approx, shade=True,color = \"g\", ax= ax1)\nsns.kdeplot(test_data[test_data.sex==\"female\"].age_approx, shade=True,color = \"r\", ax= ax1)\nsns.kdeplot(train_data[train_data.sex==\"male\"].age_approx, shade=True,color = \"g\", ax= ax2)\nsns.kdeplot(test_data[test_data.sex==\"female\"].age_approx, shade=True,color = \"r\", ax= ax2)\nax1.set_title(\"train_data\")\nax2.set_title(\"test_data\")\nax1.legend(['male','female'])\nax2.legend(['male','female'])\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_data.anatom_site_general_challenge.value_counts()\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\npd.value_counts(train_data.anatom_site_general_challenge).plot(kind = 'bar', ax=ax1)\npd.value_counts(test_data.anatom_site_general_challenge).plot(kind ='bar', ax=ax2)\nax1.set_title(\"Train Data\")\nax2.set_title(\"Test Data\")\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\npd.value_counts(train_data.benign_malignant).plot(kind = 'bar', ax=ax1)\npd.value_counts(train_data.target).plot(kind ='bar', ax=ax2)\nax1.set_title(\"Train Data\")\nax2.set_title(\"Test Data\")\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[(train_data.benign_malignant == \"benign\") & (train_data.target != 0)]\ntrain_data[(train_data.benign_malignant == \"malignant\") & (train_data.target != 1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.value_counts(train_data['diagnosis']).plot(kind = 'pie',autopct='%1.1f%%')\nax1.set_title(\"Test Data\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\npd.value_counts(train_data.loc[train_data['sex']=='male',['target','image_name']][\"target\"]).plot(kind = 'pie', ax=ax1,autopct='%1.1f%%')\npd.value_counts(train_data.loc[train_data['sex']=='female',['target','image_name']][\"target\"]).plot(kind ='pie', ax=ax2,autopct='%1.1f%%')\nax1.set_title(\"Male\")\nax2.set_title(\"Female\")\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.anatom_site_general_challenge.value_counts(dropna=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# anatom_site_general_challenge target\nfig, ax = plt.subplots(2,3 , figsize=(16, 8))\npd.value_counts(train_data.loc[train_data['anatom_site_general_challenge']=='torso',['target','image_name']][\"target\"]).plot(kind = 'pie', ax=ax[0][0],autopct='%1.1f%%')\npd.value_counts(train_data.loc[train_data['anatom_site_general_challenge']=='lower extremity',['target','image_name']][\"target\"]).plot(kind ='pie', ax=ax[0][1],autopct='%1.1f%%')\npd.value_counts(train_data.loc[train_data['anatom_site_general_challenge']=='upper extremity',['target','image_name']][\"target\"]).plot(kind = 'pie', ax=ax[0][2],autopct='%1.1f%%')\npd.value_counts(train_data.loc[train_data['anatom_site_general_challenge']=='head/neck',['target','image_name']][\"target\"]).plot(kind ='pie', ax=ax[1][0],autopct='%1.1f%%')\npd.value_counts(train_data.loc[train_data['anatom_site_general_challenge']=='palms/soles',['target','image_name']][\"target\"]).plot(kind = 'pie', ax=ax[1][1],autopct='%1.1f%%')\npd.value_counts(train_data.loc[train_data['anatom_site_general_challenge']=='oral/genital',['target','image_name']][\"target\"]).plot(kind ='pie', ax=ax[1][2],autopct='%1.1f%%')\nax[0][0].set_title(\"torso\")\nax[0][1].set_title(\"lower extremity\")\nax[0][2].set_title(\"upper extremity\")\nax[1][0].set_title(\"head/neck\")\nax[1][1].set_title(\"palms/soles\")\nax[1][2].set_title(\"oral/genital\")\nplt.suptitle(\"\",fontweight = \"bold\")\nplt.show()\n# train_data.loc[train_data['target']==0,['sex']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_baseline = train_data.copy()\ntest_data_baseline = test_data.copy()\ntrain_data_baseline = train_data_baseline.fillna(train_data_baseline.mode().iloc[0])\ntest_data_baseline = test_data_baseline.fillna(train_data_baseline.mode().iloc[0])\ntrain_data_baseline[\"male\"] = np.where(train_data_baseline[\"sex\"] == \"male\", 1,0)\ntest_data_baseline[\"male\"] = np.where(test_data_baseline[\"sex\"] == \"male\", 1,0)\ntrain_data_baseline = train_data_baseline.join(pd.get_dummies(train_data_baseline[\"anatom_site_general_challenge\"],drop_first = True))\ntest_data_baseline = test_data_baseline.join(pd.get_dummies(test_data_baseline[\"anatom_site_general_challenge\"],drop_first = True))\nscaler = MinMaxScaler()\ntrain_data_baseline[['age_approx']] = scaler.fit_transform(train_data_baseline[['age_approx']])\ntest_data_baseline[['age_approx']] = scaler.fit_transform(test_data_baseline[['age_approx']])\nX = train_data_baseline.drop(['image_name','patient_id','sex','anatom_site_general_challenge','diagnosis','benign_malignant','target'],axis = 1)\nX_pred = test_data_baseline.drop(['image_name','patient_id','sex','anatom_site_general_challenge'],axis = 1)\ny = train_data_baseline[['target']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Baseline Model\nmodel=LogisticRegression()\nskf = StratifiedKFold(shuffle=True,random_state =42)\nerror = []\nfor train_index, test_index in skf.split(X, y):\n    X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n    y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n    model.fit(X_train, y_train.values.ravel())\n    error.append(metrics.roc_auc_score(y_test, model.predict(X_test)))\n    # printing the score \n    print(\"Cross-Validation Score : %s\" % \"{0:.3%}\".format(np.mean(error)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"0.5 = This suggests no discrimination, so we might as well flip a coin.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_file['target']= model.predict(X_pred)\nsubmission_file.to_csv('submission_file.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(image_list,rows,cols,title):\n    fig,ax = plt.subplots(rows,cols,figsize = (25,5))\n    ax = ax.flatten()\n    for i, image_id in enumerate(image_list):\n        image = cv2.imread(TRAIN_IMAGES_DIR+'{}.jpg'.format(image_id))\n        image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n        ax[i].imshow(image)\n        ax[i].set_axis_off()\n        ax[i].set_title(image_id)\n    plt.suptitle(title)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(train_data[train_data.target == 0].sample(5)[\"image_name\"].values,1,5,\"Benign\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(train_data[train_data.target == 1].sample(5)[\"image_name\"].values,1,5,\"Malignant\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Do an upvote if you think this was helpful** 😬","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}