{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import required libs\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('ggplot')\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# load data\ntrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# first look at the data\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Missing values\nfig, ax = plt.subplots(1, 2, figsize= (12,6))\n\n# missing values in train\nsns.barplot(train.isnull().sum().index, train.isnull().sum()*100/ train.shape[0],\\\n            palette= 'Blues', ax= ax[0])\n#ax[0].xaxis(rotation= 90)\nax[0].set_ylabel('% Missing values')\nax[0].set_title('Missing values in train set')\n\n# missing values in test\nsns.barplot(test.isnull().sum().index, test.isnull().sum()*100/ test.shape[0],\\\n            palette= 'BrBG', ax= ax[1])\n#ax[1].set_xticks(rotation= 90)\nax[1].set_ylabel('% Missing values')\nax[1].set_title('Missing values in test set')\n\nfig.autofmt_xdate(rotation=45)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# impute missing values\n# first check the dirtribution of these variables\nfig, ax = plt.subplots(1, 2, figsize= (12, 4))\n\n\n# plot of gender\nsns.countplot(train['sex'].sort_values(ignore_index= True), \n             color= 'green',\n             label= 'Train', \n             ax= ax[0],\n             alpha= 0.3)\n\n\nsns.countplot(test['sex'].sort_values(ignore_index= True), \n             color= 'red',\n             label= 'Test', \n             ax= ax[0],\n             alpha= 0.3)\nax[0].margins(0.2)\nax[0].legend()\n\n\n# plot of location\nsns.countplot(train['anatom_site_general_challenge'],\n             color= 'red',\n             label= 'Train', \n             ax= ax[1],\n             alpha= 0.3)\n\nsns.countplot(test['anatom_site_general_challenge'],\n             color= 'blue',\n             label= 'Test', \n             ax= ax[1],\n             alpha= 0.3)\nax[1].margins(0.2)\nax[1].legend()\n\nfig.autofmt_xdate(rotation=45)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# distribution of age\nfig, ax = plt.subplots(figsize= (12, 4))\nsns.distplot(train['age_approx'], color= 'red', ax= ax, label= 'Train')\nsns.distplot(test['age_approx'], color= 'black', ax= ax, label= 'Test')\nax.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Imputing missing values\ntrain['age_approx'].fillna(train['age_approx'].mean(), inplace= True)\ntrain[train['age_approx'] == 0]['age_approx'] == train['age_approx'].mean()\ntrain['sex'].fillna('male', inplace= True)\ntrain['anatom_site_general_challenge'].fillna('unknown', inplace= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sanity check\ntrain.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# checking duplicates\n# duplicate images\nuni_img = len(train['image_name'].unique())*100/train.shape[0]\nprint('Fraction of unique images: {}%'.format(uni_img))\n\n# repeating patients\nuni_pat = len(train['patient_id'].unique())*100/train.shape[0]\nprint('Fraction of unique patients: {}%'.format(uni_pat))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are only 6% unique patients in our dataset.\n\nIt is also important to note that the dataset has few 100s [duplicate](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/161943) images. \n\nI will remove this duplicates before proceeding further.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove duplicate images\ndf_dup = pd.read_csv('../input/siim-list-of-duplicates/2020_Challenge_duplicates.csv')\ndups = df_dup[df_dup['partition'] == 'train']['ISIC_id']\n\ntrain.drop(train[train['image_name'].isin(dups)].index, inplace= True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bivariate analysis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Gender and target\ngroup = train.groupby(['sex', 'benign_malignant']).size().unstack()\ngroup['total'] = group.sum(axis= 1)\ngroup['benign'] = group['benign']/ group['total']\ngroup['malignant'] = group['malignant']/ group['total']\ngroup.drop('total', inplace= True, axis= 1)\n\ngroup.plot(kind= 'bar')\nplt.xlabel('Gender')\nplt.ylabel('% of Cancer in each Gender')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# rename columns for convenience\ntrain.columns = ['image_name', 'patient_id', 'sex', 'age', 'location', 'diagnosis', 'target_string', 'target']\ntest.columns = ['image_name', 'patient_id', 'sex', 'age', 'location']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Age and target\nfig, ax = plt.subplots(1, 2, figsize= (12, 4))\n\nsns.distplot(train[train['target'] == 0].age, color= 'red', label= 'Benign', ax= ax[0],)\nsns.distplot(train[train['target'] == 1].age, color= 'black', label= 'Malignant', ax= ax[0])\nax[0].legend()\n\n\nsns.boxplot(x= 'target_string', y= 'age', data= train, palette= 'cubehelix_r', ax= ax[1])\nax[1].set_xlabel('')\nax[1].set_ylabel('Age')\n\nfig.suptitle('Distribution of Age with Target')\nplt.margins(0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# location and target\nfig, ax = plt.subplots(figsize= (12, 4))\nsns.countplot(x= 'location', data= train, hue= 'target_string', ax= ax, palette= 'gist_heat_r')\nax.set_xlabel('Anatom Site')\nax.legend(('Malignant', 'Benign'))\nfig.autofmt_xdate()\nplt.margins(0.2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# diagnosis and target\nfig, ax = plt.subplots(figsize= (12, 4))\nsns.countplot(x= 'diagnosis', data= train, hue= 'target_string', ax= ax, palette= 'gist_ncar_r')\nax.set_xlabel('Diagnosis')\nax.legend(('Malignant', 'Benign'))\nfig.autofmt_xdate()\nplt.margins(0.2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multivariate analysis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# sex and age\nfig, ax = plt.subplots(figsize= (10, 4))\nsns.boxplot(x= 'sex', y= 'age', data= train, hue= 'target_string', ax= ax, palette= 'gist_rainbow')\nplt.margins(0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize= (12, 6))\ngroup = train.groupby(['location', 'diagnosis']).size().unstack().fillna(0).T\ngroup.plot( kind= 'bar',ax= ax, color= sns.color_palette('rocket'))\nfig.autofmt_xdate()\nplt.xlabel('Diagnosis')\nplt.ylabel('Count')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature extraction using actual images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# import libraries\nimport os\nimport gc\nimport cv2\nfrom tqdm import tqdm\nfrom skimage import measure, color\nfrom skimage.filters import threshold_otsu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dir_train = '../input/siim-isic-melanoma-classification/jpeg/train'\nimg_dir_test = '../input/siim-isic-melanoma-classification/jpeg/test'\n\ntrain_imgs = os.listdir(img_dir_train)\ntest_imgs = os.listdir(img_dir_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature extracted from images\n* Image size(total pixels)\n* Image width and height\n* Reds, blues and greens\n* Contours in the image","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"# Extract features from train images\ntrain['image_pixels'] = np.zeros(train.shape[0])\ntrain['width'] = np.zeros(train.shape[0])\ntrain['height'] = np.zeros(train.shape[0])\ntrain['red'] = np.zeros(train.shape[0])\ntrain['green'] = np.zeros(train.shape[0])\ntrain['blue'] = np.zeros(train.shape[0])\ntrain['mean_color'] = np.zeros(train.shape[0])\n#train['contours'] = np.zeros(train.shape[0])\n\nfor i in tqdm(range(train.shape[0])):\n    row = train.iloc[i, :]\n    image_name = row['image_name'] + '.jpg'\n    \n    # read image\n    img_path = img_dir_train + '/' + image_name\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n    \n    # total pixels\n    row['image_pixels'] = img.size\n    \n    # image dimensions\n    row['width'] = img.shape[1]\n    row['height'] = img.shape[0]\n    \n    # RGB channels\n    row['red'] = img[:, : , 0].mean()\n    row['green'] = img[:, :, 1].mean()\n    row['blue'] = img[:, :, 2].mean()\n    row['mean_color'] = img.mean()\n    \n    # number of contours\n    img_gray = color.rgb2gray(img)\n    thresh = threshold_otsu(img_gray)\n    thresholded_image = img_gray > thresh\n    contours = measure.find_contours(thresholded_image, 0.8)#too slow\n    \n    train.iloc[i, :] = row\n    del(img)\n    del(row)\n    gc.collect()\n    #del(imgage_gray)\n    #del(thresholded_image)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"# Extract image features in test set\ntest['image_pixels'] = np.zeros(test.shape[0])\ntest['width'] = np.zeros(test.shape[0])\ntest['height'] = np.zeros(test.shape[0])\ntest['red'] = np.zeros(test.shape[0])\ntest['green'] = np.zeros(test.shape[0])\ntest['blue'] = np.zeros(test.shape[0])\ntest['mean_color'] = np.zeros(test.shape[0])\n#test['contours'] = np.zeros(test.shape[0])\n\nfor i in tqdm(range(test.shape[0])):\n    row = test.iloc[i, :]\n    image_name = row['image_name'] + '.jpg'\n    \n    # read image\n    img_path = img_dir_test + '/' + image_name\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n    \n    # total pixels\n    row['image_pixels'] = img.size\n    \n    # image dimensions\n    row['width'] = img.shape[1]\n    row['height'] = img.shape[0]\n    \n    # RGB channels\n    row['red'] = img[:, : , 0].mean()\n    row['green'] = img[:, :, 1].mean()\n    row['blue'] = img[:, :, 2].mean()\n    row['mean_color'] = img.mean()\n    \n    # number of contours\n    img_gray = color.rgb2gray(img)\n    thresh = threshold_otsu(img_gray)\n    thresholded_image = img_gray > thresh\n    contours = measure.find_contours(thresholded_image, 0.8)#too slow\n    \n    test.iloc[i, :] = row\n    del(img)\n    del(row)\n    gc.collect()\n    #del(imgage_gray)\n    #del(thresholded_image)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/modified-data/modified_test.csv')\ntrain = pd.read_csv('../input/modified-train/modified_train.csv')\ntest.drop(test.columns[0], axis= 1, inplace= True)\ntrain.drop(train.columns[15], axis= 1, inplace= True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Model\n\nThe model I'll use is XGBoost","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# import req. libraries\nimport xgboost as xgb\nfrom xgboost import plot_importance\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split, cross_val_score\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare data for model\n\n# dummy vairables for categorical features\n# sex\ndummies = pd.get_dummies(train['sex'], prefix= 'sex')\ntrain = pd.concat([train, dummies], axis= 1)\n\ndummies = pd.get_dummies(test['sex'], prefix= 'sex')\ntest = pd.concat([test, dummies], axis= 1)\n\n# location\ndummies = pd.get_dummies(train['location'], prefix= 'anatom')\ntrain = pd.concat([train, dummies], axis= 1)\n\ndummies = pd.get_dummies(test['location'], prefix= 'anatom')\ntest = pd.concat([test, dummies], axis= 1)\n\n\n# remvoe redundant columns\nX = train.drop(['target_string', 'target', 'image_name', 'patient_id', 'sex', 'location', 'diagnosis'], axis= 1)\ny = train['target']\n\ntest.drop(['image_name', 'patient_id', 'sex', 'location'],axis= 1, inplace= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cross validation setup\nX_train, X_test, y_train, y_test = train_test_split(X, y, stratify= y, test_size= 0.2, random_state= 20)\n\ncv = StratifiedKFold(5, random_state= 20, shuffle= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 5 folds cross-validation\ncv_result = []\nfor i,( train_idx, test_idx) in tqdm(enumerate(cv.split(X_train, y_train))):\n    X_train_cv, y_train_cv = X_train.iloc[train_idx], y_train.iloc[train_idx]\n    X_test_cv, y_test_cv = X_train.iloc[test_idx], y_train.iloc[test_idx]\n    \n    \n    # model setup\n    clf = xgb.XGBClassifier(n_estimators= 100000,\n                           max_depth= 2,\n                           learning_rate= 0.001,\n                           n_jobs= -1,\n                           subsample= 0.6,\n                           colsample_bytree= 0.8,\n                           colsample_bynode= 0.8,\n                            random_state= 20,\n                            gamma=0.10\n                           )\n    # fit model\n    clf.fit(X_train_cv, y_train_cv, eval_set= [(X_test_cv, y_test_cv)],\n           eval_metric= 'auc', early_stopping_rounds= 100)\n    \n    # predict with validation set\n    roc_score = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])\n    cv_result.append(roc_score)\n    \n    # save model\n    clf.save_model('XGB_{}.txt'.format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cv_results for test set\nprint('Folds roc: ', cv_result)\nprint('Mean roc: ', np.mean(cv_result))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['anatom_unknown'] = np.zeros(len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict on test set\nimg_name = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')['image_name']\npredictions = pd.DataFrame({'image_name': img_name})\nfor i in tqdm(range(5)):\n    # load model\n    clf = xgb.XGBClassifier(n_estimators= 10000,\n                           max_depth= 2,\n                           learning_rate= 0.001,\n                           n_jobs= -1,\n                           subsample= 0.6,\n                           colsample_bytree= 0.8,\n                           colsample_bynode= 0.8,\n                            random_state= 20,\n                            gamma=0.10\n                           )\n    clf.load_model('XGB_{}.txt'.format(i))\n    \n    predictions['pred_{}'.format(i)] = clf.predict_proba(test)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission\npredictions['target'] = predictions.filter(regex='^pred').sum(axis= 1)/ 5\nsubmission_xgb = predictions[['image_name', 'target']]\n\nsubmission_xgb.to_csv('submission_xgb.csv', index= False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}