{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pydicom\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport seaborn as sns\nfrom tqdm import tqdm, tqdm_notebook\nimport gc\nfrom keras.applications.densenet import preprocess_input, DenseNet169\nfrom keras.models import Model\nfrom keras.layers import GlobalAveragePooling2D, Input, Lambda, AveragePooling1D\nimport keras.backend as K\n\nsns.set()\n\n%config InlineBackend.figure_format = 'retina'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Load Data**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We load data from csv file","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check shape of our train dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Take a look at first 5 rows of our train dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA (Exploratory Data Analysis)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We see that some values are missing. Most of the missing values are in column representing site where growth is","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":".describe doesn't help us too much, because we have only 2 numeric features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe() #Two numeric features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['sex'].hist(figsize=(10, 4))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see that there are more men in our train dataset than women","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['sex'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Age distribution looks normal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['age_approx'], bins = 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since age is one of our numeric features, let's make a boxplot for it","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.boxplot(x = 'age_approx', data = train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see outliers with age of 0","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['age_approx'] < 5].head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are two patients with age 0. Well, it may be actually newborn babys, but they are obwiously outliers, who can worse our predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['age_approx'].max()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The oldest patient is 90 years old","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['anatom_site_general_challenge'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['anatom_site_general_challenge'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"More than a half of patients have growths on torso and the same picture we see in test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10, 8))\nsns.countplot(y = 'anatom_site_general_challenge', data = train_df)\nplt.ylabel(\"Anatom site\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So most of diagnosis are labeled as 'unknown'. Let's see real diadnosis distribution","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['diagnosis'] != 'unknown']['diagnosis'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So nevus is actually benign growth, while melanoma is malignant","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"real_diagn = train_df[train_df['diagnosis'] != 'unknown']['diagnosis']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10, 8))\nsns.countplot(y = real_diagn)\nplt.ylabel(\"Diagnosis\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see very imbalanced train dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['benign_malignant'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.crosstab(train_df['benign_malignant'], train_df['target']) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So benign/malignant is actually our target. Of course, we don't have this feature in our test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we have some missing values in test dataset as well","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's compare some features with each other to see if we can find something interesting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby(['sex'])['target'].agg([np.mean, np.sum])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So men are actually more likely to get malignant growth","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10, 8))\nsns.countplot(x = 'target', hue = 'sex', data = train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.boxplot(x = 'target', y = 'age_approx', data = train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So older people are more likely to get malignant growth","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10, 8))\nsns.countplot(y = 'anatom_site_general_challenge', hue = 'target', data = train_df)\nplt.ylabel(\"Anatom site\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.crosstab(train_df['anatom_site_general_challenge'], train_df['target'], normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So malignant growths are more likely to be on torso","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Amount of unique patients in both train and test datasets:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['patient_id'].nunique(), test_df['patient_id'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jpeg_dir_train = \"../input/siim-isic-melanoma-classification/jpeg/train\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's display images for first patient in train dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"first = train_df['patient_id'].unique()[0]\nfirst","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are 115 images for first patient. Let's display only few of them","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['patient_id'] == first]['image_name']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_names = train_df[train_df['patient_id'] == first]['image_name'][:12]\nimg_names = img_names.apply(lambda x: x + '.jpg')\nimg_names","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Style of displaying images was taken from this [kernel](https://www.kaggle.com/parulpandey/melanoma-classification-eda-starter)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (12,10))\nfor i in range (len(img_names)):\n    plt.subplot(4,4, i + 1)\n    img = plt.imread(os.path.join(jpeg_dir_train, img_names.iloc[i]))\n    plt.imshow(img, cmap = 'gray')\n    plt.axis('off')\n    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's compare different types of growths with images:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_names = train_df[train_df['target'] == 0]['image_name'][:8]\nimg_names = img_names.apply(lambda x: x + '.jpg')\nprint ('Benign growths:')\n\nplt.figure(figsize = (12,10))\nfor i in range (len(img_names)):\n    plt.subplot(4,4, i + 1)\n    img = plt.imread(os.path.join(jpeg_dir_train, img_names.iloc[i]))\n    plt.imshow(img, cmap = 'gray')\n    plt.axis('off')\n    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's display malignant growths:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_names = train_df[train_df['target'] == 1]['image_name'][:8]\nimg_names = img_names.apply(lambda x: x + '.jpg')\nprint ('Malignant growths:')\n\nplt.figure(figsize = (12,10))\nfor i in range (len(img_names)):\n    plt.subplot(4,4, i + 1)\n    img = plt.imread(os.path.join(jpeg_dir_train, img_names.iloc[i]))\n    plt.imshow(img, cmap = 'gray')\n    plt.axis('off')\n    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dicom files overview","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"In addition we have .dcm files in our data, which are DICOM type of files. This format is actually widely used in different medical competitions. We already loaded pydicom library in the start, so now we can try to use it for overviewing data we have","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Interesting and helpful kernels about DICOM are [here](https://www.kaggle.com/schlerp/getting-to-know-dicom-and-the-data) and [here](https://www.kaggle.com/gpreda/visualize-ct-dicom-data). For now let's just find out if we can get valuable information from this data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dcm_path = \"../input/siim-isic-melanoma-classification/train\"\ntest_dcm_path = \"../input/siim-isic-melanoma-classification/test\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's see first dcm file","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(train_dcm_path)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_file_path = os.path.join(train_dcm_path, os.listdir(train_dcm_path)[0]) \ndicom_file = pydicom.dcmread(first_file_path)\ndicom_file","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see a lot of new data, but some of it we already know from .csv file","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dcm_info(dataset):\n    print(\"Filename.........:\", file_path)\n    print(\"Storage type.....:\", dataset.SOPClassUID)\n    print()\n\n    pat_name = dataset.PatientName\n    display_name = pat_name.family_name + \", \" + pat_name.given_name\n    print(\"Patient's name......:\", display_name)\n    print(\"Patient id..........:\", dataset.PatientID)\n    print(\"Patient's Age.......:\", dataset.PatientAge)\n    print(\"Patient's Sex.......:\", dataset.PatientSex)\n    print(\"Modality............:\", dataset.Modality)\n    print(\"Body Part Examined..:\", dataset.BodyPartExamined)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print(\"Pixel spacing....:\", dataset.PixelSpacing)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_pixel_array(dataset, figsize=(5,5)):\n    plt.figure(figsize=figsize)\n    plt.imshow(dataset.pixel_array, cmap=plt.cm.bone)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"These methods were taken from this [kernel ](https://www.kaggle.com/schlerp/getting-to-know-dicom-and-the-data)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Here we display information about patient and an image of growth","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_files = 5\nfilenames = os.listdir(train_dcm_path)[:n_files]\nfor file_name in filenames:\n    file_path = os.path.join(train_dcm_path, file_name)\n    dataset = pydicom.dcmread(file_path)\n    show_dcm_info(dataset)\n    plot_pixel_array(dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Method for extracting data from DICOM files was taken from [this](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/154658) discussion","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_DICOM_attributes():\n    images = list(os.listdir(PATH))\n    df = pd.DataFrame()\n    for image in images:\n        image_name = image.split(\".\")[0]\n        dicom_file_path = os.path.join(PATH, image)\n        dicom_file_dataset = pydicom.read_file(dicom_file_path)\n        study_date = dicom_file_dataset.StudyDate\n        modality = dicom_file_dataset.Modality\n        age = dicom_file_dataset.PatientAge\n        sex = dicom_file_dataset.PatientSex\n        body_part_examined = dicom_file_dataset.BodyPartExamined\n        patient_orientation = dicom_file_dataset.PatientOrientation\n        photometric_interpretation = dicom_file_dataset.PhotometricInterpretation\n        rows = dicom_file_dataset.Rows\n        columns = dicom_file_dataset.Columns\n\n        df = df.append(pd.DataFrame({'image_name': image_name, \n                        'dcm_modality': modality,'dcm_study_date':study_date, 'dcm_age': age, 'dcm_sex': sex,\n                        'dcm_body_part_examined': body_part_examined,'dcm_patient_orientation': patient_orientation,\n                        'dcm_photometric_interpretation': photometric_interpretation,\n                        'dcm_rows': rows, 'dcm_columns': columns}, index=[0]))\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This takes a lot of time and memory!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = train_dcm_path\ndcm_df = extract_DICOM_attributes()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dcm_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check dataframe that we got from dcm ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dcm_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dcm_df['dcm_modality'].value_counts() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So modality column doesn't help us too much","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dcm_df['dcm_sex'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Rememdering train.csv we understand, what blank space actually is, but what X means isn't clear","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.sex.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.sex.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also we got some new features like photometric interpretation, let's check them","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dcm_df['dcm_photometric_interpretation'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Another not too informative feature","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"So what is actually rows and columns features? Let's check our first image","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_to_compare = cv2.imread('../input/siim-isic-melanoma-classification/jpeg/train/ISIC_0015719.jpg')\nimg_to_compare.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rows_to_disp = ['dcm_rows', 'dcm_columns']\ndcm_df[dcm_df['image_name'] == 'ISIC_0015719'][rows_to_disp]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So it's just size of our image. Duh","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"After all, we didn't find any helpful information from .dmc files, but at least we tried. Next step is going to be data preprocessing. But before it let's clear our memory","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"del dcm_df, rows_to_disp, img_to_compare, filenames","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preprocessing","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Back to missing values:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's take a look at patients with missing values in features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_sex = train_df[train_df['sex'].isnull()]['patient_id']\nmissing_ids_sex.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Two patients have missing sex column","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[train_df['patient_id'].isin(missing_ids_sex.unique()), ['sex']]['sex'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we see any rows with their ID have missing sex. Let's check the same thing for age:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_age = train_df[train_df['age_approx'].isnull()]['patient_id']\nmissing_ids_age.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Three patients have missing age","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[train_df['patient_id'].isin(missing_ids_age.unique()), ['age_approx']]['age_approx'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we have same situation with ages. There's no data about age of these patients in dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"set(missing_ids_age) - set(missing_ids_sex)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Two patients (IP_5205991 and IP_9835712) have no age nor sex in dataset. They miss pretty helpful features for us","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_site = train_df[train_df['anatom_site_general_challenge'].isnull()]['patient_id']\nmissing_ids_site.unique() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have a quite large number of patients missing anatom site where growth is","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's drop two patients, who don't have age and sex in dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_to_drop = train_df[train_df['patient_id'].isin(missing_ids_sex.unique())].index\ntrain_df.drop(index = ind_to_drop, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Remembering two patients with age zero, we should change their age as well","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"id_w_zero = train_df[train_df['age_approx'] < 5]['patient_id']\nid_w_zero.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_zero = train_df.loc[train_df['patient_id'].isin(id_w_zero.values)].index\ntrain_df.loc[train_df['patient_id'].isin(id_w_zero.values)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So there's obviously mistake in data, patient IP_1300691 is actually 10 years  old, but in two rows his age is zero. Let's fix it","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[ind_zero, 'age_approx'] = 10.0\ntrain_df.loc[ind_zero]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Good, we fixed incorrect data about patient. Now we can input mean column values instead of missing values of age in dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"val = {'age_approx' : train_df['age_approx'].mean()}\ntrain_df.fillna(val, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's deal with categorical data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"First replace 'male' and 'female' values with numbers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mapping = {'male' : 1, 'female' : 0}\ntrain_df['sex'] = train_df['sex'].map(mapping)\ntest_df['sex'] = test_df['sex'].map(mapping)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looks like we have need to somehow imput missing values in anatom_site feature both in train and in test datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['anatom_site_general_challenge'].isna()]['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So I feel like just dropping those 518 instances of benign growths won't harm our dataset too much, while rows with malignant growths will have median value in anatom_site feature ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_to_drop = train_df[(train_df['anatom_site_general_challenge'].isna()) & (train_df['target'] == 0)].index\nind_to_drop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(ind_to_drop, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum() ,train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.fillna('torso')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It would be a good idea to drop useless columns from train dataset, since test dataset doesn't have such columns as diagnosis and benign_malignant","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(['diagnosis', 'benign_malignant'], axis = 1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Extract target from train dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, y_train = train_df.drop('target', axis = 1), train_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Things get much more complex with test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Probably taking median value from train dataset will be fine, since distribution of anatom site feature is similar in train and test datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nimp = SimpleImputer(missing_values = np.nan, strategy = 'most_frequent')\nimp.fit(X_train)\nind, col = test_df.index, test_df.columns\nX_test = pd.DataFrame(imp.transform(test_df), index = ind, columns = col)\nX_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now our train and test datasets don't have missing values. It's time to make categorical features one-hot encoded. We have only one such feature, which is anatom_site","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_features = [\"anatom_site_general_challenge\"]\nencoded = pd.get_dummies(X_train[cat_features])\nencoded.set_index(X_train.index)\nX_train.drop(cat_features, inplace = True, axis = 1)\nX_train_encoded = pd.concat([X_train,encoded], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we will do the same thing for test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"encoded = pd.get_dummies(X_test[cat_features])\nencoded.set_index(X_test.index)\nX_test.drop(cat_features, inplace = True, axis = 1)\nX_test_encoded = pd.concat([X_test,encoded], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_clean = pd.concat([X_train_encoded, y_train], axis = 1)\ntest_df_clean = X_test_encoded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_clean.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's save preprocessed data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_clean.to_csv('train_clean.csv', index=False)\ntest_df_clean.to_csv('test_clean.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Getting features from images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Define methods for resize and loading images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = 256\n\n#Paths to train and test images\ntrain_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\n\ndef resize_image(img):\n    old_size = img.shape[:2]\n    ratio = float(img_size)/max(old_size)\n    new_size = tuple([int(x*ratio) for x in old_size])\n    img = cv2.resize(img, (new_size[1],new_size[0]))\n    delta_w = img_size - new_size[1]\n    delta_h = img_size - new_size[0]\n    top, bottom = delta_h//2, delta_h-(delta_h//2)\n    left, right = delta_w//2, delta_w-(delta_w//2)\n    color = [0,0,0]\n    new_img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)\n    return new_img\n\ndef load_image(path, img_id):\n    path = os.path.join(path,img_id+'.jpg')\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    new_img = resize_image(img)\n    new_img = preprocess_input(new_img)\n    return new_img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Image processing and resizing was taken from [this](https://www.kaggle.com/anshuls235/siim-isic-melanoma-analysis-eda-prediction#2.-Studying-the-data) kernel","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 16))\nfor i,image_id in enumerate(np.random.choice(train_df[train_df['target']== 0].image_name,5)):\n    image = load_image(train_img_path,image_id)\n    fig.add_subplot(1,5,i+1)\n    plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 16))\nfor i,image_id in enumerate(np.random.choice(train_df[train_df['target']== 1].image_name,5)):\n    image = load_image(train_img_path,image_id)\n    fig.add_subplot(1,5,i+1)\n    plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We load a DenseNet neural network, which can be used to get features from our image data.  ","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"batch_size = 16\n\ntrain_img_ids = train_df.image_name.values\nn_batches = len(train_img_ids) // batch_size + 1\n\n#Model to extract image features\ninp = Input((256,256,3))\nbackbone = DenseNet169(input_tensor = inp, include_top = False)\nx = backbone.output\nx = GlobalAveragePooling2D()(x)\nx = Lambda(lambda x: K.expand_dims(x,axis=-1))(x)\nx = AveragePooling1D(4)(x)\nout = Lambda(lambda x: x[:,:,0])(x)\n\nm = Model(inp,out)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In a loop we get batches of images and extract features from them","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"features = {}\nfor b in tqdm_notebook(range(n_batches)):\n    start = b*batch_size\n    end = (b+1)*batch_size\n    batch_ids = train_img_ids[start:end]\n    batch_images = np.zeros((len(batch_ids),img_size,img_size,3))\n    for i,img_id in enumerate(batch_ids):\n        try:\n            batch_images[i] = load_image(train_img_path,img_id)\n        except:\n            pass\n    batch_preds = m.predict(batch_images)\n    for i,img_id in enumerate(batch_ids):\n        features[img_id] = batch_preds[i]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We convert features in a Dataframe","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feats = pd.DataFrame.from_dict(features, orient = 'index')\ntrain_feats.to_csv('train_img_features.csv')\ntrain_feats.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Do the same thing for test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_img_ids = test_df.image_name.values\nn_batches = len(test_img_ids) // batch_size + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = {}\nfor b in tqdm_notebook(range(n_batches)):\n    start = b*batch_size\n    end = (b+1)*batch_size\n    batch_ids = test_img_ids[start:end]\n    batch_images = np.zeros((len(batch_ids),img_size,img_size,3))\n    for i,img_id in enumerate(batch_ids):\n        try:\n            batch_images[i] = load_image(test_img_path,img_id)\n        except:\n            pass\n    batch_preds = m.predict(batch_images)\n    for i,img_id in enumerate(batch_ids):\n        features[img_id] = batch_preds[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feats = pd.DataFrame.from_dict(features, orient='index')\ntest_feats.to_csv('test_img_features.csv')\ntest_feats.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This takes lots of time, so to make it easier already did it and loaded into dataset ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feat_img = pd.read_csv ('../input/melanoma-dataset-for-images/train_img_features.csv')\ntest_feat_img = pd.read_csv ('../input/melanoma-dataset-for-images/test_img_features.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feat_img.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's make combine two of our datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feat_img = train_feat_img.set_index('Unnamed: 0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feat_img = test_feat_img.set_index('Unnamed: 0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feat_img.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feat_img.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('./train_clean.csv')\ntest_data = pd.read_csv('./test_clean.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_encoded = train_data.drop('target', axis = 1)\ny_train = train_data['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full =  X_train_encoded.merge (train_feat_img, \n                       how = 'inner',\n                      left_on = 'image_name', \n                      right_index = True,\n                      )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we have combined two datasets into a new full one","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's do the same thing for test set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full = test_data.merge (test_feat_img, \n                      how = 'inner',\n                      left_on = 'image_name', \n                      right_index = True,\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's drop not useful features so they won't bother our training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.drop(['image_name', 'patient_id'], inplace = True, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.drop('patient_id', axis = 1, inplace = True)\nX_test_full = X_test_full.set_index('image_name')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, make_scorer\nfrom sklearn import metrics\nfrom sklearn.model_selection import cross_val_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For this task I will use gradient boosting from xgboost library. After spending couple hours of training, I found this hyperparameters the best","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"boosting = xgb.XGBClassifier(max_depth = 8, \n                            reg_lambda = 1.2,\n                            subsample = 0.8, \n                            n_estimators = 400, \n                            min_child_weight = 2, \n                            learning_rate = 0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make stratified folds, so there will be equal part of benign and malignant growths in each fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits = 3)\nscore_cv = cross_val_score(boosting, X_train_full, y_train, cv = skf, scoring = 'roc_auc')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"See cross validation scores ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"score_cv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boosting.fit(X_train_full, y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After fitting make predictions. We need to predict probability, since scores in the competition is ROC-AUC","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = boosting.predict_proba(X_test_full)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Convert them to pandas Dataframe","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\n    'image_name': X_test_full.index,  \n    'target' : y_test\n})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get our submissions!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}