{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pydicom\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport seaborn as sns\nfrom tqdm import tqdm, tqdm_notebook\nimport gc\nfrom keras.applications.densenet import preprocess_input, DenseNet169\nfrom keras.models import Model\nfrom keras.layers import GlobalAveragePooling2D, Input, Lambda, AveragePooling1D\nimport keras.backend as K\n\nsns.set()\n\n%config InlineBackend.figure_format = 'retina'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_sex = train_df[train_df['sex'].isnull()]['patient_id']\nmissing_ids_sex.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[train_df['patient_id'].isin(missing_ids_sex.unique()), ['sex']]['sex'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_age = train_df[train_df['age_approx'].isnull()]['patient_id']\nmissing_ids_age.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[train_df['patient_id'].isin(missing_ids_age.unique()), ['age_approx']]['age_approx'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[train_df['patient_id'].isin(missing_ids_sex.unique()), ['sex']]['sex'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_age = train_df[train_df['age_approx'].isnull()]['patient_id']\nmissing_ids_age.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[train_df['patient_id'].isin(missing_ids_age.unique()), ['age_approx']]['age_approx'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(missing_ids_age) - set(missing_ids_sex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_ids_site = train_df[train_df['anatom_site_general_challenge'].isnull()]['patient_id']\nmissing_ids_site.unique() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_to_drop = train_df[train_df['patient_id'].isin(missing_ids_sex.unique())].index\ntrain_df.drop(index = ind_to_drop, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id_w_zero = train_df[train_df['age_approx'] < 5]['patient_id']\nid_w_zero.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_zero = train_df.loc[train_df['patient_id'].isin(id_w_zero.values)].index\ntrain_df.loc[train_df['patient_id'].isin(id_w_zero.values)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[ind_zero, 'age_approx'] = 10.0\ntrain_df.loc[ind_zero]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val = {'age_approx' : train_df['age_approx'].mean()}\ntrain_df.fillna(val, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mapping = {'male' : 1, 'female' : 0}\ntrain_df['sex'] = train_df['sex'].map(mapping)\ntest_df['sex'] = test_df['sex'].map(mapping)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['anatom_site_general_challenge'].isna()]['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_to_drop = train_df[(train_df['anatom_site_general_challenge'].isna()) & (train_df['target'] == 0)].index\nind_to_drop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(ind_to_drop, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum() ,train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.fillna('torso')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(['diagnosis', 'benign_malignant'], axis = 1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, y_train = train_df.drop('target', axis = 1), train_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nimp = SimpleImputer(missing_values = np.nan, strategy = 'most_frequent')\nimp.fit(X_train)\nind, col = test_df.index, test_df.columns\nX_test = pd.DataFrame(imp.transform(test_df), index = ind, columns = col)\nX_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_features = [\"anatom_site_general_challenge\"]\nencoded = pd.get_dummies(X_train[cat_features])\nencoded.set_index(X_train.index)\nX_train.drop(cat_features, inplace = True, axis = 1)\nX_train_encoded = pd.concat([X_train,encoded], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoded = pd.get_dummies(X_test[cat_features])\nencoded.set_index(X_test.index)\nX_test.drop(cat_features, inplace = True, axis = 1)\nX_test_encoded = pd.concat([X_test,encoded], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_clean = pd.concat([X_train_encoded, y_train], axis = 1)\ntest_df_clean = X_test_encoded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_clean.to_csv('train_clean.csv', index = False)\ntest_df_clean.to_csv('test_clean.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\n\nimg_size = 256\n\ndef resize_image(img):\n    old_size = img.shape[:2]\n    ratio = float(img_size)/max(old_size)\n    new_size = tuple([int(x*ratio) for x in old_size])\n    img = cv2.resize(img, (new_size[1],new_size[0]))\n    delta_w = img_size - new_size[1]\n    delta_h = img_size - new_size[0]\n    top, bottom = delta_h//2, delta_h-(delta_h//2)\n    left, right = delta_w//2, delta_w-(delta_w//2)\n    color = [0,0,0]\n    new_img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)\n    return new_img\n\ndef load_image(path, img_id):\n    path = os.path.join(path,img_id+'.jpg')\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    new_img = resize_image(img)\n    new_img = preprocess_input(new_img)\n    return new_img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 16))\nfor i,image_id in enumerate(np.random.choice(train_df[train_df['target']== 0].image_name,5)):\n    image = load_image(train_img_path,image_id)\n    fig.add_subplot(1,5,i+1)\n    plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = 256\nbatch_size = 16 #16 images per batch\n\ntrain_img_ids = train_df.image_name.values\nn_batches = len(train_img_ids) // batch_size + 1\n\n#Model to extract image features\ninp = Input((256,256,3))\nbackbone = DenseNet169(input_tensor = inp, include_top = False)\nx = backbone.output\nx = GlobalAveragePooling2D()(x)\nx = Lambda(lambda x: K.expand_dims(x,axis=-1))(x)\nx = AveragePooling1D(4)(x)\nout = Lambda(lambda x: x[:,:,0])(x)\n\nm = Model(inp,out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = {}\nfor b in tqdm_notebook(range(n_batches)):\n    start = b*batch_size\n    end = (b+1)*batch_size\n    batch_ids = train_img_ids[start:end]\n    batch_images = np.zeros((len(batch_ids),img_size,img_size,3))\n    for i,img_id in enumerate(batch_ids):\n        try:\n            batch_images[i] = load_image(train_img_path,img_id)\n        except:\n            pass\n    batch_preds = m.predict(batch_images)\n    for i,img_id in enumerate(batch_ids):\n        features[img_id] = batch_preds[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feats = pd.DataFrame.from_dict(features, orient = 'index')\ntrain_feats.to_csv('train_img_features.csv')\ntrain_feats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_img_ids = test_df.image_name.values\nn_batches = len(test_img_ids) // batch_size + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = {}\nfor b in tqdm_notebook(range(n_batches)):\n    start = b*batch_size\n    end = (b+1)*batch_size\n    batch_ids = test_img_ids[start:end]\n    batch_images = np.zeros((len(batch_ids),img_size,img_size,3))\n    for i,img_id in enumerate(batch_ids):\n        try:\n            batch_images[i] = load_image(test_img_path,img_id)\n        except:\n            pass\n    batch_preds = m.predict(batch_images)\n    for i,img_id in enumerate(batch_ids):\n        features[img_id] = batch_preds[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feats = pd.DataFrame.from_dict(features, orient='index')\ntest_feats.to_csv('test_img_features.csv')\ntest_feats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feat_img = pd.read_csv ('../input/melanoma-dataset-for-images/train_img_features.csv')\ntest_feat_img = pd.read_csv ('../input/melanoma-dataset-for-images/test_img_features.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feat_img.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feat_img = train_feat_img.set_index('Unnamed: 0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feat_img = test_feat_img.set_index('Unnamed: 0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feat_img.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feat_img.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('./train_clean.csv')\ntest_data = pd.read_csv('./test_clean.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_encoded = train_data.drop('target', axis = 1)\ny_train = train_data['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full =  X_train_encoded.merge (train_feat_img, \n                       how = 'inner',\n                      left_on = 'image_name', \n                      right_index = True,\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full = test_data.merge (test_feat_img, \n                      how = 'inner',\n                      left_on = 'image_name', \n                      right_index = True,\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.drop(['image_name', 'patient_id'], inplace = True, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import roc_auc_score, make_scorer\nfrom sklearn import metrics\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RandomizedSearchCV","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boosting = xgb.XGBClassifier(max_depth = 5, \n                            reg_lambda = 1.3,\n                            subsample = 0.8, \n                            n_estimators = 800, \n                            min_child_weight = 3, \n                            learning_rate = 0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scorer = make_scorer(roc_auc_score)\nskf = StratifiedKFold(n_splits = 5)\nscore_cv = cross_val_score(boosting, X_train_full, y_train, cv = skf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_cv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boost = xgb.XGBClassifier ()\n\nscorer = make_scorer(roc_auc_score)\n\nparam_grid = {\n    'n_estimators': [ 500, 800, 1000],\n    'max_depth': [5, 10, 20, 30],\n    'reg_lambda': [1.1, 1.2, 1.3],\n    \"min_child_weight\": [1,3,5]\n}\n\nskf = StratifiedKFold(n_splits = 5)\n\ngs = RandomizedSearchCV(\n        estimator = boost,\n        param_distributions = param_grid, \n        cv = skf, \n        n_jobs = -1, \n        scoring = scorer,\n        verbose = True, \n        n_iter = 10\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_est = gs.fit(X_train_full, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_est.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_est.best_params_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boosting = xgb.XGBClassifier(max_depth = 5, \n                            reg_lambda = 1.3,\n                            subsample = 0.8, \n                            n_estimators = 800, \n                            min_child_weight = 3, \n                            learning_rate = 0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boosting.fit(X_train_full, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.drop('patient_id', axis = 1, inplace = True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full = X_test_full.set_index('image_name')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = boosting.predict_proba(X_test_full)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_full.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\n    'image_name': X_test_full.index,  \n    'target' : y_test\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}