{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is innovated from [Analysis of Melanoma Metadata and EffNet Ensemble\n](https://www.kaggle.com/datafan07/analysis-of-melanoma-metadata-and-effnet-ensemble/data?), greate work and thanks!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The reason to write this is that I've already tuned my deep learning models for both tensorflow(wihout tabular data) and pytorch(with tabualr data). When I want more essence from tabular data I see this greate notebook from Ertuğrul Demir (Thanks again!). So I extract the part of XGBoost from the notebook and do some refactor for fine tuning on my local machine (works faster than kaggle if you have a good CPU). I wish this could also be helpful and bring some convenience for others.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Please refer to the original notebook if you want more data analysis and visualization.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## first things we need to do...","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport os\nimport random\nimport re\nimport math\nimport time\n\n# modules forxgboost modeling\nimport xgboost as xgb\nfrom sklearn.model_selection import StratifiedKFold, train_test_split, cross_val_score, cross_validate\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\n# ignore warnings for clearner ouputs\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"seed = 42\nrandom.seed(42)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# setting path, change path here for other melanoma dataset\n\nbase_path = '/kaggle/input/siim-isic-melanoma-classification'\ntrain_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\nimg_stats_path = '/kaggle/input/melanoma2020imgtabular'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## load data and get an impression","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(os.path.join(base_path, 'train.csv'))\ntest = pd.read_csv(os.path.join(base_path, 'test.csv'))\nsample = pd.read_csv(os.path.join(base_path, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f'train features: {train.columns.tolist()}', f'test_featuers: {test.columns.tolist()};'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# do some simple renaming for complecatec feature name\n\ntrain.columns = [\n    'img_name', 'id', 'sex', 'age', 'location', 'diagnosis',\n    'benign_malignant', 'target'\n]\ntest.columns = ['img_name', 'id', 'sex', 'age', 'location']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# data preprocessing","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Step1: fill missing value","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# now I'm using my own code for simplicity\ndef fill_missing(df):\n    '''The strategy here is:\n    1. For cat features we fill 'unknown' if catgories are many else mode\n    2. For con features we fill the median\n    '''\n    df = df.copy()\n    # fill nan values\n    df.location.fillna(value='unknown', inplace=True)\n    df.sex.fillna(value=train.sex.mode()[0], inplace=True)\n    df.age.fillna(value=df.age.median(), inplace=True)\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = fill_missing(train)\ntest = fill_missing(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().any(), test.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thanks to this great dataset by Marcelo Kittlein [here](https://www.kaggle.com/kittlein/landscape)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Step2: feature engineering - concat landscape attributes from images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading lanscape data\n\ntrain40 = pd.read_csv('../input/melanoma2020imgtabular/train40Features.csv')\ntest40 = pd.read_csv('../input/melanoma2020imgtabular/test40Features.csv')\n\ntrainmet = pd.read_csv('../input/melanoma2020imgtabular/trainMetrics.csv')\ntestmet = pd.read_csv('../input/melanoma2020imgtabular/testMetrics.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# drop duplicate data from landscape dataset\ntrain40.drop(['sex', 'age_approx', 'anatom_site_general_challenge'],\n             axis=1,\n             inplace=True)\ntest40.drop(['sex', 'age_approx', 'anatom_site_general_challenge'],\n            axis=1,\n            inplace=True)\n\n# merging both datasets\ntrain = pd.concat([train, train40, trainmet], axis=1)\ntest = pd.concat([test, test40, testmet], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Step3: label encoding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def label_encoding(df):\n#     df = df.copy()\n#     # encode labels\n#     location = LabelEncoder()\n#     sex = LabelEncoder()\n#     location_data = location.fit_transform(df.location)\n#     sex_data = sex.fit_transform(df.sex)\n#     df.location = location_data\n#     df.sex = sex_data\n    \n#     return df\n\n# train = label_encoding(train)\n# test = label_encoding(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dummy_encoding(df):\n    df = df.copy()\n    \n    # dummy encoding for label sex and location\n    sex_dummies = pd.get_dummies(df.sex, prefix='sex')\n    location_dummies = pd.get_dummies(df.location, prefix='location')\n    df = pd.concat([df, sex_dummies], axis=1)\n    df = pd.concat([df, location_dummies], axis=1)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = dummy_encoding(train)\ntest = dummy_encoding(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Step4: feature cleaning - drop uneccesary features which are not helpful after all feature engineering","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop(['sex','img_name','id','diagnosis','benign_malignant', 'location'], axis=1, inplace=True)\ntest.drop(['sex','img_name','id', 'location'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"looks like we've got what we want","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# XGBoost modelling","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Here I only use the simplified meta prediction from the original notebook as I already know the problem of overfitting and the need of more randomness. Feeling really not neccessary to the analyze and do the testing again.\n\nAlso I found the dropping process is no longer needed here as the dataset of landscape has changed  so the train/test split is no longer that obvious for our model.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## process well-preprocessed tabular data for modelling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# input and output for modelling\n\nX = train.drop('target', axis=1)\ny = train.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 5 stratified KFold with holdout validating\n\nX_train, X_test, y_train, y_test = train_test_split(X, \n                                                    y, \n                                                    test_size=0.2,\n                                                    stratify=y,\n                                                    random_state=seed)\ncv = StratifiedKFold(5, shuffle=True, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# drop features for overfitting\n# X_train.drop(['n_images', 'image_size','width','height','total_pixels','reds','blues','greens','mean_colors', 'age_min', 'age_max'], axis=1, inplace=True)\n# test.drop(['n_images', 'image_size','width','height','total_pixels','reds','blues','greens','mean_colors', 'age_min', 'age_max'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## set up model hyperparameters for fine-tuning","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"same as the original code ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"xg = xgb.XGBClassifier(\n    n_estimators=750,\n    min_child_weight=0.81,\n    learning_rate=0.025,\n    max_depth=2,\n    subsample=0.80,\n    colsample_bytree=0.42,\n    gamma=0.10,\n    random_state=42,\n    n_jobs=-1,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"estimators = [xg]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## cross validation scheme","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Great code also from the original notebook, thanks!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_check(X_train, y_train, estimators, cv):\n    model_table = pd.DataFrame()\n    row_index = 0\n    \n    for est in estimators:\n        MLA_name = est.__class__.__name__\n        model_table.loc[row_index, 'Model Name'] = MLA_name\n        \n        cv_results = cross_validate(est,\n                                    X_train,\n                                    y_train,\n                                    cv=cv,\n                                    scoring='roc_auc',\n                                    return_train_score=True,\n                                    n_jobs=-1)\n        \n        model_table.loc[row_index,\n                        'Train roc Mean'] = cv_results['train_score'].mean()\n        model_table.loc[row_index,\n                        'Test roc Mean'] = cv_results['test_score'].mean()\n        model_table.loc[row_index, 'Test Std'] = cv_results['test_score'].std()\n        model_table.loc[row_index, 'Time'] = cv_results['fit_time'].mean()\n        \n        row_index += 1\n    \n    model_table.sort_values(by=['Test roc Mean'],\n                           ascending=False,\n                           inplace=True)\n    \n    return model_table","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## make predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_models = model_check(X_train, y_train, [xg], cv)\ndisplay(raw_models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# xgboost predict\n\nxg.fit(X_train, y_train)\npredictions = xg.predict_proba(test)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create submission file with two meta features required\n\nmeta_df = pd.DataFrame(columns=['image_name', 'target'])\nmeta_df['image_name'] = sample['image_name']\nmeta_df['target'] = predictions\nmeta_df.to_csv('xgboost_meta_simplified.csv', header=True, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}