{"cells":[{"metadata":{},"cell_type":"markdown","source":"Special thanks to [allunia](https://www.kaggle.com/allunia) for notebook [Don't turn into a Smoothie after the Shake-Up](https://www.kaggle.com/allunia/don-t-turn-into-a-smoothie-after-the-shake-up)\n\n| Filename | Public LB | Private LB |\n| --- | --- | --- |\n| submission_best_mean.csv | 0.6524 | 0.7156 |\n| submission_ensemble_mean.csv | **0.6544** | **0.7167** |\n| submission_weighted_ensemble_mean.csv | 0.6538 | 0.7164 |\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 1. Initialize Environment","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport random\nimport platform\nimport itertools\nimport gc\n\nimport sklearn\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print('Python version:', platform.python_version())\nprint('Numpy version:', np.__version__)\nprint('Pandas version:', pd.__version__)\nprint('Scikit-Learn version:', sklearn.__version__)\nprint('LightGBM version:', lgb.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\n\nos.environ['PYTHONHASHSEED']=str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train2 = pd.read_csv('/kaggle/input/melanomaextendedtabular/external_upsampled_tabular.csv')\ndf_train2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\ndf_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Short EDA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train2.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['sex'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train2['sex'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['sex'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['anatom_site_general_challenge'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train2['anatom_site_general_challenge'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['anatom_site_general_challenge'].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Feature Engineering","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# get image name to create submission file\nimage_name_test = df_test['image_name']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_sex(sex):\n    if sex == 'male':\n        return 1\n    elif sex == 'female':\n        return 0\n    else:\n        return np.nan\n    \ndf_train['sex'] = df_train['sex'].apply(process_sex)\ndf_train2['sex'] = df_train2['sex'].apply(process_sex)\ndf_test['sex'] = df_test['sex'].apply(process_sex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_site(site):\n    if site == 'head/neck':\n        return 0\n    elif site == 'upper extremity':\n        return 1\n    elif site == 'lower extremity':\n        return 2\n    elif site == 'torso':\n        return 3\n    elif site == 'palms/soles':\n        return 4\n    elif site == 'oral/genital':\n        return 5\n    else:\n        return np.nan\n    \ndf_train['anatom_site_general_challenge'] = df_train['anatom_site_general_challenge'].apply(process_site)\ndf_train2['anatom_site_general_challenge'] = df_train2['anatom_site_general_challenge'].apply(process_site)\ndf_test['anatom_site_general_challenge'] = df_test['anatom_site_general_challenge'].apply(process_site)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_count_train = df_train['patient_id'].value_counts()\nimage_count_test = df_test['patient_id'].value_counts()\n\ndf_train['image_count'] = df_train['patient_id'].apply(lambda pid: image_count_train[pid])\ndf_train2['image_count'] = pd.Series([np.nan for _ in range(df_train2.shape[0])])\ndf_test['image_count'] = df_test['patient_id'].apply(lambda pid: image_count_test[pid])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"age_min_train = df_train.groupby('patient_id').age_approx.min()\nage_max_train = df_train.groupby('patient_id').age_approx.max()\nage_span_train = age_max_train - age_min_train\ndf_train['age_min'] = df_train['patient_id'].apply(lambda pid: age_min_train[pid])\ndf_train['age_max'] = df_train['patient_id'].apply(lambda pid: age_max_train[pid])\ndf_train['age_span'] = df_train['patient_id'].apply(lambda pid: age_span_train[pid])\n\ndf_train2['age_min'] = df_train['age_approx']\ndf_train2['age_max'] = df_train['age_approx']\ndf_train2['age_span'] = pd.Series([0 for _ in range(df_train2.shape[0])])\n\nage_min_test = df_test.groupby('patient_id').age_approx.min()\nage_max_test = df_test.groupby('patient_id').age_approx.max()\nage_span_test = age_max_test - age_min_test\ndf_test['age_min'] = df_test['patient_id'].apply(lambda pid: age_min_test[pid])\ndf_test['age_max'] = df_test['patient_id'].apply(lambda pid: age_max_test[pid])\ndf_test['age_span'] = df_test['patient_id'].apply(lambda pid: age_span_test[pid])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# delete unused column\ndel df_train['image_name']\ndel df_train['patient_id']\ndel df_train['diagnosis']\ndel df_train['benign_malignant']\n\ndel df_train2['image_name']\ndel df_train2['width']\ndel df_train2['height']\n\ndel df_test['image_name']\ndel df_test['patient_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get index of categorical feature\ncat_feature = ['sex', 'anatom_site_general_challenge']\ncat_feature_idx = [df_train.columns.get_loc(ct) for ct in cat_feature]\ncat_feature_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split to X and y\nX = pd.concat([df_train, df_train2], axis=0).reset_index(drop=True)\ny = X['target']\ndel X['target']\n\nX_test = df_test.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"param_dict = {\n    'boosting_type': ['gbdt', 'dart'],\n    'learning_rate': [0.1, 0.03, 0.01],\n    'n_estimators': [100, 300],\n    'feature_fraction': [5/7 + 0.01, 1.0],\n    'lambda': [\n        # l1, l2\n        [0.0, 0.0],\n        [0.001, 0.01],\n        [0.01, 0.1],\n        [1.0, 0.01],\n    ],\n}\nparam_key = list(param_dict.keys())\nparam_item = list(param_dict.values())\nparam_item","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param_list = list(itertools.product(*param_item))\nparam_list[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(param_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_model = pd.DataFrame(columns=[*param_key, *[f'model_{i}' for i in range(5)], *[f'model_{i}_auc' for i in range(5)], 'average_auc'])\ndf_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n\nfor param in param_list:\n    models = []\n    ctr = 0\n    auc_scores = []\n\n    for train_idx, val_idx in skf.split(X, y):\n        X_train, X_val = X.loc[train_idx], X.loc[val_idx]\n        y_train, y_val = y[train_idx], y[val_idx]\n\n        model = lgb.LGBMRegressor(\n            # fixed\n            is_unbalance=True,\n            seed=SEED,\n            extra_trees=True,\n            min_data_per_group=1,\n            early_stopping_round=50,\n            # tweak,\n            **{\n                param_key[0]:param[0],\n                param_key[1]:param[1],\n                param_key[2]:param[2],\n                param_key[3]:param[3],\n                'lambda_l1':param[4][0],\n                'lambda_l2':param[4][0],\n            }\n        )\n        model.fit(\n            X_train, y_train,\n            categorical_feature=cat_feature_idx,\n            eval_set=(X_val, y_val),\n            eval_metric='auc',\n            verbose=-1\n        )\n\n        y_val_pred = model.predict(X_val)\n        auc_score = roc_auc_score(y_val, y_val_pred)\n\n        models.append(model)\n        auc_scores.append(auc_score)\n        \n    df_model.loc[ df_model.shape[0] ] = [\n        *param,\n        *models,\n        *auc_scores,\n        sum(auc_scores) / len(auc_scores)\n    ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_model = df_model.sort_values(by=['average_auc', 'boosting_type', 'learning_rate', 'n_estimators'], ascending=[False, True, True, True]).reset_index(drop=True)\ndf_model.loc[:1000].to_pickle('model.pkl')\n!ls -lah","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_row', df_model.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_row', 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(X, mode='best_mean'):\n    if mode == 'best_mean':\n        y_preds = []\n        for i in range(5):\n            y_preds.append(df_model.loc[0, f'model_{i}'].predict(X))\n        y_preds = np.mean(np.array(y_preds), axis=0)\n    elif mode == 'ensemble_mean':\n        y_preds = []\n        for i in df_model.index:\n            for j in range(5):\n                y_preds.append(df_model.loc[i, f'model_{j}'].predict(X))\n        y_preds = np.mean(np.array(y_preds), axis=0)\n    elif mode == 'weighted_ensemble_mean':\n        y_preds = []\n#         model_weight = df_model['average_mcc'].apply(lambda a: a/df_model['average_mcc'].sum())\n        model_weight = []\n        for i in df_model.index:\n            model_weight.append(1 + np.log10(df_model.shape[0] - i + 1))\n        print(model_weight[:10])\n        for i in df_model.index:\n            for j in range(5):\n                y_preds.append(\n                    df_model.loc[i, f'model_{j}'].predict(X) *\n                    model_weight[i]\n                )\n        y_preds = np.array(y_preds)\n        y_preds = np.mean(y_preds, axis=0)\n    else:\n        raise ValueError(\"Mode isn't supported\")\n    \n    return y_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_pred = predict(df_test, mode='best_mean')\n\ndf_submission = pd.concat([image_name_test, pd.Series(y_test_pred, name='target')], axis=1)\ndf_submission.to_csv('submission_best_mean.csv', index=False)\n\ndf_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_pred2 = predict(df_test, mode='ensemble_mean')\n\ndf_submission2 = pd.concat([image_name_test, pd.Series(y_test_pred2, name='target')], axis=1)\ndf_submission2.to_csv('submission_ensemble_mean.csv', index=False)\n\ndf_submission2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_pred3 = predict(df_test, mode='weighted_ensemble_mean')\n\ndf_submission3 = pd.concat([image_name_test, pd.Series(y_test_pred3, name='target')], axis=1)\ndf_submission3.to_csv('submission_weighted_ensemble_mean.csv', index=False)\n\ndf_submission3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Visualize","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(df_model.loc[0, 'model_0'], ignore_zero=False, figsize=(16,9))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_tree(df_model.loc[0, 'model_0'], figsize=(32,18))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}