{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nimport category_encoders as ce\nimport matplotlib.pyplot as plt\n\nSEED = 19\nnp.random.seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n## Summary\n\nThis notebook implements quantile regression with LightGBM using only tabular data (no images). \n\n* Because LightGBM is not able to predict more than a value per model, three different models are trained for each quantile.\n* The 0.5 quantile is used for the expected value.\n* The difference between the quantiles 0.8 and 0.2 are used to estimate the confidence. \n* Training is done with group 10-fold cross validation.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"***\n## Loading data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\")\ntrain.columns = [col.lower() for col in train.columns]\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv\")\ntest.columns = [col.lower() for col in test.columns]\ntest.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\nsubmission.columns = [col.lower() for col in submission.columns]\nsubmission.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n## Minimal features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.sort_values([\"patient\",\"weeks\"], inplace=True)\ntrain[\"base_week\"] = train.groupby(\"patient\")[\"weeks\"].transform(lambda x: x.iloc[0])\ntrain[\"base_fvc\"] = train.groupby(\"patient\")[\"fvc\"].transform(lambda x: x.iloc[0])\ntrain[\"base_percent\"] = train.groupby(\"patient\")[\"percent\"].transform(lambda x: x.iloc[0])\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n## Model training: for each quantile","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"input_features = [\"weeks\", \"age\", \"sex\", \"smokingstatus\", \"base_week\", \"base_fvc\", \"base_percent\"]\ncategorical_features = [\"sex\",\"smokingstatus\"]\n\ngroup_col = \"patient\"\nn_folds = 10\ntarget = \"fvc\"\n\n# left and right quantiles used to estimate confidence\nalpha_qleft = 0.2\nalpha_qright = 0.8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = ce.OrdinalEncoder(cols=categorical_features, handle_unknown='impute')\nencoder.fit(train.loc[:, categorical_features])\ntrain.loc[:, categorical_features] = encoder.transform(train.loc[:, categorical_features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model hyperparams\nmodel_params = {\n    'objective':'quantile',\n    'metric':'quantile',\n    'max_bin': 127,\n    'num_leaves': 7,\n    'min_data_in_leaf': 15,\n    'learning_rate': 0.025,\n    'feature_fraction':0.8,\n    'bagging_fraction':0.8,\n    'bagging_freq':1,\n    'seed':SEED,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# columns where oof predictions will be saved\ntrain[\"pred_qleft\"] = None\ntrain[\"pred_expected\"] = None\ntrain[\"pred_qright\"] = None","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model training: left quantile","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_params[\"alpha\"] = alpha_qleft\ngkf = GroupKFold(n_splits=n_folds)\nall_models_qleft = list()\n\nfor fold,(train_idx, valid_idx) in enumerate(gkf.split(train, train[target], train[group_col])):\n    \n    train_data = train.loc[train_idx, :]\n    valid_data = train.loc[valid_idx, :]\n    \n    train_df_kwargs = {\n        \"data\":train_data.loc[:, input_features],\n        \"categorical_feature\":categorical_features,\n        \"free_raw_data\":False,\n        \"label\":train_data.loc[:, target].values\n    }\n    _train_data = lgb.Dataset(**train_df_kwargs)\n\n    valid_df_kwargs = {\n        \"data\":valid_data.loc[:, input_features],\n        \"categorical_feature\":categorical_features,\n        \"free_raw_data\":False,\n        \"label\":valid_data.loc[:, target].values\n    }\n    _valid_data = lgb.Dataset(**valid_df_kwargs)\n    \n    training_kwargs = {\n        \"train_set\": _train_data,\n        \"valid_sets\": _valid_data,\n        \"early_stopping_rounds\": 250,\n        \"num_boost_round\": 1000,\n        \"params\": model_params,\n        \"verbose_eval\":50,\n    }\n    model = lgb.train(**training_kwargs)\n    all_models_qleft.append(model)\n    \n    ## oof predictions\n    train.loc[valid_idx,\"pred_qleft\"] = model.predict(train.loc[valid_idx, input_features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(all_models_qleft[0], importance_type='gain', figsize=(10,8))\nplt.show()\n\nlgb.plot_importance(all_models_qleft[0], importance_type='split', figsize=(10,8))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n### Model training: expected value","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_params[\"alpha\"] = 0.5\ngkf = GroupKFold(n_splits=n_folds)\nall_models_expected = list()\n\nfor fold,(train_idx, valid_idx) in enumerate(gkf.split(train, train[target], train[group_col])):\n    \n    train_data = train.loc[train_idx, :]\n    valid_data = train.loc[valid_idx, :]\n    \n    train_df_kwargs = {\n        \"data\":train_data.loc[:, input_features],\n        \"categorical_feature\":categorical_features,\n        \"free_raw_data\":False,\n        \"label\":train_data.loc[:, target].values\n    }\n    _train_data = lgb.Dataset(**train_df_kwargs)\n\n    valid_df_kwargs = {\n        \"data\":valid_data.loc[:, input_features],\n        \"categorical_feature\":categorical_features,\n        \"free_raw_data\":False,\n        \"label\":valid_data.loc[:, target].values\n    }\n    _valid_data = lgb.Dataset(**valid_df_kwargs)\n    \n    training_kwargs = {\n        \"train_set\": _train_data,\n        \"valid_sets\": _valid_data,\n        \"early_stopping_rounds\": 250,\n        \"num_boost_round\": 1000,\n        \"params\": model_params,\n        \"verbose_eval\":50,\n    }\n    model = lgb.train(**training_kwargs)\n    all_models_expected.append(model)\n\n    ## oof predictions\n    train.loc[valid_idx,\"pred_expected\"] = model.predict(train.loc[valid_idx, input_features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(all_models_expected[0], importance_type='gain', figsize=(10,8))\nplt.show()\n\nlgb.plot_importance(all_models_expected[0], importance_type='split', figsize=(10,8))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model training: right quantile","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_params[\"alpha\"] = alpha_qright\ngkf = GroupKFold(n_splits=n_folds)\nall_models_qright = list()\n\nfor fold,(train_idx, valid_idx) in enumerate(gkf.split(train, train[target], train[group_col])):\n    \n    train_data = train.loc[train_idx, :]\n    valid_data = train.loc[valid_idx, :]\n    \n    train_df_kwargs = {\n        \"data\":train_data.loc[:, input_features],\n        \"categorical_feature\":categorical_features,\n        \"free_raw_data\":False,\n        \"label\":train_data.loc[:, target].values\n    }\n    _train_data = lgb.Dataset(**train_df_kwargs)\n\n    valid_df_kwargs = {\n        \"data\":valid_data.loc[:, input_features],\n        \"categorical_feature\":categorical_features,\n        \"free_raw_data\":False,\n        \"label\":valid_data.loc[:, target].values\n    }\n    _valid_data = lgb.Dataset(**valid_df_kwargs)\n    \n    training_kwargs = {\n        \"train_set\": _train_data,\n        \"valid_sets\": _valid_data,\n        \"early_stopping_rounds\": 250,\n        \"num_boost_round\": 1000,\n        \"params\": model_params,\n        \"verbose_eval\":50,\n    }\n    model = lgb.train(**training_kwargs)\n    all_models_qright.append(model)\n\n    ## oof predictions\n    train.loc[valid_idx,\"pred_qright\"] = model.predict(train.loc[valid_idx, input_features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(all_models_qright[0], importance_type='gain', figsize=(10,8))\nplt.show()\n\nlgb.plot_importance(all_models_qright[0], importance_type='split', figsize=(10,8))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n### Model training: cross-validation error","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def error_metric(ytrue, ypred, confidence):\n    sig_clipped = (np.clip(confidence, a_min=70., a_max=None)).astype(float)\n    delta = (np.clip(np.abs(ytrue - ypred), a_min=None, a_max=1000)).astype(float)\n    metric = -(np.sqrt(2)*delta / sig_clipped) - np.log(np.sqrt(2)*sig_clipped)\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ytrue = train.fvc.values\nypred = train.pred_expected.values\nconfidence = train.pred_qright.values - train.pred_qleft.values\nprint(f\"CV error: {error_metric(ytrue, ypred, confidence)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n## Inference: prepare predict data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test.rename({\"weeks\":\"base_week\", \"fvc\":\"base_fvc\", \"percent\":\"base_percent\"}, axis=1, inplace=True)\ntest.loc[:, categorical_features] = encoder.transform(test.loc[:, categorical_features])\ntest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[\"patient\"] = submission.patient_week.apply(lambda x: x.split(\"_\")[0])\nsubmission[\"weeks\"] = submission.patient_week.apply(lambda x: int(x.split(\"_\")[1]))\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_dataframe = (submission\n                     .merge(test, how=\"left\", on=[\"patient\"])\n                     .loc[:, input_features])\npredict_dataframe","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n## Inference: predict","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_qleft = list()\nfor model in all_models_qleft:\n    _pred = model.predict(predict_dataframe)\n    predictions_qleft.append(_pred)\n    \npred_qleft = np.mean(predictions_qleft, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_expected = list()\nfor model in all_models_expected:\n    _pred = model.predict(predict_dataframe)\n    predictions_expected.append(_pred)\n    \npred_expected = np.mean(predictions_expected, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_qright = list()\nfor model in all_models_qright:\n    _pred = model.predict(predict_dataframe)\n    predictions_qright.append(_pred)\n    \npred_qright = np.mean(predictions_qright, axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***\n## submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\nsubmission.loc[:, \"FVC\"] = pred_expected\nsubmission.loc[:, \"Confidence\"] = pred_qright - pred_qleft\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.FVC.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.Confidence.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}