{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10217487,"sourceType":"datasetVersion","datasetId":6315820}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Reference\n- {Notebook} [CMI-PIU: Features EDA](https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-features-eda)\n- {Notebook} [CMI-PIU: Actigraphy data EDA](https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-actigraphy-data-eda)\n\n<br/>\n\n- {Notebook} [CMI | Reproducible results |FixSeed,LGB-CPU|LB.492](https://www.kaggle.com/code/hideyukizushi/cmi-reproducible-results-fixseed-lgb-cpu-lb-492)\n- {Notebook} [CMI | Best Single Model](https://www.kaggle.com/code/abdmental01/cmi-best-single-model)\n- {Notebook} [CMI | Best Single Model (explained)](https://www.kaggle.com/code/sunghoshim/cmi-best-single-model-explained)\n\n<br/>\n\n- {Notebook} Optiver [[tuned LGBM] Best Public Score🚀](https://www.kaggle.com/code/kononenko/tuned-lgbm-best-public-score)\n- {Notebook} Enefit [Enefit: Target Diff](https://www.kaggle.com/code/vitalykudelya/enefit-target-diff)","metadata":{}},{"cell_type":"markdown","source":"### Version History\n\n- v27. \n- v26. categorical int (CV_merge: 0.436, CV_common: 0.463, LB: TBD)\n- v25. Fitness_Endurance-Time_Sec, BIA-BIA_Activity_Level_num (CV_merge: 0.438, CV_common: 0.463, LB: 0.420)\n- v24. v23 + lgbm params (CV_merge: 0.409, CV_common: 0.476, LB: 0.434)\n- v23. v20 + BIA-BIA_Activity_Level_num (CV_merge: 0.400, CV_common: 0.424, LB: 0.438)\n- v21. v20 + TS_ENGINEERING (CV_merge: 0.361, CV_common: 0.424, LB: 0.414)\n- v20. Separate models (CV_merge: 0.399, CV_common: 0.424, LB: 0.437)\n- **v19. v14 + Add 1 more category column (CV: 0.456, LB: 0.457)**\n- v17. v14 + Set Category Columns (CV: 0.433, LB: 0.456)\n- v16. Set Category Columns (CV: 0.394, LB: 0.358)\n- v15. Feature Engineering for timeseries #1 (CV: 0.419, LB: 0.456)\n- v14. Optimize Bins (refactored v6) (CV: 0.456, LB: 0.457)\n- v13. v6 + Recalculaute SII (CV: 0.438, LB: 0.447)\n- v12. LGBM default param(v6) + Feature Engineering for normal data (CV: 0.417, LB: 0.432)\n- v11. Another recommended LGBM param (CV: 0.447, LB: 0.435)\n- v10. Remove KNN Imputer + Use recommended LGBM param (CV: 0.460, LB: 0.446)\n- v9. StandardScaler + KNN Imputer (CV: 0.445, LB: 0.427)\n- v8. KNN Imputer (CV: 0.549, LB: 0.355)\n- v6. Optimize Bins (CV: 0.456, LB: 0.457)\n- v5. LGBM Ensemble (CV: 0.399, LB: 0.389)","metadata":{}},{"cell_type":"markdown","source":"### TODO\n- TimeSeries 에서 여러 feature 만들기\n- Optuna 돌려보기\n- TabNet\n- [x] `BIA-BIA_Activity_Level_num` 에서 5는 na로 취급 (v.22)","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nfrom tqdm import tqdm\n\nimport numpy as np\nimport polars as pl\nimport pandas as pd\n\nfrom sklearn.model_selection import StratifiedKFold, cross_validate\nfrom sklearn.metrics import make_scorer, cohen_kappa_score\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.preprocessing import StandardScaler\n\nfrom scipy.optimize import minimize\n\nimport lightgbm as lgb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-19T22:19:49.435694Z","iopub.execute_input":"2024-12-19T22:19:49.436817Z","iopub.status.idle":"2024-12-19T22:19:51.173387Z","shell.execute_reply.started":"2024-12-19T22:19:49.436732Z","shell.execute_reply":"2024-12-19T22:19:51.172007Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Config","metadata":{}},{"cell_type":"code","source":"CFG = {\n    'SEED': 42,\n\n    # LGBM default\n    'LGB_PARAMS': {\n        'num_leaves': 31,\n        'max_depth': -1,\n        'n_estimators' : 100,\n        'min_child_samples': 20,\n    },\n\n    'LGB_PARAMS_ALL': {\n        'num_leaves': 74,\n        'max_depth': 63,\n        'n_estimators': 193,\n        'min_child_samples': 55,\n        'subsample': 0.742502706293537,\n        'subsample_freq': 4,\n        'colsample_bytree': 0.8834649305034784,\n        'reg_alpha': 5.614836675689002,\n        'reg_lambda': 3.216163320817547\n    },\n\n    'LGB_PARAMS_COMMON': {\n        'num_leaves': 73,\n        'max_depth': 15,\n        'n_estimators': 90,\n        'min_child_samples': 117,\n        'subsample': 0.8744779504929355,\n        'subsample_freq': 8,\n        'colsample_bytree': 0.5915813473964658,\n        'reg_alpha': 8.255490981211558,\n        'reg_lambda': 6.9471896832656554,\n    },\n    \n    # https://www.kaggle.com/code/abdmental01/cmi-best-single-model?scriptVersionId=198691885&cellId=9\n    # 'LGB_PARAMS': {  # CV: 0.460\n    #     'learning_rate': 0.03884249148676395, 'max_depth': 12, 'num_leaves': 413, 'min_data_in_leaf': 14,\n    #     'feature_fraction': 0.7987976913702801, 'bagging_fraction': 0.7602261703576205, 'bagging_freq': 2, \n    #     'lambda_l1': 4.735462555910575, 'lambda_l2': 4.735028557007343e-06,\n    #     'verbose': -1,\n    #     'n_esitimators': 200,\n    # },\n\n    # https://www.kaggle.com/code/hideyukizushi/cmi-reproducible-results-fixseed-lgb-cpu-lb-492?scriptVersionId=205880091&cellId=35\n    # 'LGB_PARAMS': {  # CV: 0.447\n    #     'learning_rate': 0.046,\n    #     'max_depth': 12,\n    #     'num_leaves': 478,\n    #     'min_data_in_leaf': 13,\n    #     'feature_fraction': 0.893,\n    #     'bagging_fraction': 0.784,\n    #     'bagging_freq': 4,\n    #     'lambda_l1': 10,  # Increased from 6.59\n    #     'lambda_l2': 0.01,  # Increased from 2.68e-06\n    #     'verbose': -1,\n    #     'n_estimators': 300,\n    # },\n\n    'RECALCULATE_SII': False,\n    'FEATURE_ENGINEERING': True,\n    'TS_ENGINEERING': True,\n}","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:51.175880Z","iopub.execute_input":"2024-12-19T22:19:51.176614Z","iopub.status.idle":"2024-12-19T22:19:51.186911Z","shell.execute_reply.started":"2024-12-19T22:19:51.176557Z","shell.execute_reply":"2024-12-19T22:19:51.185446Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Utils for Time Series Data\n- TODO: TimeSeries 에서 여러 feature 만들기","metadata":{}},{"cell_type":"code","source":"def get_ts_simple_feature(id_path):\n    df = pl.read_parquet(id_path / 'part-0.parquet')\n    ts_feature = df.describe().filter(\n        ~pl.col(\"statistic\").is_in([\"count\", \"null_count\"])\n        # ~pl.col(\"statistic\").is_in([\"null_count\"])\n    ).select(\n        pl.all().exclude([\"statistic\", \"step\"])\n    ).to_numpy().reshape(-1)\n    \n    patient_id = id_path.name.split(\"=\")[1]\n    \n    return ts_feature, patient_id    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.188210Z","iopub.execute_input":"2024-12-19T22:19:51.188539Z","iopub.status.idle":"2024-12-19T22:19:51.207592Z","shell.execute_reply.started":"2024-12-19T22:19:51.188507Z","shell.execute_reply":"2024-12-19T22:19:51.206349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_ts_feature(id_path):\n    df = pl.read_parquet(id_path / 'part-0.parquet')\n\n    expressions = []\n    for col in ['enmo', 'anglez']:\n        for offset in [1, 3, 5]:\n            expressions.append(pl.col(col).diff(offset).alias(f\"diff{offset}_{col}\"))\n            expressions.append(pl.col(col).pct_change(offset).alias(f\"pct_change{offset}_{col}\"))\n    df = df.filter(\n        pl.col(\"non-wear_flag\") == 0\n    ).with_columns(expressions)\n\n\n    COLS = [\n        'enmo', 'anglez', 'light',\n        'diff1_enmo', 'pct_change1_enmo', 'diff3_enmo', 'pct_change3_enmo', 'diff5_enmo', 'pct_change5_enmo',\n        'diff1_anglez', 'pct_change1_anglez', 'diff3_anglez', 'pct_change3_anglez', 'diff5_anglez', 'pct_change5_anglez'    \n    ]\n    expressions = []\n    for col in COLS:\n        expressions.append(pl.col(col).mean().alias(f\"mean_{col}\"))\n        expressions.append(pl.col(col).std().alias(f\"std_{col}\"))\n        expressions.append(pl.col(col).min().alias(f\"min_{col}\"))\n        expressions.append(pl.col(col).quantile(0.25).alias(f\"25%_{col}\"))\n        expressions.append(pl.col(col).quantile(0.50).alias(f\"50%_{col}\"))\n        expressions.append(pl.col(col).quantile(0.75).alias(f\"75%_{col}\"))\n        expressions.append(pl.col(col).max().alias(f\"max_{col}\"))\n        expressions.append(pl.col(col).skew().alias(f\"skew_{col}\"))\n        expressions.append(pl.col(col).kurtosis().alias(f\"kurtosis_{col}\"))\n    \n    ts_feature = df.select(expressions).to_numpy().reshape(-1)\n    \n    patient_id = id_path.name.split(\"=\")[1]\n    \n    return ts_feature, patient_id","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:51.210249Z","iopub.execute_input":"2024-12-19T22:19:51.210672Z","iopub.status.idle":"2024-12-19T22:19:51.230746Z","shell.execute_reply.started":"2024-12-19T22:19:51.210608Z","shell.execute_reply":"2024-12-19T22:19:51.229452Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 2.1. get_ts_feature() 하나 확인","metadata":{}},{"cell_type":"code","source":"# describe example\nid_path = Path(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e\")\ndf = pl.read_parquet(id_path / 'part-0.parquet')\ndf.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.232324Z","iopub.execute_input":"2024-12-19T22:19:51.232912Z","iopub.status.idle":"2024-12-19T22:19:51.340223Z","shell.execute_reply.started":"2024-12-19T22:19:51.232867Z","shell.execute_reply":"2024-12-19T22:19:51.338985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expressions = []\nfor col in ['enmo', 'anglez']:\n    for offset in [1, 3, 5]:\n        expressions.append(pl.col(col).diff(offset).alias(f\"diff{offset}_{col}\"))\n        expressions.append(pl.col(col).pct_change(offset).alias(f\"pct_change{offset}_{col}\"))\n\ndf = df.filter(\n    pl.col(\"non-wear_flag\") == 0\n).with_columns(expressions)\n\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.341466Z","iopub.execute_input":"2024-12-19T22:19:51.341883Z","iopub.status.idle":"2024-12-19T22:19:51.362154Z","shell.execute_reply.started":"2024-12-19T22:19:51.341847Z","shell.execute_reply":"2024-12-19T22:19:51.360983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.363341Z","iopub.execute_input":"2024-12-19T22:19:51.363669Z","iopub.status.idle":"2024-12-19T22:19:51.370853Z","shell.execute_reply.started":"2024-12-19T22:19:51.363610Z","shell.execute_reply":"2024-12-19T22:19:51.369694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLS = [\n    'enmo', 'anglez', 'light',\n    'diff1_enmo', 'pct_change1_enmo', 'diff3_enmo', 'pct_change3_enmo', 'diff5_enmo', 'pct_change5_enmo',\n    'diff1_anglez', 'pct_change1_anglez', 'diff3_anglez', 'pct_change3_anglez', 'diff5_anglez', 'pct_change5_anglez'    \n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.372227Z","iopub.execute_input":"2024-12-19T22:19:51.372702Z","iopub.status.idle":"2024-12-19T22:19:51.381938Z","shell.execute_reply.started":"2024-12-19T22:19:51.372653Z","shell.execute_reply":"2024-12-19T22:19:51.380622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expressions = []\nfor col in COLS:\n    expressions.append(pl.col(col).mean().alias(f\"mean_{col}\"))\n    expressions.append(pl.col(col).std().alias(f\"std_{col}\"))\n    expressions.append(pl.col(col).min().alias(f\"min_{col}\"))\n    expressions.append(pl.col(col).quantile(0.25).alias(f\"25%_{col}\"))\n    expressions.append(pl.col(col).quantile(0.50).alias(f\"50%_{col}\"))\n    expressions.append(pl.col(col).quantile(0.75).alias(f\"75%_{col}\"))\n    expressions.append(pl.col(col).max().alias(f\"max_{col}\"))\n    expressions.append(pl.col(col).skew().alias(f\"skew_{col}\"))\n    expressions.append(pl.col(col).kurtosis().alias(f\"kurtosis_{col}\"))\n\nts_feature = df.select(expressions)\nts_feature","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.383521Z","iopub.execute_input":"2024-12-19T22:19:51.383921Z","iopub.status.idle":"2024-12-19T22:19:51.706940Z","shell.execute_reply.started":"2024-12-19T22:19:51.383866Z","shell.execute_reply":"2024-12-19T22:19:51.705693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ts_feature.to_numpy().reshape(-1).shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.710558Z","iopub.execute_input":"2024-12-19T22:19:51.710981Z","iopub.status.idle":"2024-12-19T22:19:51.718288Z","shell.execute_reply.started":"2024-12-19T22:19:51.710945Z","shell.execute_reply":"2024-12-19T22:19:51.717193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLS_TS = ts_feature.columns\nCOLS_TS[:10]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.719687Z","iopub.execute_input":"2024-12-19T22:19:51.720024Z","iopub.status.idle":"2024-12-19T22:19:51.732035Z","shell.execute_reply.started":"2024-12-19T22:19:51.719992Z","shell.execute_reply":"2024-12-19T22:19:51.730808Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 2.2. get_all_ts_feature()","metadata":{}},{"cell_type":"code","source":"def get_all_ts_feature(parquet_dir) -> pd.DataFrame:\n    items = list(Path(parquet_dir).iterdir())\n    features = []\n    ids = []\n    for id_path in tqdm(items):  # ex) \"id=00115b9f\"\n        if CFG['TS_ENGINEERING']:\n            feature, patient_id = get_ts_feature(id_path)\n        else:\n            feature, patient_id = get_ts_simple_feature(id_path)\n        features.append(feature)\n        ids.append(patient_id)\n\n    global COLS_TS\n    if CFG['TS_ENGINEERING']:\n        print(\"Use TS_ENGINEERING!!\")\n    else:\n        COLS_TS = [f\"stat_{i}\" for i in range(len(features[0]))]\n    df = pd.DataFrame(features, columns=COLS_TS, index=ids)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:51.733257Z","iopub.execute_input":"2024-12-19T22:19:51.733673Z","iopub.status.idle":"2024-12-19T22:19:51.745956Z","shell.execute_reply.started":"2024-12-19T22:19:51.733607Z","shell.execute_reply":"2024-12-19T22:19:51.744422Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Load Datasets & Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## .. 3.1. Load datasets","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ndf_test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndf_sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\ndf_train.shape, df_test.shape, df_sample.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:51.747549Z","iopub.execute_input":"2024-12-19T22:19:51.748027Z","iopub.status.idle":"2024-12-19T22:19:51.822107Z","shell.execute_reply.started":"2024-12-19T22:19:51.747978Z","shell.execute_reply":"2024-12-19T22:19:51.820911Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLS_NUM = df_test.select_dtypes(include=['float64', 'int64']).columns\nprint(len(COLS_NUM))\nCOLS_NUM","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.823453Z","iopub.execute_input":"2024-12-19T22:19:51.823820Z","iopub.status.idle":"2024-12-19T22:19:51.833295Z","shell.execute_reply.started":"2024-12-19T22:19:51.823786Z","shell.execute_reply":"2024-12-19T22:19:51.832051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_train[COLS_NUM].isna().sum().sum())\ndf_train[COLS_NUM].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:51.834570Z","iopub.execute_input":"2024-12-19T22:19:51.834922Z","iopub.status.idle":"2024-12-19T22:19:51.858924Z","shell.execute_reply.started":"2024-12-19T22:19:51.834890Z","shell.execute_reply":"2024-12-19T22:19:51.857520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_ts = get_all_ts_feature(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n# 임시로 dataset 에서 불러옴\n# df_train_ts = pd.read_csv(\"/kaggle/input/cmi-time-series-data/train_ts.csv\", index_col=0)\nprint(df_train_ts.shape)\ndf_train_ts.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:51.860365Z","iopub.execute_input":"2024-12-19T22:19:51.860814Z","iopub.status.idle":"2024-12-19T22:19:51.914166Z","shell.execute_reply.started":"2024-12-19T22:19:51.860762Z","shell.execute_reply":"2024-12-19T22:19:51.913027Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_ts = get_all_ts_feature(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ndf_test_ts.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:51.915788Z","iopub.execute_input":"2024-12-19T22:19:51.916241Z","iopub.status.idle":"2024-12-19T22:19:52.044872Z","shell.execute_reply.started":"2024-12-19T22:19:51.916192Z","shell.execute_reply":"2024-12-19T22:19:52.043473Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLS_TEST = df_test.columns.drop('id')\nlen(COLS_TEST)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.046495Z","iopub.execute_input":"2024-12-19T22:19:52.046992Z","iopub.status.idle":"2024-12-19T22:19:52.054883Z","shell.execute_reply.started":"2024-12-19T22:19:52.046944Z","shell.execute_reply":"2024-12-19T22:19:52.053694Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 3.2. Recalculate SII\n- https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-features-eda?scriptVersionId=206130660&cellId=31","metadata":{}},{"cell_type":"code","source":"PCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\nPCIAT_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.056828Z","iopub.execute_input":"2024-12-19T22:19:52.058438Z","iopub.status.idle":"2024-12-19T22:19:52.069734Z","shell.execute_reply.started":"2024-12-19T22:19:52.058395Z","shell.execute_reply":"2024-12-19T22:19:52.068424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recalculate_sii(row):\n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[PCIAT_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.071497Z","iopub.execute_input":"2024-12-19T22:19:52.071991Z","iopub.status.idle":"2024-12-19T22:19:52.085335Z","shell.execute_reply.started":"2024-12-19T22:19:52.071941Z","shell.execute_reply":"2024-12-19T22:19:52.084029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG['RECALCULATE_SII']:\n    print(\"Use RECALCULATE_SII\")\n    df_train['sii'] = df_train.apply(recalculate_sii, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.086925Z","iopub.execute_input":"2024-12-19T22:19:52.087287Z","iopub.status.idle":"2024-12-19T22:19:52.105429Z","shell.execute_reply.started":"2024-12-19T22:19:52.087252Z","shell.execute_reply":"2024-12-19T22:19:52.104130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 3.3. Feature Engineering","metadata":{}},{"cell_type":"code","source":"df_train['Fitness_Endurance-Time_Mins'].value_counts(dropna=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.107086Z","iopub.execute_input":"2024-12-19T22:19:52.107554Z","iopub.status.idle":"2024-12-19T22:19:52.128009Z","shell.execute_reply.started":"2024-12-19T22:19:52.107491Z","shell.execute_reply":"2024-12-19T22:19:52.126708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train['Fitness_Endurance-Time_Sec'].value_counts(dropna=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.129396Z","iopub.execute_input":"2024-12-19T22:19:52.129783Z","iopub.status.idle":"2024-12-19T22:19:52.146373Z","shell.execute_reply.started":"2024-12-19T22:19:52.129747Z","shell.execute_reply":"2024-12-19T22:19:52.145143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n   \n    # without season_cols - CV: 0.417, with season_cols - CV: 0.425\n    # season_cols = [col for col in df.columns if 'Season' in col]\n    # df = df.drop(season_cols, axis=1)\n\n    df['BIA-BIA_Activity_Level_num'] = df['BIA-BIA_Activity_Level_num'].replace({5: np.nan})\n\n    df['Fitness_Endurance-Time'] = df['Fitness_Endurance-Time_Mins'] * 60 + df['Fitness_Endurance-Time_Sec']\n    df.drop(columns=['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec'], inplace=True)\n\n    # df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    # df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    # df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    # df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    # df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    # df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    # df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    # df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    # df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    # df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    # df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    # df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    # df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    # df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    # df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n\n    # Set Category Columns\n    COLS_SEASON = COLS_TEST[COLS_TEST.str.contains('Season')].to_list()\n    COLS_CAT = [\n        'Basic_Demos-Sex',\n        'FGC-FGC_CU_Zone', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL_Zone',\n        # 'BIA-BIA_Activity_Level_num',  # 이것도 category 로 하면 CV 0.438, 안하면 0.456\n        # 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU_Zone',\n        # 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL_Zone',\n        # 'BIA-BIA_Frame_num', 'PreInt_EduHx-computerinternet_hoursday'\n    ]\n    for col in COLS_SEASON + COLS_CAT:\n        df[col] = df[col].astype('category')\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.148113Z","iopub.execute_input":"2024-12-19T22:19:52.148477Z","iopub.status.idle":"2024-12-19T22:19:52.158613Z","shell.execute_reply.started":"2024-12-19T22:19:52.148445Z","shell.execute_reply":"2024-12-19T22:19:52.157251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG['FEATURE_ENGINEERING']:\n    print(\"Use FEATURE_ENGINEERING\")\n    df_train = feature_engineering(df_train)\n    df_test = feature_engineering(df_test)\n\ndf_train.shape, df_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.160069Z","iopub.execute_input":"2024-12-19T22:19:52.160429Z","iopub.status.idle":"2024-12-19T22:19:52.327782Z","shell.execute_reply.started":"2024-12-19T22:19:52.160394Z","shell.execute_reply":"2024-12-19T22:19:52.326697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.329541Z","iopub.execute_input":"2024-12-19T22:19:52.330036Z","iopub.status.idle":"2024-12-19T22:19:52.360144Z","shell.execute_reply.started":"2024-12-19T22:19:52.329986Z","shell.execute_reply":"2024-12-19T22:19:52.359034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 3.4. Merge","metadata":{}},{"cell_type":"code","source":"df_train_merge = pd.merge(df_train, df_train_ts, how='inner', left_on='id', right_index=True)\ndf_test_merge = pd.merge(df_test, df_test_ts, how='inner', left_on='id', right_index=True)\ndf_train_merge.shape, df_test_merge.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.361373Z","iopub.execute_input":"2024-12-19T22:19:52.361735Z","iopub.status.idle":"2024-12-19T22:19:52.381827Z","shell.execute_reply.started":"2024-12-19T22:19:52.361700Z","shell.execute_reply":"2024-12-19T22:19:52.380626Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_diff = set(df_train_merge.columns) - set(df_test_merge.columns)\nprint(len(cols_diff))\ncols_diff","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.388221Z","iopub.execute_input":"2024-12-19T22:19:52.388605Z","iopub.status.idle":"2024-12-19T22:19:52.397063Z","shell.execute_reply.started":"2024-12-19T22:19:52.388570Z","shell.execute_reply":"2024-12-19T22:19:52.395917Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 3.5. Drop non-label rows\n- TODO: Generate sii from other features","metadata":{}},{"cell_type":"code","source":"df_train_merge['sii'].value_counts(dropna=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.398289Z","iopub.execute_input":"2024-12-19T22:19:52.398661Z","iopub.status.idle":"2024-12-19T22:19:52.414571Z","shell.execute_reply.started":"2024-12-19T22:19:52.398603Z","shell.execute_reply":"2024-12-19T22:19:52.413181Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_merge = df_train_merge.dropna(subset='sii').copy()\ndf_train_merge.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.415936Z","iopub.execute_input":"2024-12-19T22:19:52.416242Z","iopub.status.idle":"2024-12-19T22:19:52.431054Z","shell.execute_reply.started":"2024-12-19T22:19:52.416212Z","shell.execute_reply":"2024-12-19T22:19:52.429974Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLS = df_test_merge.columns.drop('id')\nlen(COLS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:52.432710Z","iopub.execute_input":"2024-12-19T22:19:52.433080Z","iopub.status.idle":"2024-12-19T22:19:52.444009Z","shell.execute_reply.started":"2024-12-19T22:19:52.433046Z","shell.execute_reply":"2024-12-19T22:19:52.442576Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. cross_validate - LGBM","metadata":{}},{"cell_type":"markdown","source":"## .. 4.1. Prepare Scorer and CV\n- {sklearn API} [make_scorer()](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html)\n- {numpy API} [numpy.digitize()](https://numpy.org/doc/stable/reference/generated/numpy.digitize.html)","metadata":{}},{"cell_type":"code","source":"def digitize_and_cohen_kappa(y_true, y_pred_raw, bins=[0.5, 1.5, 2.5]):\n    y_pred = np.digitize(y_pred_raw, bins)\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.445445Z","iopub.execute_input":"2024-12-19T22:19:52.446042Z","iopub.status.idle":"2024-12-19T22:19:52.459600Z","shell.execute_reply.started":"2024-12-19T22:19:52.445987Z","shell.execute_reply":"2024-12-19T22:19:52.458399Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kappa_scorer = make_scorer(digitize_and_cohen_kappa, bins=[0.5, 1.5, 2.5], greater_is_better=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.461054Z","iopub.execute_input":"2024-12-19T22:19:52.461441Z","iopub.status.idle":"2024-12-19T22:19:52.472919Z","shell.execute_reply.started":"2024-12-19T22:19:52.461380Z","shell.execute_reply":"2024-12-19T22:19:52.471616Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv = StratifiedKFold(5)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.474611Z","iopub.execute_input":"2024-12-19T22:19:52.475123Z","iopub.status.idle":"2024-12-19T22:19:52.490946Z","shell.execute_reply.started":"2024-12-19T22:19:52.475065Z","shell.execute_reply":"2024-12-19T22:19:52.489705Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 4.2. cross_validate() - all columns\n- {sklearn Docs} [Cross-validation: evaluating estimator performance](https://scikit-learn.org/stable/modules/cross_validation.html#the-cross-validate-function-and-multiple-metric-evaluation)\n- {sklearn API} [cross_validate()](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_validate.html#sklearn.model_selection.cross_validate)","metadata":{}},{"cell_type":"code","source":"cv_results_all = cross_validate(\n    lgb.LGBMRegressor(\n        **CFG['LGB_PARAMS_ALL'],\n        random_state=CFG['SEED'],\n        n_jobs=1,\n        force_col_wise=True,\n        verbose=-1,\n    ),\n    df_train_merge[COLS], df_train_merge['sii'],\n    cv=cv,\n    scoring={'cohen_kappa': kappa_scorer},\n    return_train_score=True,\n    return_estimator=True,\n)\n\ncv_score_all = np.mean(cv_results_all['test_cohen_kappa'])\nprint(f\"\\n- [cv_results_all] mean of test_cohen_kappa: {cv_score_all:.3f}\")\nprint(f\"- [cv_results_all] mean of train_cohen_kappa: {np.mean(cv_results_all['train_cohen_kappa']):.3f}\\n\")\ncv_results_all","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:52.492326Z","iopub.execute_input":"2024-12-19T22:19:52.492850Z","iopub.status.idle":"2024-12-19T22:19:56.715008Z","shell.execute_reply.started":"2024-12-19T22:19:52.492806Z","shell.execute_reply":"2024-12-19T22:19:56.713856Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 4.3. Feature Importance","metadata":{}},{"cell_type":"code","source":"model = cv_results_all['estimator'][0]\nmodel.importance_type = 'gain'\n\nser_imp = pd.Series(model.feature_importances_, index=model.feature_name_).sort_values(ascending=False)\nser_imp = ser_imp.sort_values(ascending=False)\nser_imp","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:56.716523Z","iopub.execute_input":"2024-12-19T22:19:56.717038Z","iopub.status.idle":"2024-12-19T22:19:56.729140Z","shell.execute_reply.started":"2024-12-19T22:19:56.716990Z","shell.execute_reply":"2024-12-19T22:19:56.727922Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_imp[:40]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:19:56.730281Z","iopub.execute_input":"2024-12-19T22:19:56.730585Z","iopub.status.idle":"2024-12-19T22:19:56.753732Z","shell.execute_reply.started":"2024-12-19T22:19:56.730554Z","shell.execute_reply":"2024-12-19T22:19:56.752366Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 4.4. cross_validate - reduced columns","metadata":{}},{"cell_type":"code","source":"COLS_remove = ser_imp[ser_imp == 0].index\nprint(len(COLS_remove))\nCOLS_remove","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:56.755146Z","iopub.execute_input":"2024-12-19T22:19:56.755514Z","iopub.status.idle":"2024-12-19T22:19:56.771125Z","shell.execute_reply.started":"2024-12-19T22:19:56.755470Z","shell.execute_reply":"2024-12-19T22:19:56.769708Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLS_REDUCED = COLS.drop(COLS_remove)\nprint(len(COLS_REDUCED))\nCOLS_REDUCED","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:56.772748Z","iopub.execute_input":"2024-12-19T22:19:56.773207Z","iopub.status.idle":"2024-12-19T22:19:56.792694Z","shell.execute_reply.started":"2024-12-19T22:19:56.773167Z","shell.execute_reply":"2024-12-19T22:19:56.791366Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_results_reduced = cross_validate(\n    lgb.LGBMRegressor(\n        **CFG['LGB_PARAMS_ALL'],\n        random_state=CFG['SEED'],\n        n_jobs=1,\n        force_col_wise=True,\n        verbose=-1,\n    ),\n    df_train_merge[COLS_REDUCED], df_train_merge['sii'],\n    cv=cv,\n    scoring={'cohen_kappa': kappa_scorer},\n    return_train_score=True,\n    return_estimator=True,\n)\n\ncv_score_reduced = np.mean(cv_results_reduced['test_cohen_kappa'])\nprint(f\"\\n- [cv_results_reduced] mean of test_cohen_kappa: {cv_score_reduced:.3f}\")\nprint(f\"- [cv_results_reduced] mean of train_cohen_kappa: {np.mean(cv_results_reduced['train_cohen_kappa']):.3f}\\n\")\ncv_results_reduced","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:19:56.794025Z","iopub.execute_input":"2024-12-19T22:19:56.794463Z","iopub.status.idle":"2024-12-19T22:20:00.936280Z","shell.execute_reply.started":"2024-12-19T22:19:56.794414Z","shell.execute_reply":"2024-12-19T22:20:00.935020Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 4.5. Choose columns and models","metadata":{}},{"cell_type":"code","source":"cv_score_all, cv_score_reduced","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:00.938106Z","iopub.execute_input":"2024-12-19T22:20:00.938457Z","iopub.status.idle":"2024-12-19T22:20:00.945124Z","shell.execute_reply.started":"2024-12-19T22:20:00.938422Z","shell.execute_reply":"2024-12-19T22:20:00.943948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if cv_score_all >= cv_score_reduced:\n    print(\"Use cv_score_all\")\n    COLS_FINAL = COLS\n    models = cv_results_all['estimator']\n    cv_score_train_merge = cv_results_all['train_cohen_kappa'].mean()\nelse:\n    print(\"Use cv_score_reduced\")\n    COLS_FINAL = COLS_REDUCED\n    models = cv_results_reduced['estimator']\n    cv_score_train_merge = cv_results_reduced['train_cohen_kappa'].mean()\n\nprint(len(COLS_FINAL))\nmodels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:00.946355Z","iopub.execute_input":"2024-12-19T22:20:00.946698Z","iopub.status.idle":"2024-12-19T22:20:00.966249Z","shell.execute_reply.started":"2024-12-19T22:20:00.946654Z","shell.execute_reply":"2024-12-19T22:20:00.965041Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Optimize Bins\n- {scipy Docs} [Optimization (scipy.optimize)](https://docs.scipy.org/doc/scipy/tutorial/optimize.html)\n- {scipy API} [scipy.optimize.minimize()](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html#scipy.optimize.minimize)","metadata":{}},{"cell_type":"markdown","source":"## .. 5.1. Prepare pred_raw","metadata":{}},{"cell_type":"code","source":"pred_raw = np.zeros(len(df_train_merge['sii']))\npred_raw.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:20:00.967672Z","iopub.execute_input":"2024-12-19T22:20:00.968178Z","iopub.status.idle":"2024-12-19T22:20:00.997142Z","shell.execute_reply.started":"2024-12-19T22:20:00.968136Z","shell.execute_reply":"2024-12-19T22:20:00.994852Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, (train_idx, test_idx) in enumerate(cv.split(df_train_merge, df_train_merge['sii'])):\n    df_test_fold = df_train_merge.iloc[test_idx]\n    model = models[i]\n    pred_raw_fold = model.predict(df_test_fold[COLS_FINAL])\n    pred_raw[test_idx] = pred_raw_fold\n    \n    score = digitize_and_cohen_kappa(df_test_fold['sii'], pred_raw_fold)\n    print(f\"[fold{i}] score: {score}\")  # 위의 cross_validate()랑 같은 점수 나오는지 확인\n\nprint(\"\\n\", pred_raw.shape)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:20:01.000791Z","iopub.execute_input":"2024-12-19T22:20:01.001327Z","iopub.status.idle":"2024-12-19T22:20:01.081341Z","shell.execute_reply.started":"2024-12-19T22:20:01.001272Z","shell.execute_reply":"2024-12-19T22:20:01.080051Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 5.2. Minimize","metadata":{}},{"cell_type":"code","source":"def evaluate_predictions(bins, y_true, y_pred_raw):\n    y_pred = np.digitize(y_pred_raw, bins)\n    return -cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:20:01.082611Z","iopub.execute_input":"2024-12-19T22:20:01.083002Z","iopub.status.idle":"2024-12-19T22:20:01.089572Z","shell.execute_reply.started":"2024-12-19T22:20:01.082967Z","shell.execute_reply":"2024-12-19T22:20:01.087621Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    res = minimize(\n        evaluate_predictions,\n        x0=[0.5, 1.5, 2.5],\n        args=(df_train_merge['sii'], pred_raw),\n        method='Nelder-Mead'\n    )\n    bins_final = res.x\nexcept Exception as e:\n    print(\"Failed to optimize bins\")\n    bins_final = [0.5, 1.5, 2.5]","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:20:01.093415Z","iopub.execute_input":"2024-12-19T22:20:01.093895Z","iopub.status.idle":"2024-12-19T22:20:01.258440Z","shell.execute_reply.started":"2024-12-19T22:20:01.093835Z","shell.execute_reply":"2024-12-19T22:20:01.257075Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 5.3. Check Score - optimized bins","metadata":{}},{"cell_type":"code","source":"models","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:01.270032Z","iopub.execute_input":"2024-12-19T22:20:01.270423Z","iopub.status.idle":"2024-12-19T22:20:01.284606Z","shell.execute_reply.started":"2024-12-19T22:20:01.270373Z","shell.execute_reply":"2024-12-19T22:20:01.283477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = []\nfor i, (train_idx, test_idx) in enumerate(cv.split(df_train_merge, df_train_merge['sii'])):\n    df_test_fold = df_train_merge.iloc[test_idx]\n    model = models[i]\n    pred_raw_fold = model.predict(df_test_fold[COLS_FINAL])\n    score = digitize_and_cohen_kappa(df_test_fold['sii'], pred_raw_fold, bins=bins_final)\n    scores.append(score)\n    print(f\"[fold{i}] score: {score}\")\n\ncv_score_optimized_bins = np.mean(scores)\nprint(f\"\\n- [Optimized Bins] mean of test_cohen_kappa: {cv_score_optimized_bins:.3f}\")","metadata":{"execution":{"iopub.status.busy":"2024-12-19T22:20:01.285957Z","iopub.execute_input":"2024-12-19T22:20:01.286278Z","iopub.status.idle":"2024-12-19T22:20:01.363419Z","shell.execute_reply.started":"2024-12-19T22:20:01.286247Z","shell.execute_reply":"2024-12-19T22:20:01.362241Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_score_all, cv_score_reduced, cv_score_optimized_bins","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:01.364696Z","iopub.execute_input":"2024-12-19T22:20:01.365036Z","iopub.status.idle":"2024-12-19T22:20:01.372220Z","shell.execute_reply.started":"2024-12-19T22:20:01.365003Z","shell.execute_reply":"2024-12-19T22:20:01.370917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Final Models","metadata":{}},{"cell_type":"code","source":"# Add a model that uses whole dataset\nmodel = lgb.LGBMRegressor(\n    **CFG['LGB_PARAMS_ALL'],\n    random_state=CFG['SEED'],\n    n_jobs=1,\n    force_col_wise=True,\n    verbose=-1,\n)\n\nmodel.fit(df_train_merge[COLS_FINAL], df_train_merge['sii'])\nmodels.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:01.373505Z","iopub.execute_input":"2024-12-19T22:20:01.373881Z","iopub.status.idle":"2024-12-19T22:20:02.358271Z","shell.execute_reply.started":"2024-12-19T22:20:01.373847Z","shell.execute_reply":"2024-12-19T22:20:02.357311Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Common Features Only Model","metadata":{}},{"cell_type":"code","source":"COLS_COMMON = df_test.columns.drop('id')\nlen(COLS_COMMON)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:02.359405Z","iopub.execute_input":"2024-12-19T22:20:02.359713Z","iopub.status.idle":"2024-12-19T22:20:02.366845Z","shell.execute_reply.started":"2024-12-19T22:20:02.359682Z","shell.execute_reply":"2024-12-19T22:20:02.365682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train.dropna(subset='sii').copy()\ndf_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:02.368106Z","iopub.execute_input":"2024-12-19T22:20:02.368388Z","iopub.status.idle":"2024-12-19T22:20:02.388876Z","shell.execute_reply.started":"2024-12-19T22:20:02.368359Z","shell.execute_reply":"2024-12-19T22:20:02.387812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_results_common_all = cross_validate(\n    lgb.LGBMRegressor(\n        **CFG['LGB_PARAMS_COMMON'],\n        random_state=CFG['SEED'],\n        n_jobs=1,\n        force_col_wise=True,\n        verbose=-1,\n    ),\n    df_train[COLS_COMMON], df_train['sii'],\n    cv=cv,\n    scoring={'cohen_kappa': kappa_scorer},\n    return_train_score=True,\n    return_estimator=True,\n)\n\ncv_score_common_all = cv_results_common_all['test_cohen_kappa'].mean()\ncv_score_train_common = cv_results_common_all['train_cohen_kappa'].mean()\nprint(f\"\\n- [cv_score_common_all] mean of test_cohen_kappa: {cv_score_common_all:.3f}\")\nprint(f\"- [cv_score_common_all] mean of train_cohen_kappa: {cv_score_train_common:.3f}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:02.390761Z","iopub.execute_input":"2024-12-19T22:20:02.391766Z","iopub.status.idle":"2024-12-19T22:20:05.023712Z","shell.execute_reply.started":"2024-12-19T22:20:02.391711Z","shell.execute_reply":"2024-12-19T22:20:05.022666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature Importance\nmodel = cv_results_common_all['estimator'][0]\nmodel.importance_type = 'gain'\n\nser_imp = pd.Series(model.feature_importances_, index=model.feature_name_).sort_values(ascending=False)\nser_imp = ser_imp.sort_values(ascending=False)\nser_imp[-5:]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:05.025411Z","iopub.execute_input":"2024-12-19T22:20:05.025887Z","iopub.status.idle":"2024-12-19T22:20:05.037940Z","shell.execute_reply.started":"2024-12-19T22:20:05.025839Z","shell.execute_reply":"2024-12-19T22:20:05.036702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models_common = cv_results_common_all['estimator']\nlen(models_common)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:05.039423Z","iopub.execute_input":"2024-12-19T22:20:05.039827Z","iopub.status.idle":"2024-12-19T22:20:05.048568Z","shell.execute_reply.started":"2024-12-19T22:20:05.039789Z","shell.execute_reply":"2024-12-19T22:20:05.047622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Optimize Bins\npred_raw = np.zeros(len(df_train['sii']))\n\nfor i, (train_idx, test_idx) in enumerate(cv.split(df_train, df_train['sii'])):\n    df_test_fold = df_train.iloc[test_idx]\n    model = models_common[i]\n    pred_raw_fold = model.predict(df_test_fold[COLS_COMMON])\n    pred_raw[test_idx] = pred_raw_fold\n    \n    score = digitize_and_cohen_kappa(df_test_fold['sii'], pred_raw_fold)\n\ntry:\n    res = minimize(\n        evaluate_predictions,\n        x0=[0.5, 1.5, 2.5],\n        args=(df_train['sii'], pred_raw),\n        method='Nelder-Mead'\n    )\n    bins_common = res.x\nexcept Exception as e:\n    print(\"Failed to optimize bins\")\n    bins_common = [0.5, 1.5, 2.5]\n\nscores = []\nfor i, (train_idx, test_idx) in enumerate(cv.split(df_train, df_train['sii'])):\n    df_test_fold = df_train.iloc[test_idx]\n    model = models_common[i]\n    pred_raw_fold = model.predict(df_test_fold[COLS_COMMON])\n    score = digitize_and_cohen_kappa(df_test_fold['sii'], pred_raw_fold, bins=bins_common)\n    scores.append(score)\n    print(f\"[fold{i}] score: {score}\")\n\ncv_score_common_optimized_bins = np.mean(scores)\nprint(f\"\\n- [Common Optimized Bins] mean of test_cohen_kappa: {cv_score_common_optimized_bins:.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:05.049908Z","iopub.execute_input":"2024-12-19T22:20:05.050291Z","iopub.status.idle":"2024-12-19T22:20:05.523969Z","shell.execute_reply.started":"2024-12-19T22:20:05.050237Z","shell.execute_reply":"2024-12-19T22:20:05.522904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add a model that uses whole dataset\nmodel = lgb.LGBMRegressor(\n    **CFG['LGB_PARAMS_COMMON'],\n    random_state=CFG['SEED'],\n    n_jobs=1,\n    force_col_wise=True,\n    verbose=-1,\n)\n\nmodel.fit(df_train[COLS_COMMON], df_train['sii'])\nmodels_common.append(model)\nlen(models_common)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:05.525383Z","iopub.execute_input":"2024-12-19T22:20:05.525795Z","iopub.status.idle":"2024-12-19T22:20:06.028127Z","shell.execute_reply.started":"2024-12-19T22:20:05.525761Z","shell.execute_reply":"2024-12-19T22:20:06.026999Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Voting","metadata":{}},{"cell_type":"code","source":"WEIGHTS = [1, 1, 1, 1, 1, 2]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.029455Z","iopub.execute_input":"2024-12-19T22:20:06.029834Z","iopub.status.idle":"2024-12-19T22:20:06.035115Z","shell.execute_reply.started":"2024-12-19T22:20:06.029795Z","shell.execute_reply":"2024-12-19T22:20:06.033722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 8.1. Common Feature Only Models","metadata":{}},{"cell_type":"code","source":"df_predict = pd.DataFrame()\n\nfor i, model in enumerate(models_common):\n    df_predict[f'model_{i}'] = model.predict(df_test[COLS_COMMON])\nprint(df_predict.shape)\ndf_predict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.036788Z","iopub.execute_input":"2024-12-19T22:20:06.037237Z","iopub.status.idle":"2024-12-19T22:20:06.134504Z","shell.execute_reply.started":"2024-12-19T22:20:06.037174Z","shell.execute_reply":"2024-12-19T22:20:06.133255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_raw = np.average(df_predict, axis=1, weights=WEIGHTS)\nprint(pred_raw.shape)\npred_raw","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.136003Z","iopub.execute_input":"2024-12-19T22:20:06.136597Z","iopub.status.idle":"2024-12-19T22:20:06.148549Z","shell.execute_reply.started":"2024-12-19T22:20:06.136546Z","shell.execute_reply":"2024-12-19T22:20:06.147276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"bins_common: {bins_common}\")\n\npred = np.digitize(pred_raw, bins_common)\nprint(pred.shape)\npred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.149801Z","iopub.execute_input":"2024-12-19T22:20:06.150147Z","iopub.status.idle":"2024-12-19T22:20:06.166603Z","shell.execute_reply.started":"2024-12-19T22:20:06.150113Z","shell.execute_reply":"2024-12-19T22:20:06.165415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_submission = pd.DataFrame({\n    'id': df_test['id'],\n    'sii': pred\n})\ndf_submission.set_index('id', inplace=True)\ndf_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.168029Z","iopub.execute_input":"2024-12-19T22:20:06.168423Z","iopub.status.idle":"2024-12-19T22:20:06.186450Z","shell.execute_reply.started":"2024-12-19T22:20:06.168374Z","shell.execute_reply":"2024-12-19T22:20:06.185126Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 8.2. Merged Feature Models","metadata":{}},{"cell_type":"code","source":"df_predict = pd.DataFrame()\n\nfor i, model in enumerate(models):\n    df_predict[f'model_{i}'] = model.predict(df_test_merge[COLS_FINAL])\nprint(df_predict.shape)\ndf_predict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.188028Z","iopub.execute_input":"2024-12-19T22:20:06.188450Z","iopub.status.idle":"2024-12-19T22:20:06.243940Z","shell.execute_reply.started":"2024-12-19T22:20:06.188415Z","shell.execute_reply":"2024-12-19T22:20:06.242824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_raw = np.average(df_predict, axis=1, weights=WEIGHTS)\nprint(pred_raw.shape)\npred_raw","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.245491Z","iopub.execute_input":"2024-12-19T22:20:06.245848Z","iopub.status.idle":"2024-12-19T22:20:06.254359Z","shell.execute_reply.started":"2024-12-19T22:20:06.245816Z","shell.execute_reply":"2024-12-19T22:20:06.253091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"bins_final: {bins_final}\")\n\npred = np.digitize(pred_raw, bins_final)\nprint(pred.shape)\npred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.255673Z","iopub.execute_input":"2024-12-19T22:20:06.256006Z","iopub.status.idle":"2024-12-19T22:20:06.268708Z","shell.execute_reply.started":"2024-12-19T22:20:06.255974Z","shell.execute_reply":"2024-12-19T22:20:06.267570Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_answer_merge = pd.DataFrame({\n    'id': df_test_merge['id'],\n    'sii': pred\n})\ndf_answer_merge.set_index('id', inplace=True)\ndf_answer_merge","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.270073Z","iopub.execute_input":"2024-12-19T22:20:06.270452Z","iopub.status.idle":"2024-12-19T22:20:06.287790Z","shell.execute_reply.started":"2024-12-19T22:20:06.270420Z","shell.execute_reply":"2024-12-19T22:20:06.286581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 8.3. Submit","metadata":{}},{"cell_type":"code","source":"df_submission.update(df_answer_merge)\ndf_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.289300Z","iopub.execute_input":"2024-12-19T22:20:06.289665Z","iopub.status.idle":"2024-12-19T22:20:06.306892Z","shell.execute_reply.started":"2024-12-19T22:20:06.289602Z","shell.execute_reply":"2024-12-19T22:20:06.305433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_submission['sii'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.308494Z","iopub.execute_input":"2024-12-19T22:20:06.308918Z","iopub.status.idle":"2024-12-19T22:20:06.322771Z","shell.execute_reply.started":"2024-12-19T22:20:06.308882Z","shell.execute_reply":"2024-12-19T22:20:06.321682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.324554Z","iopub.execute_input":"2024-12-19T22:20:06.325355Z","iopub.status.idle":"2024-12-19T22:20:06.336139Z","shell.execute_reply.started":"2024-12-19T22:20:06.325302Z","shell.execute_reply":"2024-12-19T22:20:06.334900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save time series features\ndf_train_ts.to_csv(\"train_ts.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.337568Z","iopub.execute_input":"2024-12-19T22:20:06.338832Z","iopub.status.idle":"2024-12-19T22:20:06.470711Z","shell.execute_reply.started":"2024-12-19T22:20:06.338769Z","shell.execute_reply":"2024-12-19T22:20:06.469610Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## .. 8.4. Summary","metadata":{}},{"cell_type":"code","source":"print(f\"[Merge] mean of test_cohen_kappa: {cv_score_optimized_bins:.3f}\")\nprint(f\"[Common] mean of test_cohen_kappa: {cv_score_common_optimized_bins:.3f}\")\nprint(f\"CV_merge: {cv_score_optimized_bins:.3f}, CV_common: {cv_score_common_optimized_bins:.3f}, LB: TBD\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.472547Z","iopub.execute_input":"2024-12-19T22:20:06.473053Z","iopub.status.idle":"2024-12-19T22:20:06.480422Z","shell.execute_reply.started":"2024-12-19T22:20:06.473003Z","shell.execute_reply":"2024-12-19T22:20:06.479043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"round(cv_score_train_merge, 3), round(cv_score_train_common, 3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T22:20:06.481933Z","iopub.execute_input":"2024-12-19T22:20:06.482378Z","iopub.status.idle":"2024-12-19T22:20:06.496319Z","shell.execute_reply.started":"2024-12-19T22:20:06.482330Z","shell.execute_reply":"2024-12-19T22:20:06.495231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}