{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import socket\n\nNB='test'\nDESCRIPTION='test notebook'\nHOST = socket.gethostname()\nHOST","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:34.299794Z","iopub.execute_input":"2024-09-21T20:25:34.300799Z","iopub.status.idle":"2024-09-21T20:25:34.332257Z","shell.execute_reply.started":"2024-09-21T20:25:34.300751Z","shell.execute_reply":"2024-09-21T20:25:34.331179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install polars==1.5.0 -U","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:34.334095Z","iopub.execute_input":"2024-09-21T20:25:34.334442Z","iopub.status.idle":"2024-09-21T20:25:34.338843Z","shell.execute_reply.started":"2024-09-21T20:25:34.334404Z","shell.execute_reply":"2024-09-21T20:25:34.337771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nimport os\nfrom pathlib import Path\n\nfrom dataclasses import dataclass\nfrom dataclasses import asdict\n\nimport pandas as pd\nimport polars as pl\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold\nfrom sklearn.metrics import accuracy_score, mean_squared_error, roc_auc_score, average_precision_score, mean_squared_log_error, cohen_kappa_score\nfrom sklearn.preprocessing import LabelEncoder\n\nimport pickle\n\nimport glob\n#from my_utils import get_notebook_path\n\ndef to_pickle(filename, obj):\n    with open(filename, mode='wb') as f:\n        pickle.dump(obj, f)\n        \ndef unpickle(filename):\n    with open(filename, mode='rb') as fo:\n        p = pickle.load(fo)\n    return p \n\nwarnings.simplefilter('ignore')\npd.set_option('display.max_columns', 25)\npd.set_option('display.max_rows', 25)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:34.340092Z","iopub.execute_input":"2024-09-21T20:25:34.340489Z","iopub.status.idle":"2024-09-21T20:25:38.214421Z","shell.execute_reply.started":"2024-09-21T20:25:34.340445Z","shell.execute_reply":"2024-09-21T20:25:38.213297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install lightgbm -U","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:38.216824Z","iopub.execute_input":"2024-09-21T20:25:38.217381Z","iopub.status.idle":"2024-09-21T20:25:38.223593Z","shell.execute_reply.started":"2024-09-21T20:25:38.217341Z","shell.execute_reply":"2024-09-21T20:25:38.220831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_kaggle = True","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:55.815177Z","iopub.execute_input":"2024-09-21T20:25:55.816119Z","iopub.status.idle":"2024-09-21T20:25:55.820650Z","shell.execute_reply.started":"2024-09-21T20:25:55.816071Z","shell.execute_reply":"2024-09-21T20:25:55.819400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = Path('../')\nDATA_DIR = ROOT_DIR / Path('data')\nif is_kaggle:\n    ROOT_DIR = Path('./')\n    DATA_DIR = Path(\"/kaggle/input/child-mind-institute-problematic-internet-use\")\n\nglob.glob(str(DATA_DIR) + \"/**\")","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:56.065022Z","iopub.execute_input":"2024-09-21T20:25:56.065431Z","iopub.status.idle":"2024-09-21T20:25:56.073935Z","shell.execute_reply.started":"2024-09-21T20:25:56.065391Z","shell.execute_reply":"2024-09-21T20:25:56.072784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    N_LABEL = 24\n    N_FOLD = 5\n    RANDOM_SATE = 42\n    LR = 0.15\n    PATIENCE = 30\n    EPOCH = 300\n    BATCH_SIZE = 6\n    SKIP_EVALUATE_NUM = 5","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:56.531949Z","iopub.execute_input":"2024-09-21T20:25:56.532786Z","iopub.status.idle":"2024-09-21T20:25:56.537468Z","shell.execute_reply.started":"2024-09-21T20:25:56.532734Z","shell.execute_reply":"2024-09-21T20:25:56.536355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pl.read_csv(DATA_DIR / Path('train.csv'))\ntest_df = pl.read_csv(DATA_DIR / Path('test.csv'))\n\ndisplay(train_df.shape)\ndisplay(train_df.head(5))\ndisplay(test_df.shape)\ndisplay(test_df.head(5))","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:57.535600Z","iopub.execute_input":"2024-09-21T20:25:57.536031Z","iopub.status.idle":"2024-09-21T20:25:57.659558Z","shell.execute_reply.started":"2024-09-21T20:25:57.535989Z","shell.execute_reply":"2024-09-21T20:25:57.658570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\n# TODO enmoが0をどう扱うか\n# TODO 全体の0の割合など求めてもいいかも\n\ndef split_list(lst, n):\n    return [lst[i:i + n] for i in range(0, len(lst), n)]\n\ndef agg_activity(activity_files_split):\n    agg_col = ['step', 'X', 'Y', 'Z', \"euclidean_distance\", 'enmo', 'anglez', 'non-wear_flag', 'light', 'battery_voltage']\n    \n    exprs = [pl.col(\"id\")]\n    for col in agg_col:\n        exprs += [\n            pl.col(col).max().over(\"id\").alias(f\"{col}_max\"),\n            pl.col(col).min().over(\"id\").alias(f\"{col}_min\"),\n            pl.col(col).mean().over(\"id\").alias(f\"{col}_mean\"),\n            pl.col(col).std().over(\"id\").alias(f\"{col}_std\"),\n            pl.col(col).skew().over(\"id\").alias(f\"{col}_skew\"),\n            pl.col(col).kurtosis().over(\"id\").alias(f\"{col}_kurtosis\"),\n        ]\n    \n    activity_dfs = []\n    for activity_files_split_part in tqdm(activity_files_split):\n        dfs = []\n        for activity_file in activity_files_split_part:\n            _df = pl.scan_parquet(activity_file)\n            _df = _df.with_columns(\n                pl.lit(activity_file.split(\"=\")[-1]).alias(\"id\")\n            )\n            dfs.append(_df)\n        pciat_part_df = pl.concat(dfs)\n\n        agg_df = pciat_part_df.with_columns(\n            (pl.col(\"X\")**2 + pl.col(\"Y\")**2 + pl.col(\"Z\")**2).sqrt().alias(\"euclidean_distance\")\n        ).select(\n            exprs\n        ).collect().unique()\n\n        # 測定対象曜日\n        target_day_week_df = pciat_part_df.filter(\n            pl.col(\"relative_date_PCIAT\") == 0\n        ).unique(\n            [\"relative_date_PCIAT\", \"id\"]\n        ).select(\n            pl.col(\"id\"),\n            pl.col(\"weekday\").cast(pl.Utf8)\n        ).collect()\n\n        agg_df = agg_df.join(target_day_week_df, on=\"id\", how=\"left\")\n\n        activity_dfs.append(agg_df)\n\n    activity_df = pl.concat(activity_dfs)\n    return activity_df\n","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:58.207668Z","iopub.execute_input":"2024-09-21T20:25:58.208399Z","iopub.status.idle":"2024-09-21T20:25:58.226803Z","shell.execute_reply.started":"2024-09-21T20:25:58.208354Z","shell.execute_reply":"2024-09-21T20:25:58.225432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 測定対象日だけに限った集計特徴量","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:58.516356Z","iopub.execute_input":"2024-09-21T20:25:58.517028Z","iopub.status.idle":"2024-09-21T20:25:58.521855Z","shell.execute_reply.started":"2024-09-21T20:25:58.516975Z","shell.execute_reply":"2024-09-21T20:25:58.520592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"activity_train_files_split = split_list(glob.glob(str(DATA_DIR / 'series_train.parquet/*')), 20)\ntrain_activity_df = agg_activity(activity_train_files_split)\n\ntrain_activity_df","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:25:58.822722Z","iopub.execute_input":"2024-09-21T20:25:58.823136Z","iopub.status.idle":"2024-09-21T20:34:05.575345Z","shell.execute_reply.started":"2024-09-21T20:25:58.823098Z","shell.execute_reply":"2024-09-21T20:34:05.573420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"activity_test_files_split = split_list(glob.glob(str(DATA_DIR / 'series_test.parquet/*')), 20)\ntest_activity_df = agg_activity(activity_test_files_split)\n\ntest_activity_df","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:05.578244Z","iopub.execute_input":"2024-09-21T20:34:05.578709Z","iopub.status.idle":"2024-09-21T20:34:06.326043Z","shell.execute_reply.started":"2024-09-21T20:34:05.578660Z","shell.execute_reply":"2024-09-21T20:34:06.325045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.join(train_activity_df, on=\"id\", how=\"left\")\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.327675Z","iopub.execute_input":"2024-09-21T20:34:06.328731Z","iopub.status.idle":"2024-09-21T20:34:06.352329Z","shell.execute_reply.started":"2024-09-21T20:34:06.328661Z","shell.execute_reply":"2024-09-21T20:34:06.351290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.join(test_activity_df, on=\"id\", how=\"left\")\ntest_df","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.354993Z","iopub.execute_input":"2024-09-21T20:34:06.355371Z","iopub.status.idle":"2024-09-21T20:34:06.374942Z","shell.execute_reply.started":"2024-09-21T20:34:06.355331Z","shell.execute_reply":"2024-09-21T20:34:06.373725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(list(set(train_df.columns) - set(test_df.columns)))","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.376952Z","iopub.execute_input":"2024-09-21T20:34:06.377466Z","iopub.status.idle":"2024-09-21T20:34:06.385732Z","shell.execute_reply.started":"2024-09-21T20:34:06.377414Z","shell.execute_reply":"2024-09-21T20:34:06.384530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.select(pl.col(pl.Utf8)).columns)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.387183Z","iopub.execute_input":"2024-09-21T20:34:06.387566Z","iopub.status.idle":"2024-09-21T20:34:06.403126Z","shell.execute_reply.started":"2024-09-21T20:34:06.387527Z","shell.execute_reply":"2024-09-21T20:34:06.401885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.select(\n    pl.col(\"PCIAT-PCIAT_Total\"),\n    pl.col(\"sii\")\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.404861Z","iopub.execute_input":"2024-09-21T20:34:06.405252Z","iopub.status.idle":"2024-09-21T20:34:06.414727Z","shell.execute_reply.started":"2024-09-21T20:34:06.405214Z","shell.execute_reply":"2024-09-21T20:34:06.413385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.filter(\n    pl.col(\"sii\").is_not_null()\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.416225Z","iopub.execute_input":"2024-09-21T20:34:06.416571Z","iopub.status.idle":"2024-09-21T20:34:06.448229Z","shell.execute_reply.started":"2024-09-21T20:34:06.416536Z","shell.execute_reply":"2024-09-21T20:34:06.446944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## feature and target","metadata":{}},{"cell_type":"code","source":"target = 'sii'\n\ntarget_col = [\"id\", 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_11', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_16', 'PCIAT-Season', 'PCIAT-PCIAT_10', 'sii', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_20', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_15', 'PCIAT-PCIAT_Total', 'PCIAT-PCIAT_19', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_04', 'PCIAT-PCIAT_03', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_14']\nstr_col = [\"weekday\", 'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'PCIAT-Season', 'SDS-Season', 'PreInt_EduHx-Season']\ndel_columns = target_col + str_col\n\nfeatures = list(set(train_df.columns) - set(del_columns))\nfeatures.sort()\n\nif not is_kaggle:\n    to_pickle(ROOT_DIR / Path(f'output/{HOST}_{NB}_features.pkl'), features)\n\nprint(features)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.449671Z","iopub.execute_input":"2024-09-21T20:34:06.450058Z","iopub.status.idle":"2024-09-21T20:34:06.458923Z","shell.execute_reply.started":"2024-09-21T20:34:06.450019Z","shell.execute_reply":"2024-09-21T20:34:06.457754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train lgbm","metadata":{}},{"cell_type":"markdown","source":"### metrix","metadata":{}},{"cell_type":"code","source":"def evaluation(true, pred):\n    return np.sqrt(mean_squared_error(true, pred))","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.463837Z","iopub.execute_input":"2024-09-21T20:34:06.464480Z","iopub.status.idle":"2024-09-21T20:34:06.470042Z","shell.execute_reply.started":"2024-09-21T20:34:06.464435Z","shell.execute_reply":"2024-09-21T20:34:06.468777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### encording","metadata":{}},{"cell_type":"code","source":"%%time\n_all_df = pl.concat([train_df, test_df], how=\"diagonal\")","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.471378Z","iopub.execute_input":"2024-09-21T20:34:06.471719Z","iopub.status.idle":"2024-09-21T20:34:06.484413Z","shell.execute_reply.started":"2024-09-21T20:34:06.471685Z","shell.execute_reply":"2024-09-21T20:34:06.483188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cast categorical\ncategorical_columns = [\"weekday\", 'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'PCIAT-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n_all_df = _all_df.with_columns([pl.col(c).cast(pl.Categorical).alias(f\"{c}_cate\") for c in categorical_columns])\ncate_features = [f\"{c}_cate\" for c in categorical_columns]\nfeatures = list(set(features) | set(cate_features))\n\n# count encoding\ncount_enc = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'PCIAT-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n_all_df = _all_df.with_columns([pl.col(c).count().over(c).alias(f\"{c}_count\") for c in count_enc])\ncount_features = [f\"{c}_count\" for c in count_enc]\nfeatures = list(set(features) | set(count_features))\n\ntrain_df = _all_df[:len(train_df)]\ntest_df = _all_df[len(train_df):]","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.485996Z","iopub.execute_input":"2024-09-21T20:34:06.486407Z","iopub.status.idle":"2024-09-21T20:34:06.506119Z","shell.execute_reply.started":"2024-09-21T20:34:06.486363Z","shell.execute_reply":"2024-09-21T20:34:06.505076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.filter(\n    pl.col(\"sii\").is_not_null()\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:06.507426Z","iopub.execute_input":"2024-09-21T20:34:06.507789Z","iopub.status.idle":"2024-09-21T20:34:06.514263Z","shell.execute_reply.started":"2024-09-21T20:34:06.507753Z","shell.execute_reply":"2024-09-21T20:34:06.512962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train","metadata":{}},{"cell_type":"code","source":"params = {\n    'n_estimators':5000,\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'objective': 'binary',\n    'n_jobs': -1,\n    'seed': Config.RANDOM_SATE,\n    'learning_rate': 0.01,\n}\n\n# TargetEncording Config\ntarget_enc = ['Sex', 'Ticket', 'Cabin', 'Embarked'] # 集計対象カラム\ntarget_enc_key = [] # 集計するキー\n\noof_preds = np.zeros(len(train_df))\ny_pred = np.zeros(len(test_df))\nmodels = []\ncv_scores = {}\nskf = StratifiedKFold(n_splits=Config.N_FOLD, random_state=Config.RANDOM_SATE, shuffle=True)\nfor fold, (train_index, test_index) in enumerate(skf.split(train_df[features], train_df[target])):\n\n    print(f'====== fold {fold} ======')\n\n    # TrainとTestに分割\n    x_train, x_val = train_df.select(features).clone()[train_index], train_df.select(features).clone()[test_index]\n    y_train, y_val =  train_df[target][train_index], train_df[target][test_index]\n\n    test = test_df[features]\n\n    # Target Encoding\n    if len(target_enc) > 0:\n        for t in target_enc_key:\n            for c in target_enc:\n                x_train[f'{c}_target_enc_by_{t}'] = train_df.iloc[train_index][c].map(train_df.iloc[test_index].groupby(c)[t].mean().to_dict())\n                x_val[f'{c}_target_enc_by_{t}'] = train_df.iloc[test_index][c].map(train_df.groupby(c)[t].mean().to_dict())\n                test[f'{c}_target_enc_by_{t}'] = test_df[c].map(train_df.groupby(c)[t].mean().to_dict())\n            \n    train_features = x_train.columns\n\n    # create Dataset\n    train_set = lgb.Dataset(x_train.to_pandas(), y_train.to_pandas(), categorical_feature=cate_features, free_raw_data=False)\n    val_set = lgb.Dataset(x_val.to_pandas(), y_val.to_pandas(), categorical_feature=cate_features, free_raw_data=False)\n\n    # train\n    model = lgb.train(params, train_set,\n                    valid_sets=[train_set, val_set],  # early_stoppingの評価用データ\n                    num_boost_round=10000,  # 最大学習サイクル数。early_stopping使用時は大きな値を入力\n                    callbacks=[lgb.early_stopping(stopping_rounds=100, verbose=True), # early_stopping用コールバック関数\n                               lgb.log_evaluation(100)] # コマンドライン出力用コールバック関数\n                    )\n\n\n\n    \n    models.append(model)\n\n    fold_pred = model.predict(x_val.to_pandas())\n\n    score = evaluation(y_val, fold_pred)\n    cv_scores[f'cv{fold}'] = score\n\n    oof_preds[test_index] = fold_pred\n\n    y_pred += model.predict(test.to_pandas()) / Config.N_FOLD\n\n    print(f'cv score is {score}')\n\noof_score = evaluation(train_df[target], oof_preds)\nprint(f'OOF score is {oof_score}')\n\nif not is_kaggle:\n    to_pickle(ROOT_DIR / Path(f'output/{HOST}_{NB}_lgbm_models.pkl'), models)\n    to_pickle(ROOT_DIR / Path(f'output/{HOST}_{NB}_oof_preds.pkl'), oof_preds)\n    to_pickle(ROOT_DIR / Path(f'output/{HOST}_{NB}__preds.pkl'), y_pred)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-09-21T20:34:06.515742Z","iopub.execute_input":"2024-09-21T20:34:06.516242Z","iopub.status.idle":"2024-09-21T20:34:36.125659Z","shell.execute_reply.started":"2024-09-21T20:34:06.516205Z","shell.execute_reply":"2024-09-21T20:34:36.124556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_importance = None\n\nfor i, model in enumerate(models):\n    if df_importance is None:\n        _df = pd.DataFrame([model.feature_importance(importance_type='gain'), train_features]).T\n        _df.columns = [f'model_{i}_gain', 'feature']\n        df_importance = _df\n    else:\n        _df = pd.DataFrame([model.feature_importance(importance_type='gain'), train_features]).T\n        _df.columns = [f'model_{i}_gain', 'feature']\n        df_importance = df_importance.merge(_df, how='outer', on='feature')\n\ndf_imp = df_importance\ndf_imp['mean'] = df_imp[[f'model_{i}_gain' for i in range(len(models))]].mean(axis=1)\norder = df_imp.sort_values('mean', ascending=False)['feature'].tolist()\n\ndf_imp = pd.melt(df_imp, id_vars=['feature'], value_vars=[f'model_{i}_gain' for i in range(len(models))])\ndf_imp['value'] = df_imp['value'].astype(float)\n\nfig, ax = plt.subplots(figsize=(len(df_imp['feature'].drop_duplicates()) * .4, 5))\nsns.boxenplot(x=\"feature\", y=\"value\", data=df_imp, order=order)\nax.tick_params(axis='x', rotation=90)\nax.set_title('feature importance')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:36.126897Z","iopub.execute_input":"2024-09-21T20:34:36.127216Z","iopub.status.idle":"2024-09-21T20:34:39.111029Z","shell.execute_reply.started":"2024-09-21T20:34:36.127183Z","shell.execute_reply":"2024-09-21T20:34:39.109659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.typing import ArrayLike, NDArray\nfrom functools import partial\nimport optuna\n\nclass OptimizedRounder:\n    \"\"\"\n    A class for optimizing the rounding of continuous predictions into discrete class labels using Optuna.\n    The optimization process maximizes the Quadratic Weighted Kappa score by learning thresholds that separate\n    continuous predictions into class intervals.\n\n    Args:\n        n_classes (int): The number of discrete class labels.\n        n_trials (int, optional): The number of trials for the Optuna optimization. Defaults to 100.\n\n    Attributes:\n        n_classes (int): The number of discrete class labels.\n        labels (NDArray[np.int_]): An array of class labels from 0 to `n_classes - 1`.\n        n_trials (int): The number of optimization trials.\n        metric (Callable): The Quadratic Weighted Kappa score metric used for optimization.\n        thresholds (List[float]): The optimized thresholds learned after calling `fit()`.\n\n    Methods:\n        fit(y_pred: NDArray[np.float_], y_true: NDArray[np.int_]) -> None:\n            Fits the rounding thresholds based on continuous predictions and ground truth labels.\n\n            Args:\n                y_pred (NDArray[np.float_]): Continuous predictions that need to be rounded.\n                y_true (NDArray[np.int_]): Ground truth class labels.\n\n            Returns:\n                None\n\n        predict(y_pred: NDArray[np.float_]) -> NDArray[np.int_]:\n            Predicts discrete class labels by rounding continuous predictions using the fitted thresholds.\n            `fit()` must be called before `predict()`.\n\n            Args:\n                y_pred (NDArray[np.float_]): Continuous predictions to be rounded.\n\n            Returns:\n                NDArray[np.int_]: Predicted class labels.\n\n        _normalize(y: NDArray[np.float_]) -> NDArray[np.float_]:\n            Normalizes the continuous values to the range [0, `n_classes - 1`].\n\n            Args:\n                y (NDArray[np.float_]): Continuous values to be normalized.\n\n            Returns:\n                NDArray[np.float_]: Normalized values.\n\n    References:\n        - This implementation uses Optuna for threshold optimization.\n        - Quadratic Weighted Kappa is used as the evaluation metric.\n    \"\"\"\n\n    def __init__(self, n_classes: int, n_trials: int = 100):\n        self.n_classes = n_classes\n        self.labels = np.arange(n_classes)\n        self.n_trials = n_trials\n        self.metric = partial(cohen_kappa_score, weights=\"quadratic\")\n\n    def fit(self, y_pred: NDArray[np.float_], y_true: NDArray[np.int_]) -> None:\n        y_pred = self._normalize(y_pred)\n\n        def objective(trial: optuna.Trial) -> float:\n            thresholds = []\n            for i in range(self.n_classes - 1):\n                low = max(thresholds) if i > 0 else min(self.labels)\n                high = max(self.labels)\n                th = trial.suggest_float(f\"threshold_{i}\", low, high)\n                thresholds.append(th)\n            try:\n                y_pred_rounded = np.digitize(y_pred, thresholds)\n            except ValueError:\n                return -100\n            return self.metric(y_true, y_pred_rounded)\n\n        optuna.logging.disable_default_handler()\n        study = optuna.create_study(direction=\"maximize\")\n        study.optimize(\n            objective,\n            n_trials=self.n_trials,\n        )\n        self.thresholds = [study.best_params[f\"threshold_{i}\"] for i in range(self.n_classes - 1)]\n\n    def predict(self, y_pred: NDArray[np.float_]) -> NDArray[np.int_]:\n        assert hasattr(self, \"thresholds\"), \"fit() must be called before predict()\"\n        y_pred = self._normalize(y_pred)\n        return np.digitize(y_pred, self.thresholds)\n\n    def _normalize(self, y: NDArray[np.float_]) -> NDArray[np.float_]:\n        # normalize y_pred to [0, n_classes - 1]\n        return (y - y.min()) / (y.max() - y.min()) * (self.n_classes - 1)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:39.112717Z","iopub.execute_input":"2024-09-21T20:34:39.113206Z","iopub.status.idle":"2024-09-21T20:34:39.293077Z","shell.execute_reply.started":"2024-09-21T20:34:39.113153Z","shell.execute_reply":"2024-09-21T20:34:39.291904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimize thresholds\noptimizer = OptimizedRounder(n_classes=4, n_trials=500)\noptimizer.fit(oof_preds, train_df[\"sii\"])\ny_pred_rounded = optimizer.predict(oof_preds)\n\n# Calculate QWK\nqwk = cohen_kappa_score(train_df[\"sii\"], y_pred_rounded, weights=\"quadratic\")\nprint(f\"Cross-Validated QWK Score: {qwk}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:39.294600Z","iopub.execute_input":"2024-09-21T20:34:39.294987Z","iopub.status.idle":"2024-09-21T20:34:50.140600Z","shell.execute_reply.started":"2024-09-21T20:34:39.294949Z","shell.execute_reply":"2024-09-21T20:34:50.139414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:50.142282Z","iopub.execute_input":"2024-09-21T20:34:50.143256Z","iopub.status.idle":"2024-09-21T20:34:50.150408Z","shell.execute_reply.started":"2024-09-21T20:34:50.143201Z","shell.execute_reply":"2024-09-21T20:34:50.149272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_rounded = optimizer.predict(y_pred)\ntest_pred_rounded","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:34:50.152272Z","iopub.execute_input":"2024-09-21T20:34:50.152766Z","iopub.status.idle":"2024-09-21T20:34:50.163907Z","shell.execute_reply.started":"2024-09-21T20:34:50.152706Z","shell.execute_reply":"2024-09-21T20:34:50.162832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = test_df.select(\n    pl.col(\"id\"),\n    pl.Series(\"sii\", test_pred_rounded)\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:51:17.097718Z","iopub.execute_input":"2024-09-21T20:51:17.098233Z","iopub.status.idle":"2024-09-21T20:51:17.105266Z","shell.execute_reply.started":"2024-09-21T20:51:17.098190Z","shell.execute_reply":"2024-09-21T20:51:17.103999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:51:17.705367Z","iopub.execute_input":"2024-09-21T20:51:17.705788Z","iopub.status.idle":"2024-09-21T20:51:17.713755Z","shell.execute_reply.started":"2024-09-21T20:51:17.705746Z","shell.execute_reply":"2024-09-21T20:51:17.712756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.write_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-09-21T20:51:25.784460Z","iopub.execute_input":"2024-09-21T20:51:25.784889Z","iopub.status.idle":"2024-09-21T20:51:25.790663Z","shell.execute_reply.started":"2024-09-21T20:51:25.784837Z","shell.execute_reply":"2024-09-21T20:51:25.789390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}