{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9629349,"sourceType":"datasetVersion","datasetId":5878331}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Thanks for some work earlier**\n\n* https://www.kaggle.com/code/ambrosm/piu-eda-which-makes-sense\n* https://www.kaggle.com/code/abdmental01/cmi-best-single-model\n* https://www.kaggle.com/code/lennarthaupts/cmi-detecting-problematic-digital-behavior\n\n","metadata":{}},{"cell_type":"code","source":"import os\nimport random\n\nfrom sklearn.base import clone\nfrom concurrent.futures import ThreadPoolExecutor\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nfrom torch.backends.mkl import verbose\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nimport pandas as pd\nimport numpy as np\nimport lightgbm\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.optimize import minimize\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score, ConfusionMatrixDisplay\nfrom catboost import CatBoostRegressor, Pool, CatBoostClassifier\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom lightgbm import LGBMRegressor, LGBMClassifier, Booster\nimport warnings\n\nfrom colorama import Fore, Style  # output style lib\n\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-23T12:32:34.854692Z","iopub.execute_input":"2024-10-23T12:32:34.855152Z","iopub.status.idle":"2024-10-23T12:32:41.712493Z","shell.execute_reply.started":"2024-10-23T12:32:34.855112Z","shell.execute_reply":"2024-10-23T12:32:41.711197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\ndef threshold_rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1, np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:32:41.714856Z","iopub.execute_input":"2024-10-23T12:32:41.715713Z","iopub.status.idle":"2024-10-23T12:32:41.723951Z","shell.execute_reply.started":"2024-10-23T12:32:41.715655Z","shell.execute_reply":"2024-10-23T12:32:41.722612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load datasets\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:32:41.725734Z","iopub.execute_input":"2024-10-23T12:32:41.726670Z","iopub.status.idle":"2024-10-23T12:32:41.826875Z","shell.execute_reply.started":"2024-10-23T12:32:41.726626Z","shell.execute_reply":"2024-10-23T12:32:41.825608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_funcs = {\n    'mean': 'mean',\n    'std': 'std',\n    'min': 'min',\n    'max': 'max',\n    lambda x: x.quantile(0.25): 'Q1',\n    lambda x: x.quantile(0.50): 'Q2',\n    lambda x: x.quantile(0.75): 'Q3'\n}\n\n\ndef process_file(filename, dirname):\n    ts = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    ts = ts[ts['non-wear_flag'] == 0]\n\n    ts['id'] = filename.split('=')[1]\n    num_cols = ['X', 'Y', 'Z', 'enmo', 'anglez', 'light']\n    for col in num_cols:\n        ts[col] = ts[col].abs()\n\n    ts[\"hours\"] = ts[\"time_of_day\"] // (3_600 * 1_000_000_000)\n    ts = ts[((ts[\"hours\"] >= 22) | (ts[\"hours\"] <= 5)) | ((ts[\"hours\"] >= 7) & (ts[\"hours\"] <= 20))]\n    ts['day_night'] = np.where((ts[\"hours\"] >= 22) | (ts[\"hours\"] <= 5), 'night', 'day')\n\n    # 为每个季度分别计算聚合值\n    quarter_data = []\n    for q in range(1, 5):  # 遍历 1 到 4 个季度\n        for dn in ['day', 'night']:\n            quarter_stats = ts[(ts['quarter'] == q) & (ts['day_night'] == dn)].groupby('id').agg({\n                'X': agg_funcs.keys(),\n                'Y': agg_funcs.keys(),\n                'Z': agg_funcs.keys(),\n                'enmo': agg_funcs.keys(),\n                'anglez': agg_funcs.keys(),\n                'light': agg_funcs.keys()\n            })\n\n            # 简化列名，增加季度标识\n            quarter_stats.columns = [f'Q{q}_{dn}_{col[0]}_{col[1]}' for col in quarter_stats.columns]\n            quarter_data.append(quarter_stats)\n\n    return pd.concat(quarter_data, axis=1).reset_index()\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    # ts_feats, indexes = zip(*results)\n    df = pd.concat(results)\n    # df['id'] = indexes\n    return df\n\n\n# Load time series data\n# train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntrain_ts = pd.read_csv('/kaggle/input/cmi-train-time-series/train_ts.csv')\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntrain_ts","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:32:49.858217Z","iopub.execute_input":"2024-10-23T12:32:49.858701Z","iopub.status.idle":"2024-10-23T12:32:50.738081Z","shell.execute_reply.started":"2024-10-23T12:32:49.858659Z","shell.execute_reply":"2024-10-23T12:32:50.736727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['PCIAT-Season', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', 'PCIAT-PCIAT_04',\n             'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09',\n             'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14',\n             'PCIAT-PCIAT_15', 'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19',\n             'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total']\n\ntrain.drop(drop_cols, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:33:04.655712Z","iopub.execute_input":"2024-10-23T12:33:04.656186Z","iopub.status.idle":"2024-10-23T12:33:04.665982Z","shell.execute_reply.started":"2024-10-23T12:33:04.656135Z","shell.execute_reply":"2024-10-23T12:33:04.664281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(train_ts, how='left', on='id')\ntest = test.merge(test_ts, how='left', on='id')\n\ntrain.drop('id', inplace=True, axis=1)\ntest.drop('id', inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:33:17.113399Z","iopub.execute_input":"2024-10-23T12:33:17.113901Z","iopub.status.idle":"2024-10-23T12:33:17.163645Z","shell.execute_reply.started":"2024-10-23T12:33:17.113855Z","shell.execute_reply":"2024-10-23T12:33:17.162466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.dropna(subset=['sii']).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:33:26.852465Z","iopub.execute_input":"2024-10-23T12:33:26.852985Z","iopub.status.idle":"2024-10-23T12:33:26.883203Z","shell.execute_reply.started":"2024-10-23T12:33:26.852940Z","shell.execute_reply":"2024-10-23T12:33:26.882095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = []\n\nfor col in train.columns:\n    dtype = train[col].dtype\n    if pd.api.types.is_object_dtype(dtype):\n        cat_cols.append(col)\n\ncat_cols","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:33:35.299732Z","iopub.execute_input":"2024-10-23T12:33:35.300205Z","iopub.status.idle":"2024-10-23T12:33:35.330359Z","shell.execute_reply.started":"2024-10-23T12:33:35.300161Z","shell.execute_reply":"2024-10-23T12:33:35.328933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns = train.columns.str.replace('<lambda_0>', 'quant1', regex=False)\ntrain.columns = train.columns.str.replace('<lambda_1>', 'median', regex=False)\ntrain.columns = train.columns.str.replace('<lambda_2>', 'quant3', regex=False)\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:33:42.559155Z","iopub.execute_input":"2024-10-23T12:33:42.559610Z","iopub.status.idle":"2024-10-23T12:33:42.603270Z","shell.execute_reply.started":"2024-10-23T12:33:42.559570Z","shell.execute_reply":"2024-10-23T12:33:42.601860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 42\nn_splits = 5\n\nnp.random.seed(42)\n\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n\ndef train_model(model_class, data):\n    data = data.copy()\n    X = data.drop('sii', axis=1)\n    y = data['sii']\n\n    X[cat_cols] = X[cat_cols].fillna('missing')\n\n    for col in cat_cols:\n        X[col] = X[col].astype('category')\n        mapping = create_mapping(col, X)\n        X[col] = X[col].replace(mapping).astype(int)\n\n    train_scores = []\n    valid_scores = []\n\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    oof_non_rounded = np.zeros(len(y), dtype=float)\n    oof_rounded = np.zeros(len(y), dtype=int)\n\n    models = []\n\n    for fold, (train_idx, valid_idx) in enumerate(skf.split(X, y)):\n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n\n        model = clone(model_class)\n\n        if isinstance(model_class, (CatBoostClassifier, CatBoostRegressor)):\n            train_pool = Pool(X_train, y_train, cat_features=cat_cols)\n            val_pool = Pool(X_valid, y_valid, cat_features=cat_cols)\n            model.fit(train_pool, eval_set=val_pool, early_stopping_rounds=100)\n\n        elif isinstance(model_class, (LGBMRegressor, LGBMClassifier)):\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)])\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_valid)\n\n        oof_non_rounded[valid_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[valid_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_valid, y_val_pred_rounded)\n\n        train_scores.append(train_kappa)\n        valid_scores.append(val_kappa)\n        models.append(model)\n\n        # print(f\"Fold {fold + 1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        # clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_scores):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(valid_scores):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded),\n                              method='Nelder-Mead')  # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    opt_weight = KappaOPtimizer.x\n    oof_tuned = threshold_rounder(oof_non_rounded, opt_weight)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    return models, opt_weight, np.mean(train_scores), np.mean(valid_scores)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:34:03.856628Z","iopub.execute_input":"2024-10-23T12:34:03.857631Z","iopub.status.idle":"2024-10-23T12:34:03.879387Z","shell.execute_reply.started":"2024-10-23T12:34:03.857580Z","shell.execute_reply":"2024-10-23T12:34:03.877975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\n\n\n# 定义 Optuna 的优化目标函数\ndef objective(trial):\n    lgb_params = {\n        'objective': 'poisson',\n        'n_estimators': trial.suggest_int('n_estimators', 200, 5000, step=20),  \n        'learning_rate': trial.suggest_discrete_uniform('learning_rate', 0.02, 0.08, 0.01),  \n        'max_depth': trial.suggest_int('max_depth', 5, 15), \n        'num_leaves': trial.suggest_int('num_leaves', 100, 500, step=10),  \n        'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 100, step=5),  \n        'feature_fraction': trial.suggest_discrete_uniform('feature_fraction', 0.4, 1.0, 0.05),  \n        'bagging_fraction': trial.suggest_discrete_uniform('bagging_fraction', 0.4, 1.0, 0.05),  \n        'bagging_freq': trial.suggest_int('bagging_freq', 1, 10),  \n        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),  \n        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),  \n        'min_child_samples': trial.suggest_int('min_child_samples', 20, 100, step=5),  \n        'subsample': trial.suggest_discrete_uniform('subsample', 0.7, 1.0, 0.05),  \n        'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.7, 1.0, 0.05),  \n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-4, 10.0),  \n        'min_split_gain': trial.suggest_loguniform('min_split_gain', 1e-5, 1.0),  \n        'min_child_weight': trial.suggest_loguniform('min_child_weight', 1e-4, 1.0),  \n        'random_state': seed,\n        'verbose': -1\n    }\n    lgb_model = LGBMRegressor(**lgb_params)\n\n#     cat_params = {\n#         'objective': 'Tweedie:variance_power=1.5',\n#         'iterations': trial.suggest_int('iterations', 200, 2000, step=10),\n#         'learning_rate': trial.suggest_uniform('learning_rate', 0.02, 0.08),\n#         # 'rsm': trial.suggest_uniform('rsm', 0.4, 1.0),\n#         'random_seed': seed,\n#         'verbose': 0,\n#         'devices': '0',\n#         'task_type': 'GPU'\n#     }\n#     cat_model = CatBoostRegressor(**cat_params)\n\n    models, opt_weight, train_score, valid_score = train_model(lgb_model, train)\n    return valid_score\n\n\nstudy = optuna.create_study(direction='maximize')\nstudy.optimize(objective, n_trials=600)\n\nprint(\"Best parameters: \", study.best_params)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T12:34:43.420394Z","iopub.execute_input":"2024-10-23T12:34:43.420881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}