{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":211391753,"sourceType":"kernelVersion"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":25918.26698,"end_time":"2024-12-11T21:05:08.350179","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-11T13:53:10.083199","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"bd64a15a","cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os \nimport polars as pl\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport gc\nfrom sklearn.metrics import r2_score\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom tqdm.auto import tqdm\nimport joblib","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-12-11T13:53:12.370590Z","iopub.status.busy":"2024-12-11T13:53:12.370345Z","iopub.status.idle":"2024-12-11T13:53:17.995899Z","shell.execute_reply":"2024-12-11T13:53:17.995199Z"},"papermill":{"duration":5.631692,"end_time":"2024-12-11T13:53:17.997852","exception":false,"start_time":"2024-12-11T13:53:12.366160","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e867152c","cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    feature_cols = [\"symbol_id\", \"time_id\"] \\\n        + [f\"feature_{idx:02d}\" for idx in range(79)] \\\n        + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    categorical_cols = []\n    Debug=True\n\ndef reduce_mem_usage(df, float16_as32=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    for col in df.columns:  # 遍历每列的列名\n        col_type = df[col].dtype  # 列名的类型\n        if col_type != object and str(col_type) != 'category':\n            c_min, c_max = df[col].min(), df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df.loc[:, col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df.loc[:, col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df.loc[:, col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df.loc[:, col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32:\n                        df.loc[:, col] = df[col].astype(np.float32)\n                    else:\n                        df.loc[:, col] = df[col].astype(np.float16)  \n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df.loc[:, col] = df[col].astype(np.float32)\n                else:\n                    df.loc[:, col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:53:18.005307Z","iopub.status.busy":"2024-12-11T13:53:18.004832Z","iopub.status.idle":"2024-12-11T13:53:18.019294Z","shell.execute_reply":"2024-12-11T13:53:18.018464Z"},"papermill":{"duration":0.020465,"end_time":"2024-12-11T13:53:18.021481","exception":false,"start_time":"2024-12-11T13:53:18.001016","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"19755dc2","cell_type":"code","source":"if CONFIG.Debug:\n    file_paths = [f\"/kaggle/input/janestree-process-data/train_folder/partition_id={i}/train_data_0.parquet\" for i in range(3, 9)]\n    # dfs = [pl.read_parquet(file).drop(\"partition_id\") for file in file_paths]\n    dfs = [pl.read_parquet(file) for file in file_paths]\n    train_df = pl.concat(dfs)\n    del file_paths, dfs\n    gc.collect()\n    print(\"Final shape:\", train_df.shape)\nelse:\n    train_df = pl.read_parquet(f\"/kaggle/input/janestree-process-data/train_folder\")\n    print(train_df.shape)\n    train_df=train_df.drop(\"partition_id\")\n    print(\"after drop\",train_df.shape)","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:53:18.029651Z","iopub.status.busy":"2024-12-11T13:53:18.028997Z","iopub.status.idle":"2024-12-11T13:53:52.884806Z","shell.execute_reply":"2024-12-11T13:53:52.883953Z"},"papermill":{"duration":34.864776,"end_time":"2024-12-11T13:53:52.889534","exception":false,"start_time":"2024-12-11T13:53:18.024758","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"eccbb6c0","cell_type":"code","source":"supervised_usable = (train_df.filter(pl.col('responder_6').is_not_null()))\nmissing_count = (supervised_usable.null_count().transpose(include_header=True,\n                   header_name='feature',\n                   column_names=['null_count']).sort('null_count', descending=True).with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n       \nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n            1 - missing_count.get_column('null_ratio'),\n            left=missing_count.get_column('null_ratio'),\n            color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()\n\ndel missing_count,supervised_usable\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:53:52.896604Z","iopub.status.busy":"2024-12-11T13:53:52.895977Z","iopub.status.idle":"2024-12-11T13:53:54.030133Z","shell.execute_reply":"2024-12-11T13:53:54.029308Z"},"papermill":{"duration":1.140712,"end_time":"2024-12-11T13:53:54.033066","exception":false,"start_time":"2024-12-11T13:53:52.892354","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"886c10b4","cell_type":"code","source":"df_lazy = train_df.lazy()\n\nfill_exprs = [pl.col(c).fill_null(0) for c in train_df.columns]\ndf_lazy = df_lazy.with_columns(*fill_exprs)\ntrain_df = df_lazy.collect()  # 在最后收集计算结果\ndel df_lazy, fill_exprs\ngc.collect()\n    \nprint(\"Total null values:\", train_df.null_count().select(pl.all().sum()).to_series()[0])\n\ntrain_size = int(len(train_df) * 0.85)\ntest = train_df[train_size:]\ntrain = train_df[:train_size]\ndel train_df,train_size\ngc.collect()\nprint(train.shape,test.shape)","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:53:54.045239Z","iopub.status.busy":"2024-12-11T13:53:54.044788Z","iopub.status.idle":"2024-12-11T13:53:55.414457Z","shell.execute_reply":"2024-12-11T13:53:55.413489Z"},"papermill":{"duration":1.377573,"end_time":"2024-12-11T13:53:55.416292","exception":false,"start_time":"2024-12-11T13:53:54.038719","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"76e52811","cell_type":"code","source":"train=train.to_pandas()\ntest=test.to_pandas()\ntrain.dtypes.value_counts()","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:53:55.431687Z","iopub.status.busy":"2024-12-11T13:53:55.430977Z","iopub.status.idle":"2024-12-11T13:54:09.587403Z","shell.execute_reply":"2024-12-11T13:54:09.586611Z"},"papermill":{"duration":14.166876,"end_time":"2024-12-11T13:54:09.589952","exception":false,"start_time":"2024-12-11T13:53:55.423076","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"475bb54c","cell_type":"code","source":"X_train = train[CONFIG.feature_cols].copy()\ny_train = train[CONFIG.target_col].copy()\nw_train = train[\"weight\"].copy()\ndel train\nX_valid = test[CONFIG.feature_cols]\ny_valid = test[CONFIG.target_col]\nw_valid = test[\"weight\"]\ndel test\ngc.collect()\n\nX_train.shape, y_train.shape, w_train.shape, X_valid.shape, y_valid.shape, w_valid.shape","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:54:09.602498Z","iopub.status.busy":"2024-12-11T13:54:09.602205Z","iopub.status.idle":"2024-12-11T13:54:26.442476Z","shell.execute_reply":"2024-12-11T13:54:26.441590Z"},"papermill":{"duration":16.848439,"end_time":"2024-12-11T13:54:26.444161","exception":false,"start_time":"2024-12-11T13:54:09.595722","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b8a85379","cell_type":"code","source":"import optuna\nfrom sklearn.metrics import mean_squared_error\n\n# 定义目标函数\ndef objective(trial):\n    # 定义参数搜索空间\n    params = {\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True), \n        'max_depth': trial.suggest_int('max_depth', 3, 8), \n        'n_estimators': trial.suggest_int('n_estimators', 100, 800),  \n        'subsample': trial.suggest_float('subsample', 0.3, 1.0), \n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.3, 1.0),  \n        'reg_alpha': trial.suggest_float('reg_alpha', 1e-3, 5, log=True), \n        'reg_lambda': trial.suggest_float('reg_lambda', 1e-3, 5, log=True), \n        'random_state': 1212,\n        'tree_method': 'gpu_hist', \n        'device': 'cuda', \n        'n_gpus': 2, \n    }\n\n    print(f\"Trial {trial.number} - Params: {params}\")\n    model = XGBRegressor(**params)\n\n\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        eval_metric=\"rmse\",\n        early_stopping_rounds=30,\n        verbose=30\n    )\n\n    y_pred_valid = model.predict(X_valid)\n    rmse = mean_squared_error(y_valid, y_pred_valid, squared=False)\n    return rmse\n\nstudy = optuna.create_study(direction='minimize')  \n\nstudy.optimize(objective, n_trials=50)\n\nprint(\"Best Trial:\")\nprint(study.best_trial.params)\nbest_params = study.best_trial.params","metadata":{"execution":{"iopub.execute_input":"2024-12-11T13:54:26.457071Z","iopub.status.busy":"2024-12-11T13:54:26.456526Z","iopub.status.idle":"2024-12-11T20:25:09.661823Z","shell.execute_reply":"2024-12-11T20:25:09.660898Z"},"papermill":{"duration":23443.213783,"end_time":"2024-12-11T20:25:09.663866","exception":false,"start_time":"2024-12-11T13:54:26.450083","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"859fb9cf","cell_type":"code","source":"# Custom R2 metric for XGBoost\ndef r2_xgb(y_true, y_pred, sample_weight=None):\n    # 计算加权均值\n    if sample_weight is not None:\n        y_mean = np.average(y_true, weights=sample_weight)\n    else:\n        y_mean = np.mean(y_true)\n    numerator = np.sum(sample_weight * (y_pred - y_true) ** 2) if sample_weight is not None else np.sum((y_pred - y_true) ** 2)\n    # 总平方和 (TSS)\n    denominator = np.sum(sample_weight * (y_true - y_mean) ** 2) if sample_weight is not None else np.sum((y_true - y_mean) ** 2)\n    # 避免分母为零\n    denominator = max(denominator, 1e-38)\n    # R² 计算\n    r2 = 1 - numerator / denominator\n    return r2\n    \nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 500,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': 1212,\n    'tree_method': 'gpu_hist',\n    'device' : 'cuda',\n    'n_gpus' : 2,\n    }\n    \n# model_xgb = XGBRegressor(**XGB_Params,eval_metric=r2_xgb, disable_default_eval_metric=True)\n# model_xgb.fit(X_train, y_train,sample_weight=w_train,\n#               eval_set=[(X_valid, y_valid)], \n#               sample_weight_eval_set=[w_valid], verbose=5)\nmodel_xgb = XGBRegressor(**best_params,eval_metric=\"rmse\")\nmodel_xgb.fit(X_train, y_train,\n              eval_set=[(X_valid, y_valid)],early_stopping_rounds=30,\n              verbose=15)\n\ny_pred_valid = model_xgb.predict(X_valid)\nvalid_score = r2_score(y_valid, y_pred_valid, sample_weight=w_valid )\ndel X_train,y_train, w_train, X_valid, y_valid, w_valid\ngc.collect()\nvalid_score","metadata":{"execution":{"iopub.execute_input":"2024-12-11T20:25:09.761113Z","iopub.status.busy":"2024-12-11T20:25:09.760494Z","iopub.status.idle":"2024-12-11T21:04:35.378687Z","shell.execute_reply":"2024-12-11T21:04:35.377810Z"},"papermill":{"duration":2365.717451,"end_time":"2024-12-11T21:04:35.429745","exception":false,"start_time":"2024-12-11T20:25:09.712294","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7e207f0b","cell_type":"code","source":"os.system('mkdir models')\njoblib.dump(model_xgb, '/kaggle/working/models/xgb.model')\nprint(\"finish training\")\n# models=joblib.load(\"/kaggle/working/models/xgb.model\")","metadata":{"execution":{"iopub.execute_input":"2024-12-11T21:04:35.529555Z","iopub.status.busy":"2024-12-11T21:04:35.528866Z","iopub.status.idle":"2024-12-11T21:04:35.599038Z","shell.execute_reply":"2024-12-11T21:04:35.596888Z"},"papermill":{"duration":0.122836,"end_time":"2024-12-11T21:04:35.601603","exception":false,"start_time":"2024-12-11T21:04:35.478767","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}