{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":215104393,"sourceType":"kernelVersion"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade polars","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:11:37.071497Z","iopub.execute_input":"2024-12-28T06:11:37.071830Z","iopub.status.idle":"2024-12-28T06:11:40.794807Z","shell.execute_reply.started":"2024-12-28T06:11:37.071802Z","shell.execute_reply":"2024-12-28T06:11:40.793877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import necessary libraries\nimport pandas as pd\nimport polars as pl\nimport numpy as np\nimport os\n\nfrom matplotlib import pyplot as plt\n\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import r2_score  # Evaluation metric\nimport pickle  # Used for saving model results\n\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:11:43.550488Z","iopub.execute_input":"2024-12-28T06:11:43.550796Z","iopub.status.idle":"2024-12-28T06:11:43.555698Z","shell.execute_reply.started":"2024-12-28T06:11:43.550772Z","shell.execute_reply":"2024-12-28T06:11:43.554841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    seed = 42\n    target_col = \"responder_6\"\n    feature_cols = ['date_id', 'time_id', 'symbol_id','feature_00', 'feature_01', 'feature_02', 'feature_03', 'feature_04', 'feature_05', 'feature_06', 'feature_07', 'feature_08', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_32', 'feature_33', 'feature_34', 'feature_35', 'feature_36', 'feature_37', 'feature_38', 'feature_39', 'feature_40', 'feature_41', 'feature_42', 'feature_43', 'feature_44', 'feature_45', 'feature_46', 'feature_47', 'feature_48', 'feature_49', 'feature_50', 'feature_51', 'feature_52', 'feature_53', 'feature_54', 'feature_55', 'feature_56', 'feature_57', 'feature_58', 'feature_59', 'feature_60', 'feature_62', 'feature_63', 'feature_64', 'feature_65', 'feature_66', 'feature_67', 'feature_68', 'feature_69', 'feature_70', 'feature_71', 'feature_72', 'feature_73', 'feature_74', 'feature_75', 'feature_76', 'feature_77', 'feature_78']\n    categorical_cols = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:11:49.057479Z","iopub.execute_input":"2024-12-28T06:11:49.057813Z","iopub.status.idle":"2024-12-28T06:11:49.063862Z","shell.execute_reply.started":"2024-12-28T06:11:49.057785Z","shell.execute_reply":"2024-12-28T06:11:49.062757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pl.scan_parquet(\"/kaggle/input/js24-preprocessing/training.parquet\").collect().to_pandas()\nvalid = pl.scan_parquet(\"/kaggle/input/js24-preprocessing/validation.parquet\").collect().to_pandas()\ntrain.shape, valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:14:25.331667Z","iopub.execute_input":"2024-12-28T06:14:25.332009Z","iopub.status.idle":"2024-12-28T06:14:43.628077Z","shell.execute_reply.started":"2024-12-28T06:14:25.331985Z","shell.execute_reply":"2024-12-28T06:14:43.627195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Trick of boosting LB score: 0.45->0.49\ntrain = pd.concat([train, valid]).reset_index(drop=True)\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:15:23.888766Z","iopub.execute_input":"2024-12-28T06:15:23.889109Z","iopub.status.idle":"2024-12-28T06:15:31.881740Z","shell.execute_reply.started":"2024-12-28T06:15:23.889080Z","shell.execute_reply":"2024-12-28T06:15:31.880932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = train[ CONFIG.feature_cols ]\ny_train = train[ CONFIG.target_col ]\nw_train = train[ \"weight\" ]\n\nX_valid = valid[ CONFIG.feature_cols ]\ny_valid = valid[ CONFIG.target_col ]\nw_valid = valid[ \"weight\" ]\n\nX_train.shape, y_train.shape, w_train.shape, X_valid.shape, y_valid.shape, w_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:15:44.393989Z","iopub.execute_input":"2024-12-28T06:15:44.394289Z","iopub.status.idle":"2024-12-28T06:15:45.590831Z","shell.execute_reply.started":"2024-12-28T06:15:44.394266Z","shell.execute_reply":"2024-12-28T06:15:45.589802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(seed):\n    # XGBoost parameters for CPU\n    XGB_Params = {\n        'learning_rate': 0.05,\n        'max_depth': 8,\n        'n_estimators': 500,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'reg_alpha': 1,\n        'reg_lambda': 5,\n        'random_state': seed,\n        'tree_method': 'gpu_hist',\n        'device' : 'cuda',\n        'n_gpus' : 2,\n    }\n    \n    XGB_Model = XGBRegressor(**XGB_Params)\n    return XGB_Model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:16:23.617334Z","iopub.execute_input":"2024-12-28T06:16:23.617636Z","iopub.status.idle":"2024-12-28T06:16:23.622023Z","shell.execute_reply.started":"2024-12-28T06:16:23.617613Z","shell.execute_reply":"2024-12-28T06:16:23.621188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n# Fit the model with early stopping\nmodel = get_model(CONFIG.seed)\n\nmodel.fit(\n    X_train,\n    y_train,\n    sample_weight=w_train,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:16:47.593079Z","iopub.execute_input":"2024-12-28T06:16:47.593384Z","iopub.status.idle":"2024-12-28T06:20:45.730045Z","shell.execute_reply.started":"2024-12-28T06:16:47.593361Z","shell.execute_reply":"2024-12-28T06:20:45.729173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_train1 = model.predict(X_train.iloc[:X_train.shape[0]//2])\ny_pred_train2 = model.predict(X_train.iloc[X_train.shape[0]//2:])\ntrain_score = r2_score(y_train, np.concatenate([y_pred_train1, y_pred_train2], axis=0), sample_weight=w_train )\ntrain_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:21:35.782671Z","iopub.execute_input":"2024-12-28T06:21:35.783085Z","iopub.status.idle":"2024-12-28T06:21:54.897895Z","shell.execute_reply.started":"2024-12-28T06:21:35.783054Z","shell.execute_reply":"2024-12-28T06:21:54.896984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_valid = model.predict(X_valid)\nvalid_score = r2_score(y_valid, y_pred_valid, sample_weight=w_valid )\nvalid_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:22:06.209676Z","iopub.execute_input":"2024-12-28T06:22:06.210008Z","iopub.status.idle":"2024-12-28T06:22:09.989332Z","shell.execute_reply.started":"2024-12-28T06:22:06.209979Z","shell.execute_reply":"2024-12-28T06:22:09.988417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_means = { symbol_id : -1 for symbol_id in range(39) }\nfor symbol_id, gdf in train[[\"symbol_id\", CONFIG.target_col]].groupby(\"symbol_id\"):\n    y_mean = gdf[ CONFIG.target_col ].mean()\n    y_means[symbol_id] = y_mean\n    print(f\"symbol_id = {symbol_id}, y_means = {y_mean:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:22:22.285573Z","iopub.execute_input":"2024-12-28T06:22:22.285853Z","iopub.status.idle":"2024-12-28T06:22:23.014132Z","shell.execute_reply.started":"2024-12-28T06:22:22.285831Z","shell.execute_reply":"2024-12-28T06:22:23.013079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_detail = { symbol_id : 0 for symbol_id in range(39) }\nfor symbol_id, gdf in valid.groupby(\"symbol_id\"):\n    X_valid = gdf[ CONFIG.feature_cols ]\n    y_valid = gdf[ CONFIG.target_col ]\n    w_valid = gdf[ \"weight\" ]\n    y_pred_valid = model.predict(X_valid)\n    score = r2_score(y_valid, y_pred_valid, sample_weight=w_valid )\n    cv_detail[symbol_id] = score\n    \n    print(f\"symbol_id = {symbol_id}, score = {score:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:22:34.158479Z","iopub.execute_input":"2024-12-28T06:22:34.158802Z","iopub.status.idle":"2024-12-28T06:22:39.208688Z","shell.execute_reply.started":"2024-12-28T06:22:34.158767Z","shell.execute_reply":"2024-12-28T06:22:39.207780Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sids = list(cv_detail.keys())\nplt.bar(sids, [cv_detail[sid] for sid in sids])\nplt.grid()\nplt.xlabel(\"symbol_id\")\nplt.ylabel(\"CV score\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:22:48.962250Z","iopub.execute_input":"2024-12-28T06:22:48.962529Z","iopub.status.idle":"2024-12-28T06:22:49.265414Z","shell.execute_reply.started":"2024-12-28T06:22:48.962506Z","shell.execute_reply":"2024-12-28T06:22:49.264477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result = {\n    \"model\" : model,\n    \"cv\" : valid_score,\n    \"cv_detail\" : cv_detail,\n    \"y_mean\" : y_means,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:23:26.333326Z","iopub.execute_input":"2024-12-28T06:23:26.333779Z","iopub.status.idle":"2024-12-28T06:23:26.338754Z","shell.execute_reply.started":"2024-12-28T06:23:26.333740Z","shell.execute_reply":"2024-12-28T06:23:26.337881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"result.pkl\", \"wb\") as fp:\n    pickle.dump(result, fp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:23:28.653191Z","iopub.execute_input":"2024-12-28T06:23:28.653524Z","iopub.status.idle":"2024-12-28T06:23:28.703906Z","shell.execute_reply.started":"2024-12-28T06:23:28.653488Z","shell.execute_reply":"2024-12-28T06:23:28.703005Z"}},"outputs":[],"execution_count":null}]}