{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.6"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"7ff3a8e4","cell_type":"code","source":"import shap\nimport optuna\nimport joblib\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"313c6ec8","cell_type":"code","source":"class DATA:\n    train_path = \"train.parquet\"\n    test_path = \"test.parquet\"","metadata":{},"outputs":[],"execution_count":null},{"id":"66b89098","cell_type":"code","source":"train = pd.read_parquet(DATA.train_path).reset_index(drop=True)\ntrain.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"a9a888f5","cell_type":"code","source":"train.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"e5ecd286","cell_type":"code","source":"np.isinf(train).sum().sum()","metadata":{},"outputs":[],"execution_count":null},{"id":"ab76946d","cell_type":"code","source":"train = train.replace([np.inf, -np.inf], np.nan)","metadata":{},"outputs":[],"execution_count":null},{"id":"94559852","cell_type":"code","source":"train = train.fillna(train.mean())","metadata":{},"outputs":[],"execution_count":null},{"id":"b4e36560","cell_type":"code","source":"train.isna().sum().sum()","metadata":{},"outputs":[],"execution_count":null},{"id":"97bbea79","cell_type":"code","source":"train = train.dropna(axis=1, how='all')","metadata":{},"outputs":[],"execution_count":null},{"id":"d0576b84","cell_type":"code","source":"train.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"20e57fdc","cell_type":"code","source":"train = train.astype(np.float32)","metadata":{},"outputs":[],"execution_count":null},{"id":"a8a89f09","cell_type":"code","source":"train.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"aa5f9038","cell_type":"code","source":"X = train.drop(\"label\", axis=1)\ny = train[\"label\"]","metadata":{},"outputs":[],"execution_count":null},{"id":"c8c937bc","cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=24)","metadata":{},"outputs":[],"execution_count":null},{"id":"87e49974","cell_type":"code","source":"params = {\n    \"n_estimators\": 100,\n    \"learning_rate\": 0.05,\n    \"min_child_samples\": 5,\n    \"num_leaves\": 64,\n    \"min_split_gain\": 0.001,\n    \"reg_alpha\": 0.0,\n    \"reg_lambda\": 0.0\n}\n\nmodel = lgb.LGBMRegressor(**params)  \nmodel.fit(X_train, y_train)\n\nexplainer = shap.TreeExplainer(model, feature_perturbation=\"tree_path_dependent\", model_output=\"raw\")\nshap_values = explainer.shap_values(X_train)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"367a5dc6","cell_type":"code","source":"shap.summary_plot(shap_values, X_train)","metadata":{},"outputs":[],"execution_count":null},{"id":"b339e1b6","cell_type":"code","source":"selected_features = [\n    \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n    \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n    \"X888\",\"X421\",\"X333\",\n    \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n]","metadata":{},"outputs":[],"execution_count":null},{"id":"63ebe1a3","cell_type":"code","source":" FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\", \"bid_qty\",\n        \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\"X817\", \n        \"X586\",  \"X292\"\n    ]","metadata":{},"outputs":[],"execution_count":null},{"id":"ccec5aff","cell_type":"code","source":"X = X[selected_features]","metadata":{},"outputs":[],"execution_count":null},{"id":"b57acc43","cell_type":"code","source":"model = None ","metadata":{},"outputs":[],"execution_count":null},{"id":"e753ad92","cell_type":"code","source":"def objective(trial):\n    params = {\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.005, 0.1),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 6, 20),\n        \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 10),\n        \"subsample\": trial.suggest_float(\"subsample\", 0.5, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 0.9),\n        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.3, 0.9),\n        \"colsample_bynode\": trial.suggest_float(\"colsample_bynode\", 0.5, 0.9),\n        \"gamma\": trial.suggest_float(\"gamma\", 0, 2),\n        \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0, 20),\n        \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0, 20),\n        \"max_leaves\": trial.suggest_int(\"max_leaves\", 8, 64),\n        \"n_estimators\": 1500,\n        \"tree_method\": \"hist\",\n        \"random_state\": 42,\n        \"n_jobs\": -1,\n        \"eval_metric\": \"rmse\", \n        \"early_stopping_rounds\": 50,\n    }\n\n    X_train, X_valid, y_train, y_valid = train_test_split(\n        X, y, test_size=0.2, random_state=24\n    )\n\n    model = xgb.XGBRegressor(**params)\n\n    model.fit(\n    X_train, y_train,\n    eval_set=[(X_valid, y_valid)],\n    \n    verbose=100\n)\n\n\n    preds = model.predict(X_valid)\n    mse = mean_squared_error(y_valid, preds)\n    score = np.sqrt(mse)\n    return -score\n\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=50)\n\nprint(\"Best trial:\")\nprint(study.best_trial)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"a889b3e8","cell_type":"code","source":"params = {\n    'learning_rate': 0.09228887167539337,\n    'max_depth': 19,\n    'min_child_weight': 3,\n    'subsample': 0.7039412968134979,\n    'colsample_bytree': 0.8749337558444557,\n    'colsample_bylevel': 0.5018412064526978,\n    'colsample_bynode': 0.7707383740364354,\n    'gamma': 0.5942966824160839,\n    'reg_alpha': 11.320878455430973,\n    'reg_lambda': 4.9307231703527705,\n    'max_leaves': 60,\n    'n_estimators': 1500,\n    'tree_method': 'hist',\n    'random_state': 42,\n    'n_jobs': -1,\n}\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"d45e57a4","cell_type":"code","source":"XGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": 42,\n    \"n_jobs\": -1\n}\n\n#0.104","metadata":{},"outputs":[],"execution_count":null},{"id":"0f05fb70","cell_type":"code","source":"XGB_PARAMS = {\n    # Core parameters\n    'tree_method': 'hist',  # Changed from 'hist' for determinism\n    'device': 'gpu',\n    'n_jobs': -1,  # Single thread\n    \n    # Model parameters\n    'colsample_bytree': 0.4111224922845363, \n    'colsample_bynode': 0.28869302181383194,\n    'gamma': 1.4665430311056709, \n    'learning_rate': 0.014053505540364681, \n    'max_depth': 7, \n    'max_leaves': 40, \n    'n_estimators': 500,\n    'reg_alpha': 27.791606770656145, \n    'reg_lambda': 84.90603428439086,\n    'subsample': 0.06567,\n    \n    # Deterministic parameters\n    'random_state': 42,\n\n    'verbosity': 0,\n}","metadata":{},"outputs":[],"execution_count":null},{"id":"7c8d5b2d","cell_type":"code","source":"\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\noof_preds = np.zeros(len(X))\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n    X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n\n    model = xgb.XGBRegressor(\n        **XGB_PARAMS,\n        early_stopping_rounds=50\n    )\n\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        \n        verbose=0\n    )\n\n    preds = model.predict(X_val)\n    oof_preds[val_idx] = preds\n\n    fold_rmse = mean_squared_error(y_val, preds)\n    fold_rmse = np.sqrt(fold_rmse)\n\n    fold_pearson, _ = pearsonr(y_val, preds)\n\n   \n    print(f\"Fold {fold+1} | RMSE: {fold_rmse:.4f} | Pearson: {fold_pearson:.4f}\")\n\noverall_rmse = mean_squared_error(y, oof_preds)\noverall_rmse = np.sqrt(overall_rmse)\noverall_pearson, _ = pearsonr(y, oof_preds)\nprint(f\"\\nOverall CV RMSE: {overall_rmse:.4f}\")\nprint(f\"Overall CV Pearson: {overall_pearson:.4f}\")\n","metadata":{},"outputs":[],"execution_count":null},{"id":"5d7ea394","cell_type":"code","source":"\nfinal_model = xgb.XGBRegressor(\n    **XGB_PARAMS,\n)\n\nfinal_model.fit(X, y)\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"194414f4","cell_type":"code","source":"test = pd.read_parquet(DATA.test_path)\ntest.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"ef615807","cell_type":"code","source":"test.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"ca17b00c","cell_type":"code","source":"np.isinf(test).sum().sum()","metadata":{},"outputs":[],"execution_count":null},{"id":"c0f823d2","cell_type":"code","source":"test = test.replace([np.inf, -np.inf], np.nan)\ntest = test.fillna(test.mean())","metadata":{},"outputs":[],"execution_count":null},{"id":"ec7bb376","cell_type":"code","source":"test.isna().sum().sum()","metadata":{},"outputs":[],"execution_count":null},{"id":"541b6702","cell_type":"code","source":"test = test.astype(np.float32)\ntest.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"273277c5","cell_type":"code","source":"test = test[selected_features]\ntest.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"bab20753","cell_type":"code","source":"y_test_preds = final_model.predict(test)\n\n\njoblib.dump(final_model, \"final_xgb_model.pkl\")\n\n\nimport pandas as pd\nsubmission = pd.DataFrame({\n    \"ID\": test.index,  \n    \"prediction\": y_test_preds\n})\nsubmission.to_csv(\"submission2.csv\", index=False)","metadata":{},"outputs":[],"execution_count":null}]}