{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DRW XGBRegressor w/Pearson","metadata":{"papermill":{"duration":0.009232,"end_time":"2023-04-04T16:08:25.219577","exception":false,"start_time":"2023-04-04T16:08:25.210345","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:30:18.882006Z","iopub.execute_input":"2025-05-29T13:30:18.884006Z","iopub.status.idle":"2025-05-29T13:30:23.821065Z","shell.execute_reply.started":"2025-05-29T13:30:18.88393Z","shell.execute_reply":"2025-05-29T13:30:23.819578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport optuna\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport xgboost as xgb\nfrom sklearn.preprocessing import scale\nimport yaml","metadata":{"papermill":{"duration":13.400642,"end_time":"2023-04-04T16:08:38.628644","exception":false,"start_time":"2023-04-04T16:08:25.228002","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-29T13:30:23.824197Z","iopub.execute_input":"2025-05-29T13:30:23.82464Z","iopub.status.idle":"2025-05-29T13:30:23.837532Z","shell.execute_reply.started":"2025-05-29T13:30:23.824602Z","shell.execute_reply":"2025-05-29T13:30:23.836274Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Processed Data","metadata":{"papermill":{"duration":0.007927,"end_time":"2023-04-04T16:08:38.645324","exception":false,"start_time":"2023-04-04T16:08:38.637397","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train=pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\nTEST=pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:30:23.838423Z","iopub.execute_input":"2025-05-29T13:30:23.838831Z","iopub.status.idle":"2025-05-29T13:31:13.063971Z","shell.execute_reply.started":"2025-05-29T13:30:23.838772Z","shell.execute_reply":"2025-05-29T13:31:13.061855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N = list(range(len(train)))\nrandom.shuffle(N) \ntrain=train.iloc[N[0:10000]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:13.065972Z","iopub.execute_input":"2025-05-29T13:31:13.06631Z","iopub.status.idle":"2025-05-29T13:31:13.888137Z","shell.execute_reply.started":"2025-05-29T13:31:13.066281Z","shell.execute_reply":"2025-05-29T13:31:13.887023Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/drw-label-distribution","metadata":{}},{"cell_type":"code","source":"def func(x):\n    return np.sign(x) * np.log1p(np.abs(x))\n    \ndef inverse_func(y):\n    return np.sign(y) * (np.expm1(np.abs(y)))    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:13.89102Z","iopub.execute_input":"2025-05-29T13:31:13.891448Z","iopub.status.idle":"2025-05-29T13:31:13.898746Z","shell.execute_reply.started":"2025-05-29T13:31:13.891418Z","shell.execute_reply":"2025-05-29T13:31:13.897188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target='label'\ntrainX = train.drop('label',axis=1)\ntrainY = train['label'].apply(func)\ntestX = TEST.drop('label',axis=1)\ncolumns=trainX.columns.tolist()","metadata":{"papermill":{"duration":0.020777,"end_time":"2023-04-04T16:08:38.860315","exception":false,"start_time":"2023-04-04T16:08:38.839538","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:13.900565Z","iopub.execute_input":"2025-05-29T13:31:13.901025Z","iopub.status.idle":"2025-05-29T13:31:15.307854Z","shell.execute_reply.started":"2025-05-29T13:31:13.900983Z","shell.execute_reply":"2025-05-29T13:31:15.30654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainY.min(),trainY.max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:15.308989Z","iopub.execute_input":"2025-05-29T13:31:15.309283Z","iopub.status.idle":"2025-05-29T13:31:15.32107Z","shell.execute_reply.started":"2025-05-29T13:31:15.309261Z","shell.execute_reply":"2025-05-29T13:31:15.320006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainX = np.where(trainX == -np.inf, -10000, trainX)\ntestX = np.where(testX == -np.inf, -10000, testX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:15.322363Z","iopub.execute_input":"2025-05-29T13:31:15.322839Z","iopub.status.idle":"2025-05-29T13:31:21.089559Z","shell.execute_reply.started":"2025-05-29T13:31:15.322779Z","shell.execute_reply":"2025-05-29T13:31:21.088354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ntrainX = scaler.fit_transform(trainX)\ntestX = scaler.transform(testX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:21.090638Z","iopub.execute_input":"2025-05-29T13:31:21.090958Z","iopub.status.idle":"2025-05-29T13:31:24.461505Z","shell.execute_reply.started":"2025-05-29T13:31:21.090933Z","shell.execute_reply":"2025-05-29T13:31:24.46026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df=pd.DataFrame(data=trainX,columns=columns)\ntest_df=pd.DataFrame(data=testX,columns=columns)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.019227,"end_time":"2023-04-04T16:08:38.934219","exception":false,"start_time":"2023-04-04T16:08:38.914992","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:24.462739Z","iopub.execute_input":"2025-05-29T13:31:24.463086Z","iopub.status.idle":"2025-05-29T13:31:24.472052Z","shell.execute_reply.started":"2025-05-29T13:31:24.463059Z","shell.execute_reply":"2025-05-29T13:31:24.470904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit=pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:24.473408Z","iopub.execute_input":"2025-05-29T13:31:24.473758Z","iopub.status.idle":"2025-05-29T13:31:24.850533Z","shell.execute_reply.started":"2025-05-29T13:31:24.473733Z","shell.execute_reply":"2025-05-29T13:31:24.849149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.018903,"end_time":"2023-04-04T16:08:39.086493","exception":false,"start_time":"2023-04-04T16:08:39.06759","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:24.852112Z","iopub.execute_input":"2025-05-29T13:31:24.852401Z","iopub.status.idle":"2025-05-29T13:31:24.860031Z","shell.execute_reply.started":"2025-05-29T13:31:24.852379Z","shell.execute_reply":"2025-05-29T13:31:24.857668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.022983,"end_time":"2023-04-04T16:08:39.118671","exception":false,"start_time":"2023-04-04T16:08:39.095688","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:24.861248Z","iopub.execute_input":"2025-05-29T13:31:24.861582Z","iopub.status.idle":"2025-05-29T13:31:24.887077Z","shell.execute_reply.started":"2025-05-29T13:31:24.861557Z","shell.execute_reply":"2025-05-29T13:31:24.885623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.021902,"end_time":"2023-04-04T16:08:39.150722","exception":false,"start_time":"2023-04-04T16:08:39.12882","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:24.891553Z","iopub.execute_input":"2025-05-29T13:31:24.892076Z","iopub.status.idle":"2025-05-29T13:31:24.915189Z","shell.execute_reply.started":"2025-05-29T13:31:24.892044Z","shell.execute_reply":"2025-05-29T13:31:24.913887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\ntest_feat_df = to_feature(test_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.037519,"end_time":"2023-04-04T16:08:39.197304","exception":false,"start_time":"2023-04-04T16:08:39.159785","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T13:31:24.916675Z","iopub.execute_input":"2025-05-29T13:31:24.917083Z","execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.009289,"end_time":"2023-04-04T16:08:39.216375","exception":false,"start_time":"2023-04-04T16:08:39.207086","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from scipy.stats import pearsonr\nimport gc\n\ndef pearson_metric(y_true, y_pred):\n    pearson_r, _ = pearsonr(y_true, y_pred)\n    return pearson_r, True\n    \n# Create validation set once outside the objective function\ntrain_x, test_x, train_y, test_y = train_test_split(trainX, trainY, test_size=0.2, random_state=42)","metadata":{"papermill":{"duration":0.025357,"end_time":"2023-04-04T16:08:39.280702","exception":false,"start_time":"2023-04-04T16:08:39.255345","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial):\n    params =   {\n      'random_state'          : trial.suggest_int('random_state', 50, 70), \n      'colsample_bytree'      : trial.suggest_float('colsample_bytree', 0.2, 0.4),\n      'learning_rate'         : trial.suggest_float('learning_rate', 0.1, 0.18),\n      'max_depth'             : trial.suggest_int( 'max_depth', 2, 10), \n                   \n      'reg_alpha'             : trial.suggest_float('reg_alpha', 0.02, 0.12),\n      'reg_lambda'            : trial.suggest_float('reg_lambda', 0.6, 0.8),\n      'min_child_weight'      : trial.suggest_int('min_child_weight', 20, 30),     \n\n      #fixed part    \n      'n_estimators'          : 100, \n      'objective'             : 'reg:squarederror',\n      'eval_metric'           : \"rmse\",\n      'verbosity'             : 0,\n    }\n    \n    model = xgb.XGBRegressor(**params)\n    model.fit(train_x, train_y, \n             eval_set=[(test_x, test_y)],\n             )\n    \n    preds = model.predict(test_x)\n    pearson_score,_ = pearson_metric(test_y, preds)\n    \n    return pearson_score\n","metadata":{"papermill":{"duration":0.025357,"end_time":"2023-04-04T16:08:39.280702","exception":false,"start_time":"2023-04-04T16:08:39.255345","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=10)\nBest_trial=study.best_trial.params\nprint(Best_trial)","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport optuna.visualization.matplotlib as vis_matplotlib\nimport matplotlib.pyplot as plt\n\ndisplay(vis_matplotlib.plot_optimization_history(study))\ndisplay(vis_matplotlib.plot_slice(study))\ndisplay(vis_matplotlib.plot_param_importances(study))","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(trainX.shape)\nprint(trainY.shape)\nprint(testX.shape)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Best_trial=study.best_trial.params\n\nfix_dict =  {        \n      #fixed part       \n      'n_estimators'          : 100, \n      'objective'             : 'reg:squarederror',\n      'eval_metric'           : \"rmse\",\n      'verbosity'             : 0,\n}\n\nBest_trial.update(fix_dict)","metadata":{"papermill":{"duration":0.259343,"end_time":"2023-04-04T16:10:39.412607","exception":false,"start_time":"2023-04-04T16:10:39.153264","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.DataFrame(data=trainX,columns=columns)\ntest = pd.DataFrame(data=testX,columns=columns)\ntarget = pd.DataFrame(data=trainY)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.811Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    oof = np.zeros(len(train)) #######################\n    oof[val_idx]=val_preds #######################\n    np.save('oof.npy',oof) #######################\n","metadata":{}},{"cell_type":"code","source":"# Initialize array for predictions\noof = np.zeros(len(train)) #######################\npreds = np.zeros((submit.shape[0]))\nkf = KFold(n_splits=5, random_state=48, shuffle=True)\n\nfor i, (trn_idx, test_idx) in enumerate(kf.split(train[columns], target)):\n    print(f\"\\nFold {i+1}\")\n    \n    # Get data for current fold\n    X_tr = train[columns].iloc[trn_idx]\n    X_val = train[columns].iloc[test_idx]\n    y_tr = target.iloc[trn_idx]\n    y_val = target.iloc[test_idx]\n    \n    # Configure model with memory-efficient settings\n    model = xgb.XGBRegressor(\n        **Best_trial,\n        n_jobs=1,  # Reduce parallel processing\n        device='cpu'  # Ensure using CPU\n    )\n    \n    # Train with early stopping if possible\n    model.fit(\n        X_tr, y_tr,\n        eval_set=[(X_val, y_val)],\n    )\n    val_preds = model.predict(X_val)\n    oof[test_idx]=val_preds #######################\n    pearson_score, _ = pearsonr(y_val, val_preds)\n    print(pearson_score)\n    \n    # Make predictions and update ensemble\n    fold_preds = model.predict(test[columns])\n    preds += fold_preds / kf.n_splits\n    ","metadata":{"_kg_hide-output":true,"papermill":{"duration":1.417213,"end_time":"2023-04-04T16:10:41.584587","exception":false,"start_time":"2023-04-04T16:10:40.167374","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.save('oof.npy',oof) #######################","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_y_pred = inverse_func(preds)\n\nsubmit.iloc[:,1]= TEST_y_pred\nsubmit.to_csv('submission.csv',index=False)\ndisplay(submit)","metadata":{"papermill":{"duration":0.301116,"end_time":"2023-04-04T16:10:42.68397","exception":false,"start_time":"2023-04-04T16:10:42.382854","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-29T13:31:31.812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}