{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DRW CatBoostRegressor w/Pearson ","metadata":{"papermill":{"duration":0.009232,"end_time":"2023-04-04T16:08:25.219577","exception":false,"start_time":"2023-04-04T16:08:25.210345","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:05:13.423726Z","iopub.execute_input":"2025-05-26T11:05:13.423984Z","iopub.status.idle":"2025-05-26T11:05:18.848778Z","shell.execute_reply.started":"2025-05-26T11:05:13.42396Z","shell.execute_reply":"2025-05-26T11:05:18.847642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport optuna\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, Pool\nfrom sklearn.preprocessing import scale\nimport yaml","metadata":{"papermill":{"duration":13.400642,"end_time":"2023-04-04T16:08:38.628644","exception":false,"start_time":"2023-04-04T16:08:25.228002","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:05:18.851576Z","iopub.execute_input":"2025-05-26T11:05:18.851972Z","iopub.status.idle":"2025-05-26T11:05:45.099998Z","shell.execute_reply.started":"2025-05-26T11:05:18.851921Z","shell.execute_reply":"2025-05-26T11:05:45.098828Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Processed Data","metadata":{"papermill":{"duration":0.007927,"end_time":"2023-04-04T16:08:38.645324","exception":false,"start_time":"2023-04-04T16:08:38.637397","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train=pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\nTEST=pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:05:45.101053Z","iopub.execute_input":"2025-05-26T11:05:45.101986Z","iopub.status.idle":"2025-05-26T11:06:43.397475Z","shell.execute_reply.started":"2025-05-26T11:05:45.101942Z","shell.execute_reply":"2025-05-26T11:06:43.393599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N = list(range(len(train)))\nrandom.shuffle(N) \ntrain=train.iloc[N[0:10000]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:43.402253Z","iopub.execute_input":"2025-05-26T11:06:43.403297Z","iopub.status.idle":"2025-05-26T11:06:49.132059Z","shell.execute_reply.started":"2025-05-26T11:06:43.403184Z","shell.execute_reply":"2025-05-26T11:06:49.127388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/drw-label-distribution","metadata":{}},{"cell_type":"code","source":"def func(x):\n    return np.sign(x) * np.log1p(np.abs(x))\n    \ndef inverse_func(y):\n    return np.sign(y) * (np.expm1(np.abs(y)))  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:49.135994Z","iopub.execute_input":"2025-05-26T11:06:49.136433Z","iopub.status.idle":"2025-05-26T11:06:49.15493Z","shell.execute_reply.started":"2025-05-26T11:06:49.136406Z","shell.execute_reply":"2025-05-26T11:06:49.150468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pearson_metric(y_true, y_pred):\n    pearson_r, _ = pearsonr(y_true, y_pred)\n    return 'pearson', pearson_r, True","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target='label'\ntrainX = train.drop('label',axis=1)\ntrainY = train['label'].apply(func)\ntestX = TEST.drop('label',axis=1)\ncolumns=trainX.columns.tolist()\n","metadata":{"papermill":{"duration":0.020777,"end_time":"2023-04-04T16:08:38.860315","exception":false,"start_time":"2023-04-04T16:08:38.839538","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:06:49.156852Z","iopub.execute_input":"2025-05-26T11:06:49.15817Z","iopub.status.idle":"2025-05-26T11:06:53.912139Z","shell.execute_reply.started":"2025-05-26T11:06:49.158094Z","shell.execute_reply":"2025-05-26T11:06:53.910988Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainX = np.where(trainX == -np.inf, -10000, trainX)\ntestX = np.where(testX == -np.inf, -10000, testX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:53.918584Z","iopub.execute_input":"2025-05-26T11:06:53.91911Z","iopub.status.idle":"2025-05-26T11:06:59.452367Z","shell.execute_reply.started":"2025-05-26T11:06:53.919067Z","shell.execute_reply":"2025-05-26T11:06:59.451403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ntrainX = scaler.fit_transform(trainX)\ntestX = scaler.transform(testX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:59.453539Z","iopub.execute_input":"2025-05-26T11:06:59.453916Z","iopub.status.idle":"2025-05-26T11:07:03.101192Z","shell.execute_reply.started":"2025-05-26T11:06:59.453884Z","shell.execute_reply":"2025-05-26T11:07:03.099999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainX=pd.DataFrame(data=trainX,columns=columns)\ntestX=pd.DataFrame(data=testX,columns=columns)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.019227,"end_time":"2023-04-04T16:08:38.934219","exception":false,"start_time":"2023-04-04T16:08:38.914992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.102752Z","iopub.execute_input":"2025-05-26T11:07:03.103668Z","iopub.status.idle":"2025-05-26T11:07:03.110703Z","shell.execute_reply.started":"2025-05-26T11:07:03.103631Z","shell.execute_reply":"2025-05-26T11:07:03.109382Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df=trainX\ntest_df=testX","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit=pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\nprint(submit.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:03.112203Z","iopub.execute_input":"2025-05-26T11:07:03.112515Z","iopub.status.idle":"2025-05-26T11:07:03.497448Z","shell.execute_reply.started":"2025-05-26T11:07:03.11249Z","shell.execute_reply":"2025-05-26T11:07:03.496587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.018903,"end_time":"2023-04-04T16:08:39.086493","exception":false,"start_time":"2023-04-04T16:08:39.06759","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.498634Z","iopub.execute_input":"2025-05-26T11:07:03.499009Z","iopub.status.idle":"2025-05-26T11:07:03.523724Z","shell.execute_reply.started":"2025-05-26T11:07:03.498983Z","shell.execute_reply":"2025-05-26T11:07:03.522579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.022983,"end_time":"2023-04-04T16:08:39.118671","exception":false,"start_time":"2023-04-04T16:08:39.095688","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.524974Z","iopub.execute_input":"2025-05-26T11:07:03.525364Z","iopub.status.idle":"2025-05-26T11:07:03.54777Z","shell.execute_reply.started":"2025-05-26T11:07:03.52533Z","shell.execute_reply":"2025-05-26T11:07:03.546424Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.021902,"end_time":"2023-04-04T16:08:39.150722","exception":false,"start_time":"2023-04-04T16:08:39.12882","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.549368Z","iopub.execute_input":"2025-05-26T11:07:03.549703Z","iopub.status.idle":"2025-05-26T11:07:03.579543Z","shell.execute_reply.started":"2025-05-26T11:07:03.549648Z","shell.execute_reply":"2025-05-26T11:07:03.578188Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\ntest_feat_df = to_feature(test_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.037519,"end_time":"2023-04-04T16:08:39.197304","exception":false,"start_time":"2023-04-04T16:08:39.159785","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.581159Z","iopub.execute_input":"2025-05-26T11:07:03.58147Z","iopub.status.idle":"2025-05-26T11:07:10.051358Z","shell.execute_reply.started":"2025-05-26T11:07:03.581448Z","shell.execute_reply":"2025-05-26T11:07:10.050139Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.009289,"end_time":"2023-04-04T16:08:39.216375","exception":false,"start_time":"2023-04-04T16:08:39.207086","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from scipy.stats import pearsonr\nimport gc\n\ndef objective(trial, data=trainX, target=trainY):\n    # 1. Use more memory-efficient split\n    train_x, valid_x, train_y, valid_y = train_test_split(\n        data, target, \n        test_size=0.2,\n        random_state=42\n    )\n    \n    # 2. Suggest parameters with tighter ranges where possible\n    param = {\n        \"iterations\": trial.suggest_int(\"iterations\", 500, 1500),  # Reduced upper bound\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.2, log=True),  # Reduced upper bound\n        \"depth\": trial.suggest_int(\"depth\", 4, 8),  # Reduced upper bound\n        \"l2_leaf_reg\": trial.suggest_float(\"l2_leaf_reg\", 0.1, 5.0, log=True),  # Tighter range\n        \n        # 3. Use more conservative parameter ranges\n        \"random_strength\": trial.suggest_float(\"random_strength\", 1e-5, 5.0, log=True),\n        \"bagging_temperature\": trial.suggest_float(\"bagging_temperature\", 0.1, 0.9),\n        \"border_count\": trial.suggest_int(\"border_count\", 64, 192),  # Reduced range\n        \"od_wait\": trial.suggest_int(\"od_wait\", 20, 40),  # Reduced range\n\n        # Fixed parameters\n        \"bootstrap_type\": \"Bayesian\",\n        \"od_type\": \"Iter\",\n        \"random_seed\": 42,\n        \"verbose\": False,  # Changed from 0 to False for clarity\n        \"allow_writing_files\": False  # Prevents temporary file writing\n    }\n\n    # 4. Create pools with less memory overhead\n    train_pool = Pool(train_x, train_y, has_header=False)\n    valid_pool = Pool(valid_x, valid_y, has_header=False)\n    \n    # 5. Configure model with memory-saving options\n    model = CatBoostRegressor(**param)\n    \n    try:\n        model.fit(\n            train_pool,\n            eval_set=valid_pool,\n            early_stopping_rounds=30,  # Reduced from 50\n            verbose=0,\n            use_best_model=True\n        )\n        \n        # 6. Predict in batches if dataset is large\n        preds = model.predict(valid_x)\n        pearson_score, _ = pearsonr(valid_y, preds)\n        \n        # 7. Explicit cleanup\n        del train_pool, valid_pool, model\n        gc.collect()\n        \n        return pearson_score\n        \n    except Exception as e:\n        # Ensure resources are cleaned up even if error occurs\n        del train_pool, valid_pool, model\n        gc.collect()\n        return float('-inf')  # Return worst possible score","metadata":{"papermill":{"duration":0.025357,"end_time":"2023-04-04T16:08:39.280702","exception":false,"start_time":"2023-04-04T16:08:39.255345","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:10.052881Z","iopub.execute_input":"2025-05-26T11:07:10.053735Z","iopub.status.idle":"2025-05-26T11:07:10.06706Z","shell.execute_reply.started":"2025-05-26T11:07:10.053666Z","shell.execute_reply":"2025-05-26T11:07:10.065807Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=100)\nBest_trial=study.best_trial.params\nprint(Best_trial)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:10.068865Z","iopub.execute_input":"2025-05-26T11:07:10.069249Z","iopub.status.idle":"2025-05-26T11:07:11.686928Z","shell.execute_reply.started":"2025-05-26T11:07:10.06922Z","shell.execute_reply":"2025-05-26T11:07:11.685788Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport optuna.visualization.matplotlib as vis_matplotlib\nimport matplotlib.pyplot as plt\n\ndisplay(vis_matplotlib.plot_optimization_history(study))\ndisplay(vis_matplotlib.plot_slice(study))\ndisplay(vis_matplotlib.plot_param_importances(study))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:11.689008Z","iopub.execute_input":"2025-05-26T11:07:11.689628Z","iopub.status.idle":"2025-05-26T11:07:15.342705Z","shell.execute_reply.started":"2025-05-26T11:07:11.689593Z","shell.execute_reply":"2025-05-26T11:07:15.341737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(trainX.shape)\nprint(trainY.shape)\nprint(testX.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:15.349949Z","iopub.execute_input":"2025-05-26T11:07:15.350301Z","iopub.status.idle":"2025-05-26T11:07:15.378669Z","shell.execute_reply.started":"2025-05-26T11:07:15.350269Z","shell.execute_reply":"2025-05-26T11:07:15.377384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Best_trial=study.best_trial.params\n\nfix_dict =  {        \n    #fixed part\n    \"bootstrap_type\": \"Bayesian\",\n    \"od_type\": \"Iter\",\n    \"random_seed\": 42,\n    \"verbose\": 0\n}\n\nBest_trial.update(fix_dict)","metadata":{"papermill":{"duration":0.259343,"end_time":"2023-04-04T16:10:39.412607","exception":false,"start_time":"2023-04-04T16:10:39.153264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:15.379959Z","iopub.execute_input":"2025-05-26T11:07:15.380321Z","iopub.status.idle":"2025-05-26T11:07:15.404374Z","shell.execute_reply.started":"2025-05-26T11:07:15.380296Z","shell.execute_reply":"2025-05-26T11:07:15.40343Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.DataFrame(data=trainX,columns=columns)\ntest = pd.DataFrame(data=testX,columns=columns)\ntarget = pd.DataFrame(data=trainY)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:15.405723Z","iopub.execute_input":"2025-05-26T11:07:15.406164Z","iopub.status.idle":"2025-05-26T11:07:15.436061Z","shell.execute_reply.started":"2025-05-26T11:07:15.406138Z","shell.execute_reply":"2025-05-26T11:07:15.434769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    oof = np.zeros(len(train)) #######################\n    oof[val_idx]=val_preds #######################\n    np.save('oof.npy',oof) #######################","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\noof = np.zeros(len(train)) #######################\n# Initialize predictions array more efficiently\npreds = np.zeros(submit.shape[0], dtype=np.float32)  # Use float32 to save memory\n\n# Configure KFold with potentially less memory overhead\nkf = KFold(n_splits=5, random_state=48, shuffle=True)\n\nfor i, (trn_idx, test_idx) in enumerate(kf.split(train[columns], target)):\n    # 1. Use memory-efficient data splitting\n    X_tr = train[columns].iloc[trn_idx].copy()  # Explicit copy to avoid memory leaks\n    X_val = train[columns].iloc[test_idx].copy()\n    y_tr = target.iloc[trn_idx].copy()\n    y_val = target.iloc[test_idx].copy()\n    \n    # 2. Configure model with memory-saving parameters\n    model = CatBoostRegressor(\n        **Best_trial,\n        allow_writing_files=False,  # Prevent temporary file writing\n        thread_count=2  # Limit threads to control memory (adjust based on your system)\n    )\n    \n    # 3. Train with early stopping and memory monitoring\n    try:\n        model.fit(\n            X_tr, y_tr,\n            eval_set=[(X_val, y_val)],\n            early_stopping_rounds=20,  # Helps prevent overfitting and long runs\n            use_best_model=True\n        )\n        \n        # 4. Make predictions in memory-efficient way\n        fold_preds = model.predict(test[columns])\n        preds += fold_preds / kf.n_splits\n        \n        # 5. Calculate and report metrics\n        val_preds = model.predict(X_val)\n        oof[test_idx]=val_preds #######################\n        pearson_score, _ = pearsonr(y_val, val_preds)\n        print(pearson_score)\n        \n        # 6. Explicit memory cleanup\n        del X_tr, X_val, y_tr, y_val, val_preds, fold_preds\n        gc.collect()\n        \n    except Exception as e:\n        print(f\"Error in fold {i+1}: {str(e)}\")\n        del X_tr, X_val, y_tr, y_val\n        gc.collect()\n        continue\n\n# Final cleanup\ngc.collect()","metadata":{"_kg_hide-output":true,"papermill":{"duration":1.417213,"end_time":"2023-04-04T16:10:41.584587","exception":false,"start_time":"2023-04-04T16:10:40.167374","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:15.440401Z","iopub.execute_input":"2025-05-26T11:07:15.440723Z","iopub.status.idle":"2025-05-26T11:08:46.761845Z","shell.execute_reply.started":"2025-05-26T11:07:15.440678Z","shell.execute_reply":"2025-05-26T11:08:46.760299Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.save('oof.npy',oof) #######################","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_y_pred = inverse_func(preds)\n\nsubmit.iloc[:,1]= TEST_y_pred\nsubmit.to_csv('submission.csv',index=False)\ndisplay(submit)","metadata":{"papermill":{"duration":0.301116,"end_time":"2023-04-04T16:10:42.68397","exception":false,"start_time":"2023-04-04T16:10:42.382854","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:08:46.76247Z","iopub.status.idle":"2025-05-26T11:08:46.762789Z","shell.execute_reply.started":"2025-05-26T11:08:46.762628Z","shell.execute_reply":"2025-05-26T11:08:46.762641Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}