{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DRW LGBMRegressor w/Pearson","metadata":{"papermill":{"duration":0.009232,"end_time":"2023-04-04T16:08:25.219577","exception":false,"start_time":"2023-04-04T16:08:25.210345","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:05:13.423726Z","iopub.execute_input":"2025-05-26T11:05:13.423984Z","iopub.status.idle":"2025-05-26T11:05:18.848778Z","shell.execute_reply.started":"2025-05-26T11:05:13.42396Z","shell.execute_reply":"2025-05-26T11:05:18.847642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport optuna\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport lightgbm as lgb\nfrom sklearn.preprocessing import scale\nimport yaml","metadata":{"papermill":{"duration":13.400642,"end_time":"2023-04-04T16:08:38.628644","exception":false,"start_time":"2023-04-04T16:08:25.228002","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:05:18.851576Z","iopub.execute_input":"2025-05-26T11:05:18.851972Z","iopub.status.idle":"2025-05-26T11:05:45.099998Z","shell.execute_reply.started":"2025-05-26T11:05:18.851921Z","shell.execute_reply":"2025-05-26T11:05:45.098828Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Processed Data","metadata":{"papermill":{"duration":0.007927,"end_time":"2023-04-04T16:08:38.645324","exception":false,"start_time":"2023-04-04T16:08:38.637397","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train=pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\nTEST=pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:05:45.101053Z","iopub.execute_input":"2025-05-26T11:05:45.101986Z","iopub.status.idle":"2025-05-26T11:06:43.397475Z","shell.execute_reply.started":"2025-05-26T11:05:45.101942Z","shell.execute_reply":"2025-05-26T11:06:43.393599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N = list(range(len(train)))\nrandom.shuffle(N) \ntrain=train.iloc[N[0:10000]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:43.402253Z","iopub.execute_input":"2025-05-26T11:06:43.403297Z","iopub.status.idle":"2025-05-26T11:06:49.132059Z","shell.execute_reply.started":"2025-05-26T11:06:43.403184Z","shell.execute_reply":"2025-05-26T11:06:49.127388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/drw-label-distribution","metadata":{}},{"cell_type":"code","source":"def func(x):\n    return np.sign(x) * np.log1p(np.abs(x))\n    \ndef inverse_func(y):\n    return np.sign(y) * (np.expm1(np.abs(y)))  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:49.135994Z","iopub.execute_input":"2025-05-26T11:06:49.136433Z","iopub.status.idle":"2025-05-26T11:06:49.15493Z","shell.execute_reply.started":"2025-05-26T11:06:49.136406Z","shell.execute_reply":"2025-05-26T11:06:49.150468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pearson_metric(y_true, y_pred):\n    pearson_r, _ = pearsonr(y_true, y_pred)\n    return 'pearson', pearson_r, True","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target='label'\ntrainX = train.drop('label',axis=1)\ntrainY = train['label'].apply(func)\ntestX = TEST.drop('label',axis=1)\ncolumns=trainX.columns.tolist()","metadata":{"papermill":{"duration":0.020777,"end_time":"2023-04-04T16:08:38.860315","exception":false,"start_time":"2023-04-04T16:08:38.839538","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:06:49.156852Z","iopub.execute_input":"2025-05-26T11:06:49.15817Z","iopub.status.idle":"2025-05-26T11:06:53.912139Z","shell.execute_reply.started":"2025-05-26T11:06:49.158094Z","shell.execute_reply":"2025-05-26T11:06:53.910988Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainX = np.where(trainX == -np.inf, -10000, trainX)\ntestX = np.where(testX == -np.inf, -10000, testX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:53.918584Z","iopub.execute_input":"2025-05-26T11:06:53.91911Z","iopub.status.idle":"2025-05-26T11:06:59.452367Z","shell.execute_reply.started":"2025-05-26T11:06:53.919067Z","shell.execute_reply":"2025-05-26T11:06:59.451403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ntrainX = scaler.fit_transform(trainX)\ntestX = scaler.transform(testX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:06:59.453539Z","iopub.execute_input":"2025-05-26T11:06:59.453916Z","iopub.status.idle":"2025-05-26T11:07:03.101192Z","shell.execute_reply.started":"2025-05-26T11:06:59.453884Z","shell.execute_reply":"2025-05-26T11:07:03.099999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df=pd.DataFrame(data=trainX,columns=columns)\ntest_df=pd.DataFrame(data=testX,columns=columns)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.019227,"end_time":"2023-04-04T16:08:38.934219","exception":false,"start_time":"2023-04-04T16:08:38.914992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.102752Z","iopub.execute_input":"2025-05-26T11:07:03.103668Z","iopub.status.idle":"2025-05-26T11:07:03.110703Z","shell.execute_reply.started":"2025-05-26T11:07:03.103631Z","shell.execute_reply":"2025-05-26T11:07:03.109382Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit=pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:03.112203Z","iopub.execute_input":"2025-05-26T11:07:03.112515Z","iopub.status.idle":"2025-05-26T11:07:03.497448Z","shell.execute_reply.started":"2025-05-26T11:07:03.11249Z","shell.execute_reply":"2025-05-26T11:07:03.496587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.018903,"end_time":"2023-04-04T16:08:39.086493","exception":false,"start_time":"2023-04-04T16:08:39.06759","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.498634Z","iopub.execute_input":"2025-05-26T11:07:03.499009Z","iopub.status.idle":"2025-05-26T11:07:03.523724Z","shell.execute_reply.started":"2025-05-26T11:07:03.498983Z","shell.execute_reply":"2025-05-26T11:07:03.522579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.022983,"end_time":"2023-04-04T16:08:39.118671","exception":false,"start_time":"2023-04-04T16:08:39.095688","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.524974Z","iopub.execute_input":"2025-05-26T11:07:03.525364Z","iopub.status.idle":"2025-05-26T11:07:03.54777Z","shell.execute_reply.started":"2025-05-26T11:07:03.52533Z","shell.execute_reply":"2025-05-26T11:07:03.546424Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.021902,"end_time":"2023-04-04T16:08:39.150722","exception":false,"start_time":"2023-04-04T16:08:39.12882","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.549368Z","iopub.execute_input":"2025-05-26T11:07:03.549703Z","iopub.status.idle":"2025-05-26T11:07:03.579543Z","shell.execute_reply.started":"2025-05-26T11:07:03.549648Z","shell.execute_reply":"2025-05-26T11:07:03.578188Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\ntest_feat_df = to_feature(test_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.037519,"end_time":"2023-04-04T16:08:39.197304","exception":false,"start_time":"2023-04-04T16:08:39.159785","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:03.581159Z","iopub.execute_input":"2025-05-26T11:07:03.58147Z","iopub.status.idle":"2025-05-26T11:07:10.051358Z","shell.execute_reply.started":"2025-05-26T11:07:03.581448Z","shell.execute_reply":"2025-05-26T11:07:10.050139Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.009289,"end_time":"2023-04-04T16:08:39.216375","exception":false,"start_time":"2023-04-04T16:08:39.207086","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from scipy.stats import pearsonr\nimport gc\n\n# Create validation set once outside the objective function\ntrain_x, test_x, train_y, test_y = train_test_split(trainX, trainY, test_size=0.2, random_state=42)\n\ndef objective(trial):\n    param = {\n        'lambda_l1': trial.suggest_float('lambda_l1', 0.001, 0.002),\n        'lambda_l2': trial.suggest_float('lambda_l2', 6, 12),\n        \"bagging_freq\": trial.suggest_int('bagging_freq', 3, 5),\n        \"bagging_fraction\": trial.suggest_float('bagging_fraction', 0.56, 0.58),\n        \"feature_fraction\": trial.suggest_float('feature_fraction', 0.75, 0.76),\n        'learning_rate': trial.suggest_float('learning_rate', 0.04, 0.12),\n        \"verbosity\": trial.suggest_int('verbosity', 6, 8),  \n        \"max_depth\": trial.suggest_int('max_depth', 3, 6),  \n\n        # Memory optimizations\n        'n_jobs': 1,  # Use only one core\n        'device': 'cpu',  # Ensure using CPU\n        \n        # Fixed parameters\n        'random_state': 64, \n        'num_leaves': 64,        \n        'objective': 'rmse', \n        'reg_lambda': 1.,\n        'reg_alpha': .1,\n        'n_estimators': 10, \n        'colsample_bytree': .5, \n        'min_child_samples': 10,\n        'subsample_freq': 3,\n        'subsample': .9,\n        'importance_type': 'gain', \n    }\n\n    def pearson_metric(y_true, y_pred):\n        pearson_r, _ = pearsonr(y_true, y_pred)\n        return 'pearson', pearson_r, True\n    \n    model = lgb.LGBMRegressor(**param)\n    \n    try:\n        model.fit(train_x, train_y, \n                 eval_set=[(test_x, test_y)],\n                 eval_metric=pearson_metric)\n        \n        preds = model.predict(test_x)\n        pearson_score, _ = pearsonr(test_y, preds)\n        \n        # Clean up\n        del model, preds\n        gc.collect()\n        \n        return pearson_score\n    except Exception as e:\n        # If training fails, return a bad score\n        del model\n        gc.collect()\n        return -1  # Or use float('-inf') depending on your optuna setup","metadata":{"papermill":{"duration":0.025357,"end_time":"2023-04-04T16:08:39.280702","exception":false,"start_time":"2023-04-04T16:08:39.255345","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:10.052881Z","iopub.execute_input":"2025-05-26T11:07:10.053735Z","iopub.status.idle":"2025-05-26T11:07:10.06706Z","shell.execute_reply.started":"2025-05-26T11:07:10.053666Z","shell.execute_reply":"2025-05-26T11:07:10.065807Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='maximize')\nstudy.optimize(objective, n_trials=100)\nBest_trial=study.best_trial.params\nprint(Best_trial)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:10.068865Z","iopub.execute_input":"2025-05-26T11:07:10.069249Z","iopub.status.idle":"2025-05-26T11:07:11.686928Z","shell.execute_reply.started":"2025-05-26T11:07:10.06922Z","shell.execute_reply":"2025-05-26T11:07:11.685788Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport optuna.visualization.matplotlib as vis_matplotlib\nimport matplotlib.pyplot as plt\n\ndisplay(vis_matplotlib.plot_optimization_history(study))\ndisplay(vis_matplotlib.plot_slice(study))\ndisplay(vis_matplotlib.plot_param_importances(study))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:11.689008Z","iopub.execute_input":"2025-05-26T11:07:11.689628Z","iopub.status.idle":"2025-05-26T11:07:15.342705Z","shell.execute_reply.started":"2025-05-26T11:07:11.689593Z","shell.execute_reply":"2025-05-26T11:07:15.341737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(trainX.shape)\nprint(trainY.shape)\nprint(testX.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:15.349949Z","iopub.execute_input":"2025-05-26T11:07:15.350301Z","iopub.status.idle":"2025-05-26T11:07:15.378669Z","shell.execute_reply.started":"2025-05-26T11:07:15.350269Z","shell.execute_reply":"2025-05-26T11:07:15.377384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Best_trial=study.best_trial.params\n\nfix_dict =  {        \n    'random_state': 64, \n    'num_leaves': 64,        \n    'objective': 'rmse', \n    'reg_lambda': 1.,\n    'reg_alpha': .1,\n    'n_estimators': 1000, \n    'colsample_bytree': .5, \n    'min_child_samples': 10,\n    'subsample_freq': 3,\n    'subsample': .9,\n    'importance_type': 'gain', \n}\n\nBest_trial.update(fix_dict)","metadata":{"papermill":{"duration":0.259343,"end_time":"2023-04-04T16:10:39.412607","exception":false,"start_time":"2023-04-04T16:10:39.153264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:15.379959Z","iopub.execute_input":"2025-05-26T11:07:15.380321Z","iopub.status.idle":"2025-05-26T11:07:15.404374Z","shell.execute_reply.started":"2025-05-26T11:07:15.380296Z","shell.execute_reply":"2025-05-26T11:07:15.40343Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.DataFrame(data=trainX,columns=columns)\ntest = pd.DataFrame(data=testX,columns=columns)\ntarget = pd.DataFrame(data=trainY)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T11:07:15.405723Z","iopub.execute_input":"2025-05-26T11:07:15.406164Z","iopub.status.idle":"2025-05-26T11:07:15.436061Z","shell.execute_reply.started":"2025-05-26T11:07:15.406138Z","shell.execute_reply":"2025-05-26T11:07:15.434769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    oof = np.zeros(len(train)) #######################\n    oof[val_idx]=val_preds #######################\n    np.save('oof.npy',oof) #######################\n    \n","metadata":{}},{"cell_type":"code","source":"# Initialize array for predictions\noof = np.zeros(len(train)) #######################\npreds = np.zeros((submit.shape[0]))\nkf = KFold(n_splits=5, random_state=48, shuffle=True)\n\nfor i, (trn_idx, test_idx) in enumerate(kf.split(train[columns], target)):\n    print(f\"\\nFold {i+1}\")\n    \n    # Get data for current fold\n    X_tr = train[columns].iloc[trn_idx]\n    X_val = train[columns].iloc[test_idx]\n    y_tr = target.iloc[trn_idx]\n    y_val = target.iloc[test_idx]\n    \n    # Configure model with memory-efficient settings\n    model = lgb.LGBMRegressor(\n        **Best_trial,\n        n_jobs=1,  # Reduce parallel processing\n        device='cpu'  # Ensure using CPU\n    )\n    \n    # Train with early stopping if possible\n    model.fit(\n        X_tr, y_tr,\n        eval_set=[(X_val, y_val)],\n        eval_metric='rmse',\n        callbacks=[lgb.early_stopping(50, verbose=False)]  # Add early stopping\n    )\n    \n    # Make predictions and update ensemble\n    fold_preds = model.predict(test[columns])\n    preds += fold_preds / kf.n_splits\n    \n    # Calculate validation metrics\n    val_preds = model.predict(X_val)\n    oof[test_idx]=val_preds #######################\n    \n    pearson_score, _ = pearsonr(y_val, val_preds)\n    print(pearson_score)\n    \n    # Clean up memory\n    del X_tr, X_val, y_tr, y_val, model, fold_preds, val_preds\n    gc.collect()","metadata":{"_kg_hide-output":true,"papermill":{"duration":1.417213,"end_time":"2023-04-04T16:10:41.584587","exception":false,"start_time":"2023-04-04T16:10:40.167374","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:07:15.440401Z","iopub.execute_input":"2025-05-26T11:07:15.440723Z","iopub.status.idle":"2025-05-26T11:08:46.761845Z","shell.execute_reply.started":"2025-05-26T11:07:15.440678Z","shell.execute_reply":"2025-05-26T11:08:46.760299Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.save('oof.npy',oof) #######################","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_y_pred = inverse_func(preds)\n\nsubmit.iloc[:,1]= TEST_y_pred\nsubmit.to_csv('submission.csv',index=False)\ndisplay(submit)","metadata":{"papermill":{"duration":0.301116,"end_time":"2023-04-04T16:10:42.68397","exception":false,"start_time":"2023-04-04T16:10:42.382854","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-05-26T11:08:46.76247Z","iopub.status.idle":"2025-05-26T11:08:46.762789Z","shell.execute_reply.started":"2025-05-26T11:08:46.762628Z","shell.execute_reply":"2025-05-26T11:08:46.762641Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}