{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install koolbox scikit-learn==1.5.2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:15:51.728691Z","iopub.execute_input":"2025-09-23T03:15:51.728974Z","iopub.status.idle":"2025-09-23T03:16:01.481003Z","shell.execute_reply.started":"2025-09-23T03:15:51.728946Z","shell.execute_reply":"2025-09-23T03:16:01.480076Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Importing Packages","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt \nimport warnings \nimport seaborn as sns \nimport joblib \nimport optuna \nfrom koolbox import Trainer   \nfrom sklearn.model_selection import KFold \nfrom sklearn.linear_model import Ridge  \nfrom lightgbm import LGBMRegressor \nfrom xgboost import XGBRegressor  \nfrom scipy.stats import pearsonr as pr \nfrom sklearn.base import clone \nimport gc \n\nwarnings.filterwarnings('ignore')   ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:16:01.482994Z","iopub.execute_input":"2025-09-23T03:16:01.483232Z","iopub.status.idle":"2025-09-23T03:16:07.759462Z","shell.execute_reply.started":"2025-09-23T03:16:01.483206Z","shell.execute_reply":"2025-09-23T03:16:07.758906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configs","metadata":{}},{"cell_type":"code","source":"class CFG:\n    train_path='/kaggle/input/drw-crypto-market-prediction/train.parquet'\n    test_path='/kaggle/input/drw-crypto-market-prediction/test.parquet'\n    submission_path='/kaggle/input/drw-crypto-market-prediction/sample_submission.csv' \n    target = \"label\"\n    n_folds = 5\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 500","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:36:30.939251Z","iopub.execute_input":"2025-09-23T03:36:30.939632Z","iopub.status.idle":"2025-09-23T03:36:30.955671Z","shell.execute_reply.started":"2025-09-23T03:36:30.939616Z","shell.execute_reply":"2025-09-23T03:36:30.955192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data=pd.read_parquet(CFG.train_path)\ntest_data=pd.read_parquet(CFG.test_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:16:07.765021Z","iopub.execute_input":"2025-09-23T03:16:07.765250Z","iopub.status.idle":"2025-09-23T03:16:49.351909Z","shell.execute_reply.started":"2025-09-23T03:16:07.765230Z","shell.execute_reply":"2025-09-23T03:16:49.351345Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading and processing","metadata":{}},{"cell_type":"code","source":"def reduce_memory_usage(df): \n    initial_mem=df.memory_usage().sum()/1024**3\n    print(f\"starting memory:{initial_mem:.2f} GB\")   \n    for col in df.columns:  \n        col_type=df[col].dtype\n        \n        col_min=df[col].min() \n        col_max=df[col].max()\n\n        if str(col_type)[:3]=='int': \n            if col_min > np.iinfo(np.int8).min and col_max < np.iinfo(np.int8).max: \n                df[col] = df[col].astype(np.int8) \n            elif col_min > np.iinfo(np.int16).min and col_max < np.iinfo(np.int16).max: \n                df[col] = df[col].astype(np.int16) \n            elif col_min > np.iinfo(np.int32).min and col_max < np.iinfo(np.int32).max: \n                df[col] = df[col].astype(np.int32) \n            elif col_min > np.iinfo(np.int64).min and col_max < np.iinfo(np.int64).max: \n                df[col] = df[col].astype(np.int64)   \n        else :\n            if col_min > np.finfo(np.float16).min and col_max < np.finfo(np.float16).max: \n                df[col] = df[col].astype(np.float16)  \n            elif col_min > np.finfo(np.float32).min and col_max < np.finfo(np.float32).max: \n                df[col] = df[col].astype(np.float32)   \n            else : \n                df[col] = df[col].astype(np.float64)     \n    final_mem=df.memory_usage().sum()/1024**3 \n    reduced_by=100*(initial_mem - final_mem)/initial_mem\n    print(f'initial memory : {initial_mem:.2f}GB') \n    print(f'final memory : {final_mem:.2f}GB') \n    print(f'reduced memory by : {reduced_by:.2f}%')\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:16:49.352972Z","iopub.execute_input":"2025-09-23T03:16:49.353221Z","iopub.status.idle":"2025-09-23T03:16:49.360598Z","shell.execute_reply.started":"2025-09-23T03:16:49.353196Z","shell.execute_reply":"2025-09-23T03:16:49.360105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = reduce_memory_usage(train_data)\ntest_data = reduce_memory_usage(test_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:16:49.361544Z","iopub.execute_input":"2025-09-23T03:16:49.361782Z","iopub.status.idle":"2025-09-23T03:16:57.852177Z","shell.execute_reply.started":"2025-09-23T03:16:49.361759Z","shell.execute_reply":"2025-09-23T03:16:57.851351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_features(df):   \n    df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty'] \n    df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n    df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n    \n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty']+1e-10)\n    df['sell_pressure'] = df['sell_qty'] / (df['volume']+1e-10)\n    df['log_volume'] = np.log1p(df['volume'])   \n\n    df['effective_spread_proxy'] = np.abs(df['buy_qty']-df['sell_qty'])/(df['volume']+1e-10)\n    df['bid_ask_imbalance'] = (df['bid_qty']-df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] \n                                                              + 1e-10)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) /( df['buy_qty'] + df['sell_qty']\n                                                                  + 1e-10)\n    df['liquidity_ratio']=(df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-10)\n    \n    #Price Pressure indiactors\n    \n    df['net_order_flow'] = df['buy_qty']-df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] /( df['volume'] + 1e-10)\n    df['volume_weighted_buy'] =df['volume'] * df['buy_qty']\n    \n    #Liquidity Depth Measures\n    \n    df['total_depth']=df['bid_qty']+ df['ask_qty']   \n    df['depth_imbalance'] = (df['bid_qty']-df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + \n                                                                    1e-10)  \n    df['log_depth']=np.log1p(df['total_depth'])\n    \n    # Order flow Toxicity \n    \n    df['kyle_lambda']=np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity']= np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth']\n                                                                     + 1e-10)\n    # Market ACTIVITY Indicators\n    df['volume_depth_ratio'] = df['volume'] /(df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty']) \n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['bid_qty']) \n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    df['realized_spread_proxy'] = 2*np.abs(df['net_order_flow'])/(df['volume'] + 1e-10)\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['quote_volatility_proxy'] = np.abs(df['depth_imbalance'])  \n    \n    df['flow_depth_interaction'] = df['net_order_flow'] * df['total_depth']\n    df['imbalance_volume_interaction'] = df['order_flow_imbalance'] * df['volume']\n    df['depth_volume_interaction'] = df['total_depth'] * df['volume']\n    df['buy_sell_spread'] = np.abs(df['buy_qty'] - df['sell_qty'])\n    df['bid_ask_spread'] = np.abs(df['bid_qty'] - df['ask_qty'])  \n    \n    df['trade_informativeness'] = df['net_order_flow']/(df['bid_qty'] + df['ask_qty'] +1e-10) \n    df['execution_shortfall_proxy'] = df['buy_sell_spread'] / (df['volume'] + 1e-10) \n    df['adverse_selection_proxy'] = df['net_order_flow'] / (df['total_depth'] +1e-10) * df['volume'] \n    \n    df['fill_probability'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + \n                                                                      1e-10) \n    df['execution_rate'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + \n                                                               1e-10)\n    df['market_efficiency'] = df['volume'] / (df['bid_ask_spread'] + 1e-10) \n    \n    \n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['sqrt_depth'] = np.sqrt(df['total_depth'])\n    df['volume_squared'] = df['volume'] ** 2\n    df['imbalance_squared'] = df['order_flow_imbalance'] ** 2  \n   \n    df['bid_ratio'] = df['bid_qty'] / (df['total_depth'] + 1e-10)\n    df['ask_ratio'] = df['ask_qty'] / (df['total_depth'] + 1e-10)\n    df['buy_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['sell_ratio'] = df['sell_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10) \n\n\n    df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_stress'] = df['volume'] / (df['total_depth'] + 1e-10) * np.abs(df['order_flow_imbalance'])\n    df['depth_depletion'] = df['volume'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n\n\n    df['net_buying_ratio'] = df['net_order_flow']/(df['volume']+1e-10) \n    df['directional_volume'] = df['net_order_flow'] * np.log1p(df['volume']) \n    df['signed_volume'] = np.sign(df['net_order_flow']) * df['volume'] \n    \n    return df.replace([np.inf,-np.inf],0).fillna(0)   \n\ndef create_aggregated_features(df,feature_list,prefix): \n    df[f'{prefix}_sum'] = df[feature_list].sum(axis=1)\n    df[f'{prefix}_mean'] = df[feature_list].mean(axis=1)\n    df[f'{prefix}_median'] = df[feature_list].median(axis=1)\n    df[f'{prefix}_max'] = df[feature_list].max(axis=1)\n    df[f'{prefix}_min'] = df[feature_list].min(axis=1)\n    df[f'{prefix}_std'] = df[feature_list].std(axis=1)\n    return df\n\ndef extract_time_features(df): \n    df['timestamp'] = pd.to_datetime(df['timestamp']) \n    df['hour'] = df['timestamp'].dt.hour \n    dt['minute'] = df['timestamp'].dt.minute\n    dt['dayofweek'] = df['dayofweek'].dt.dayofweek\n    dt['Y_M_D_H'] = df['timestamp'].dt.strftime('%Y-%m-%d-%H') \n    dt['Y_M_D_H_M'] = df['timestamp'].dt.strftime('%Y-%m-%d-%H-%M') \n    return df  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:16:57.855009Z","iopub.execute_input":"2025-09-23T03:16:57.855251Z","iopub.status.idle":"2025-09-23T03:16:57.873270Z","shell.execute_reply.started":"2025-09-23T03:16:57.855231Z","shell.execute_reply":"2025-09-23T03:16:57.872629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = add_features(train_data)\ntest_data = add_features(test_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:16:57.873917Z","iopub.execute_input":"2025-09-23T03:16:57.874155Z","iopub.status.idle":"2025-09-23T03:17:14.438417Z","shell.execute_reply.started":"2025-09-23T03:16:57.874135Z","shell.execute_reply":"2025-09-23T03:17:14.437612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Negative_features_list = ['X563', 'X560', 'X486', 'X215', 'X7', 'X290', 'X581', 'X492',\n       'X569', 'X590', 'X201', 'X625', 'X247', 'X283', 'X261', 'X626',\n       'X193', 'X545', 'X195', 'X291', 'X703', 'X243', 'X297', 'X38',\n       'X490', 'X252', 'X42', 'X41', 'X254', 'X706', 'X237', 'X40', 'X16',\n       'X331', 'X47', 'X571', 'X302', 'X711', 'X280',\n       'activity_intensity', 'X412', 'X580', 'X54', 'X679', 'X245',\n       'X700', 'X622', 'X646', 'X223', 'X608', 'X695', 'X48', 'X319',\n       'X621', 'X125', 'X539', 'X259', 'X651', 'X648', 'X278', 'X682',\n       'X441', 'X5', 'X378', 'X565', 'X343', 'X654', 'X629', 'X14',\n       'X448', 'ask_qty', 'bid_qty', 'X95', 'X127', 'X463', 'X698',\n       'X482', 'X123', 'X46', 'X488', 'X572', 'X521', 'X30', 'X562',\n       'X421', 'X578', 'X380', 'X303', 'X171', 'X731', 'X452', 'X690',\n       'X221', 'X226', 'X456', 'X32', 'X643', 'X115', 'X676', 'X217',\n       'X602', 'X329', 'X765', 'X507', 'X722', 'X708', 'X179', 'X636',\n       'X210', 'X670', 'X767', 'X416', 'X218', 'X317', 'X204', 'X461',\n       'X250', 'X208', 'X159', 'X385', 'X198', 'X450', 'X28', 'X457',\n       'X400', 'X373', 'X272', 'X77', 'X154', 'X777', 'X256', 'X203',\n       'X20', 'X326', 'X407', 'X476', 'X274', 'X424', 'X33', 'X267',\n       'X443', 'X499', 'X113', 'X547', 'X382', 'X554', 'X583', 'X76',\n       'X639', 'X81']  \n\nPositive_features_list = ['X557', 'X566', 'X485', 'X194', 'X575', 'X584', 'X587', 'X493',\n       'X216', 'X44', 'X253', 'X627', 'X50', 'X707', 'X36', 'X6', 'X287',\n       'X262', 'X551', 'X244', 'X8', 'X286', 'X214', 'X236', 'X624',\n       'X623', 'X483', 'X337', 'X15', 'X240', 'X200', 'X289', 'X39',\n       'X699', 'X533', 'X222', 'X260', 'X325', 'X647', 'X299', 'X285',\n       'X650', 'X577', 'X284', 'X559', 'X43', 'X202', 'X702', 'X251',\n       'X704', 'X246', 'X418', 'X710', 'X683', 'X55', 'X620', 'X694',\n       'X298', 'X31', 'X292', 'X574', 'X239', 'X605', 'X288', 'X494',\n       'X121', 'X449', 'X462', 'X89', 'X696', 'X489', 'total_depth',\n       'X678', 'X165', 'X628', 'X406', 'X675', 'X568', 'X294', 'X384',\n       'X131', 'X481', 'X374', 'X119', 'X56', 'X133', 'X442', 'X277',\n       'X422', 'X655', 'X9', 'X451', 'X455', 'X209', 'X440', 'X652',\n       'X379', 'X323', 'X279', 'X611', 'X735', 'X766', 'X527', 'X205',\n       'X129', 'X644', 'X21', 'X224', 'X196', 'X642', 'X313', 'X293',\n       'X427', 'X258', 'X586', 'X225', 'X726', 'X691', 'X219', 'X335',\n       'X4', 'X635', 'X506', 'X415', 'X173', 'X458', 'X117', 'X372',\n       'X197', 'X27', 'X281', 'X672', 'X275', 'X500', 'X775', 'X469',\n       'X680', 'X599', 'X35', 'X29', 'X477', 'X401', 'X330', 'X160',\n       'X370', 'X266', 'X408', 'X640', 'X87', 'X70']         \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:17:14.439367Z","iopub.execute_input":"2025-09-23T03:17:14.439580Z","iopub.status.idle":"2025-09-23T03:17:14.448310Z","shell.execute_reply.started":"2025-09-23T03:17:14.439563Z","shell.execute_reply":"2025-09-23T03:17:14.447592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in [5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150]:\n    train_data = create_aggregated_features(train_data, Negative_features_list[:i], f\"Negative_features_{i}\")\n    test_data = create_aggregated_features(test_data, Negative_features_list[:i], f\"Negative_features_{i}\")\n    train_data = create_aggregated_features(train_data, Positive_features_list[:i], f\"Positive_features_{i}\")\n    test_data = create_aggregated_features(test_data, Positive_features_list[:i], f\"Positive_features_{i}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:17:14.449469Z","iopub.execute_input":"2025-09-23T03:17:14.449719Z","iopub.status.idle":"2025-09-23T03:22:47.905545Z","shell.execute_reply.started":"2025-09-23T03:17:14.449698Z","shell.execute_reply":"2025-09-23T03:22:47.904939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Features = ['X455', 'X557', 'X6', 'X259', 'X627', 'X450', 'X48', 'X44', 'X448', 'X291',\n            'X563', 'X218', 'X574', 'X695', 'X41', 'X418', 'X644', 'X621', 'X648', 'X451',\n            'X443', 'X643', 'Negative_features_10_sum', 'X722', 'X290', 'X200', \n            'Positive_features_10_sum', 'X458', 'X286', 'X571', 'X640', 'X559', 'X625',\n            'X222', 'X201', 'X214', 'X193', 'X639', 'X197', 'X31', 'X323', 'X456', \n            'X319', 'X278', 'X260', 'X251', 'X651', 'X5', 'X652', 'X285', 'X584', \n            'X317', 'X412', 'X647', 'Positive_features_100_std', 'X42', 'X440', 'X225',\n            'X575', 'X203', 'X683', 'X289', 'X204', 'X35', 'X562', 'X566', 'X605', \n            'X33', 'X293', 'Negative_features_5_sum', 'X219', 'X45', 'X587', 'X36',\n            'X16', 'X449', 'X569', 'X287', 'X246', 'X56', 'X726', 'X8', 'X379', 'X292',\n            'X679', 'X507', 'X406', 'Negative_features_150_std', 'X165', 'X329', 'X376', \n            'X15', 'X608', 'X626', 'X678', 'X628', 'X696', 'X457', 'X539', 'X469', \n            'X442', 'Positive_features_5_sum', 'Positive_features_60_std',\n            'Positive_features_90_std', 'X580', 'X694', 'X491', 'X258', 'X454', 'X46', \n            'X196', 'X123', 'X765', 'X370', 'X215', 'Negative_features_100_std', \n            'X331', 'X125', 'X119', 'X254', 'X279', 'X4', 'X298', 'X551', 'X768', 'X675',\n            'X226', 'X223', 'X43', 'X568', 'X444', 'X337', 'X320', \n            'Negative_features_60_std', 'X335', 'X581', 'X623', 'X294', 'X325', 'X572',\n            'X554', 'X416', 'X486', 'X602', 'X484', 'X577', 'X735', 'X288', 'X250', 'X691',\n            'X776', 'X766', 'X113', 'X611', 'X635','X666', 'X401', 'X489', 'X326',\n            'X699', 'X718', 'X636', 'X767', 'X476', 'Negative_features_70_std', 'X154',\n            'X221', 'X706', 'X506', 'X470', 'Positive_features_30_sum', 'X34', 'X21',\n            'X284', 'X38', 'X245', 'X700', 'X620', 'X415', 'X407',\n            'Positive_features_90_median', 'X32', 'X244', 'X670', 'X95', 'X343', 'X299',\n            'X129', 'X421', 'X482', 'X14', 'X668', 'X704', 'X9', 'X216', 'X77', 'X247',\n            'X155', 'Negative_features_90_std', 'Negative_features_50_sum', 'X477',\n            'X642', 'X465', 'X160', 'X373', 'X629', 'X777', 'X609', 'X724', 'X422', \n            'X672', 'X441', 'X302', 'X646', 'X527', 'X707', 'X71', 'X39', 'X194', \n            'X728', 'X410', 'X682', 'Positive_features_70_std', 'X49', 'X127', 'X198', \n            'X229', 'X579', 'X599', 'X622', 'X601', 'X461', 'fill_probability', 'X121',\n            'Positive_features_20_std', 'X676', 'Negative_features_30_sum', 'X573', \n            'X720', 'X50', 'X209', 'X690', 'X173', 'X89', 'X313', 'X439', 'X275', \n            'X171', 'X20', 'X30', 'X708', 'X237', 'X710', 'X217', 'X488', \n            'Negative_features_50_std', 'X133', 'X404', 'X281', 'X427', 'X28', 'X70', \n            'X650', 'X311', 'X731', 'X764', 'X385', 'X274', 'X51', \n            'Negative_features_150_median', 'X205', 'X283', 'X266', 'X408', 'X424', \n            'X610', 'X277', 'X261', 'X362', 'X230', 'X256', 'X280', 'X624', 'X447', \n            'X303', 'X175', 'X485', 'X462', 'X262', 'X115', 'Positive_features_50_sum', \n            'X117', 'Negative_features_30_std', 'X775', 'X565', 'X27',\n            'Negative_features_80_std', 'X377', 'X400', 'X272', 'X698', 'X578', \n            'Negative_features_60_sum', 'X159', 'X664', 'X269', 'X87', 'X680', 'X179',\n            'X508', 'X481', 'X499', 'X114', 'X545', 'X604', 'Positive_features_40_median',\n            'Positive_features_50_std', 'X267', 'X253', 'X459', 'X118', 'X547', 'X76', \n            'X464', 'Positive_features_150_std', 'X479', 'X295', 'X161', 'X192', 'X7', \n            'X316', 'Positive_features_90_max', 'X23', 'X548', 'X494', 'X500', 'X174', \n            'X521', 'X533', 'X655', 'X236', 'X24', 'ask_ratio', 'X382', 'X492', 'X380',\n            'X368', 'X29', 'X242', 'X148', 'X463', 'X238', 'X662', 'X598', 'X149', 'X124',\n            'X81', 'Negative_features_70_sum', 'X606', 'Negative_features_40_sum', \n            'X243', 'X723', 'X169', 'X314', 'X157', 'X84', 'X692', \n            'Negative_features_5_mean', 'X402', 'X328', 'X612', 'X613', 'X252', 'X769', \n            'Positive_features_5_mean', 'X210', 'X322', 'X235', 'X334', 'X271', 'X656',\n            'X409', 'Positive_features_5_std', 'X395', 'X255', 'X213', 'X107', 'X498', \n            'X446', 'X338', 'X530', 'X600', 'X112', 'X151', 'X583', 'X228', 'X78', 'X384',\n            'X372', 'X139', 'Negative_features_80_sum', 'X371', 'sell_ratio', 'X211', \n            'X120', 'X341', 'net_order_flow', 'X550', 'X90', 'Positive_features_40_std', \n            'X300', 'X667', 'X344', 'X40', 'X65', 'X497', 'X413', 'X490', 'X483', \n            'Negative_features_40_median', 'X212', 'X398', 'X73', 'X231', 'buy_qty', \n            'X734', 'Positive_features_30_std', 'Positive_features_80_max', 'X475', \n            'X163', 'X386', 'X542', 'X472', 'Negative_features_90_sum', 'X619', 'X282', \n            'activity_intensity', 'X576', 'X336', 'X241', 'X560', 'X375', 'X417', 'X333',\n            'X330', 'Positive_features_50_median', 'X590', 'Negative_features_10_mean',\n            'X240', 'Negative_features_40_std']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:22:47.906387Z","iopub.execute_input":"2025-09-23T03:22:47.907106Z","iopub.status.idle":"2025-09-23T03:22:47.918856Z","shell.execute_reply.started":"2025-09-23T03:22:47.907078Z","shell.execute_reply":"2025-09-23T03:22:47.918187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train_data[Features] \ny = train_data[CFG.target]\nX_test = test_data[Features]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:22:47.919761Z","iopub.execute_input":"2025-09-23T03:22:47.920037Z","iopub.status.idle":"2025-09-23T03:22:49.612008Z","shell.execute_reply.started":"2025-09-23T03:22:47.920015Z","shell.execute_reply":"2025-09-23T03:22:49.611336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pearsonr(y_true, y_pred):\n    return pr(y_true, y_pred)[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T04:48:44.127579Z","iopub.execute_input":"2025-09-23T04:48:44.128316Z","iopub.status.idle":"2025-09-23T04:48:44.131501Z","shell.execute_reply.started":"2025-09-23T04:48:44.128289Z","shell.execute_reply":"2025-09-23T04:48:44.130936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_params = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bytree\": 0.5625888953382505,\n    \"learning_rate\": 0.029312951475451557,\n    \"min_child_samples\": 63,\n    \"min_child_weight\": 0.11456572852335424,\n    \"n_estimators\": 126,\n    \"n_jobs\": -1,\n    \"num_leaves\": 37,\n    \"random_state\": 42,\n    \"reg_alpha\": 85.2476527854083,\n    \"reg_lambda\": 99.38305361388907, \n    \"subsample\" : 0.450669817684892,\n    \"verbose\" : -1\n} \n\nlgbm_goss_params ={\n    \"boosting_type\":\"goss\",\n    'colsample_bytree': 0.34695458228489784,\n    'learning_rate':0.031023014900595287, \n    'min_child_samples':30, \n    'min_child_weight':0.4727729225033618,\n    'n_estimators':220, \n    \"n_jobs\":-1, \n    \"num_leaves\":58, \n    \"random_state\":42, \n    \"reg_alpha\":38.665994901468224,\n    \"reg_lambda\":92.76991677464294, \n    \"subsample\":0.4810891284493255,\n    \"verbose\":-1\n} \n\nxgb_params={\n    'colsample_bylevel':0.4778015829774066,\n    \"colsample_bynode\": 0.362764358742407,\n    \"colsample_bytree\": 0.7107423488010493, \n    \"gamma\": 1.7094857725240398,\n    \"learning_rate\": 0.02213323588455387,\n    \"max_depth\":20, \n    \"max_leaves\":12, \n    \"min_child_weight\":16,\n    \"n_estimators\":200, \n    \"n_jobs\":-1, \n    \"random_state\":42, \n    \"reg_alpha\":39.352415706891264,\n    \"reg_lambda\":75.44843704068275,\n    \"subsample\": 0.06566669853471274,\n    \"verbosity\":0\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:22:49.617346Z","iopub.execute_input":"2025-09-23T03:22:49.617585Z","iopub.status.idle":"2025-09-23T03:22:49.637652Z","shell.execute_reply.started":"2025-09-23T03:22:49.617569Z","shell.execute_reply":"2025-09-23T03:22:49.637029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fold_scores = {}\noverall_scores = {}\n\noof_preds = {}\ntest_preds = {}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:22:49.638504Z","iopub.execute_input":"2025-09-23T03:22:49.639184Z","iopub.status.idle":"2025-09-23T03:22:49.664427Z","shell.execute_reply.started":"2025-09-23T03:22:49.639162Z","shell.execute_reply":"2025-09-23T03:22:49.663926Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training Base Models","metadata":{}},{"cell_type":"markdown","source":"# LightGBM (gbdt)","metadata":{}},{"cell_type":"code","source":"lgbm_trainer = Trainer(\n    LGBMRegressor(**lgbm_params),\n    cv=KFold(n_splits=5, shuffle=False),\n    metric=pearsonr,\n    task=\"regression\",\n    metric_precision=6\n)\n\nlgbm_trainer.fit(X, y)\n\nfold_scores['LightGBM (gbdt)'] = lgbm_trainer.fold_scores  \noverall_scores['LightGBM (gbdt)'] = [pearsonr(lgbm_trainer.oof_preds,y)] \noof_preds['LightGBM (gbdt)'] = lgbm_trainer.oof_preds \ntest_preds['LightGBM (gbdt)'] = lgbm_trainer.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:22:49.665062Z","iopub.execute_input":"2025-09-23T03:22:49.665278Z","iopub.status.idle":"2025-09-23T03:26:23.487213Z","shell.execute_reply.started":"2025-09-23T03:22:49.665263Z","shell.execute_reply":"2025-09-23T03:26:23.486637Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# LightGBM (goss)","metadata":{}},{"cell_type":"code","source":"lgbm_goss_trainer=Trainer(\n    LGBMRegressor(**lgbm_goss_params),\n    cv=KFold(n_splits=5,shuffle=False), \n    metric=pearsonr,\n    task='regression',\n    metric_precision=6\n) \nlgbm_goss_trainer.fit(X,y) \n\nfold_scores['LightGBM (goss)'] = lgbm_goss_trainer.fold_scores \noverall_scores['LightGBM (goss)'] =[pearsonr(lgbm_goss_trainer.oof_preds,y)] \noof_preds['LightGBM (goss)'] = lgbm_goss_trainer.oof_preds \ntest_preds['LightGBM (goss)'] = lgbm_goss_trainer.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:26:23.487668Z","iopub.execute_input":"2025-09-23T03:26:23.487835Z","iopub.status.idle":"2025-09-23T03:30:29.248571Z","shell.execute_reply.started":"2025-09-23T03:26:23.487819Z","shell.execute_reply":"2025-09-23T03:30:29.248010Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"xgb_trainer=Trainer(\n    XGBRegressor(**xgb_params), \n    cv=KFold(n_splits=5,shuffle=False),\n    metric=pearsonr,\n    task='regression',\n    metric_precision=6\n) \nxgb_trainer.fit(X,y) \n\nfold_scores['XGBoost'] = xgb_trainer.fold_scores \noverall_scores['XGBoost'] = [pearsonr(xgb_trainer.oof_preds,y)] \noof_preds['XGBoost'] = xgb_trainer.oof_preds \ntest_preds['XGBoost'] = xgb_trainer.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:30:29.249054Z","iopub.execute_input":"2025-09-23T03:30:29.249229Z","iopub.status.idle":"2025-09-23T03:36:30.835156Z","shell.execute_reply.started":"2025-09-23T03:30:29.249213Z","shell.execute_reply":"2025-09-23T03:36:30.834595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X=pd.DataFrame(oof_preds)  \nX_test=pd.DataFrame(test_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T03:36:30.835913Z","iopub.execute_input":"2025-09-23T03:36:30.836213Z","iopub.status.idle":"2025-09-23T03:36:30.929564Z","shell.execute_reply.started":"2025-09-23T03:36:30.836195Z","shell.execute_reply":"2025-09-23T03:36:30.929002Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensembling With Ridge","metadata":{}},{"cell_type":"code","source":"def plot_weights(weights,title): \n    sorted_indices=np.argsort(weights[0])[::-1]\n    sorted_coeffs=np.array(weights[0])[sorted_indices]\n    sorted_model_names=np.array(list(oof_preds.keys()))[sorted_indices] \n    \n    plt.figure(figsize=(10,weights.shape[1]*0.5))\n    ax=sns.barplot(x=sorted_coeffs,y=sorted_model_names,palette='RdYlGn_r') \n    \n    \n    for i,(value,name) in enumerate(zip(sorted_coeffs,sorted_model_names)): \n        if value>=0: \n            ax.text(value,i,f'{value:.3f}',va='center',ha='left',color='black')\n        else:\n            ax.text(value,i,f'{value:.3f}',va='center',ha='right',color='black') \n        xlim=ax.get_xlim() \n        ax.set_xlim(xlim[0]-0.1*abs(xlim[0]),xlim[1]+0.1*abs(xlim[1]))\n    \n    plt.title(title)\n    plt.xlabel(\"\") \n    plt.ylabel(\"\") \n    plt.tight_layout() \n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T05:25:51.944141Z","iopub.execute_input":"2025-09-23T05:25:51.944821Z","iopub.status.idle":"2025-09-23T05:25:51.950964Z","shell.execute_reply.started":"2025-09-23T05:25:51.944795Z","shell.execute_reply":"2025-09-23T05:25:51.950223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial):\n    params={\n        'random_state':CFG.seed,\n        'alpha': trial.suggest_float('alpha',0,1), \n        'tol': trial.suggest_float('tol',1e-6,1e-2), \n        'fit_intercept' : trial.suggest_categorical('fit_intercept',[True,False]), \n        'positive':trial.suggest_categorical('positive',[True,False])\n    } \n    trainer=Trainer(\n        Ridge(**params),\n        cv=KFold(n_splits=5,shuffle=False), \n        metric=pearsonr,\n        task='regression',\n        verbose=False\n    ) \n    trainer.fit(X,y)\n    \n    return pearsonr(trainer.oof_preds,y) \n\nif CFG.run_optuna: \n    sampler = optuna.samplers.TPESampler(seed=CFG.seed,multivariate = True,n_startup_trials=CFG.n_optuna_trials//10) \n    study = optuna.create_study(direction='maximize',sampler=sampler) \n    study.optimize(objective,n_trials=CFG.n_optuna_trials,n_jobs=-1) \n    best_params=study.best_params  \n    ridge_params={\n        'random_state':CFG.seed, \n        'alpha':best_params[\"alpha\"],\n        'tol':best_params['tol'], \n        'fit_intercept':best_params['fit_intercept'], \n        'positive':best_params['positive']\n    } \nelse:\n    ridge_params={\n        random_state:CFG.seed\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T04:56:40.915393Z","iopub.execute_input":"2025-09-23T04:56:40.915657Z","iopub.status.idle":"2025-09-23T05:02:01.465737Z","shell.execute_reply.started":"2025-09-23T04:56:40.915637Z","shell.execute_reply":"2025-09-23T05:02:01.465088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ridge_trainer=Trainer(\n    Ridge(**ridge_params),\n    cv=KFold(n_splits=5,shuffle=False),\n    metric=pearsonr,\n    task='regression', \n    metric_precision=6,\n)   \nridge_trainer.fit(X,y) \n\nfold_scores['Ridge (ensemble)'] = ridge_trainer.fold_scores \noverall_scores['Ridge (ensemble)'] = [pearsonr(ridge_trainer.oof_preds,y)] \nridge_test_preds = ridge_trainer.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T05:11:53.324833Z","iopub.execute_input":"2025-09-23T05:11:53.325520Z","iopub.status.idle":"2025-09-23T05:11:54.078259Z","shell.execute_reply.started":"2025-09-23T05:11:53.325495Z","shell.execute_reply":"2025-09-23T05:11:54.077526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ridge_coeffs = np.zeros((1,X.shape[1])) \nfor m in ridge_trainer.estimators:\n    ridge_coeffs +=m.coef_ \nridge_coeffs = ridge_coeffs / len(ridge_trainer.estimators)     \n\nplot_weights(ridge_coeffs,\"Ridge Coefficients\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T05:25:57.306828Z","iopub.execute_input":"2025-09-23T05:25:57.307455Z","iopub.status.idle":"2025-09-23T05:25:57.630150Z","shell.execute_reply.started":"2025-09-23T05:25:57.307430Z","shell.execute_reply":"2025-09-23T05:25:57.629527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv(CFG.submission_path) \nsub['prediction']=ridge_test_preds  \nsub.to_csv(f'submission.csv',index=False)\nsub.head() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T05:30:12.892153Z","iopub.execute_input":"2025-09-23T05:30:12.892421Z","iopub.status.idle":"2025-09-23T05:30:14.128504Z","shell.execute_reply.started":"2025-09-23T05:30:12.892401Z","shell.execute_reply":"2025-09-23T05:30:14.127871Z"}},"outputs":[],"execution_count":null}]}