{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nimport lightgbm as lgb\nfrom typing import List\nimport gc\n\nimport os\nimport shutil\nimport glob\n\nimport matplotlib.pyplot as plt \nimport seaborn as sns\n\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\nfrom wandb.integration.lightgbm import _WandbCallback, log_summary","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:03:10.441235Z","iopub.execute_input":"2024-11-13T11:03:10.441635Z","iopub.status.idle":"2024-11-13T11:03:16.098255Z","shell.execute_reply.started":"2024-11-13T11:03:10.441575Z","shell.execute_reply":"2024-11-13T11:03:16.097415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"config = {\n    'model': 'LGBM',\n    'dataset': '',\n    'save_code': False,\n    'note': 'Walk Forward Validation - Baseline'\n}\n\nuser_secrets = UserSecretsClient()\nwandb_key = user_secrets.get_secret(\"wandb_key\")\n\nwandb.login(key=wandb_key)\nrun = wandb.init(project=\"JSRTMDF\", job_type='Model training', config=config)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:03:16.100055Z","iopub.execute_input":"2024-11-13T11:03:16.100767Z","iopub.status.idle":"2024-11-13T11:03:21.410215Z","shell.execute_reply.started":"2024-11-13T11:03:16.100714Z","shell.execute_reply":"2024-11-13T11:03:21.409225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FILES = [\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8',\n    '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9'\n\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:03:21.411657Z","iopub.execute_input":"2024-11-13T11:03:21.411954Z","iopub.status.idle":"2024-11-13T11:03:22.512165Z","shell.execute_reply.started":"2024-11-13T11:03:21.411922Z","shell.execute_reply":"2024-11-13T11:03:22.511193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WalkForwardingValidation():\n    def __init__(self, files: List[str], x_column:List[str], y_column: str, lgb_params: dict):\n        self.files = files\n        self.x_column = x_column\n        self.y_column = y_column\n        self.lgb_params = lgb_params\n\n    def _create_folder(self, folder_name)->str:\n        folder_path = f'/kaggle/working/model/{folder_name}'\n        # Check if the folder exists\n        if os.path.exists(folder_path):\n            # If it exists, delete all contents\n            shutil.rmtree(folder_path)\n        # Create the folder (it will be empty)\n        os.makedirs(folder_path)\n\n        return folder_path\n\n    def _calculate_r2(self, y_true, y_pred, weights):\n        \"\"\"\n        Calculate the sample weighted zero-mean R-squared score (R2).\n    \n        Parameters:\n        - y_true (pd.Series or np.array): Ground truth values.\n        - y_pred (pd.Series or np.array): Predicted values.\n        - weights (pd.Series or np.array): Sample weights.\n    \n        Returns:\n        - float: R2 score.\n        \"\"\"\n        numerator = np.sum(weights * (y_true - y_pred) ** 2)\n        denominator = np.sum(weights * (y_true ** 2))\n        r2_score = 1 - (numerator / denominator)\n        return r2_score\n        \n\n    def run(self) -> List[dict]:\n        n_files = len(self.files)\n\n        wfv_results = []\n        r2_score = []\n        for i in range(0, n_files - 1):\n            print(f\"Initializing file {self.files[i]}\")\n            folder_path = self._create_folder(f'booster_{i}')\n            \n            df_train = pd.read_parquet(self.files[i])\n            df_valid = pd.read_parquet(self.files[i+1])\n\n            ds_train = lgb.Dataset(\n                data = df_train[self.x_column],\n                label = df_train[self.y_column]\n            )\n\n            ds_valid = lgb.Dataset(\n                data = df_valid[self.x_column],\n                label = df_valid[self.y_column]\n            )\n\n            del df_train\n            gc.collect()\n\n            #avoid override params in every fold\n            log_params = True if i == 0 else False\n\n            evals_result = {}\n\n            booster = lgb.train(\n                params = self.lgb_params,\n                train_set = ds_train,\n                valid_sets = [ds_valid],\n                callbacks = [\n                    lgb.log_evaluation(20), \n                    lgb.early_stopping(20), \n                    _WandbCallback(log_params = log_params)\n                ]\n            )\n\n            wfv_results.append(booster.best_score['valid_0']['rmse'])\n\n            booster.save_model(folder_path+\"/model.txt\", num_iteration=booster.best_iteration)\n\n            \"\"\"\n            Calculate R2 score\n            \"\"\"\n            y_valid_pred = booster.predict(df_valid[self.x_column])\n            r2_score.append(\n                self._calculate_r2(df_valid[self.y_column], y_valid_pred, df_valid['weight'])\n            )\n\n        return wfv_results, r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:05:54.161792Z","iopub.execute_input":"2024-11-13T11:05:54.162254Z","iopub.status.idle":"2024-11-13T11:05:54.679289Z","shell.execute_reply.started":"2024-11-13T11:05:54.162216Z","shell.execute_reply":"2024-11-13T11:05:54.678533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_params = {   \n    'boosting_type': 'gbdt',\n    'objective': 'regression_l2',\n    'metric': 'rmse',\n    \"max_depth\": 8,\n    'max_bin': 224,\n    'verbose': -1,\n    'learning_rate': 0.02,\n    \"n_estimators\": 7000,  \n    \"extra_trees\":True,\n    'num_leaves':1600,\n    'min_data_in_leaf': 4400,\n    'lambda_l1': 10,\n    'lambda_l2':50,\n    'min_gain_to_split': 2,\n    'bagging_fraction': 0.2,\n    'feature_fraction': 0.8999999999999999,\n    'random_state': 42,\n    'device': 'gpu'\n}\n\ntemp = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0')\nfeature_columns = temp.drop(columns=['responder_0', 'responder_1','responder_2','responder_3','responder_4','responder_5',\n                                    'responder_6','responder_7','responder_8']).columns.to_list()\ntarget_column = 'responder_6'\ndel temp \ngc.collect()\n\n\nwfv = WalkForwardingValidation(\n    FILES,\n    feature_columns,\n    target_column,\n    lgbm_params\n)\n\nresult, r2_score = wfv.run()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:05:54.851567Z","iopub.execute_input":"2024-11-13T11:05:54.852412Z","iopub.status.idle":"2024-11-13T11:06:03.232058Z","shell.execute_reply.started":"2024-11-13T11:05:54.852372Z","shell.execute_reply":"2024-11-13T11:06:03.231091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Walk Forwarding Validation RMSE: {np.mean(result)}')\nprint(f'Walk Forwarding Validation R2: {np.mean(r2_score)}')\n\nwandb.log({'WFV RMSE': np.mean(result)})\nwandb.log({'WFV R2': np.mean(r2_score)})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:07:17.971765Z","iopub.execute_input":"2024-11-13T11:07:17.972681Z","iopub.status.idle":"2024-11-13T11:07:18.627977Z","shell.execute_reply.started":"2024-11-13T11:07:17.972638Z","shell.execute_reply":"2024-11-13T11:07:18.626799Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"code","source":"def get_feature_importance()-> pd.DataFrame:\n    feature_importance_array = np.empty((0,3))\n    i = 0\n    for filename in glob.iglob('/kaggle/working/model/*/*.txt', recursive=True):\n    \n        booster = lgb.Booster(model_file=filename)\n        \n        feature_name = booster.feature_name()\n        feature_importance = booster.feature_importance()\n    \n        temp = np.hstack((\n            np.full((len(feature_name),1), fill_value=i),\n            np.array(feature_name).reshape((-1,1)),\n            np.array(feature_importance).reshape((-1,1))\n        ))\n        \n        feature_importance_array = np.vstack((\n            feature_importance_array,\n            temp\n        ))\n    \n        i+=1\n    \n    feature_importance_df = pd.DataFrame({\n        'timestep':feature_importance_array[:,0],\n        'feature_name':feature_importance_array[:,1],\n        'feature_importance':feature_importance_array[:,2]\n    })\n\n    feature_importance_df['timestep'] = feature_importance_df['timestep'].astype(int)\n    feature_importance_df['feature_importance'] = feature_importance_df['feature_importance'].astype(int)\n    \n\n    return feature_importance_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:07:20.963763Z","iopub.execute_input":"2024-11-13T11:07:20.964133Z","iopub.status.idle":"2024-11-13T11:07:21.377393Z","shell.execute_reply.started":"2024-11-13T11:07:20.964096Z","shell.execute_reply":"2024-11-13T11:07:21.376671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_importance_df = get_feature_importance()\ngrouped_feature_importance_df = feature_importance_df.groupby('feature_name').agg(\n    median_feature_importance=pd.NamedAgg(column=\"feature_importance\", aggfunc=\"median\"),\n    std_feature_importance=pd.NamedAgg(column=\"feature_importance\", aggfunc=\"std\"),\n    min_feature_importance=pd.NamedAgg(column=\"feature_importance\", aggfunc=\"min\"),\n    max_feature_importance=pd.NamedAgg(column=\"feature_importance\", aggfunc=\"max\")\n).reset_index().sort_values(by=['min_feature_importance','median_feature_importance','std_feature_importance'], ascending=[False, False,True]).head(100)\n\n#Save csv for further analysis\nfeature_importance_df.to_csv('feature_importance.csv', index=False)\ngrouped_feature_importance_df.to_csv('grouped_feature_importance.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:07:22.392219Z","iopub.execute_input":"2024-11-13T11:07:22.392955Z","iopub.status.idle":"2024-11-13T11:07:22.873628Z","shell.execute_reply.started":"2024-11-13T11:07:22.392912Z","shell.execute_reply":"2024-11-13T11:07:22.872867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,20))\nsns.boxplot(\n    data=feature_importance_df.loc[feature_importance_df['feature_name'].isin(grouped_feature_importance_df['feature_name'].to_list())],\n    x='feature_importance',\n    y='feature_name',\n    order=grouped_feature_importance_df['feature_name'].to_list(),\n    ax=ax,\n    palette='flare'\n)\nax.set_title('Distribution of top 100 features over all time')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:07:23.958460Z","iopub.execute_input":"2024-11-13T11:07:23.959185Z","iopub.status.idle":"2024-11-13T11:07:26.045264Z","shell.execute_reply.started":"2024-11-13T11:07:23.959142Z","shell.execute_reply":"2024-11-13T11:07:26.044495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wandb.finish()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:08:16.081130Z","iopub.execute_input":"2024-11-13T11:08:16.081519Z","iopub.status.idle":"2024-11-13T11:08:19.688981Z","shell.execute_reply.started":"2024-11-13T11:08:16.081481Z","shell.execute_reply":"2024-11-13T11:08:19.688162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}