{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9801075,"sourceType":"datasetVersion","datasetId":6006872},{"sourceId":9806342,"sourceType":"datasetVersion","datasetId":6010899},{"sourceId":143336629,"sourceType":"kernelVersion"},{"sourceId":203900450,"sourceType":"kernelVersion"},{"sourceId":171905,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":146319,"modelId":168862}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv, pd.read_parquet )\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n\nimport os, gc\nfrom tqdm.auto import tqdm\nimport pickle # module to serialize and deserialize objects\nimport re # for Regular expression operations \n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.optimizers import Adam\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data  import Dataset, DataLoader\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\n\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import VotingRegressor\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor\n\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-17T06:29:39.971242Z","iopub.execute_input":"2024-12-17T06:29:39.97213Z","iopub.status.idle":"2024-12-17T06:29:58.683493Z","shell.execute_reply.started":"2024-12-17T06:29:39.972092Z","shell.execute_reply":"2024-12-17T06:29:58.682762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \n\n# Load a data from each file:\nr = range(2)\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pd.read_parquet(file_path)\n    samples.append(part)\n    \nsample_df = pd.concat(samples, ignore_index=True) # Concatenate all samples into one DataFrame if needed\n\nsample_df.round(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T06:00:02.338354Z","iopub.execute_input":"2024-12-17T06:00:02.338776Z","iopub.status.idle":"2024-12-17T06:00:06.688122Z","shell.execute_reply.started":"2024-12-17T06:00:02.338709Z","shell.execute_reply":"2024-12-17T06:00:06.687182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ENSEMBLE_SOLUTIONS = ['SOLUTION_14','SOLUTION_5']\nOPTION,__WTS = 'option 91',[0.899, 0.28]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T06:00:06.689187Z","iopub.execute_input":"2024-12-17T06:00:06.689536Z","iopub.status.idle":"2024-12-17T06:00:06.695448Z","shell.execute_reply.started":"2024-12-17T06:00:06.689499Z","shell.execute_reply":"2024-12-17T06:00:06.694153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if 'SOLUTION_14' in ENSEMBLE_SOLUTIONS:    \n    \n    class CONFIG:\n        seed = 42\n        target_col = \"responder_6\"\n        # feature_cols = [\"symbol_id\", \"time_id\"] + [f\"feature_{idx:02d}\" for idx in range(79)]+ [f\"responder_{idx}_lag_1\" for idx in range(9)]\n        feature_cols = [f\"feature_{idx:02d}\" for idx in range(79)]+ [f\"responder_{idx}_lag_1\" for idx in range(9)]\n\n        model_paths = [\n            #\"/kaggle/input/js24-train-gbdt-model-with-lags-singlemodel/result.pkl\",\n            #\"/kaggle/input/js24-trained-gbdt-model/result.pkl\",\n            \"/kaggle/input/js-xs-nn-trained-model\",\n            \"/kaggle/input/js-with-lags-trained-xgb/result.pkl\",\n        ]\n\nif 'SOLUTION_14' in ENSEMBLE_SOLUTIONS:\n    \n    valid = pl.scan_parquet(\n        f\"/kaggle/input/js24-preprocessing-create-lags/validation.parquet/\"\n    ).collect().to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T06:32:28.864317Z","iopub.execute_input":"2024-12-17T06:32:28.864695Z","iopub.status.idle":"2024-12-17T06:32:30.949496Z","shell.execute_reply.started":"2024-12-17T06:32:28.864663Z","shell.execute_reply":"2024-12-17T06:32:30.948773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport statsmodels.api as sm\nimport joblib\n\n# データの読み込み (Jane Streetコンペ用の例)\n# 実際のパスやファイル名は環境に合わせて変更してください\ntrain_data = pd.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet')\n\n# 特徴量カラムを定義\n# 例として 'feature_'で始まるカラムを全て特徴量とする\nfeature_cols = [col for col in sample_df.columns if col.startswith('feature_')]\n\n# 目的変数を定義\n# ここでは単純にrespを目的変数としてみる\ny_train = train_data['responder_6']\n\n# 特徴量データを抽出\nX_train = train_data[feature_cols]\n\n# 欠損値処理 (必要に応じて方法を変更)\nX_train = X_train.fillna(0)\n\n# statsmodelsでOLSモデルを作成\n# OLSは定数項を明示的に加える必要がある\nX_train_const = sm.add_constant(X_train)\n\n# モデルの学習\nmodel = sm.OLS(y_train, X_train_const).fit()\n\n# モデルの結果を表示\nprint(model.summary())\n\n# 学習済みモデルの保存\n# joblib.dump(model, 'stats_model_results.pkl')\n# print(\"Model saved successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T06:33:06.953413Z","iopub.execute_input":"2024-12-17T06:33:06.954303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nfrom IPython.display import display\n\n# ここでは statsmodels の学習済みモデルを仮に読み込んでいるとする\n# 例：\n# import joblib\n# stats_model_results = joblib.load('statsmodel_ols_results.pkl') \n# CONFIG.feature_cols = [... 予測に使う特徴量のリスト ...]\n\n# ENSEMBLE設定\nENSEMBLE_SOLUTIONS = ['SOLUTION_14','SOLUTION_5']\nOPTION,__WTS = 'option 91',[0.899, 0.28]\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:    \n    pdB = predict_14(test,lags).to_pandas()\n    pdC = predict_5 (test,lags).to_pandas()\n\n    pdB = pdB.rename(columns={'responder_6':'responder_B'})\n    pdC = pdC.rename(columns={'responder_6':'responder_C'})\n    pds = pd.merge(pdB,pdC, on=['row_id'])\n    pds['responder_6'] =\\\n        pds['responder_B'] * __WTS[0] +\\\n        pds['responder_C'] * __WTS[1] \n\n    display(pds)\n    predictions = test.select('row_id', pl.lit(0.0).alias('responder_6'))\n    pred = pds['responder_6'].to_numpy()\n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n    return predictions\n\n# ここで使用するモデルは statsmodels によるモデルを想定\n# predict_5は元々存在していることを仮定（SOLUTION_5）\n# SOLUTION_5に対するモデル置き換えも必要であれば同様に行う\n# ここでは現状の記述を維持し、predict_14のみstatsmodelsに変更する。\n\nif 'SOLUTION_14' in ENSEMBLE_SOLUTIONS:    \n    lags_: pl.DataFrame | None = None\n\n    def predict_14(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n        global lags_\n        if lags is not None:\n            lags_ = lags\n\n        predictions_14 = test.select(\n            'row_id',\n            pl.lit(0.0).alias('responder_6'),\n        )\n\n        # lags処理（元コード踏襲）\n        symbol_ids = test.select('symbol_id').to_numpy()[:, 0]\n        if lags is not None:\n            lags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last() # 前日最終行を取得\n            test = test.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n        else:\n            # lagsがない場合に、responder_X_lag_1列を0埋め\n            test = test.with_columns(\n                ( pl.lit(0.0).alias(f'responder_{idx}_lag_1') for idx in range(9) )\n            )\n\n        # 以下、statsmodelsを用いて予測する処理に切り替え\n        # 特徴量列を抽出\n        test_input = test.select[CONFIG.feature_cols].to_pandas()\n        test_input = test_input.fillna(method='ffill').fillna(0)\n\n        # statsmodelsで予測\n        # 事前学習済みOLSモデルを用いることを想定（モデルはグローバルに`stats_model_results`として存在）\n        # OLSモデルはたいてい定数項を含むデザイン行列で学習しているため、\n        # 必要に応じて add_constant が必要。ただし学習時に add_constant していれば、ここも同様に合わせる必要がある。\n        import statsmodels.api as sm\n        test_input_with_const = sm.add_constant(test_input, has_constant='add')  # モデル学習時の定数項処理に合わせて変更\n        preds = stats_model_results.predict(test_input_with_const)\n        # predsは1次元のnumpy arrayが返る想定\n\n        # 予測値をクリップ(元コードが -5~5でクリップしていた)\n        preds = np.clip(preds, a_min=-5, a_max=5)\n\n        predictions_14 = \\\n        test.select('row_id').\\\n        with_columns(\n            pl.Series(\n                name   = 'responder_6', \n                values = preds,\n                dtype  = pl.Float64,\n            )\n        )\n\n        return predictions_14","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T06:31:29.101176Z","iopub.execute_input":"2024-12-17T06:31:29.101962Z","iopub.status.idle":"2024-12-17T06:31:29.113361Z","shell.execute_reply.started":"2024-12-17T06:31:29.101926Z","shell.execute_reply":"2024-12-17T06:31:29.112465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}