{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv, pd.read_parquet )\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n\nimport os, gc\nfrom tqdm.auto import tqdm\nimport pickle # module to serialize and deserialize objects\nimport re # for Regular expression operations \n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.optimizers import Adam\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data  import Dataset, DataLoader\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\n\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import VotingRegressor\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor\n\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:43:23.174821Z","iopub.execute_input":"2024-12-25T13:43:23.175222Z","iopub.status.idle":"2024-12-25T13:43:23.183658Z","shell.execute_reply.started":"2024-12-25T13:43:23.175185Z","shell.execute_reply":"2024-12-25T13:43:23.182122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gridColor = 'lightgrey'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:43:24.755162Z","iopub.execute_input":"2024-12-25T13:43:24.755576Z","iopub.status.idle":"2024-12-25T13:43:24.760359Z","shell.execute_reply.started":"2024-12-25T13:43:24.755544Z","shell.execute_reply":"2024-12-25T13:43:24.758859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \n\n# Load a data from each file:\nr = range(2)\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pd.read_parquet(file_path)\n    samples.append(part)\n    \ntrain_df = pd.concat(samples, ignore_index=True) # Concatenate all samples into one DataFrame if needed\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:45:10.261364Z","iopub.execute_input":"2024-12-25T13:45:10.261714Z","iopub.status.idle":"2024-12-25T13:45:14.138113Z","shell.execute_reply.started":"2024-12-25T13:45:10.261684Z","shell.execute_reply":"2024-12-25T13:45:14.136745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load a data from each file:\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\ntest = [] \n\nfile_path = f\"{path}/test.parquet/date_id=0/part-0.parquet\"\npart = pd.read_parquet(file_path)\ntest.append(part)\n    \ntest_df = pd.concat(test, ignore_index=True) # Concatenate all samples into one DataFrame if needed\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:43:27.826621Z","iopub.execute_input":"2024-12-25T13:43:27.826959Z","iopub.status.idle":"2024-12-25T13:43:27.845305Z","shell.execute_reply.started":"2024-12-25T13:43:27.826932Z","shell.execute_reply":"2024-12-25T13:43:27.84411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.round(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:45:24.180531Z","iopub.execute_input":"2024-12-25T13:45:24.181454Z","iopub.status.idle":"2024-12-25T13:45:25.338947Z","shell.execute_reply.started":"2024-12-25T13:45:24.181405Z","shell.execute_reply":"2024-12-25T13:45:25.337852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# 各特徴量を最頻値で補完\nfor column in train_df.columns:\n    if train_df[column].isnull().sum() > 0:  # 欠損値が存在する場合のみ処理\n        if not train_df[column].mode().empty:  # 最頻値が存在するか確認\n            mode_value = train_df[column].mode()[0]  # 最頻値を取得\n            train_df[column].fillna(mode_value, inplace=True)\n        else:\n            print(f\"列 {column} に最頻値が存在しません。スキップします。\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:45:27.93902Z","iopub.execute_input":"2024-12-25T13:45:27.93941Z","iopub.status.idle":"2024-12-25T13:46:42.071393Z","shell.execute_reply.started":"2024-12-25T13:45:27.939378Z","shell.execute_reply":"2024-12-25T13:46:42.070355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\n# 正規化を行う列\ncolumns_to_normalize = ['feature_11', 'feature_09','feature_10']\n\n# MinMaxScalerを使用して正規化\nscaler = MinMaxScaler()\n\ntrain_df[columns_to_normalize] = scaler.fit_transform(train_df[columns_to_normalize])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:46:42.072783Z","iopub.execute_input":"2024-12-25T13:46:42.073212Z","iopub.status.idle":"2024-12-25T13:46:42.312384Z","shell.execute_reply.started":"2024-12-25T13:46:42.073176Z","shell.execute_reply":"2024-12-25T13:46:42.311334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 対象の列をリストに指定\ncolumns_to_fill = ['feature_21', 'feature_26' , 'feature_27' , 'feature_31']\n\n# 特定の列を確認して、丸ごと欠損値であれば0に変換\nfor column in columns_to_fill:\n    if train_df[column].isnull().all():  # 列が丸ごと欠損値か確認\n        train_df[column] = 0  # 欠損値を0に変換\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:46:42.314067Z","iopub.execute_input":"2024-12-25T13:46:42.314469Z","iopub.status.idle":"2024-12-25T13:46:42.385894Z","shell.execute_reply.started":"2024-12-25T13:46:42.314426Z","shell.execute_reply":"2024-12-25T13:46:42.3849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntest_df.round(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:44:46.309842Z","iopub.execute_input":"2024-12-25T13:44:46.310205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.info(())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:41:24.936642Z","iopub.execute_input":"2024-12-25T13:41:24.937105Z","iopub.status.idle":"2024-12-25T13:41:24.969421Z","shell.execute_reply.started":"2024-12-25T13:41:24.937065Z","shell.execute_reply":"2024-12-25T13:41:24.968102Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 欠損値を0に置き換えた列を記録するリスト\nreplaced_columns = []\n\n# 全ての列を対象に処理\nfor column in test_df.columns:\n    if test_df[column].isnull().all():  # 列が丸ごと欠損値か確認\n        test_df[column] = 0  # 欠損値を0に変換\n        replaced_columns.append(column)  # 置き換えた列を記録\n\n# 置き換えた列を表示\nprint(\"Replaced columns:\", replaced_columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T13:44:35.093047Z","iopub.execute_input":"2024-12-25T13:44:35.093553Z","iopub.status.idle":"2024-12-25T13:44:35.122439Z","shell.execute_reply.started":"2024-12-25T13:44:35.093512Z","shell.execute_reply":"2024-12-25T13:44:35.121159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}