{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport glob\nimport seaborn as sns\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-02T16:11:15.118481Z","iopub.execute_input":"2024-11-02T16:11:15.118989Z","iopub.status.idle":"2024-11-02T16:11:15.125811Z","shell.execute_reply.started":"2024-11-02T16:11:15.118936Z","shell.execute_reply":"2024-11-02T16:11:15.124633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.svm import SVR\nfrom xgboost import XGBRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:15.128459Z","iopub.execute_input":"2024-11-02T16:11:15.128939Z","iopub.status.idle":"2024-11-02T16:11:15.140433Z","shell.execute_reply.started":"2024-11-02T16:11:15.128895Z","shell.execute_reply":"2024-11-02T16:11:15.139032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = \"/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\"\nfeatures=pd.read_csv(features)\nresponders = \"/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv\"\nresponders = pd.read_csv(responders)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:15.142135Z","iopub.execute_input":"2024-11-02T16:11:15.143106Z","iopub.status.idle":"2024-11-02T16:11:15.161775Z","shell.execute_reply.started":"2024-11-02T16:11:15.143060Z","shell.execute_reply":"2024-11-02T16:11:15.160457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testing = pd.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\navailableFields = testing.columns","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:15.163186Z","iopub.execute_input":"2024-11-02T16:11:15.163619Z","iopub.status.idle":"2024-11-02T16:11:15.180605Z","shell.execute_reply.started":"2024-11-02T16:11:15.163575Z","shell.execute_reply":"2024-11-02T16:11:15.179290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lagz = pd.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:15.183492Z","iopub.execute_input":"2024-11-02T16:11:15.183900Z","iopub.status.idle":"2024-11-02T16:11:15.194825Z","shell.execute_reply.started":"2024-11-02T16:11:15.183857Z","shell.execute_reply":"2024-11-02T16:11:15.193419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainings = \"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\ntrainings = glob.glob(trainings+\"/*/*\")","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:15.196580Z","iopub.execute_input":"2024-11-02T16:11:15.197072Z","iopub.status.idle":"2024-11-02T16:11:15.210369Z","shell.execute_reply.started":"2024-11-02T16:11:15.197016Z","shell.execute_reply":"2024-11-02T16:11:15.208848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\ntrain = pl.concat([pl.read_parquet(x) for x in trainings])","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:15.212150Z","iopub.execute_input":"2024-11-02T16:11:15.212588Z","iopub.status.idle":"2024-11-02T16:11:31.604614Z","shell.execute_reply.started":"2024-11-02T16:11:15.212543Z","shell.execute_reply":"2024-11-02T16:11:31.603117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import shuffle\ntrain = shuffle(train, random_state=0, n_samples=100000).drop_nulls()","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:31.606289Z","iopub.execute_input":"2024-11-02T16:11:31.606774Z","iopub.status.idle":"2024-11-02T16:11:41.976524Z","shell.execute_reply.started":"2024-11-02T16:11:31.606706Z","shell.execute_reply":"2024-11-02T16:11:41.975226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(train[\"date_id\"], train[\"responder_6\"])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:41.978216Z","iopub.execute_input":"2024-11-02T16:11:41.978643Z","iopub.status.idle":"2024-11-02T16:11:42.438532Z","shell.execute_reply.started":"2024-11-02T16:11:41.978600Z","shell.execute_reply":"2024-11-02T16:11:42.437293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr6=train.corr()[\"responder_6\"]\ngudz=[np.where(abs(corr6)>np.percentile(corr6,75))][0][0]\ntrain=train[[train.columns[i] for i in gudz]]","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:42.440214Z","iopub.execute_input":"2024-11-02T16:11:42.440743Z","iopub.status.idle":"2024-11-02T16:11:42.545477Z","shell.execute_reply.started":"2024-11-02T16:11:42.440683Z","shell.execute_reply":"2024-11-02T16:11:42.543788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train[[k for k in train.columns if k in availableFields]]\ny = train['responder_6']","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:42.549793Z","iopub.execute_input":"2024-11-02T16:11:42.550809Z","iopub.status.idle":"2024-11-02T16:11:42.565077Z","shell.execute_reply.started":"2024-11-02T16:11:42.550728Z","shell.execute_reply":"2024-11-02T16:11:42.560199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:42.568934Z","iopub.execute_input":"2024-11-02T16:11:42.576386Z","iopub.status.idle":"2024-11-02T16:11:42.768994Z","shell.execute_reply.started":"2024-11-02T16:11:42.576271Z","shell.execute_reply":"2024-11-02T16:11:42.767679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:42.770860Z","iopub.execute_input":"2024-11-02T16:11:42.771324Z","iopub.status.idle":"2024-11-02T16:11:42.798874Z","shell.execute_reply.started":"2024-11-02T16:11:42.771278Z","shell.execute_reply":"2024-11-02T16:11:42.797690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X,y\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:42.800386Z","iopub.execute_input":"2024-11-02T16:11:42.800781Z","iopub.status.idle":"2024-11-02T16:11:42.926665Z","shell.execute_reply.started":"2024-11-02T16:11:42.800735Z","shell.execute_reply":"2024-11-02T16:11:42.925097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slr = LinearRegression() #R^2 train: 0.005, test: 0.006\n#slr = XGBRegressor() #R^2 train: 0.362, test: -0.074\n#slr = DecisionTreeRegressor() #R^2 train: 1.000, test: -1.193\nslr.fit(X_train, y_train)\ny_train_pred = slr.predict(X_train)\ny_test_pred = slr.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:42.928628Z","iopub.execute_input":"2024-11-02T16:11:42.929167Z","iopub.status.idle":"2024-11-02T16:11:43.002117Z","shell.execute_reply.started":"2024-11-02T16:11:42.929108Z","shell.execute_reply":"2024-11-02T16:11:43.000655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('R^2 train: %.3f, test: %.3f' % (r2_score(y_train, y_train_pred), r2_score(y_test, y_test_pred)))","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:11:43.003971Z","iopub.execute_input":"2024-11-02T16:11:43.006156Z","iopub.status.idle":"2024-11-02T16:11:43.020628Z","shell.execute_reply.started":"2024-11-02T16:11:43.006097Z","shell.execute_reply":"2024-11-02T16:11:43.019299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub=pd.read_csv(\"/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:23:13.709066Z","iopub.execute_input":"2024-11-02T16:23:13.710186Z","iopub.status.idle":"2024-11-02T16:23:13.731779Z","shell.execute_reply.started":"2024-11-02T16:23:13.710129Z","shell.execute_reply":"2024-11-02T16:23:13.730706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"responder_6\"]=slr.predict(testing[X_train.columns].fillna(0))","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:24:34.682177Z","iopub.execute_input":"2024-11-02T16:24:34.683432Z","iopub.status.idle":"2024-11-02T16:24:34.693728Z","shell.execute_reply.started":"2024-11-02T16:24:34.683374Z","shell.execute_reply":"2024-11-02T16:24:34.692534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/working/\"\nsub.to_csv(path+\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T16:25:59.715264Z","iopub.execute_input":"2024-11-02T16:25:59.716712Z","iopub.status.idle":"2024-11-02T16:25:59.734526Z","shell.execute_reply.started":"2024-11-02T16:25:59.716656Z","shell.execute_reply":"2024-11-02T16:25:59.733204Z"},"trusted":true},"execution_count":null,"outputs":[]}]}