{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9849268,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Real-time market data forecasting","metadata":{}},{"cell_type":"markdown","source":"## Importing relevent libraries and data","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-15T18:57:33.045823Z","iopub.execute_input":"2024-10-15T18:57:33.046373Z","iopub.status.idle":"2024-10-15T18:57:34.490983Z","shell.execute_reply.started":"2024-10-15T18:57:33.046309Z","shell.execute_reply":"2024-10-15T18:57:34.489844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.read_csv(\"/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\")\nresponders = pd.read_csv(\"/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\")\n\n# Data is stored within parquet files and it's too big to open and load all at once in the memory. \ntrain = pl.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-10-15T18:57:34.493412Z","iopub.execute_input":"2024-10-15T18:57:34.494694Z","iopub.status.idle":"2024-10-15T18:58:40.952075Z","shell.execute_reply.started":"2024-10-15T18:57:34.494624Z","shell.execute_reply":"2024-10-15T18:58:40.950465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2024-10-15T19:05:49.986082Z","iopub.execute_input":"2024-10-15T19:05:49.986828Z","iopub.status.idle":"2024-10-15T19:05:50.028285Z","shell.execute_reply.started":"2024-10-15T19:05:49.986778Z","shell.execute_reply":"2024-10-15T19:05:50.026996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Exploration","metadata":{}},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-10-15T18:58:40.954218Z","iopub.execute_input":"2024-10-15T18:58:40.954766Z","iopub.status.idle":"2024-10-15T18:58:40.997278Z","shell.execute_reply.started":"2024-10-15T18:58:40.954714Z","shell.execute_reply":"2024-10-15T18:58:40.995805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts = train.select([pl.col(col).is_null().sum().alias(col) for col in train.columns])\nnull_counts*100/len(train)","metadata":{"execution":{"iopub.status.busy":"2024-10-15T18:58:41.000565Z","iopub.execute_input":"2024-10-15T18:58:41.001280Z","iopub.status.idle":"2024-10-15T18:58:41.067418Z","shell.execute_reply.started":"2024-10-15T18:58:41.001205Z","shell.execute_reply":"2024-10-15T18:58:41.065950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"feature 21, 26, 27, 31 have about 17% of data missing, that's a lot of missing data. ","metadata":{}},{"cell_type":"code","source":"print(\"The trading period for the assets is\",train['date_id'].n_unique(), \"days. During this period\", train['symbol_id'].n_unique(), \"different assets were traded.\")","metadata":{"execution":{"iopub.status.busy":"2024-10-15T18:59:32.848187Z","iopub.execute_input":"2024-10-15T18:59:32.848676Z","iopub.status.idle":"2024-10-15T18:59:33.390208Z","shell.execute_reply.started":"2024-10-15T18:59:32.848633Z","shell.execute_reply":"2024-10-15T18:59:33.388848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distinct Finite data","metadata":{}},{"cell_type":"code","source":"print(\"features 9, 10, 11 contain\" ,train['feature_09'].n_unique(),',', train['feature_10'].n_unique(),',', train['feature_11'].n_unique(), \"unique features each respectively signifying distinct nature of data\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-15T19:04:20.371583Z","iopub.execute_input":"2024-10-15T19:04:20.372037Z","iopub.status.idle":"2024-10-15T19:04:21.629928Z","shell.execute_reply.started":"2024-10-15T19:04:20.371993Z","shell.execute_reply":"2024-10-15T19:04:21.628703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}