{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-28T02:58:20.236289Z","iopub.execute_input":"2024-10-28T02:58:20.236704Z","iopub.status.idle":"2024-10-28T02:58:20.319244Z","shell.execute_reply.started":"2024-10-28T02:58:20.236662Z","shell.execute_reply":"2024-10-28T02:58:20.318101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Packages:\n\nimport polars as pl\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:28:52.297403Z","iopub.execute_input":"2024-10-28T03:28:52.297842Z","iopub.status.idle":"2024-10-28T03:28:52.303366Z","shell.execute_reply.started":"2024-10-28T03:28:52.297801Z","shell.execute_reply":"2024-10-28T03:28:52.301853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nclass LoadData:\n    \n    def __init__(self, file_paths):\n        self.file_paths = file_paths\n        \n    def load_and_concat(self):\n        partitioned_data = [pl.read_parquet(file_path) for file_path in self.file_paths]\n        df = pl.concat(partitioned_data)\n        \n        return df\n    \nfile_paths = file_paths = sorted(glob.glob('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/*/*.parquet'))\n    \nloader = LoadData(file_paths)\ndf_train = loader.load_and_concat()\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-27T19:41:44.014350Z","iopub.execute_input":"2024-10-27T19:41:44.014940Z","iopub.status.idle":"2024-10-27T19:42:31.842769Z","shell.execute_reply.started":"2024-10-27T19:41:44.014884Z","shell.execute_reply":"2024-10-27T19:42:31.841163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Performing EDA on the first partition of the training data df0","metadata":{}},{"cell_type":"code","source":"df0 = pl.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-10-28T02:59:06.599599Z","iopub.execute_input":"2024-10-28T02:59:06.600069Z","iopub.status.idle":"2024-10-28T02:59:08.934310Z","shell.execute_reply.started":"2024-10-28T02:59:06.600026Z","shell.execute_reply":"2024-10-28T02:59:08.933217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df0.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:27:49.816980Z","iopub.execute_input":"2024-10-28T03:27:49.817681Z","iopub.status.idle":"2024-10-28T03:27:49.824795Z","shell.execute_reply.started":"2024-10-28T03:27:49.817638Z","shell.execute_reply":"2024-10-28T03:27:49.823504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df0.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-28T02:59:24.383560Z","iopub.execute_input":"2024-10-28T02:59:24.384697Z","iopub.status.idle":"2024-10-28T02:59:31.307146Z","shell.execute_reply.started":"2024-10-28T02:59:24.384642Z","shell.execute_reply":"2024-10-28T02:59:31.305929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate null counts for each column\nnull_counts = df0.null_count().unpivot(variable_name=\"column\", value_name=\"null_count\")\n\n# Filter for columns with non-zero null counts\nnon_zero_null_counts = null_counts.filter(pl.col(\"null_count\") > 0)\n\n# Display the columns with non-zero null counts\nfor row in non_zero_null_counts.rows():\n    print(row)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:27:00.632031Z","iopub.execute_input":"2024-10-28T03:27:00.633336Z","iopub.status.idle":"2024-10-28T03:27:00.642499Z","shell.execute_reply.started":"2024-10-28T03:27:00.633286Z","shell.execute_reply":"2024-10-28T03:27:00.641132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feature in [f'feature_{i:02d}' for i in range(9, 14)]:\n    plt.hist(df0[feature].to_list(), bins=50)\n    plt.title(f'Distribution of {feature}')\n    plt.xlabel(feature)\n    plt.ylabel('Frequency')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:34:57.071990Z","iopub.execute_input":"2024-10-28T03:34:57.072411Z","iopub.status.idle":"2024-10-28T03:35:24.969122Z","shell.execute_reply.started":"2024-10-28T03:34:57.072370Z","shell.execute_reply":"2024-10-28T03:35:24.967480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df0['responder_6'].to_list(), bins=50)\nplt.title(f'Distribution of responder_6')\nplt.xlabel('Responder 6')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:36:47.428877Z","iopub.execute_input":"2024-10-28T03:36:47.429302Z","iopub.status.idle":"2024-10-28T03:36:53.033139Z","shell.execute_reply.started":"2024-10-28T03:36:47.429262Z","shell.execute_reply":"2024-10-28T03:36:53.032025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_time = df0[['date_id', 'time_id', 'symbol_id', 'weight', 'responder_6']]","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:42:02.869594Z","iopub.execute_input":"2024-10-28T03:42:02.870514Z","iopub.status.idle":"2024-10-28T03:42:02.877088Z","shell.execute_reply.started":"2024-10-28T03:42:02.870468Z","shell.execute_reply":"2024-10-28T03:42:02.875791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_time.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:42:41.294901Z","iopub.execute_input":"2024-10-28T03:42:41.295325Z","iopub.status.idle":"2024-10-28T03:42:41.304083Z","shell.execute_reply.started":"2024-10-28T03:42:41.295284Z","shell.execute_reply":"2024-10-28T03:42:41.302841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_time.tail(20)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:46:27.832224Z","iopub.execute_input":"2024-10-28T03:46:27.832671Z","iopub.status.idle":"2024-10-28T03:46:27.843167Z","shell.execute_reply.started":"2024-10-28T03:46:27.832632Z","shell.execute_reply":"2024-10-28T03:46:27.841826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_day = df_time.filter(pl.col(\"date_id\") == 0)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:47:29.225147Z","iopub.execute_input":"2024-10-28T03:47:29.225583Z","iopub.status.idle":"2024-10-28T03:47:29.232577Z","shell.execute_reply.started":"2024-10-28T03:47:29.225542Z","shell.execute_reply":"2024-10-28T03:47:29.231147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_day.schema","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:47:35.667196Z","iopub.execute_input":"2024-10-28T03:47:35.667856Z","iopub.status.idle":"2024-10-28T03:47:35.676495Z","shell.execute_reply.started":"2024-10-28T03:47:35.667805Z","shell.execute_reply":"2024-10-28T03:47:35.675157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_day.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:47:38.965174Z","iopub.execute_input":"2024-10-28T03:47:38.965694Z","iopub.status.idle":"2024-10-28T03:47:38.974518Z","shell.execute_reply.started":"2024-10-28T03:47:38.965640Z","shell.execute_reply":"2024-10-28T03:47:38.973146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_day.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:47:41.925648Z","iopub.execute_input":"2024-10-28T03:47:41.926729Z","iopub.status.idle":"2024-10-28T03:47:41.935301Z","shell.execute_reply.started":"2024-10-28T03:47:41.926681Z","shell.execute_reply":"2024-10-28T03:47:41.933905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_day.tail(20)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T03:47:45.262811Z","iopub.execute_input":"2024-10-28T03:47:45.264090Z","iopub.status.idle":"2024-10-28T03:47:45.273482Z","shell.execute_reply.started":"2024-10-28T03:47:45.264009Z","shell.execute_reply":"2024-10-28T03:47:45.271746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}