{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"- EDA v1 : https://www.kaggle.com/code/motono0223/eda-jane-street-real-time-market-data-forecasting\n- EDA v2 : this notebook (visualization null_count for each date_id)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-26T11:10:41.563170Z","iopub.execute_input":"2024-10-26T11:10:41.563773Z","iopub.status.idle":"2024-10-26T11:10:41.570007Z","shell.execute_reply.started":"2024-10-26T11:10:41.563724Z","shell.execute_reply":"2024-10-26T11:10:41.568814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:10:41.572596Z","iopub.execute_input":"2024-10-26T11:10:41.573007Z","iopub.status.idle":"2024-10-26T11:10:41.592874Z","shell.execute_reply.started":"2024-10-26T11:10:41.572968Z","shell.execute_reply":"2024-10-26T11:10:41.591527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count null(NaN) for eatch columns (group by date_id)\nnull_count_per_date_id = train.group_by(\"date_id\").agg(pl.all().null_count()).collect()\nnull_count_per_date_id","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:10:41.594416Z","iopub.execute_input":"2024-10-26T11:10:41.594868Z","iopub.status.idle":"2024-10-26T11:12:00.067322Z","shell.execute_reply.started":"2024-10-26T11:10:41.594826Z","shell.execute_reply":"2024-10-26T11:12:00.064827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Counter number of records group by date_id\nrecords_date_id = train.group_by(\"date_id\").agg(pl.count().alias(\"num_records\")).collect()\nrecords_date_id","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:12:00.070407Z","iopub.execute_input":"2024-10-26T11:12:00.071573Z","iopub.status.idle":"2024-10-26T11:12:01.510048Z","shell.execute_reply.started":"2024-10-26T11:12:00.071463Z","shell.execute_reply":"2024-10-26T11:12:01.507450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count null(NaN) for all features columns (group by date_id)\nfeatures = [f\"feature_{i:02d}\" for i in range(79) ]\nsum_null_count_per_date_id = null_count_per_date_id.with_columns(\n    null_count=pl.sum_horizontal(features)\n).select(\n    \"date_id\", \"null_count\"\n).join( records_date_id, on=\"date_id\", how=\"inner\").to_pandas()\nsum_null_count_per_date_id[\"null_ratio\"] = sum_null_count_per_date_id[\"null_count\"] / sum_null_count_per_date_id[\"num_records\"] / 79\nsum_null_count_per_date_id","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:12:01.518200Z","iopub.execute_input":"2024-10-26T11:12:01.520363Z","iopub.status.idle":"2024-10-26T11:12:01.569403Z","shell.execute_reply.started":"2024-10-26T11:12:01.520192Z","shell.execute_reply":"2024-10-26T11:12:01.566720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"offline_strt_dt = 530\ny_max = sum_null_count_per_date_id[\"null_ratio\"].max()\n\nplt.scatter( sum_null_count_per_date_id[\"date_id\"], sum_null_count_per_date_id[\"null_ratio\"] )\nplt.plot([offline_strt_dt, offline_strt_dt], [0, y_max], c=\"black\")\nplt.xlabel(\"date_id\")\nplt.ylabel(\"null_ratio\")\nplt.grid()\nplt.show()\n\nprint(f\"If threshold is {offline_strt_dt}, \")\n\ntmp = sum_null_count_per_date_id[ sum_null_count_per_date_id[\"date_id\"] > offline_strt_dt ]\nplt.scatter( tmp[\"date_id\"], tmp[\"null_ratio\"] )\nplt.plot([offline_strt_dt, offline_strt_dt], [0, y_max], c=\"black\")\nplt.xlabel(\"date_id\")\nplt.ylabel(\"null_ratio\")\nplt.grid()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:13:23.025540Z","iopub.execute_input":"2024-10-26T11:13:23.026596Z","iopub.status.idle":"2024-10-26T11:13:23.598933Z","shell.execute_reply.started":"2024-10-26T11:13:23.026528Z","shell.execute_reply":"2024-10-26T11:13:23.597554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter( sum_null_count_per_date_id[\"date_id\"], sum_null_count_per_date_id[\"num_records\"] )\nplt.xlabel(\"date_id\")\nplt.ylabel(\"num_records\")\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:12:02.193410Z","iopub.execute_input":"2024-10-26T11:12:02.194537Z","iopub.status.idle":"2024-10-26T11:12:02.659772Z","shell.execute_reply.started":"2024-10-26T11:12:02.194402Z","shell.execute_reply":"2024-10-26T11:12:02.658290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter( sum_null_count_per_date_id[\"date_id\"], sum_null_count_per_date_id[\"null_count\"] )\nplt.xlabel(\"date_id\")\nplt.ylabel(\"null_count\")\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:12:02.661611Z","iopub.execute_input":"2024-10-26T11:12:02.662116Z","iopub.status.idle":"2024-10-26T11:12:02.965245Z","shell.execute_reply.started":"2024-10-26T11:12:02.662058Z","shell.execute_reply":"2024-10-26T11:12:02.963968Z"},"trusted":true},"execution_count":null,"outputs":[]}]}