{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":4.669361,"end_time":"2024-10-10T13:05:46.686069","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-10T13:05:42.016708","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport polars as pl\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-12-26T05:01:46.601687Z","iopub.execute_input":"2024-12-26T05:01:46.602141Z","iopub.status.idle":"2024-12-26T05:01:46.607836Z","shell.execute_reply.started":"2024-12-26T05:01:46.602104Z","shell.execute_reply":"2024-12-26T05:01:46.606580Z"},"papermill":{"duration":1.223703,"end_time":"2024-10-10T13:05:45.825911","exception":false,"start_time":"2024-10-10T13:05:44.602208","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \n\n# Load a data from each file:\nr = range(10)\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pd.read_parquet(file_path)\n    part = part[['date_id','time_id','symbol_id','feature_24','responder_6']]\n    samples.append(part)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:01:46.610228Z","iopub.execute_input":"2024-12-26T05:01:46.610671Z","iopub.status.idle":"2024-12-26T05:02:36.025803Z","shell.execute_reply.started":"2024-12-26T05:01:46.610624Z","shell.execute_reply":"2024-12-26T05:02:36.024403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df = pd.concat(samples, ignore_index=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:36.027018Z","iopub.execute_input":"2024-12-26T05:02:36.027318Z","iopub.status.idle":"2024-12-26T05:02:36.326518Z","shell.execute_reply.started":"2024-12-26T05:02:36.027288Z","shell.execute_reply":"2024-12-26T05:02:36.325279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:36.328966Z","iopub.execute_input":"2024-12-26T05:02:36.329295Z","iopub.status.idle":"2024-12-26T05:02:36.344150Z","shell.execute_reply.started":"2024-12-26T05:02:36.329262Z","shell.execute_reply":"2024-12-26T05:02:36.342829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#选取一部分进行分析\nb = sample_df[sample_df['date_id']>=1698-180].reset_index(drop=True)\na = b[b['symbol_id']==38].reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:36.345614Z","iopub.execute_input":"2024-12-26T05:02:36.346081Z","iopub.status.idle":"2024-12-26T05:02:36.857716Z","shell.execute_reply.started":"2024-12-26T05:02:36.346034Z","shell.execute_reply":"2024-12-26T05:02:36.856468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"b['symbol_id'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:36.859317Z","iopub.execute_input":"2024-12-26T05:02:36.859792Z","iopub.status.idle":"2024-12-26T05:02:36.926326Z","shell.execute_reply.started":"2024-12-26T05:02:36.859724Z","shell.execute_reply":"2024-12-26T05:02:36.924881Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 对a分析","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30, 6))\nsns.scatterplot(x='feature_24', y='responder_6', data=a)\nplt.xlabel('feature_24')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:36.927869Z","iopub.execute_input":"2024-12-26T05:02:36.928227Z","iopub.status.idle":"2024-12-26T05:02:37.663874Z","shell.execute_reply.started":"2024-12-26T05:02:36.928194Z","shell.execute_reply":"2024-12-26T05:02:37.662525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(30, 6))\nplt.plot(a['feature_24'],a['responder_6'])\nplt.xlabel('feature_24')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:37.665288Z","iopub.execute_input":"2024-12-26T05:02:37.665636Z","iopub.status.idle":"2024-12-26T05:02:37.986489Z","shell.execute_reply.started":"2024-12-26T05:02:37.665602Z","shell.execute_reply":"2024-12-26T05:02:37.985299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"a['feature_24'].corr(a['responder_6'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:37.990689Z","iopub.execute_input":"2024-12-26T05:02:37.991648Z","iopub.status.idle":"2024-12-26T05:02:38.003443Z","shell.execute_reply.started":"2024-12-26T05:02:37.991592Z","shell.execute_reply":"2024-12-26T05:02:38.002316Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"通过绘制散点图,折线图和计算相关系数，可得'feature_24'和'responder_6'之间不存在线性关系","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.histplot(a['feature_24'], bins=50, kde=True)\nplt.xlabel('feature_24')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:38.004858Z","iopub.execute_input":"2024-12-26T05:02:38.005190Z","iopub.status.idle":"2024-12-26T05:02:39.124092Z","shell.execute_reply.started":"2024-12-26T05:02:38.005159Z","shell.execute_reply":"2024-12-26T05:02:39.122839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.boxplot(x=a['feature_24'])\nplt.xlabel('feature_24')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:39.125493Z","iopub.execute_input":"2024-12-26T05:02:39.125899Z","iopub.status.idle":"2024-12-26T05:02:39.329990Z","shell.execute_reply.started":"2024-12-26T05:02:39.125863Z","shell.execute_reply":"2024-12-26T05:02:39.328745Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"箱型图中存在异常值，根据直方图分析，可能是由于数据略微偏斜导致中位数偏小","metadata":{}},{"cell_type":"code","source":"a['feature_24_diff'] = a['feature_24'].diff()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:39.331232Z","iopub.execute_input":"2024-12-26T05:02:39.331553Z","iopub.status.idle":"2024-12-26T05:02:39.337723Z","shell.execute_reply.started":"2024-12-26T05:02:39.331521Z","shell.execute_reply":"2024-12-26T05:02:39.336818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"a['feature_24'].isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:39.339166Z","iopub.execute_input":"2024-12-26T05:02:39.339477Z","iopub.status.idle":"2024-12-26T05:02:39.352983Z","shell.execute_reply.started":"2024-12-26T05:02:39.339447Z","shell.execute_reply":"2024-12-26T05:02:39.351799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.histplot(a['feature_24_diff'], bins=50, kde=True)\nplt.xlabel('feature_24_diff')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:39.354202Z","iopub.execute_input":"2024-12-26T05:02:39.354513Z","iopub.status.idle":"2024-12-26T05:02:40.470175Z","shell.execute_reply.started":"2024-12-26T05:02:39.354483Z","shell.execute_reply":"2024-12-26T05:02:40.468948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.boxplot(x=a['feature_24_diff'])\nplt.xlabel('feature_24_diff')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:40.471698Z","iopub.execute_input":"2024-12-26T05:02:40.472158Z","iopub.status.idle":"2024-12-26T05:02:40.684080Z","shell.execute_reply.started":"2024-12-26T05:02:40.472112Z","shell.execute_reply":"2024-12-26T05:02:40.682903Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"根据直方图可以看出，差分后数据分布特别集中，导致在箱型图中异常值较多，所以不适合直接差分","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 对b分析","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.histplot(b['feature_24'], bins=50, kde=True)\nplt.xlabel('feature_24')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:02:40.685445Z","iopub.execute_input":"2024-12-26T05:02:40.685809Z","iopub.status.idle":"2024-12-26T05:03:11.081357Z","shell.execute_reply.started":"2024-12-26T05:02:40.685744Z","shell.execute_reply":"2024-12-26T05:03:11.080271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.boxplot(x=b['feature_24'])\nplt.xlabel('feature_24')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:03:11.082894Z","iopub.execute_input":"2024-12-26T05:03:11.083249Z","iopub.status.idle":"2024-12-26T05:03:11.960832Z","shell.execute_reply.started":"2024-12-26T05:03:11.083215Z","shell.execute_reply":"2024-12-26T05:03:11.959514Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"与a类似且更明显","metadata":{}},{"cell_type":"code","source":"b['feature_24_diff'] = b['feature_24'].diff()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:21:15.943073Z","iopub.execute_input":"2024-12-26T05:21:15.944263Z","iopub.status.idle":"2024-12-26T05:21:15.966003Z","shell.execute_reply.started":"2024-12-26T05:21:15.944216Z","shell.execute_reply":"2024-12-26T05:21:15.964849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.histplot(b['feature_24_diff'], bins=50, kde=True)\nplt.xlabel('feature_24_diff')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:21:16.213164Z","iopub.execute_input":"2024-12-26T05:21:16.214314Z","iopub.status.idle":"2024-12-26T05:21:48.530110Z","shell.execute_reply.started":"2024-12-26T05:21:16.214253Z","shell.execute_reply":"2024-12-26T05:21:48.528838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.boxplot(x=b['feature_24_diff'])\nplt.xlabel('feature_24_diff')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:03:44.080843Z","iopub.execute_input":"2024-12-26T05:03:44.081266Z","iopub.status.idle":"2024-12-26T05:03:44.794837Z","shell.execute_reply.started":"2024-12-26T05:03:44.081219Z","shell.execute_reply":"2024-12-26T05:03:44.793642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"差分后分布数据呈现近似正态分布，差分后数据分布集中在中部，导致箱型图中存在偏小异常值和偏大异常值且近似对称","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30, 6))\nsns.scatterplot(x='feature_24_diff', y='responder_6', data=b)\nplt.xlabel('feature_24_diff')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:03:44.796232Z","iopub.execute_input":"2024-12-26T05:03:44.796669Z","iopub.status.idle":"2024-12-26T05:03:59.100192Z","shell.execute_reply.started":"2024-12-26T05:03:44.796619Z","shell.execute_reply":"2024-12-26T05:03:59.099010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.rcParams['agg.path.chunksize'] = 200\nplt.rcParams['path.simplify_threshold'] = 0.1\nplt.figure(figsize=(30, 6))\nplt.plot(b['feature_24_diff'],b['responder_6'])\nplt.xlabel('feature_24_diff')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T05:08:36.808852Z","iopub.execute_input":"2024-12-26T05:08:36.809256Z","iopub.status.idle":"2024-12-26T05:12:31.966281Z","shell.execute_reply.started":"2024-12-26T05:08:36.809219Z","shell.execute_reply":"2024-12-26T05:12:31.965013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}