{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9865643,"sourceType":"datasetVersion","datasetId":6055566}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport os\n\nimport pandas as pd\nimport polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server\n\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:47:39.490839Z","iopub.execute_input":"2024-11-14T10:47:39.491424Z","iopub.status.idle":"2024-11-14T10:47:41.407961Z","shell.execute_reply.started":"2024-11-14T10:47:39.491355Z","shell.execute_reply":"2024-11-14T10:47:41.406576Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_inference_server.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/templates.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/base_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/relay.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/kaggle_evaluation.proto\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2_grpc.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/__init__.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/responders.csv\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/sample_submission.csv\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/features.csv\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_gateway.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_inference_server.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/__init__.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/templates.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/base_gateway.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/relay.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/kaggle_evaluation.proto\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/__init__.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2_grpc.py\n/kaggle/input/dataset-jamlee/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/__init__.py\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv')\ndf[:3]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-12T15:08:12.698035Z","iopub.execute_input":"2024-11-12T15:08:12.699084Z","iopub.status.idle":"2024-11-12T15:08:12.734425Z","shell.execute_reply.started":"2024-11-12T15:08:12.699024Z","shell.execute_reply":"2024-11-12T15:08:12.733232Z"}},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":" responder tag_0 tag_1 tag_2 tag_3 tag_4\n0 responder_0 True False True False False\n1 responder_1 True False False True False\n2 responder_2 True True False False False","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
respondertag_0tag_1tag_2tag_3tag_4
0responder_0TrueFalseTrueFalseFalse
1responder_1TrueFalseFalseTrueFalse
2responder_2TrueTrueFalseFalseFalse
\n
"},"metadata":{}}],"execution_count":4},{"cell_type":"code","source":"table = pq.read_table('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ndf1 = table.to_pandas()\ndf1[:3]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:48:05.356056Z","iopub.execute_input":"2024-11-14T10:48:05.357225Z","iopub.status.idle":"2024-11-14T10:48:07.161048Z","shell.execute_reply.started":"2024-11-14T10:48:05.357174Z","shell.execute_reply":"2024-11-14T10:48:07.160019Z"}},"outputs":[{"execution_count":2,"output_type":"execute_result","data":{"text/plain":" date_id time_id symbol_id weight feature_00 feature_01 feature_02 \\\n0 0 0 1 3.889038 NaN NaN NaN \n1 0 0 7 1.370613 NaN NaN NaN \n2 0 0 9 2.285698 NaN NaN NaN \n\n feature_03 feature_04 feature_05 ... responder_0 responder_1 \\\n0 NaN NaN 0.851033 ... 0.738489 -0.069556 \n1 NaN NaN 0.676961 ... 2.965889 1.190077 \n2 NaN NaN 1.056285 ... -0.864488 -0.280303 \n\n responder_2 responder_3 responder_4 responder_5 responder_6 \\\n0 1.380875 2.005353 0.186018 1.218368 0.775981 \n1 -0.523998 3.849921 2.626981 5.000000 0.703665 \n2 -0.326697 0.375781 1.271291 0.099793 2.109352 \n\n responder_7 responder_8 partition_id \n0 0.346999 0.095504 0 \n1 0.216683 0.778639 0 \n2 0.670881 0.772828 0 \n\n[3 rows x 93 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
date_idtime_idsymbol_idweightfeature_00feature_01feature_02feature_03feature_04feature_05...responder_0responder_1responder_2responder_3responder_4responder_5responder_6responder_7responder_8partition_id
00013.889038NaNNaNNaNNaNNaN0.851033...0.738489-0.0695561.3808752.0053530.1860181.2183680.7759810.3469990.0955040
10071.370613NaNNaNNaNNaNNaN0.676961...2.9658891.190077-0.5239983.8499212.6269815.0000000.7036650.2166830.7786390
20092.285698NaNNaNNaNNaNNaN1.056285...-0.864488-0.280303-0.3266970.3757811.2712910.0997932.1093520.6708810.7728280
\n

3 rows × 93 columns

\n
"},"metadata":{}}],"execution_count":2},{"cell_type":"code","source":"table2 = pq.read_table('/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet')\ndf2 = table2.to_pandas()\ndf2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:48:54.490452Z","iopub.execute_input":"2024-11-14T10:48:54.490964Z","iopub.status.idle":"2024-11-14T10:48:57.512036Z","shell.execute_reply.started":"2024-11-14T10:48:54.490915Z","shell.execute_reply":"2024-11-14T10:48:57.510886Z"}},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":" date_id time_id symbol_id weight feature_00 feature_01 \\\n0 170 0 0 2.112212 NaN NaN \n1 170 0 1 2.760715 NaN NaN \n2 170 0 2 1.813596 NaN NaN \n3 170 0 3 0.926893 NaN NaN \n4 170 0 7 1.665231 NaN NaN \n... ... ... ... ... ... ... \n2804242 339 848 19 4.384988 -0.357850 -1.135620 \n2804243 339 848 30 0.913199 0.013418 -1.095379 \n2804244 339 848 33 1.226038 -0.378911 -1.521467 \n2804245 339 848 34 1.267464 -0.343313 -1.548126 \n2804246 339 848 38 2.266878 -0.414920 -1.261129 \n\n feature_02 feature_03 feature_04 feature_05 ... responder_0 \\\n0 NaN NaN NaN 1.060330 ... -0.293646 \n1 NaN NaN NaN 0.482468 ... -0.075267 \n2 NaN NaN NaN 1.020798 ... -5.000000 \n3 NaN NaN NaN 0.510098 ... 3.336086 \n4 NaN NaN NaN 0.547458 ... -0.707027 \n... ... ... ... ... ... ... \n2804242 -0.375377 0.005456 -1.241660 -0.139730 ... 0.163901 \n2804243 -0.315413 -0.227001 -1.131748 -0.119023 ... 0.637513 \n2804244 0.209378 -0.241914 -1.597019 -0.082501 ... -3.343947 \n2804245 -0.188658 0.174160 -1.160716 -0.150091 ... 0.966054 \n2804246 0.046846 -0.117544 -1.687853 -0.096446 ... 0.344730 \n\n responder_1 responder_2 responder_3 responder_4 responder_5 \\\n0 -0.061842 -0.305413 -0.419151 -0.111796 -0.535104 \n1 -0.359360 -1.270054 -0.018332 -0.040286 -1.417509 \n2 -5.000000 0.194658 -5.000000 -5.000000 -5.000000 \n3 2.051951 2.400644 0.962730 -0.939277 1.845870 \n4 -0.344866 -1.248052 -0.129645 -3.145927 -0.452708 \n... ... ... ... ... ... \n2804242 0.134067 1.124346 0.484879 0.255225 0.479602 \n2804243 -0.011480 1.779785 0.757132 0.274228 0.741818 \n2804244 -1.012429 -2.968987 -1.112956 -0.598563 -1.537665 \n2804245 0.551264 0.175104 -0.035818 -0.015150 -0.425369 \n2804246 0.556149 2.404575 0.893058 0.378054 1.023353 \n\n responder_6 responder_7 responder_8 partition_id \n0 -0.044332 -0.039061 -0.744789 1 \n1 0.085840 0.487232 -0.124533 1 \n2 1.583400 0.018712 -1.055035 1 \n3 -2.372452 -1.663179 -4.585349 1 \n4 0.300044 0.489202 0.242737 1 \n... ... ... ... ... \n2804242 0.008540 0.027126 0.000452 1 \n2804243 1.014812 0.573904 2.359970 1 \n2804244 -0.259630 -0.105986 -0.480081 1 \n2804245 -0.115926 -0.012685 -0.144224 1 \n2804246 0.276847 0.197048 0.424032 1 \n\n[2804247 rows x 93 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
date_idtime_idsymbol_idweightfeature_00feature_01feature_02feature_03feature_04feature_05...responder_0responder_1responder_2responder_3responder_4responder_5responder_6responder_7responder_8partition_id
0170002.112212NaNNaNNaNNaNNaN1.060330...-0.293646-0.061842-0.305413-0.419151-0.111796-0.535104-0.044332-0.039061-0.7447891
1170012.760715NaNNaNNaNNaNNaN0.482468...-0.075267-0.359360-1.270054-0.018332-0.040286-1.4175090.0858400.487232-0.1245331
2170021.813596NaNNaNNaNNaNNaN1.020798...-5.000000-5.0000000.194658-5.000000-5.000000-5.0000001.5834000.018712-1.0550351
3170030.926893NaNNaNNaNNaNNaN0.510098...3.3360862.0519512.4006440.962730-0.9392771.845870-2.372452-1.663179-4.5853491
4170071.665231NaNNaNNaNNaNNaN0.547458...-0.707027-0.344866-1.248052-0.129645-3.145927-0.4527080.3000440.4892020.2427371
..................................................................
2804242339848194.384988-0.357850-1.135620-0.3753770.005456-1.241660-0.139730...0.1639010.1340671.1243460.4848790.2552250.4796020.0085400.0271260.0004521
2804243339848300.9131990.013418-1.095379-0.315413-0.227001-1.131748-0.119023...0.637513-0.0114801.7797850.7571320.2742280.7418181.0148120.5739042.3599701
2804244339848331.226038-0.378911-1.5214670.209378-0.241914-1.597019-0.082501...-3.343947-1.012429-2.968987-1.112956-0.598563-1.537665-0.259630-0.105986-0.4800811
2804245339848341.267464-0.343313-1.548126-0.1886580.174160-1.160716-0.150091...0.9660540.5512640.175104-0.035818-0.015150-0.425369-0.115926-0.012685-0.1442241
2804246339848382.266878-0.414920-1.2611290.046846-0.117544-1.687853-0.096446...0.3447300.5561492.4045750.8930580.3780541.0233530.2768470.1970480.4240321
\n

2804247 rows × 93 columns

\n
"},"metadata":{}}],"execution_count":4},{"cell_type":"code","source":"dfm = pd.merge(df1,df2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:48:59.754665Z","iopub.execute_input":"2024-11-14T10:48:59.755133Z","iopub.status.idle":"2024-11-14T10:51:21.794563Z","shell.execute_reply.started":"2024-11-14T10:48:59.755088Z","shell.execute_reply":"2024-11-14T10:51:21.793498Z"}},"outputs":[],"execution_count":5},{"cell_type":"code","source":"dfm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#看看给的因子长什么β样\nplt.figure(figsize=(10, 6))\nplt.plot(df2['date_id'], df2['feature_60'], marker='o', linestyle='-', color='b')\nplt.xlabel('Date ID')\nplt.ylabel('Feature 70')\nplt.title('Feature 02 Over Time')\nplt.xticks(rotation=45)\nplt.grid()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-12T15:08:17.661823Z","iopub.status.idle":"2024-11-12T15:08:17.662298Z","shell.execute_reply.started":"2024-11-12T15:08:17.662075Z","shell.execute_reply":"2024-11-12T15:08:17.662098Z"}},"outputs":[],"execution_count":null}]}