{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":10978571,"sourceType":"competition"}],"dockerImageVersionId":30839,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n# Janestreet Extract Train Data by Symbol","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport sys\nimport copy\nfrom pathlib import Path\nfrom datetime import datetime, timedelta, date\nimport time\nfrom dateutil.relativedelta import relativedelta \n\nimport pyarrow.parquet as pq\nimport pyarrow as pa\n\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:42:06.475885Z","iopub.execute_input":"2025-02-05T11:42:06.476328Z","iopub.status.idle":"2025-02-05T11:42:06.944920Z","shell.execute_reply.started":"2025-02-05T11:42:06.476296Z","shell.execute_reply":"2025-02-05T11:42:06.943948Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Training Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_parquet(r\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:06.475385Z","iopub.execute_input":"2025-02-05T11:43:06.475789Z","iopub.status.idle":"2025-02-05T11:43:07.524890Z","shell.execute_reply.started":"2025-02-05T11:43:06.475757Z","shell.execute_reply":"2025-02-05T11:43:07.523726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:08.919947Z","iopub.execute_input":"2025-02-05T11:43:08.920339Z","iopub.status.idle":"2025-02-05T11:43:08.958853Z","shell.execute_reply.started":"2025-02-05T11:43:08.920305Z","shell.execute_reply":"2025-02-05T11:43:08.957913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:12.440316Z","iopub.execute_input":"2025-02-05T11:43:12.440678Z","iopub.status.idle":"2025-02-05T11:43:12.448770Z","shell.execute_reply.started":"2025-02-05T11:43:12.440649Z","shell.execute_reply":"2025-02-05T11:43:12.447864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_cols = []\nbase_cols = []\nfeat_cols = []\nfor c in df.columns:\n    if c.startswith('responder_'):\n        target_cols.append(c)\n    elif c.startswith('feature_'):\n        feat_cols.append(c)\n    else:\n        base_cols.append(c)\nprint(f\"target: {target_cols}\\nfeatures: {feat_cols}\\nbase: {base_cols}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:15.945208Z","iopub.execute_input":"2025-02-05T11:43:15.945574Z","iopub.status.idle":"2025-02-05T11:43:15.952312Z","shell.execute_reply.started":"2025-02-05T11:43:15.945546Z","shell.execute_reply":"2025-02-05T11:43:15.951154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['date_id'].nunique(), df['time_id'].nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:18.580462Z","iopub.execute_input":"2025-02-05T11:43:18.580920Z","iopub.status.idle":"2025-02-05T11:43:18.613601Z","shell.execute_reply.started":"2025-02-05T11:43:18.580874Z","shell.execute_reply":"2025-02-05T11:43:18.612368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['symbol_id'].value_counts().sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:26.485485Z","iopub.execute_input":"2025-02-05T11:43:26.485888Z","iopub.status.idle":"2025-02-05T11:43:26.508459Z","shell.execute_reply.started":"2025-02-05T11:43:26.485846Z","shell.execute_reply":"2025-02-05T11:43:26.507179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"na_cnt = df.isna().sum()\nna_cnt.sort_values(inplace=True, ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:28.924981Z","iopub.execute_input":"2025-02-05T11:43:28.925345Z","iopub.status.idle":"2025-02-05T11:43:29.375177Z","shell.execute_reply.started":"2025-02-05T11:43:28.925316Z","shell.execute_reply":"2025-02-05T11:43:29.374231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"keep_feats = na_cnt[(na_cnt.index.str.contains('feature_', regex=False)) & (na_cnt<100)].index.values.tolist()\nprint(keep_feats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:31.094892Z","iopub.execute_input":"2025-02-05T11:43:31.095282Z","iopub.status.idle":"2025-02-05T11:43:31.102517Z","shell.execute_reply.started":"2025-02-05T11:43:31.095252Z","shell.execute_reply":"2025-02-05T11:43:31.101367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# responder_6\ndf[base_cols + keep_feats + target_cols]['symbol_id'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:37.975221Z","iopub.execute_input":"2025-02-05T11:43:37.975600Z","iopub.status.idle":"2025-02-05T11:43:38.170801Z","shell.execute_reply.started":"2025-02-05T11:43:37.975568Z","shell.execute_reply":"2025-02-05T11:43:38.169742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.loc[df['symbol_id']==30, keep_feats]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:42.187131Z","iopub.execute_input":"2025-02-05T11:43:42.187476Z","iopub.status.idle":"2025-02-05T11:43:42.370297Z","shell.execute_reply.started":"2025-02-05T11:43:42.187449Z","shell.execute_reply":"2025-02-05T11:43:42.368950Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Select one symbol","metadata":{}},{"cell_type":"code","source":"sid= 30\nkeep_cols = base_cols + keep_feats + target_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:52.405691Z","iopub.execute_input":"2025-02-05T11:43:52.406084Z","iopub.status.idle":"2025-02-05T11:43:52.410908Z","shell.execute_reply.started":"2025-02-05T11:43:52.406049Z","shell.execute_reply":"2025-02-05T11:43:52.409574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Extract Data","metadata":{}},{"cell_type":"code","source":"train_folder = r\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n# next(os.walk(train_folder))[1]\nall_train_files = []\n\nif Path(train_folder).exists():\n    all_train_files = list(Path(train_folder).rglob(\"*.parquet\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:43:55.764986Z","iopub.execute_input":"2025-02-05T11:43:55.765356Z","iopub.status.idle":"2025-02-05T11:43:55.821690Z","shell.execute_reply.started":"2025-02-05T11:43:55.765330Z","shell.execute_reply":"2025-02-05T11:43:55.820508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dfs = []\n\nfor train_file in tqdm(all_train_files):\n    df = pd.read_parquet(train_file)\n    train_dfs.append(df.loc[df['symbol_id']==30, keep_cols].copy(deep=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:46:33.268688Z","iopub.execute_input":"2025-02-05T11:46:33.269570Z","iopub.status.idle":"2025-02-05T11:47:40.412790Z","shell.execute_reply.started":"2025-02-05T11:46:33.269514Z","shell.execute_reply":"2025-02-05T11:47:40.411675Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all = pd.concat(train_dfs)\ndf_all.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:49:50.900502Z","iopub.execute_input":"2025-02-05T11:49:50.900996Z","iopub.status.idle":"2025-02-05T11:49:51.032232Z","shell.execute_reply.started":"2025-02-05T11:49:50.900962Z","shell.execute_reply":"2025-02-05T11:49:51.031109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del train_dfs\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:49:53.145446Z","iopub.execute_input":"2025-02-05T11:49:53.145807Z","iopub.status.idle":"2025-02-05T11:49:53.273722Z","shell.execute_reply.started":"2025-02-05T11:49:53.145777Z","shell.execute_reply":"2025-02-05T11:49:53.272393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:49:54.330166Z","iopub.execute_input":"2025-02-05T11:49:54.330520Z","iopub.status.idle":"2025-02-05T11:49:54.533376Z","shell.execute_reply.started":"2025-02-05T11:49:54.330492Z","shell.execute_reply":"2025-02-05T11:49:54.532404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pq.write_table(pa.Table.from_pandas(df_all), f'{sid}.parquet', compression = 'GZIP')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-05T11:50:04.130209Z","iopub.execute_input":"2025-02-05T11:50:04.130582Z","iopub.status.idle":"2025-02-05T11:50:25.801360Z","shell.execute_reply.started":"2025-02-05T11:50:04.130556Z","shell.execute_reply":"2025-02-05T11:50:25.800226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}