{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv, pd.read_parquet )\nimport polars as pl\nimport time \n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n\nimport os, gc\nfrom tqdm.auto import tqdm\nimport pickle # module to serialize and deserialize objects\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, binarize\nfrom sklearn.preprocessing import OrdinalEncoder, PolynomialFeatures, OneHotEncoder, LabelEncoder \n\nimport kaggle_evaluation.jane_street_inference_server\n\npd.set_option('display.max_rows', 250)\npd.set_option('display.max_columns', 800)\npd.set_option('display.max_seq_items', 800)\npd.set_option('display.width', 1000)\npl.Config.set_tbl_width_chars(120)\npl.Config.set_tbl_rows(120)\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:13:17.283241Z","iopub.execute_input":"2024-12-06T22:13:17.283624Z","iopub.status.idle":"2024-12-06T22:13:19.099476Z","shell.execute_reply.started":"2024-12-06T22:13:17.283588Z","shell.execute_reply":"2024-12-06T22:13:19.098460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:13:19.101906Z","iopub.execute_input":"2024-12-06T22:13:19.102960Z","iopub.status.idle":"2024-12-06T22:13:19.113248Z","shell.execute_reply.started":"2024-12-06T22:13:19.102923Z","shell.execute_reply":"2024-12-06T22:13:19.112058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \n\n# Load a data from each file:\nr = range(10)\nfor i in r:\n    print(i)\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pl.read_parquet(file_path)\n    samples.append(part)\n    \ndf = pl.concat(samples) \n\n# df.round(1)\ndf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:13:19.114581Z","iopub.execute_input":"2024-12-06T22:13:19.114986Z","iopub.status.idle":"2024-12-06T22:14:18.493444Z","shell.execute_reply.started":"2024-12-06T22:13:19.114953Z","shell.execute_reply":"2024-12-06T22:14:18.492047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 47127338/10\n# 5800000+5800000+5800000+5800000+5800000+5800000+5800000+6527338\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:14:18.495523Z","iopub.execute_input":"2024-12-06T22:14:18.496080Z","iopub.status.idle":"2024-12-06T22:14:18.504615Z","shell.execute_reply.started":"2024-12-06T22:14:18.496029Z","shell.execute_reply":"2024-12-06T22:14:18.503483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time \n\ndf[:5800000].write_parquet('js_data_processed_1.parquet')\ndf[5800000:11600000].write_parquet('js_data_processed_2.parquet')\ndf[11600000:17400000].write_parquet('js_data_processed_3.parquet')\ndf[17400000:23200000].write_parquet('js_data_processed_4.parquet')\ndf[23200000:29000000].write_parquet('js_data_processed_5.parquet')\ndf[29000000:34800000].write_parquet('js_data_processed_6.parquet')\ndf[34800000:40600000].write_parquet('js_data_processed_7.parquet')\ndf[40600000:47127338].write_parquet('js_data_processed_8.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:14:18.508739Z","iopub.execute_input":"2024-12-06T22:14:18.509160Z","iopub.status.idle":"2024-12-06T22:15:22.457764Z","shell.execute_reply.started":"2024-12-06T22:14:18.509126Z","shell.execute_reply":"2024-12-06T22:15:22.456708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %%time\n\ndf = df.sample(fraction=0.16, seed=31) \n\ndf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:22.459240Z","iopub.execute_input":"2024-12-06T22:15:22.459688Z","iopub.status.idle":"2024-12-06T22:15:41.439969Z","shell.execute_reply.started":"2024-12-06T22:15:22.459630Z","shell.execute_reply":"2024-12-06T22:15:41.438908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.write_parquet('js_data_sample1.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:41.441530Z","iopub.execute_input":"2024-12-06T22:15:41.442275Z","iopub.status.idle":"2024-12-06T22:15:54.859132Z","shell.execute_reply.started":"2024-12-06T22:15:41.442203Z","shell.execute_reply":"2024-12-06T22:15:54.858119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.860465Z","iopub.execute_input":"2024-12-06T22:15:54.860844Z","iopub.status.idle":"2024-12-06T22:15:54.870020Z","shell.execute_reply.started":"2024-12-06T22:15:54.860784Z","shell.execute_reply":"2024-12-06T22:15:54.866519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df['time_id'].min()\n# # 968 possible time_id values\n# # factors of 968:\n# # 1, 2, 4, 8, 11, 22, 44, 88, 121, 242, 484, 968\n\n# # add standard temporal features\n# df = df.with_columns(\n#     (np.pi * 2 * (pl.col(\"time_id\") +1)/ 968).sin().alias(\"sin_time_id\"),\n#     (np.pi * 2 * (pl.col(\"time_id\") +1)/ 968).cos().alias(\"cos_time_id\"),\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.871019Z","iopub.status.idle":"2024-12-06T22:15:54.871421Z","shell.execute_reply.started":"2024-12-06T22:15:54.871248Z","shell.execute_reply":"2024-12-06T22:15:54.871267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# norm_dist = ['feature_01', 'feature_04', 'feature_08', 'feature_18', 'feature_19',\n#             'feature_24', 'feature_32', 'feature_34', 'feature_35', 'feature_36',\n#              'feature_45', 'feature_46', 'feature_50', 'feature_52', 'feature_53',\n#              'feature_55', 'feature_56', 'feature_57', 'feature_65', 'feature_66',\n            \n#              'responder_0', 'responder_1', 'responder_2', 'responder_3', 'responder_4',\n#              'responder_5', 'responder_7', 'responder_8',\n             \n#              'feature_26', 'feature_31'\n#             ]\n\n# norm_dist_longtails = ['feature_05', 'feature_06', 'feature_07', 'feature_37', \n#                        'feature_38','feature_47', 'feature_48', 'feature_49', \n#                        'feature_58', 'feature_59','feature_60']\n     \n# multimodal_dist2 = [ 'feature_20', 'feature_28', 'feature_33', 'feature_39', \n#                      'feature_40', 'feature_41', 'feature_42', 'feature_43', \n#                      'feature_44','feature_51', 'feature_54']\n\n# multimodal_dist3 =['feature_00', 'feature_02', 'feature_03', 'feature_23', 'feature_27']\n\n# multi_n_dist = ['feature_61']\n\n# skewed_right = ['feature_12', 'feature_13', 'feature_14', 'feature_15', \n#                 'feature_16', 'feature_17', 'feature_22', 'feature_25', \n#                 'feature_29', 'feature_30', 'feature_62', 'feature_63', \n#                'feature_64', 'feature_67', 'feature_68', 'feature_69',\n#                'feature_70', 'feature_71', 'feature_72', 'feature_73', \n#                 'feature_74', 'feature_75', 'feature_76', 'feature_77', \n#                 'feature_78', \n#                 'feature_21',\n#                ]\n\n# categorical = ['feature_09', 'feature_10', 'feature_11', \n#                'symbol_id'\n#               ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.873202Z","iopub.status.idle":"2024-12-06T22:15:54.873635Z","shell.execute_reply.started":"2024-12-06T22:15:54.873420Z","shell.execute_reply":"2024-12-06T22:15:54.873439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# symbol_list = df['symbol_id'].unique()\n# symbol_list\n# feature_cols = []\n# for i in range(79):\n#     feature_cols.append(f\"feature_{i:02d}\")\n\n# for ii in feature_cols:\n#     print(ii, df[ii].is_null().sum())\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.875298Z","iopub.status.idle":"2024-12-06T22:15:54.875728Z","shell.execute_reply.started":"2024-12-06T22:15:54.875543Z","shell.execute_reply":"2024-12-06T22:15:54.875563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %%time\n\n# ## fill na with mean\n\n\n# print('fill na norm_dist')\n# for i in norm_dist:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(\n#             pl.col(i).mean().over(\"symbol_id\")\n#         )\n#     )\n\n# print('fill na norm_dist_longtails')\n# for i in norm_dist_longtails:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(\n#             pl.col(i).mean().over(\"symbol_id\")\n#         )\n#     )\n\n# print('fill na multimodal_dist2')\n# for i in multimodal_dist2:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(\n#             pl.col(i).mean().over(\"symbol_id\")\n#         )\n#     )\n\n# print('fill na multimodal_dist3')\n# for i in multimodal_dist3:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(\n#             pl.col(i).mean().over(\"symbol_id\")\n#         )\n#     )\n\n# print('fill na multi_n_dist')\n# for i in multi_n_dist:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(\n#             pl.col(i).mean().over(\"symbol_id\")\n#         )\n#     )\n\n# print('fill na skewed_right')\n# for i in skewed_right:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(\n#             pl.col(i).mean().over(\"symbol_id\")\n#         )\n#     )\n\n# print('fill na categorical')\n# for i in categorical:\n#     df = df.with_columns(\n#         pl.col(i).fill_null(999)\n#     )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.876791Z","iopub.status.idle":"2024-12-06T22:15:54.877199Z","shell.execute_reply.started":"2024-12-06T22:15:54.877029Z","shell.execute_reply":"2024-12-06T22:15:54.877048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# %%time\n\n# responder_list = ['responder_0', 'responder_1', 'responder_2', 'responder_3', \n#                   'responder_4','responder_5', 'responder_7', 'responder_7', 'responder_8']\n\n# for ii in responder_list:\n#     print(ii, df[ii].is_null().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.879560Z","iopub.status.idle":"2024-12-06T22:15:54.879990Z","shell.execute_reply.started":"2024-12-06T22:15:54.879754Z","shell.execute_reply":"2024-12-06T22:15:54.879773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print ('donzo')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T22:15:54.909422Z","iopub.status.idle":"2024-12-06T22:15:54.909744Z","shell.execute_reply.started":"2024-12-06T22:15:54.909587Z","shell.execute_reply":"2024-12-06T22:15:54.909603Z"}},"outputs":[],"execution_count":null}]}