{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":207221536,"sourceType":"kernelVersion"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":217.735984,"end_time":"2024-11-09T02:26:21.512992","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-11-09T02:22:43.777008","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Jane Street Real-Time Market Data Forecasting: Computing Feature Statistics\n\n\nThis notebook processes a large dataset of market data in multiple partitions to calculate essential feature statistics for model training. The primary steps involve reading data in chunks, performing incremental calculations, and ultimately computing the mean, standard deviation, minimum, and maximum values for each feature. This structured approach minimizes memory use and efficiently calculates these statistics without loading the entire dataset at once. Key steps include:\n1. Partitioned Data Loading: Each partition of the dataset is loaded one at a time, enabling large-scale data processing in manageable portions.\n2. Incremental Calculation: For each partition, the code calculates and accumulates the sum, squared sum, minimum, and maximum values for each feature.\n3. Mean and Standard Deviation Calculation: After processing all partitions, the mean and standard deviation are derived for each feature.\n4. Handling Missing Data: Missing values are checked for and addressed in the minimum, maximum, and standard deviation arrays.\n5. Result Serialization: The final mean, standard deviation, minimum, and maximum values are serialized and saved for later use in model normalization and scaling, ensuring the model receives consistent and normalized input features.\n\nThis approach is optimized for large datasets and provides a scalable, memory-efficient solution to prepare statistical values for machine learning models.\n","metadata":{"papermill":{"duration":0.004571,"end_time":"2024-11-09T02:22:46.746040","exception":false,"start_time":"2024-11-09T02:22:46.741469","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\n\nimport pandas as pd\n\nimport polars as pl\n\nimport numpy as np\n\nimport pickle","metadata":{"papermill":{"duration":14.998955,"end_time":"2024-11-09T02:23:01.749058","exception":false,"start_time":"2024-11-09T02:22:46.750103","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T16:36:37.792738Z","iopub.execute_input":"2024-11-16T16:36:37.793138Z","iopub.status.idle":"2024-11-16T16:36:39.202574Z","shell.execute_reply.started":"2024-11-16T16:36:37.793099Z","shell.execute_reply":"2024-11-16T16:36:39.201635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_columns =  [f\"feature_{i:02d}\" for i in range(79)]\n\ntarget_column = \"responder_6\"","metadata":{"execution":{"iopub.status.busy":"2024-11-16T16:36:41.810663Z","iopub.execute_input":"2024-11-16T16:36:41.811172Z","iopub.status.idle":"2024-11-16T16:36:41.816960Z","shell.execute_reply.started":"2024-11-16T16:36:41.811133Z","shell.execute_reply":"2024-11-16T16:36:41.815738Z"},"papermill":{"duration":0.013241,"end_time":"2024-11-09T02:23:01.765381","exception":false,"start_time":"2024-11-09T02:23:01.752140","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Load training data","metadata":{"papermill":{"duration":0.002493,"end_time":"2024-11-09T02:23:01.770846","exception":false,"start_time":"2024-11-09T02:23:01.768353","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X_train = None\n\ny_train = None\n\nn_samples = 0\n\nfeature_sum = None\n\nfeature_sq_sum = None\n\nfeature_min = None\n\nfeature_max = None\n\nfor i in range(9):\n\n    print(\"=\" * 30)\n\n    print(f\"Partition {i}\")\n\n    print(\"=\" * 30)\n\n    df = pd.read_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\")\n\n    print(df.head())\n\n    X_part = df[feature_columns]\n\n    # Initialize sums if first partition\n\n    feature_part_min = X_part.min(axis=0).to_numpy()\n\n    feature_part_max = X_part.max(axis=0).to_numpy()\n\n    if feature_sum is None:\n\n        feature_sum = X_part.sum()\n\n        feature_sq_sum = (X_part ** 2).sum()\n\n        n_samples = X_part.notnull().sum()\n\n        X_train = X_part\n\n        y_train = df[target_column]\n\n        feature_min = feature_part_min\n\n        feature_max = feature_part_max\n\n\n    else:\n\n        feature_sum += X_part.sum()\n\n        feature_sq_sum += (X_part ** 2).sum()\n\n        n_samples += X_part.notnull().sum()\n\n        X_train = pd.concat([X_train, X_part])\n\n        y_train = pd.concat([y_train, df[target_column]])\n\n        for j in range(len(feature_columns)):\n\n            if feature_part_min[j] < feature_min[j] or pd.isna(feature_min[j]):\n\n                feature_min[j] = feature_part_min[j]\n\n            if feature_part_max[j] > feature_max[j] or pd.isna(feature_max[j]):\n\n                feature_max[j] = feature_part_max[j]\n\n# Calculate mean and standard deviation\n\nprint(f\"Number of samples:{n_samples}\")\n\nmean_values = feature_sum / n_samples\n\nvariance = (feature_sq_sum / n_samples) - (mean_values ** 2)\n\nstd_values = np.sqrt(variance)\n\nprint(\"Mean values:\\n\", mean_values)\n\nprint(\"\\nStandard deviation values:\\n\", std_values)\n\nmean_values = mean_values.astype(np.float32)\nwith open(\"mean_pandas.pkl\", \"wb\") as f:\n    pickle.dump(mean_values, f)\nmean_values = mean_values.to_numpy()\n\nstd_values = std_values.astype(np.float32).to_numpy()\n\nfor j, val in enumerate(std_values):\n\n    if pd.isna(val):\n\n        std_values[j] = X_train[feature_columns[j]].std()\n\n\nfor j, val in enumerate(feature_min):\n\n    if pd.isna(val):\n\n        feature_min[j] = X_train[feature_columns[j]].min()\n\nfor j, val in enumerate(feature_max):\n\n    if pd.isna(val):\n\n        feature_max[j] = X_train[feature_columns[j]].max()\n\nwith open(\"mean.pkl\", \"wb\") as f:\n\n    pickle.dump(mean_values, f)\n\nwith open(\"std.pkl\", \"wb\") as f:\n\n    pickle.dump(std_values, f)\n\nwith open(\"min.pkl\", \"wb\") as f:\n\n    pickle.dump(feature_min, f)\n\nwith open(\"max.pkl\", \"wb\") as f:\n\n    pickle.dump(feature_max, f)    ","metadata":{"execution":{"iopub.status.busy":"2024-11-16T16:36:50.721694Z","iopub.execute_input":"2024-11-16T16:36:50.722146Z","iopub.status.idle":"2024-11-16T16:39:39.185894Z","shell.execute_reply.started":"2024-11-16T16:36:50.722102Z","shell.execute_reply":"2024-11-16T16:39:39.184804Z"},"papermill":{"duration":196.672357,"end_time":"2024-11-09T02:26:18.445802","exception":false,"start_time":"2024-11-09T02:23:01.773445","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}