{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install pmdarima","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T23:58:22.068304Z","iopub.execute_input":"2024-12-15T23:58:22.068814Z","iopub.status.idle":"2024-12-15T23:58:32.874317Z","shell.execute_reply.started":"2024-12-15T23:58:22.068771Z","shell.execute_reply":"2024-12-15T23:58:32.872669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport pandas as pd\nimport numpy as np\nfrom statsmodels.tsa.arima.model import ARIMA\nfrom statsmodels.tsa.stattools import adfuller\nfrom sklearn.metrics import r2_score\nimport matplotlib.pyplot as plt\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nfrom pmdarima import auto_arima\n\n\n# 1: Load Data\ntrain_dir = r'/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet'\ntest_file = r'/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet'\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-16T00:02:14.231200Z","iopub.execute_input":"2024-12-16T00:02:14.231654Z","iopub.status.idle":"2024-12-16T00:02:14.239227Z","shell.execute_reply.started":"2024-12-16T00:02:14.231619Z","shell.execute_reply":"2024-12-16T00:02:14.237777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load training data (reduced partitions and rows)\ndef load_train_data(train_dir, columns=None, max_partitions=2):\n    dataframes = []\n    for i in range(max_partitions):\n        file_path = f\"{train_dir}/partition_id={i}/part-0.parquet\"\n        print(f\"Loading: {file_path}\")\n        df = pq.read_table(file_path, columns=columns).to_pandas()\n        dataframes.append(df)\n    return pd.concat(dataframes, ignore_index=True)\n\n# Load only necessary columns and downsample data\ntrain_data = load_train_data(train_dir, columns=['date_id', 'weight'], max_partitions=2)\ntrain_data = train_data.dropna().iloc[-5000:]  # Use the most recent 5000 rows\ntrain_data['weight'] = train_data['weight'].astype('float32')\n\n# 2: Preprocess Training Data\ntarget_series = train_data['weight']\n\n# Smooth the series to reduce noise\ntarget_series_smooth = target_series.rolling(window=5).mean().dropna()\n\n# 3: Ensure Stationarity (Log transformation + Differencing)\ntarget_series_log = np.log(target_series_smooth[target_series_smooth > 0])  # Log transformation\n\ntarget_series_diff = target_series_log.diff().dropna()\nresult_diff = adfuller(target_series_diff)\nprint(f\"ADF Statistic (after differencing): {result_diff[0]}, p-value: {result_diff[1]}\")\nif result_diff[1] > 0.05:\n    print(\"Series is still non-stationary after differencing!\")\n\n# 4: Split Data into Train and Validation Sets\nsplit_idx = int(len(target_series_diff) * 0.7)  # 70% training, 30% validation\ntrain_data = target_series_diff[:split_idx]\nval_data = target_series_diff[split_idx:]\n\n# 5: Auto-Tune ARIMA Model Parameters\nprint(\"Tuning ARIMA parameters...\")\nauto_model = auto_arima(train_data, seasonal=False, stepwise=True, trace=True)\nprint(f\"Optimal ARIMA order: {auto_model.order}\")\n\n# Train ARIMA Model with Optimal Parameters\ntrain_data_limited = train_data[-2000:]  # Limit to last 2000 points for memory efficiency\nmodel = ARIMA(train_data_limited, order=auto_model.order)\nmodel_fit = model.fit()\nprint(model_fit.summary())\n\n# 6: Compute Training Score for Auto-Tuned ARIMA\ntrain_predictions = model_fit.fittedvalues\ntrain_r2 = r2_score(train_data[-len(train_predictions):], train_predictions)\nprint(f\"Auto-Tuned ARIMA Training R²: {train_r2}\")\n\n# Validate Auto-Tuned ARIMA Model\nval_predictions = model_fit.forecast(steps=len(val_data))\nval_r2 = r2_score(val_data, val_predictions)\nprint(f\"Auto-Tuned ARIMA Validation R²: {val_r2}\")\n\n# Run ARIMA(1, 1, 1) Model\nprint(\"\\nRunning ARIMA(1,1,1) model...\")\nmodel_fixed = ARIMA(train_data_limited, order=(1, 1, 1))\nmodel_fit_fixed = model_fixed.fit()\nprint(model_fit_fixed.summary())\n\n# Training Score for ARIMA(1,1,1)\ntrain_predictions_fixed = model_fit_fixed.fittedvalues\ntrain_r2_fixed = r2_score(train_data[-len(train_predictions_fixed):], train_predictions_fixed)\nprint(f\"ARIMA(1,1,1) Training R²: {train_r2_fixed}\")\n\n# Validate ARIMA(1,1,1) Model\nval_predictions_fixed = model_fit_fixed.forecast(steps=len(val_data))\nval_r2_fixed = r2_score(val_data, val_predictions_fixed)\nprint(f\"ARIMA(1,1,1) Validation R²: {val_r2_fixed}\")\n\n\n# 7: Forecast Using Test Data\n# Load test data with safe schema enforcement\ndef safe_load_parquet(file_path, columns=None):\n    print(f\"Loading test data from: {file_path}\")\n\n    # Define schema to enforce column types\n    schema = pa.schema([\n        ('date_id', pa.int32()),  # Force date_id to int32\n        ('row_id', pa.int64()),   # Force row_id to int64\n    ])\n\n    # Read Parquet with enforced schema\n    if columns:\n        table = pq.read_table(file_path, columns=columns, schema=schema)\n    else:\n        table = pq.read_table(file_path, schema=schema)\n\n    df = table.to_pandas()\n    return df\n\n# Call the function correctly (columns must be a list)\ntest_data = safe_load_parquet(test_file, columns=['row_id'])\n\n# Forecast for the first 1000 rows\ntest_data = test_data[['row_id']].iloc[:1000]\ntest_data['weight_pred'] = model_fit_fixed.forecast(steps=len(test_data))\n\n# 8: Save Test Predictions for Submission\nsubmission_df = test_data[['row_id', 'weight_pred']].rename(columns={'weight_pred': 'weight'})\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T00:02:16.344649Z","iopub.execute_input":"2024-12-16T00:02:16.345611Z","iopub.status.idle":"2024-12-16T00:02:29.264878Z","shell.execute_reply.started":"2024-12-16T00:02:16.345564Z","shell.execute_reply":"2024-12-16T00:02:29.263368Z"}},"outputs":[],"execution_count":null}]}