{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":221399,"sourceType":"modelInstanceVersion","modelInstanceId":188834,"modelId":210842},{"sourceId":221562,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":188980,"modelId":210985}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Mainly for model training\n\nDepending on the size of your training set, you will need an [inference notebook](https://www.kaggle.com/code/regisvargas/inference-jane-street-a-beginner-s-notebook).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:09:03.802358Z","iopub.execute_input":"2025-01-07T10:09:03.802741Z","iopub.status.idle":"2025-01-07T10:09:03.807861Z","shell.execute_reply.started":"2025-01-07T10:09:03.802693Z","shell.execute_reply":"2025-01-07T10:09:03.806864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n# Initialize a list to hold samples from each file\nsamples = []\n# Load a sample from each file\nfor i in range(10):\n    file_path = f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    chunk = pd.read_parquet(file_path)\n    \n    # Take a sample of the data (adjust sample size as needed)\n    sample_chunk = chunk.sample(n=500000, random_state=42)  # For example, 100 rows\n    samples.append(sample_chunk)\n# Concatenate all samples into one DataFrame if needed\nsample_df = pd.concat(samples, ignore_index=True)\n","metadata":{"execution":{"iopub.status.busy":"2025-01-07T10:09:05.871466Z","iopub.execute_input":"2025-01-07T10:09:05.872111Z","iopub.status.idle":"2025-01-07T10:10:49.300086Z","shell.execute_reply.started":"2025-01-07T10:09:05.872076Z","shell.execute_reply":"2025-01-07T10:10:49.299156Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:55:36.490626Z","iopub.execute_input":"2025-01-07T08:55:36.491027Z","iopub.status.idle":"2025-01-07T08:55:37.105000Z","shell.execute_reply.started":"2025-01-07T08:55:36.490999Z","shell.execute_reply":"2025-01-07T08:55:37.104066Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare data","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nimport os\n# Separate features and responders\nfeatures = sample_df.filter(regex='^feature_')\nresponders = sample_df.filter(regex='^responder_')\nweights = sample_df['weight']\n# Convert to numpy arrays for TensorFlow\nX = features.values  # Features for input\n#y = responders.values  # Responders for output\n# Assuming you have a DataFrame `y_train` with all responders\ny = responders[['responder_6']].values  # Keep only responder_6\nprint(y)\nX = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\ny = np.nan_to_num(y, nan=0.0, posinf=0.0, neginf=0.0)\nprint(y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:55:48.686254Z","iopub.execute_input":"2025-01-07T08:55:48.686607Z","iopub.status.idle":"2025-01-07T08:56:13.942543Z","shell.execute_reply.started":"2025-01-07T08:55:48.686575Z","shell.execute_reply":"2025-01-07T08:56:13.941588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:56:13.944375Z","iopub.execute_input":"2025-01-07T08:56:13.945330Z","iopub.status.idle":"2025-01-07T08:56:14.554201Z","shell.execute_reply.started":"2025-01-07T08:56:13.945268Z","shell.execute_reply":"2025-01-07T08:56:14.553218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:56:14.555697Z","iopub.execute_input":"2025-01-07T08:56:14.556041Z","iopub.status.idle":"2025-01-07T08:56:14.590702Z","shell.execute_reply.started":"2025-01-07T08:56:14.556001Z","shell.execute_reply":"2025-01-07T08:56:14.589786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:56:14.592615Z","iopub.execute_input":"2025-01-07T08:56:14.592948Z","iopub.status.idle":"2025-01-07T08:56:30.456220Z","shell.execute_reply.started":"2025-01-07T08:56:14.592912Z","shell.execute_reply":"2025-01-07T08:56:30.455356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nnu_va = features.isnull().sum()\nprint(nu_va)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:56:30.457408Z","iopub.execute_input":"2025-01-07T08:56:30.457890Z","iopub.status.idle":"2025-01-07T08:56:30.882150Z","shell.execute_reply.started":"2025-01-07T08:56:30.457848Z","shell.execute_reply":"2025-01-07T08:56:30.881232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(79):\n    if i<10:\n        features[f'feature_0{i}'] = features[f'feature_0{i}'].fillna(features[f'feature_0{i}'].mean())\n    else:\n        features[f'feature_{i}'] = features[f'feature_{i}'].fillna(features[f'feature_{i}'].mean())        \n        \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:56:30.883147Z","iopub.execute_input":"2025-01-07T08:56:30.883422Z","iopub.status.idle":"2025-01-07T08:56:33.135046Z","shell.execute_reply.started":"2025-01-07T08:56:30.883395Z","shell.execute_reply":"2025-01-07T08:56:33.134064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nu_va = features.isnull().sum()\nprint(nu_va)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:56:38.366100Z","iopub.execute_input":"2025-01-07T08:56:38.366468Z","iopub.status.idle":"2025-01-07T08:56:38.894273Z","shell.execute_reply.started":"2025-01-07T08:56:38.366435Z","shell.execute_reply":"2025-01-07T08:56:38.893305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T09:37:34.612834Z","iopub.execute_input":"2025-01-07T09:37:34.613216Z","iopub.status.idle":"2025-01-07T09:37:36.958245Z","shell.execute_reply.started":"2025-01-07T09:37:34.613184Z","shell.execute_reply":"2025-01-07T09:37:36.957274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:02:33.593193Z","iopub.execute_input":"2025-01-05T06:02:33.594009Z","iopub.status.idle":"2025-01-05T06:02:33.600812Z","shell.execute_reply.started":"2025-01-05T06:02:33.593963Z","shell.execute_reply":"2025-01-05T06:02:33.599669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Is_keras = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T11:43:15.587512Z","iopub.execute_input":"2024-12-02T11:43:15.587758Z","iopub.status.idle":"2024-12-02T11:43:15.597040Z","shell.execute_reply.started":"2024-12-02T11:43:15.587734Z","shell.execute_reply":"2024-12-02T11:43:15.596304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()  #scaleing \nx = scaler.fit_transform(X)\n\n# ensure the dateset length divide by Timestamp like 10\ntimesteps = 1\nx = x[:len(x) - len(x) % timesteps] \ny = y[:len(y) - len(y) % timesteps] \n\n# now reshaping the data for LSTM\nn_features = x.shape[1]\nx_lstm = x.reshape(-1,timesteps,n_features)\ny_lstm = y.reshape(-1,timesteps)\nx_train, x_val, y_train, y_val = train_test_split(x_lstm, y_lstm, test_size=0.2, random_state=42)\nprint(\"x_train shape:\", x_train.shape)  # (samples, timesteps, features)\nprint(\"x_val shape:\", x_val.shape)\nprint(\"y_train shape:\", y_train.shape)  # (samples, timesteps)\nprint(\"y_val shape:\", y_val.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:11:52.114688Z","iopub.execute_input":"2025-01-07T10:11:52.115043Z","iopub.status.idle":"2025-01-07T10:12:04.135276Z","shell.execute_reply.started":"2025-01-07T10:11:52.115015Z","shell.execute_reply":"2025-01-07T10:12:04.134385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:02:53.694892Z","iopub.execute_input":"2025-01-05T06:02:53.695257Z","iopub.status.idle":"2025-01-05T06:02:53.701967Z","shell.execute_reply.started":"2025-01-05T06:02:53.695223Z","shell.execute_reply":"2025-01-05T06:02:53.700981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:02:56.745009Z","iopub.execute_input":"2025-01-05T06:02:56.746033Z","iopub.status.idle":"2025-01-05T06:02:56.752267Z","shell.execute_reply.started":"2025-01-05T06:02:56.745992Z","shell.execute_reply":"2025-01-05T06:02:56.751304Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training with Prepare Data for LSTM ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import LSTM, Dropout, Dense","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:03:00.148521Z","iopub.execute_input":"2025-01-05T06:03:00.149225Z","iopub.status.idle":"2025-01-05T06:03:00.157412Z","shell.execute_reply.started":"2025-01-05T06:03:00.149167Z","shell.execute_reply":"2025-01-05T06:03:00.156514Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Two stacked LSTM layers with units (e.g., 64 each), followed by dropout layers for regularization.\n## Dense layers with non-linear activation functions (e.g., Swish or ReLU).\n## An output layer with a sigmoid activation for binary prediction (action).\n## Optimizer: Adam; Loss: Binary Crossentropy; Metric: AUC-ROC or accuracy.","metadata":{}},{"cell_type":"code","source":"# so , we have almost 50 lakhs rows sample data for train and validation \nmodel = Sequential(\n    [\n        # 1st lstm layer\n        LSTM(128,activation='tanh',return_sequences=True,input_shape=(timesteps,n_features)),\n        Dropout(0.2),  # use for overfitting can be change to reduce or remove overfitting   also For Regualarizatioin\n\n        # 2nd LSTM Layer\n\n        LSTM(64,activation='tanh',return_sequences=False),\n        Dropout(0.2),  # Droout the data for Regularization\n        \n    \n        # Dense layer for non-linear activation function\n        Dense(16,activation='swish'),\n        Dropout(0.1),\n\n        Dense(1)  # output layer , \n\n        \n    ]\n)\n\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:03:03.910820Z","iopub.execute_input":"2025-01-05T06:03:03.911454Z","iopub.status.idle":"2025-01-05T06:03:05.800393Z","shell.execute_reply.started":"2025-01-05T06:03:03.911419Z","shell.execute_reply":"2025-01-05T06:03:05.799417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy', 'AUC'])\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:03:13.270357Z","iopub.execute_input":"2025-01-05T06:03:13.271253Z","iopub.status.idle":"2025-01-05T06:03:13.299417Z","shell.execute_reply.started":"2025-01-05T06:03:13.271214Z","shell.execute_reply":"2025-01-05T06:03:13.298615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=25, batch_size=32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T06:03:37.257303Z","iopub.execute_input":"2025-01-05T06:03:37.257659Z","execution_failed":"2025-01-05T10:08:37.952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def weighted_r2(y_val, y_pred, weights):\n\n#     # Zero-mean adjustment for predictions\n#     y_pred_mean = (y_pred * weights).sum() / weights.sum()\n#     y_pred_adjusted = y_pred - y_pred_mean\n\n#     # Total error (Weighted Residual Sum of Squares)\n#     total_error = (weights * (y_val - y_pred_adjusted) ** 2).sum()\n\n#     # Variance (Weighted Total Sum of Squares)\n#     variance = (weights * (y_val - y_val.mean()) ** 2).sum()\n\n#     # Weighted R² score\n#     weighted_r2_score = 1 - (total_error / variance)\n#     return weighted_r2_score\n\n\n# epochs = [15,20,25]\n# batch_size= [32,64,128]\n\n# for i in range(3):\n#     history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=epochs[i], batch_size=batch_size[i])\n#     print(f\"Batch Size {batch_size[i]}, epochs :- {epochs[i]}\\n\")\n    \n#     x_val = x_val.reshape(-1, timesteps, n_features)\n#     y_pred = model.predict(x_val)\n#     # Assuming you have y_test (true values), y_pred (predicted values), and weights\n#     weights = np.random.rand(len(y_val))  # Example weights; replace with actual weights\n#     r2_score = weighted_r2(y_val.flatten(), y_pred.flatten(), weights)\n#     print(f\"R2 Score \")\n#     print(f\"Weighted R² Score {i} ::-- {r2_score:.4f}\") \n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T11:43:26.747903Z","iopub.execute_input":"2024-12-02T11:43:26.748546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nx_val = x_val.reshape(-1, timesteps, n_features)\ny_pred = model.predict(x_val)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:12:14.185905Z","iopub.execute_input":"2025-01-07T10:12:14.186253Z","iopub.status.idle":"2025-01-07T10:13:13.844993Z","shell.execute_reply.started":"2025-01-07T10:12:14.186223Z","shell.execute_reply":"2025-01-07T10:13:13.844201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:13:13.847781Z","iopub.execute_input":"2025-01-07T10:13:13.848587Z","iopub.status.idle":"2025-01-07T10:13:13.855169Z","shell.execute_reply.started":"2025-01-07T10:13:13.848538Z","shell.execute_reply":"2025-01-07T10:13:13.854343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\ndef weighted_r2(y_val, y_pred, weights):\n\n    # Zero-mean adjustment for predictions\n    y_pred_mean = (y_pred * weights).sum() / weights.sum()\n    y_pred_adjusted = y_pred - y_pred_mean\n\n    # Total error (Weighted Residual Sum of Squares)\n    total_error = (weights * (y_val - y_pred_adjusted) ** 2).sum()\n\n    # Variance (Weighted Total Sum of Squares)\n    variance = (weights * (y_val - y_val.mean()) ** 2).sum()\n\n    # Weighted R² score\n    weighted_r2_score = 1 - (total_error / variance)\n    return weighted_r2_score\n\n# Assuming you have y_test (true values), y_pred (predicted values), and weights\nweights = np.random.rand(len(y_val))  # Example weights; replace with actual weights\nr2_score = weighted_r2(y_val.flatten(), y_pred.flatten(), weights)\n\nprint(f\"Weighted R² Score: {r2_score:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:13:13.856409Z","iopub.execute_input":"2025-01-07T10:13:13.856797Z","iopub.status.idle":"2025-01-07T10:13:13.891722Z","shell.execute_reply.started":"2025-01-07T10:13:13.856739Z","shell.execute_reply":"2025-01-07T10:13:13.890782Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGBoost\n\nSee [Feature engineering, xgboost](https://www.kaggle.com/code/dlarionov/feature-engineering-xgboost#Part-2,-xgboost) and [🥇🥇Jane Street Baseline lgb, xgb and catboost🥇🥇](https://www.kaggle.com/code/yuanzhezhou/jane-street-baseline-lgb-xgb-and-catboost)for details.","metadata":{}},{"cell_type":"code","source":"import joblib\nmodel.save('mymodel.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T10:18:37.745843Z","iopub.execute_input":"2025-01-05T10:18:37.746153Z","iopub.status.idle":"2025-01-05T10:18:37.762534Z","shell.execute_reply.started":"2025-01-05T10:18:37.746126Z","shell.execute_reply":"2025-01-05T10:18:37.761520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.301988Z","iopub.execute_input":"2025-01-07T10:10:49.302369Z","iopub.status.idle":"2025-01-07T10:10:49.309019Z","shell.execute_reply.started":"2025-01-07T10:10:49.302330Z","shell.execute_reply":"2025-01-07T10:10:49.307533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = \"/kaggle/input/jane_street/tensorflow2/version-1.0/1/mymodel.h5\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.310529Z","iopub.execute_input":"2025-01-07T10:10:49.310897Z","iopub.status.idle":"2025-01-07T10:10:49.318721Z","shell.execute_reply.started":"2025-01-07T10:10:49.310855Z","shell.execute_reply":"2025-01-07T10:10:49.317692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.models import load_model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.321049Z","iopub.execute_input":"2025-01-07T10:10:49.321336Z","iopub.status.idle":"2025-01-07T10:10:49.335772Z","shell.execute_reply.started":"2025-01-07T10:10:49.321310Z","shell.execute_reply":"2025-01-07T10:10:49.334687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = load_model(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.336919Z","iopub.execute_input":"2025-01-07T10:10:49.337427Z","iopub.status.idle":"2025-01-07T10:10:49.488521Z","shell.execute_reply.started":"2025-01-07T10:10:49.337388Z","shell.execute_reply":"2025-01-07T10:10:49.487644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.489706Z","iopub.execute_input":"2025-01-07T10:10:49.490033Z","iopub.status.idle":"2025-01-07T10:10:49.494921Z","shell.execute_reply.started":"2025-01-07T10:10:49.489995Z","shell.execute_reply":"2025-01-07T10:10:49.494006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.496084Z","iopub.execute_input":"2025-01-07T10:10:49.496406Z","iopub.status.idle":"2025-01-07T10:10:49.508460Z","shell.execute_reply.started":"2025-01-07T10:10:49.496370Z","shell.execute_reply":"2025-01-07T10:10:49.507709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import kaggle_evaluation.jane_street_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:10:49.531537Z","iopub.execute_input":"2025-01-07T10:10:49.531872Z","iopub.status.idle":"2025-01-07T10:10:49.540002Z","shell.execute_reply.started":"2025-01-07T10:10:49.531837Z","shell.execute_reply":"2025-01-07T10:10:49.539114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T08:53:33.815072Z","iopub.execute_input":"2025-01-07T08:53:33.815318Z","iopub.status.idle":"2025-01-07T08:53:33.823697Z","shell.execute_reply.started":"2025-01-07T08:53:33.815283Z","shell.execute_reply":"2025-01-07T08:53:33.822706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\n# Assuming features required by the model are named 'feature_00', 'feature_01', etc.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_\n    if lags is not None:\n        lags_ = lags\n    # Extract the features for the model input\n    feature_columns = [col for col in test.columns if col.startswith(\"feature_\")]\n    features = test.select(feature_columns).to_numpy()  # Convert to numpy array for model input\n    features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    # Generate predictions using the model.0\n    #model_predictions = model.predict(features)\n    print(\"features :- \",features)\n    x = features\n    n_features = x.shape[1]\n    x_lstm = x.reshape(-1,1,n_features)\n    print(x_lstm.shape)\n    responder_6_predictions = model.predict(x_lstm)\n   # print(responder_6_predictions)    \n    #responder_6_predictions = model_predictions[:, 6]  # Assuming responder_6 is at index 6\n    # Create a new Polars DataFrame with row_id and responder_6 predictions\n    predictions = test.select(\"row_id\").with_columns(\n        pl.Series(\"responder_6\", responder_6_predictions)\n    )\n    print(predictions)\n    # Ensure the output format and length requirements\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    \n    assert len(predictions) == len(test)\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:16:34.479213Z","iopub.execute_input":"2025-01-07T10:16:34.480208Z","iopub.status.idle":"2025-01-07T10:16:34.490712Z","shell.execute_reply.started":"2025-01-07T10:16:34.480159Z","shell.execute_reply":"2025-01-07T10:16:34.489737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server\n\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T10:16:38.006687Z","iopub.execute_input":"2025-01-07T10:16:38.007033Z","iopub.status.idle":"2025-01-07T10:16:38.270982Z","shell.execute_reply.started":"2025-01-07T10:16:38.007002Z","shell.execute_reply":"2025-01-07T10:16:38.270082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build the Autoencoder Model","metadata":{}},{"cell_type":"markdown","source":"Gradient Centralization for Better Training Performance\n\nSee https://keras.io/examples/vision/gradient_centralization/ for details. ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Autoencoder Model","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission\n\nSee [Jane Street RMF Demo Submission](https://www.kaggle.com/code/ryanholbrook/jane-street-rmf-demo-submission) for details.\n\nDepending on the size of your training set, you will need an [inference notebook](https://www.kaggle.com/code/regisvargas/inference-jane-street-a-beginner-s-notebook).","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}