{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":182654,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":155681,"modelId":178142}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Mainly for model training\n\nDepending on the size of your training set, you will need an [inference notebook](https://www.kaggle.com/code/regisvargas/inference-jane-street-a-beginner-s-notebook).","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import models\npd.set_option('display.max_rows', None)  # No limit on rows\npd.set_option('display.max_columns', None)  # No limit on columns\nimport cupy as cp\nTEST = True  # Change to True for testing, False for training\n\nimport polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T12:45:50.384092Z","iopub.execute_input":"2024-12-03T12:45:50.384464Z","iopub.status.idle":"2024-12-03T12:45:50.389801Z","shell.execute_reply.started":"2024-12-03T12:45:50.384432Z","shell.execute_reply":"2024-12-03T12:45:50.388995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = \"/kaggle/input/janestreet-demo/keras/default/1/best_model_fold_0.h5.keras\"\nmodel = tf.keras.models.load_model(model_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T12:45:56.874329Z","iopub.execute_input":"2024-12-03T12:45:56.875034Z","iopub.status.idle":"2024-12-03T12:45:57.333096Z","shell.execute_reply.started":"2024-12-03T12:45:56.875003Z","shell.execute_reply":"2024-12-03T12:45:57.332399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: pl.DataFrame | None = None) -> pl.DataFrame:\n    \"\"\"Generate predictions using the pre-trained model.\"\"\"\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    # Preprocess the test data using Polars\n    test = test.fill_null(0).fill_nan(0)\n\n    # Extract features and prepare input matrix\n    feat_cols = [col for col in test.columns if col.startswith('feature_')]\n    features = test.select(feat_cols).to_numpy()\n    X = np.hstack([\n        test.select(['time_id', 'symbol_id']).to_numpy(), \n        features\n    ])\n\n    # Generate predictions\n    pred = model.predict(X)[1].ravel()\n\n    # Prepare and return predictions as a Polars DataFrame\n    predictions = pl.DataFrame({\n        'row_id': test['row_id'].to_numpy(),\n        'responder_6': pred\n    })\n\n    return predictions\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T12:45:58.512444Z","iopub.execute_input":"2024-12-03T12:45:58.512770Z","iopub.status.idle":"2024-12-03T12:45:58.519159Z","shell.execute_reply.started":"2024-12-03T12:45:58.512742Z","shell.execute_reply":"2024-12-03T12:45:58.518177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T12:45:33.136707Z","iopub.execute_input":"2024-12-03T12:45:33.137024Z","iopub.status.idle":"2024-12-03T12:45:33.254915Z","shell.execute_reply.started":"2024-12-03T12:45:33.136995Z","shell.execute_reply":"2024-12-03T12:45:33.254269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def predict(test: pd.DataFrame, lags: pd.DataFrame | None = None) -> pd.DataFrame:\n#     \"\"\"Generate predictions using the pre-trained model.\"\"\"\n#     # Preprocess the test data to prepare features (matching the training preprocessing)\n#     global lags_\n#     if lags is not None:\n#         lags_ = lags\n\n#     test = test.to_pandas()\n#     test.ffill(inplace=True)\n#     test.fillna(0, inplace=True)\n\n#     feat_cols = [col for col in test.columns if col.startswith('feature_')]\n#     features = test[feat_cols].values\n#     time_id = test['time_id'].values.reshape(-1, 1)  # Ensure it is (n_samples, 1)\n#     symbol_id = test['symbol_id'].values.reshape(-1, 1)  # Ensure it is (n_samples, 1)\n#     X = np.hstack([time_id, symbol_id, features])\n\n\n#     # Load the pre-trained model\n    \n#     # model_path2 = \"/kaggle/input/janestreet-demo/keras/default/1/best_model_fold_0.h5.keras\"\n#     # model_path3 = \"/kaggle/input/janestreet-demo/keras/default/1/best_model_fold_0.h5.keras\"\n   \n    \n\n#     # Generate predictions\n#     # pred = [0]  # Assuming responder_6 is the first output\n#     pred = model.predict(X)[1].reshape(-1)  # Assuming responder_6 is the first output\n   \n#     # print(pred)\n#     # Prepare the predictions DataFrame\n#     predictions = pd.DataFrame({\n#         'row_id': test['row_id'],\n#         'responder_6': pred\n#     })\n#     predictions=pl.from_pandas(predictions)\n#     # predictions = test.select(\"row_id\").with_columns(\n#     #     pl.Series(\"responder_6\", pred)\n#     # )\n\n#     # Ensure the output format and length requirements\n#     if isinstance(predictions, pl.DataFrame):\n#         assert predictions.columns == ['row_id', 'responder_6']\n#     elif isinstance(predictions, pd.DataFrame):\n#         assert (predictions.columns == ['row_id', 'responder_6']).all()\n#     else:\n#         raise TypeError('The predict function must return a DataFrame')\n    \n#     assert len(predictions) == len(test)\n#     return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T12:42:12.384623Z","iopub.execute_input":"2024-12-03T12:42:12.385196Z","iopub.status.idle":"2024-12-03T12:42:12.393897Z","shell.execute_reply.started":"2024-12-03T12:42:12.385156Z","shell.execute_reply":"2024-12-03T12:42:12.392843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}