{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":201579302,"sourceType":"kernelVersion"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### NEED FOR HELP\n\nI cannot submit with TF model : [supporting discussion](https://www.kaggle.com/competitions/jane-street-real-time-market-data-forecasting/discussion/541396).","metadata":{"execution":{"iopub.status.busy":"2024-10-18T15:37:03.579529Z","iopub.execute_input":"2024-10-18T15:37:03.581126Z","iopub.status.idle":"2024-10-18T15:37:04.078382Z","shell.execute_reply.started":"2024-10-18T15:37:03.581065Z","shell.execute_reply":"2024-10-18T15:37:04.076807Z"}}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server\nimport os\nfrom tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:30:19.302662Z","iopub.execute_input":"2024-10-20T16:30:19.303174Z","iopub.status.idle":"2024-10-20T16:30:19.983657Z","shell.execute_reply.started":"2024-10-20T16:30:19.303116Z","shell.execute_reply":"2024-10-20T16:30:19.982297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.models as M\nimport tensorflow.keras.layers as L","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:30:19.985514Z","iopub.execute_input":"2024-10-20T16:30:19.986125Z","iopub.status.idle":"2024-10-20T16:30:24.124528Z","shell.execute_reply.started":"2024-10-20T16:30:19.986061Z","shell.execute_reply":"2024-10-20T16:30:24.123175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"USE_STYNTHETIC = False\n\nif USE_STYNTHETIC:\n    syn_dir = '/kaggle/input/js24-rmf-generate-synthetic-test-data'\n    test_parquet = f'{syn_dir}/synthetic_test.parquet'\n    lag_parquet = f'{syn_dir}/synthetic_lag.parquet'\n    total_time_steps = pl.scan_parquet(test_parquet).select(\n        (pl.col(\"date_id\")*10000+pl.col('time_id')).n_unique()   \n        ).collect().item()\nelse:\n    test_parquet = \"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet\"\n    lag_parquet =  \"/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet\"\n    total_time_steps = 1\n    \nprint(\"Test parquet:\", test_parquet)\nprint(\"Lag parquet:\", lag_parquet)\nprint(\"Total time steps:\", total_time_steps)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:30:24.125970Z","iopub.execute_input":"2024-10-20T16:30:24.126694Z","iopub.status.idle":"2024-10-20T16:30:24.155286Z","shell.execute_reply.started":"2024-10-20T16:30:24.126649Z","shell.execute_reply":"2024-10-20T16:30:24.153777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_model(nh=50):\n    x_in = L.Input(name=\"inps\", shape=(79,) )\n    x = L.BatchNormalization(name=\"norm\")(x_in)\n    #x = x_in\n    \n    act = \"relu\"\n    x = L.Dense(nh, name=\"d1\", activation=act)(x)\n    x = L.Dense(nh, name=\"d2\", activation=act)(x)\n    preds = L.Dense(1, name=\"preds\", activation=\"linear\")(x) \n    \n    model = M.Model(x_in, preds, name='ANN')\n    model.compile(loss=\"mse\", optimizer='adam')\n    \n    return model\n#======================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:30:24.156866Z","iopub.execute_input":"2024-10-20T16:30:24.157299Z","iopub.status.idle":"2024-10-20T16:30:24.166660Z","shell.execute_reply.started":"2024-10-20T16:30:24.157257Z","shell.execute_reply":"2024-10-20T16:30:24.165257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FE = [f\"feature_{str(i).zfill(2)}\" for i in range(79)]\nmodel = make_model()\nprint(model.summary())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:30:24.168352Z","iopub.execute_input":"2024-10-20T16:30:24.168746Z","iopub.status.idle":"2024-10-20T16:30:24.303254Z","shell.execute_reply.started":"2024-10-20T16:30:24.168706Z","shell.execute_reply":"2024-10-20T16:30:24.301909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRY = True\nif TRY:\n    x = np.random.normal(0,1,size=(100,79))\n    p = model.predict(x, batch_size=512, verbose=0)[:,0]\n    print(p.shape)\n    print(p[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:30:24.304662Z","iopub.execute_input":"2024-10-20T16:30:24.305062Z","iopub.status.idle":"2024-10-20T16:30:24.509331Z","shell.execute_reply.started":"2024-10-20T16:30:24.305024Z","shell.execute_reply":"2024-10-20T16:30:24.507856Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### FUNCTION","metadata":{}},{"cell_type":"code","source":"\npbar = tqdm(total=total_time_steps, disable=(total_time_steps == 1))\n\ndef predict(test,lags):\n    global FE, pbar\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    x = test[FE].to_pandas().fillna(3).values\n    preds = model.predict(x, batch_size=512, verbose=0)[:, 0]\n    preds = np.clip(preds, -5, 5)\n\n    predictions = predictions.with_columns(pl.Series('responder_6', preds.ravel()))\n\n    # sanity check\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n    assert len(predictions) == len(test)\n\n    pbar.update(1)\n    \n    return predictions","metadata":{"execution":{"iopub.status.busy":"2024-10-20T16:30:24.511577Z","iopub.execute_input":"2024-10-20T16:30:24.512124Z","iopub.status.idle":"2024-10-20T16:30:24.528230Z","shell.execute_reply.started":"2024-10-20T16:30:24.512070Z","shell.execute_reply":"2024-10-20T16:30:24.526693Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SUBMIT","metadata":{}},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            test_parquet, lag_parquet, \n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-10-20T16:30:24.532709Z","iopub.execute_input":"2024-10-20T16:30:24.533211Z","iopub.status.idle":"2024-10-20T16:36:28.626604Z","shell.execute_reply.started":"2024-10-20T16:30:24.533151Z","shell.execute_reply":"2024-10-20T16:36:28.624883Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.path.isfile('submission.parquet'):\n    pl_sub = pl.read_parquet('submission.parquet')\n    print(len(pl_sub))\n    display(pl_sub)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T16:36:28.629836Z","iopub.execute_input":"2024-10-20T16:36:28.630248Z","iopub.status.idle":"2024-10-20T16:36:28.649162Z","shell.execute_reply.started":"2024-10-20T16:36:28.630209Z","shell.execute_reply":"2024-10-20T16:36:28.647752Z"}},"outputs":[],"execution_count":null}]}