{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":10245336,"sourceType":"datasetVersion","datasetId":6336300}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"21c2a0ee-f57b-4650-96c7-fc6eafa6ea0d","_cell_guid":"1415df9a-5724-4f95-af5d-73338dc8e0c5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T06:50:29.839234Z","iopub.execute_input":"2024-12-29T06:50:29.839858Z","iopub.status.idle":"2024-12-29T06:50:30.201982Z","shell.execute_reply.started":"2024-12-29T06:50:29.839806Z","shell.execute_reply":"2024-12-29T06:50:30.200600Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nfrom sklearn.metrics import r2_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import pearsonr, spearmanr","metadata":{"_uuid":"240291de-9cbf-4014-9b5d-858d88eb65b6","_cell_guid":"68bbf573-d662-43a4-9382-581b06b06ccf","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T08:02:21.421439Z","iopub.execute_input":"2024-12-29T08:02:21.421834Z","iopub.status.idle":"2024-12-29T08:02:21.427454Z","shell.execute_reply.started":"2024-12-29T08:02:21.421803Z","shell.execute_reply":"2024-12-29T08:02:21.426219Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom typing import Optional\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_uuid":"3b26ec40-abfc-4726-bc44-08bb0f525b23","_cell_guid":"f9fdb3b6-50c7-43fb-8298-32f42477ec72","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T08:02:21.599267Z","iopub.execute_input":"2024-12-29T08:02:21.599632Z","iopub.status.idle":"2024-12-29T08:02:21.604324Z","shell.execute_reply.started":"2024-12-29T08:02:21.599604Z","shell.execute_reply":"2024-12-29T08:02:21.603259Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train=pl.scan_parquet(f\"/kaggle/input/20241219-data/training.parquet\").collect()","metadata":{"_uuid":"82448873-98ff-4c89-bdc3-0e2276125cb5","_cell_guid":"571971d7-3825-48e4-b546-0b9e8d1257ee","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T08:02:26.483539Z","iopub.execute_input":"2024-12-29T08:02:26.483917Z","iopub.status.idle":"2024-12-29T08:02:31.280349Z","shell.execute_reply.started":"2024-12-29T08:02:26.483883Z","shell.execute_reply":"2024-12-29T08:02:31.279191Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"cc03f075-4cb1-4130-8256-726f6b9553c2","_cell_guid":"fb49e76b-573b-487f-beec-9652ae03433f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = train[['id','symbol_id','feature_06','responder_6']]","metadata":{"_uuid":"8415fb34-e8a4-42a9-97f8-d358f862af99","_cell_guid":"5f1f3be9-b1a3-4c83-9931-71c69d81fe04","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T08:02:31.281824Z","iopub.execute_input":"2024-12-29T08:02:31.282085Z","iopub.status.idle":"2024-12-29T08:02:31.287838Z","shell.execute_reply.started":"2024-12-29T08:02:31.282064Z","shell.execute_reply":"2024-12-29T08:02:31.286811Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df=df.to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T08:04:12.516797Z","iopub.execute_input":"2024-12-29T08:04:12.517211Z","iopub.status.idle":"2024-12-29T08:04:12.760731Z","shell.execute_reply.started":"2024-12-29T08:04:12.517176Z","shell.execute_reply":"2024-12-29T08:04:12.759691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T08:04:14.758023Z","iopub.execute_input":"2024-12-29T08:04:14.758352Z","iopub.status.idle":"2024-12-29T08:04:14.790345Z","shell.execute_reply.started":"2024-12-29T08:04:14.758326Z","shell.execute_reply":"2024-12-29T08:04:14.789435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation = df[['feature_06', 'responder_6']].corr()\nprint(correlation)\n\nplt.figure(figsize=(8, 6))\nsns.heatmap(correlation, annot=True, cmap='coolwarm')\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T08:04:17.349801Z","iopub.execute_input":"2024-12-29T08:04:17.350138Z","iopub.status.idle":"2024-12-29T08:04:17.809098Z","shell.execute_reply.started":"2024-12-29T08:04:17.350111Z","shell.execute_reply":"2024-12-29T08:04:17.807978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['feature_06_bin'] = pd.cut(df['feature_06'], bins=5)\n\ngrouped = df.groupby('feature_06_bin')['responder_6'].mean().reset_index()\nprint(grouped)\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x='feature_06_bin', y='responder_6', data=grouped)\nplt.title('Mean responder_6 by feature_06 bins')\nplt.xlabel('feature_06 bins')\nplt.ylabel('Mean responder_6')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T08:04:19.108336Z","iopub.execute_input":"2024-12-29T08:04:19.108724Z","iopub.status.idle":"2024-12-29T08:04:19.671566Z","shell.execute_reply.started":"2024-12-29T08:04:19.108693Z","shell.execute_reply":"2024-12-29T08:04:19.670270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['responder_6_bin'] = pd.cut(df['responder_6'], bins=[-float('inf'), df['responder_6'].median(), float('inf')], labels=['Low', 'High'])\n\nplt.figure(figsize=(10, 6))\nsns.boxplot(x='responder_6_bin', y='feature_06', data=df)\nplt.title('Distribution of feature_06 by responder_6 groups')\nplt.xlabel('responder_6 groups')\nplt.ylabel('feature_06')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T07:00:14.005299Z","iopub.execute_input":"2024-12-29T07:00:14.005686Z","iopub.status.idle":"2024-12-29T07:00:15.785537Z","shell.execute_reply.started":"2024-12-29T07:00:14.005643Z","shell.execute_reply":"2024-12-29T07:00:15.784429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.regplot(x='feature_06', y='responder_6', data=df, order=2, scatter_kws={'alpha':0.5})\nplt.title('Polynomial Regression: feature_06 vs responder_6')\nplt.xlabel('feature_06')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T07:00:15.786882Z","iopub.execute_input":"2024-12-29T07:00:15.787174Z","iopub.status.idle":"2024-12-29T07:20:49.054347Z","shell.execute_reply.started":"2024-12-29T07:00:15.787146Z","shell.execute_reply":"2024-12-29T07:20:49.053004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.boxplot(x=df['feature_06'])\nplt.title('Boxplot of feature_06')\nplt.show()\n\nplt.figure(figsize=(12, 6))\nsns.boxplot(x=df['responder_6'])\nplt.title('Boxplot of responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:55:25.680005Z","iopub.execute_input":"2024-12-29T06:55:25.680287Z","iopub.status.idle":"2024-12-29T06:55:28.328664Z","shell.execute_reply.started":"2024-12-29T06:55:25.680254Z","shell.execute_reply":"2024-12-29T06:55:28.327538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.scatterplot(x='feature_06', y='responder_6', data=df)\nplt.title('Scatter plot of feature_06 vs responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:55:28.330346Z","iopub.execute_input":"2024-12-29T06:55:28.330739Z","iopub.status.idle":"2024-12-29T06:55:41.780211Z","shell.execute_reply.started":"2024-12-29T06:55:28.330705Z","shell.execute_reply":"2024-12-29T06:55:41.779066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val=pl.scan_parquet(f\"/kaggle/input/20241219-data/validation.parquet\").collect()","metadata":{"_uuid":"c5427df4-5737-4219-a18d-a4c78a2800e4","_cell_guid":"d3918ea5-1948-40a4-a07d-a440f1e73f73","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T06:50:38.492271Z","iopub.execute_input":"2024-12-29T06:50:38.492602Z","iopub.status.idle":"2024-12-29T06:50:38.831308Z","shell.execute_reply.started":"2024-12-29T06:50:38.492574Z","shell.execute_reply":"2024-12-29T06:50:38.830190Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val=val.to_pandas()","metadata":{"_uuid":"40743976-aa58-4e03-973c-c4852656e8b3","_cell_guid":"8180caf3-7571-4eac-8359-8e95b9f1dc3d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T06:50:38.832844Z","iopub.execute_input":"2024-12-29T06:50:38.833208Z","iopub.status.idle":"2024-12-29T06:50:39.145223Z","shell.execute_reply.started":"2024-12-29T06:50:38.833177Z","shell.execute_reply":"2024-12-29T06:50:39.143811Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val","metadata":{"_uuid":"bc59995f-8c04-4d48-956f-f155db7f30b1","_cell_guid":"0c9e503e-98b7-46a8-ab82-3dd5abe96233","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T06:50:39.146661Z","iopub.execute_input":"2024-12-29T06:50:39.147060Z","iopub.status.idle":"2024-12-29T06:50:39.232593Z","shell.execute_reply.started":"2024-12-29T06:50:39.147025Z","shell.execute_reply":"2024-12-29T06:50:39.231540Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"64ce6ed7-ca9e-49da-964a-94586dfbefd3","_cell_guid":"fe5c1c5a-a460-4ea1-8b57-aff36e9134b9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"fb985bde-f62c-477d-aa7b-70e1aa70cf8b","_cell_guid":"3aef9efa-44b6-4c95-986c-dcefc1f8936c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"f5e0b966-f19e-44d3-a933-78ed5946df97","_cell_guid":"ddcc13aa-fb05-4e78-a71c-cbc6526c56c5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"7ec023b3-e38c-41db-951d-74a74e2d6226","_cell_guid":"428768e2-af8d-4a5d-a6fe-7a402485c926","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"6d966d5a-e553-4989-bd24-6edf629f2a51","_cell_guid":"63f6cb44-9641-4951-95f5-2b03a7094747","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: Optional[pl.DataFrame] = None) -> pl.DataFrame:\n    \"\"\"\n    Simple prediction function for time series competition.\n    \n    Args:\n        test (pl.DataFrame): Current test data\n        lags (pl.DataFrame, optional): Historical lag data\n    \n    Returns:\n        pl.DataFrame: Predictions for responder_6\n    \"\"\"\n    # Ensure required columns exist\n    required_columns = ['row_id', 'date_id', 'time_id', 'symbol_id']\n    for col in required_columns:\n        if col not in test.columns:\n            raise ValueError(f\"Missing required column: {col}\")\n    \n    # If lags exist, compute mean of responder_6\n    if lags is not None and 'responder_6' in lags.columns:\n        historical_median = lags['responder_6'].median()\n    else:\n        historical_median = 0.0\n    \n    # Generate predictions (with small noise for randomness)\n    predictions = test.with_columns([\n        pl.lit(historical_median + np.random.normal(0, 0.1)).alias('responder_6')\n    ])\n    print(predictions.select(['row_id', 'responder_6']))\n    return predictions.select(['row_id', 'responder_6'])","metadata":{"_uuid":"744d5888-d9a8-4ad5-905f-5477053c027f","_cell_guid":"9ea3b2f4-bb4e-4744-839d-0d8e84f6fb53","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T06:51:13.324565Z","iopub.execute_input":"2024-12-29T06:51:13.324932Z","iopub.status.idle":"2024-12-29T06:51:13.331816Z","shell.execute_reply.started":"2024-12-29T06:51:13.324898Z","shell.execute_reply":"2024-12-29T06:51:13.330503Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"_uuid":"7ac8c73f-0370-411e-8e19-84e6d1bd36c1","_cell_guid":"91027b51-6321-437b-9407-d4475c4f60cc","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T06:51:13.332741Z","iopub.execute_input":"2024-12-29T06:51:13.333115Z","iopub.status.idle":"2024-12-29T06:51:13.382343Z","shell.execute_reply.started":"2024-12-29T06:51:13.333078Z","shell.execute_reply":"2024-12-29T06:51:13.381398Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}