{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-27T20:20:29.602090Z","iopub.execute_input":"2024-10-27T20:20:29.603053Z","iopub.status.idle":"2024-10-27T20:20:29.630766Z","shell.execute_reply.started":"2024-10-27T20:20:29.603006Z","shell.execute_reply":"2024-10-27T20:20:29.629611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nfrom sklearn.linear_model import Ridge\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport kaggle_evaluation.jane_street_inference_server\n\nimport random\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:29.632760Z","iopub.execute_input":"2024-10-27T20:20:29.633125Z","iopub.status.idle":"2024-10-27T20:20:29.639503Z","shell.execute_reply.started":"2024-10-27T20:20:29.633088Z","shell.execute_reply":"2024-10-27T20:20:29.638232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:29.640976Z","iopub.execute_input":"2024-10-27T20:20:29.641333Z","iopub.status.idle":"2024-10-27T20:20:29.651279Z","shell.execute_reply.started":"2024-10-27T20:20:29.641288Z","shell.execute_reply":"2024-10-27T20:20:29.650157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pl.read_parquet(DATA_PATH + \"/test.parquet\")\nlags = pl.read_parquet(DATA_PATH + \"/lags.parquet\")\ntrain = pl.read_parquet(DATA_PATH + \"/train.parquet\",n_rows=1_000_000)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:29.653310Z","iopub.execute_input":"2024-10-27T20:20:29.653701Z","iopub.status.idle":"2024-10-27T20:20:29.994407Z","shell.execute_reply.started":"2024-10-27T20:20:29.653655Z","shell.execute_reply":"2024-10-27T20:20:29.993446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:29.995591Z","iopub.execute_input":"2024-10-27T20:20:29.995984Z","iopub.status.idle":"2024-10-27T20:20:30.014213Z","shell.execute_reply.started":"2024-10-27T20:20:29.995947Z","shell.execute_reply":"2024-10-27T20:20:30.013027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.015451Z","iopub.execute_input":"2024-10-27T20:20:30.015808Z","iopub.status.idle":"2024-10-27T20:20:30.028647Z","shell.execute_reply.started":"2024-10-27T20:20:30.015772Z","shell.execute_reply":"2024-10-27T20:20:30.027580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.031781Z","iopub.execute_input":"2024-10-27T20:20:30.032217Z","iopub.status.idle":"2024-10-27T20:20:30.052091Z","shell.execute_reply.started":"2024-10-27T20:20:30.032180Z","shell.execute_reply":"2024-10-27T20:20:30.050926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_lags(df,on='date_id'):\n    \n    df = df.with_columns((pl.col(on) - 1))\n    \n    df = df.with_columns(\n    pl.when(pl.col(\"time_id\") > 0)\n    .then(0).alias(\"time_id\"))\n    \n    df = df.filter(pl.all_horizontal(pl.col(\"time_id\").is_not_null()))\n\n    df = df.filter(pl.all_horizontal(pl.col(\"date_id\").gt(-1)))\n    \n    df = df.select([\"date_id\", \"time_id\", \"responder_0\", \"responder_1\", \"responder_2\", \"responder_3\", \"responder_4\", \"responder_5\", \n                   \"responder_6\", \"responder_7\", \"responder_8\"])\n    \n    df = df.rename({\"responder_0\": \"responder_0_lag_1\",\"responder_1\": \"responder_1_lag_1\",\n                   \"responder_2\": \"responder_2_lag_1\",\"responder_3\": \"responder_3_lag_1\",\n                   \"responder_4\": \"responder_4_lag_1\",\"responder_5\": \"responder_5_lag_1\",\n                   \"responder_6\": \"responder_6_lag_1\",\"responder_7\": \"responder_7_lag_1\",\n                   \"responder_8\": \"responder_8_lag_1\",})\n\n    \n \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.053848Z","iopub.execute_input":"2024-10-27T20:20:30.054233Z","iopub.status.idle":"2024-10-27T20:20:30.063061Z","shell.execute_reply.started":"2024-10-27T20:20:30.054184Z","shell.execute_reply":"2024-10-27T20:20:30.062001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags_emulate = make_lags(train)\nlags_emulate","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.064545Z","iopub.execute_input":"2024-10-27T20:20:30.065626Z","iopub.status.idle":"2024-10-27T20:20:30.394084Z","shell.execute_reply.started":"2024-10-27T20:20:30.065577Z","shell.execute_reply":"2024-10-27T20:20:30.392896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_date = lags_emulate.select('date_id').unique(maintain_order= True)\nunique_date.select('date_id')","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.395322Z","iopub.execute_input":"2024-10-27T20:20:30.395669Z","iopub.status.idle":"2024-10-27T20:20:30.416805Z","shell.execute_reply.started":"2024-10-27T20:20:30.395620Z","shell.execute_reply":"2024-10-27T20:20:30.415538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.with_columns(pl.col(\"date_id\").is_in(unique_date.to_series().to_list()).alias('matching_dates'))\ntrain = train.filter(pl.all_horizontal(pl.col(\"matching_dates\")==True)).drop('matching_dates')\ntrain\n","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.418079Z","iopub.execute_input":"2024-10-27T20:20:30.418419Z","iopub.status.idle":"2024-10-27T20:20:30.456479Z","shell.execute_reply.started":"2024-10-27T20:20:30.418385Z","shell.execute_reply":"2024-10-27T20:20:30.455432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\n# Replace this function with your inference code.\n# You can return either a Pandas or Polars dataframe, though Polars is recommended.\n# Each batch of predictions (except the very first) must be returned within 10 minutes of the batch features being provided.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None, emulate=False) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    # All the responders from the previous day are passed in at time_id == 0. We save them in a global variable for access at every time_id.\n    # Use them as extra features, if you like.\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    if not emulate:\n        predictions = test.select(\n                'row_id', 'date_id',\n                pl.lit(0.0).alias('responder_6'),\n            )\n    else:\n        predictions = test.select(\n                 'date_id',\n                pl.lit(0.0).alias('responder_6'),\n            )\n    \n    print('predictions')\n    print(predictions)\n    \n    mean_by_date = (lags_.group_by(\"date_id\", maintain_order=True).median().\n                    select('date_id','responder_6_lag_1'))\n\n    print('mean_by_date1')\n    print(mean_by_date)\n\n    #mean_by_date = mean_by_date.select(pl.all().repeat_by(predictions.height).arr.explode())\n    mean_by_date = mean_by_date.rename({\"responder_6_lag_1\": \"responder_6\"})\n\n    print('mean_by_date2')\n    print(mean_by_date)\n\n    #mean_by_date = mean_by_date.select(pl.all().repeat_by(predictions.height).arr.explode())\n\n    mean_by_date = predictions.join(mean_by_date, on=\"date_id\")\n\n    print('mean_by_date3')\n    print(mean_by_date)\n    \n    \n    #print(mean_by_date)\n    if not emulate:\n        predictions = predictions.replace_column(1, mean_by_date.to_series(3)).drop('responder_6')\n    else:\n        predictions = predictions.replace_column(1, mean_by_date.to_series(2))\n\n    predictions = predictions.rename({\"responder_6_right\": \"responder_6\"})\n    \n    predictions = predictions.fill_nan(0)\n    \n    print('predictions')\n    print(predictions)\n\n    \n    if not emulate:\n\n        # The predict function must return a DataFrame\n        assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n        # with columns 'row_id', 'responer_6'\n        assert predictions.columns == ['row_id', 'responder_6']\n        # and as many rows as the test data.\n        assert len(predictions) == len(test)\n\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.458225Z","iopub.execute_input":"2024-10-27T20:20:30.458577Z","iopub.status.idle":"2024-10-27T20:20:30.469893Z","shell.execute_reply.started":"2024-10-27T20:20:30.458541Z","shell.execute_reply":"2024-10-27T20:20:30.468680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_emulat = predict(train,lags_emulate, emulate=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.471527Z","iopub.execute_input":"2024-10-27T20:20:30.471928Z","iopub.status.idle":"2024-10-27T20:20:30.556671Z","shell.execute_reply.started":"2024-10-27T20:20:30.471890Z","shell.execute_reply":"2024-10-27T20:20:30.555507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def weighted_zero_mean_r2(y_true, y_pred, weights):\n    \"\"\"\n    Calculate the sample weighted zero-mean R-squared score.\n\n    Parameters:\n    y_true (numpy.ndarray): Ground-truth values for responder_6.\n    y_pred (numpy.ndarray): Predicted values for responder_6.\n    weights (numpy.ndarray): Sample weight vector.\n\n    Returns:\n    float: The weighted zero-mean R-squared score.\n    \"\"\"\n    \n\n    \n    numerator = np.sum(weights * (y_true - y_pred)**2)\n    denominator = np.sum(weights * y_true**2)\n    \n    r2_score = 1 - numerator / denominator\n    return r2_score","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.558422Z","iopub.execute_input":"2024-10-27T20:20:30.558806Z","iopub.status.idle":"2024-10-27T20:20:30.564917Z","shell.execute_reply.started":"2024-10-27T20:20:30.558767Z","shell.execute_reply":"2024-10-27T20:20:30.563679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score =  weighted_zero_mean_r2(train.select('responder_6').to_series(0).to_pandas(), \n                               predictions_emulat.select('responder_6').to_series(0).to_pandas(), \n                               train.select('weight').to_series(0).to_pandas())\nscore","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.569614Z","iopub.execute_input":"2024-10-27T20:20:30.570086Z","iopub.status.idle":"2024-10-27T20:20:30.593601Z","shell.execute_reply.started":"2024-10-27T20:20:30.570039Z","shell.execute_reply":"2024-10-27T20:20:30.592564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.594887Z","iopub.execute_input":"2024-10-27T20:20:30.595217Z","iopub.status.idle":"2024-10-27T20:20:30.653073Z","shell.execute_reply.started":"2024-10-27T20:20:30.595184Z","shell.execute_reply":"2024-10-27T20:20:30.651964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pl.read_parquet(\"/kaggle/working/submission.parquet\")\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-10-27T20:20:30.654399Z","iopub.execute_input":"2024-10-27T20:20:30.654777Z","iopub.status.idle":"2024-10-27T20:20:30.663076Z","shell.execute_reply.started":"2024-10-27T20:20:30.654739Z","shell.execute_reply":"2024-10-27T20:20:30.661813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}