{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom pathlib import Path\n\nfrom catboost import CatBoostRegressor, EShapCalcType, EFeaturesSelectionAlgorithm, Pool\nimport kaggle_evaluation.jane_street_inference_server\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n\nDATA_DIR = Path(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:28:13.385127Z","iopub.execute_input":"2025-01-15T19:28:13.385310Z","iopub.status.idle":"2025-01-15T19:28:16.431693Z","shell.execute_reply.started":"2025-01-15T19:28:13.385292Z","shell.execute_reply":"2025-01-15T19:28:16.430731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_09_dict = {\n    2: 0,\n    4: 1,\n    9: 2,\n    11: 3,\n    12: 4,\n    14: 5,\n    15: 6,\n    25: 7,\n    26: 8,\n    30: 9,\n    34: 10,\n    42: 11,\n    44: 12,\n    46: 13,\n    49: 14,\n    50: 15,\n    57: 16,\n    64: 17,\n    68: 18,\n    70: 19,\n    81: 20,\n    82: 21\n}\nfeature_10_dict = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 10: 7, 12: 8}\nfeature_11_dict = {\n    9: 0,\n    11: 1,\n    13: 2,\n    16: 3,\n    24: 4,\n    25: 5,\n    34: 6,\n    40: 7,\n    48: 8,\n    50: 9,\n    59: 10,\n    62: 11,\n    63: 12,\n    66: 13,\n    76: 14,\n    150: 15,\n    158: 16,\n    159: 17,\n    171: 18,\n    195: 19,\n    214: 20,\n    230: 21,\n    261: 22,\n    297: 23,\n    336: 24,\n    376: 25,\n    388: 26,\n    410: 27,\n    522: 28,\n    534: 29,\n    539: 30\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:28:16.432737Z","iopub.execute_input":"2025-01-15T19:28:16.433123Z","iopub.status.idle":"2025-01-15T19:28:16.439536Z","shell.execute_reply.started":"2025-01-15T19:28:16.433098Z","shell.execute_reply":"2025-01-15T19:28:16.438680Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_memory_usage() -> pl.Expr:\n    expressions = [\n        pl.col(pl.Float64).cast(pl.Float32),\n        pl.col(\"date_id\", \"time_id\").cast(pl.Int16),\n        pl.col(\"symbol_id\").cast(pl.Int8),\n        ]\n    return expressions\n\ndef map_category() -> pl.Expr:\n    expressions = [\n        pl.col('symbol_id').replace({i: i for i in range(39)}).fill_null(99),\n        pl.col('feature_09').replace(feature_09_dict).fill_null(99),\n        pl.col('feature_10').replace(feature_10_dict).fill_null(99),\n        pl.col('feature_11').replace(feature_11_dict).fill_null(99),\n    ]\n    return expressions\n    \ndef get_temporal_features() -> pl.Expr:\n    expressions = [\n        (pl.col('date_id') % 170).alias('day'),\n        (pl.col('date_id') * 2 * np.pi / 170).sin().cast(pl.Float32).alias('date_sin'),\n        (pl.col('date_id') * 2 * np.pi / 170).cos().cast(pl.Float32).alias('date_cos'),\n        #(pl.col('time_id') * 2 * np.pi / 967).sin().cast(pl.Float32).alias('time_id_sin'),\n        #(pl.col('time_id') * 2 * np.pi / 967).cos().cast(pl.Float32).alias('time_id_cos')\n    ]\n    return expressions\n\ndef get_lag_stats_per_day() -> pl.Expr:\n    group = [\"date_id\", \"symbol_id\"]\n    # cols = [f\"responder_{i}_lag_1\" for i in range(9)]\n    cols = ['responder_6_lag_1']\n    expressions = []\n    for col in cols:\n        exprs = [\n            # pl.col(col).min().over(group).alias(f\"{col}_min\"),\n            pl.col(col).max().over(group).alias(f\"{col}_max\"),\n            # pl.col(col).std().over(group).alias(f\"{col}_std\")\n            # pl.col(col).median().over(group).alias(f\"{col}_median_per_day\")\n        ]\n        expressions.extend(exprs)\n    return expressions\n\ndef get_lag_features() -> pl.Expr:\n    group = ['symbol_id', 'date_id']\n    expressions = [\n        pl.col('feature_07').shift(2).over(group).alias('feature_07_lag'),\n        pl.col('feature_06').shift(1).over(group).alias('feature_06_lag'),\n        pl.col('feature_60').shift(2).over(group).alias('feature_60_lag'),\n        # pl.col('feature_04').shift(4).over(group).alias('feature_04_lag'),\n        # pl.col('feature_05').shift(5).over(group).alias('feature_05_lag'),\n        # pl.col('feature_36').shift(4).over(group).alias('feature_36_lag'),\n        # pl.col('feature_58').shift(4).over(group).alias('feature_58_lag'),\n        # pl.col('feature_59').shift(1).over(group).alias('feature_59_lag'),\n        # pl.col('feature_38').shift(8).over(group).alias('feature_38_lag'),\n        # pl.col('feature_52').shift(3).over(group).alias('feature_52_lag'),\n    ]\n    return expressions\n\ndef generate_features(df, df_lags):\n    exprs_1 = [\n        map_category(),\n        get_temporal_features(),\n        get_lag_features(),\n    ]\n    expressions = [e for sublist in exprs_1 for e in sublist]\n    df = df.with_columns(reduce_memory_usage())\n    df = df.with_columns(\n        pl.col('feature_09').cast(pl.Int8), \n        pl.col('feature_10').cast(pl.Int8), \n        pl.col('feature_11').cast(pl.Int16), \n    )\n    df = df.with_columns(expressions)\n    \n    df_lags = df_lags.with_columns(reduce_memory_usage())\n    df_lags = df_lags.with_columns(get_lag_stats_per_day())\n    df = df.join(\n        df_lags,\n        on=[\"date_id\", \"time_id\", \"symbol_id\"], how=\"left\"\n    )\n    ign_cols = [f\"responder_{i}_lag_1\" for i in range(9)] + ['date_id']\n    return df.select(pl.all().exclude(ign_cols))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:28:16.440391Z","iopub.execute_input":"2025-01-15T19:28:16.440576Z","iopub.status.idle":"2025-01-15T19:28:16.459994Z","shell.execute_reply.started":"2025-01-15T19:28:16.440559Z","shell.execute_reply":"2025-01-15T19:28:16.459077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_lag_responders() -> pl.Expr:\n    cols = [f\"responder_{i}\" for i in range(9)]\n    expressions = [\n        pl.col(col)\n        .shift(i)\n        .over('symbol_id', 'time_id')\n        .alias(f\"{col}_lag_{i}\")\n        for col in cols for i in [1] # lags\n        ]\n    return expressions\n\ntrain = pl.scan_parquet(DATA_DIR).filter(pl.col('partition_id')>5)\ntrain = train.with_columns(get_lag_responders()).collect()\n# remove nulls resulting from day 1 lags\nstart_date = train.select('date_id')[0]\ntrain = train.filter(pl.col('date_id') > start_date)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:28:16.460770Z","iopub.execute_input":"2025-01-15T19:28:16.461058Z","iopub.status.idle":"2025-01-15T19:29:12.705627Z","shell.execute_reply.started":"2025-01-15T19:28:16.461009Z","shell.execute_reply":"2025-01-15T19:29:12.704613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = train.filter(\n    pl.col('date_id')>1576,\n    pl.col('symbol_id')==0\n    ).select('feature_07').to_numpy().reshape(-1)\nplot_acf(data, lags = 10, title='Autocorrelation'); print()\nplot_pacf(data, lags = 10, title='Partial Autocorrelation'); print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:29:12.707858Z","iopub.execute_input":"2025-01-15T19:29:12.708986Z","iopub.status.idle":"2025-01-15T19:29:16.207086Z","shell.execute_reply.started":"2025-01-15T19:29:12.708926Z","shell.execute_reply":"2025-01-15T19:29:16.206045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features = [f\"feature_{i:02}\" for i in range(79)]\nlag_responders = [f\"responder_{i}_lag_1\" for i in range(9)]\n\nparams = {\n    'iterations': 1200,\n    'learning_rate': 0.02,\n    'depth': 8,\n    'l2_leaf_reg': 5,\n    'bootstrap_type': 'Bernoulli',\n    'subsample': 0.9,\n    'loss_function': 'RMSE',\n    'eval_metric': 'MAE',\n    'metric_period': 100,\n    'od_type': 'Iter',\n    'od_wait': 30,\n    'task_type': 'GPU',\n    'allow_writing_files': False,\n    'use_best_model': False\n}\n\nmodel = CatBoostRegressor(**params)\n\nx_train = generate_features(\n    train.filter(pl.col('date_id')<1576).select(['date_id', 'time_id', 'symbol_id', 'weight'] + features),\n    train.filter(pl.col('date_id')<1576).select(['date_id', 'time_id', 'symbol_id'] + lag_responders)\n    ).to_numpy()\ny_train = train.filter(pl.col('date_id')<1576).select('responder_6').to_numpy()\n\nx_valid = generate_features(\n    train.filter(pl.col('date_id')>1576).select(['date_id', 'time_id', 'symbol_id', 'weight'] + features),\n    train.filter(pl.col('date_id')>1576).select(['date_id', 'time_id', 'symbol_id'] + lag_responders)\n    ).to_numpy()\ny_valid = train.filter(pl.col('date_id')>1576).select('responder_6').to_numpy()\n\nmodel.fit(\n    Pool(x_train, y_train),\n    # verbose=False,\n    eval_set=[(x_valid, y_valid)]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:29:16.208087Z","iopub.execute_input":"2025-01-15T19:29:16.208327Z","iopub.status.idle":"2025-01-15T19:32:02.393871Z","shell.execute_reply.started":"2025-01-15T19:29:16.208301Z","shell.execute_reply":"2025-01-15T19:32:02.392928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_r2(y_true, y_pred, sample_weight):\n    r2 = (1 - np.average((y_true - y_pred)**2, weights=sample_weight) /\n          (np.average(y_true**2, weights=sample_weight) + 1e-38)\n    )\n    return r2\n\nr2_score = weighted_r2(\n    y_valid.reshape(-1),\n    model.predict(x_valid),\n    train.filter(pl.col('date_id')>1576).select('weight').to_numpy().reshape(-1)\n)\nprint(\"r2_score:\",  r2_score.round(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:32:02.395548Z","iopub.execute_input":"2025-01-15T19:32:02.395847Z","iopub.status.idle":"2025-01-15T19:32:06.854557Z","shell.execute_reply.started":"2025-01-15T19:32:02.395821Z","shell.execute_reply":"2025-01-15T19:32:06.853585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}