{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time\nimport pickle\nimport torch\nimport torch.nn as nn\nimport shutil\nimport polars as pl\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-18T16:45:37.584384Z","iopub.execute_input":"2023-07-18T16:45:37.584766Z","iopub.status.idle":"2023-07-18T16:45:42.957615Z","shell.execute_reply.started":"2023-07-18T16:45:37.584736Z","shell.execute_reply":"2023-07-18T16:45:42.956641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = {\n    \"session_id\": pl.Int64,\n    \"elapsed_time\": pl.Int64,\n    \"level\": pl.Int8,\n    \"page\": pl.Float32,\n    \"room_coor_x\": pl.Float32,\n    \"room_coor_y\": pl.Float32,\n    \"screen_coor_x\": pl.Float32,\n    \"screen_coor_y\": pl.Float32,\n    \"hover_duration\": pl.Float32,\n    \"fullscreen\": pl.Int8,\n    \"hq\": pl.Int8,\n    \"music\": pl.Int8,\n}\ndtypes = [pl.col(key).cast(value) for key, value in dtypes.items()]\nfills = [\n    pl.col(\"page\").fill_null(-1),\n    pl.col(\"fqid\").fill_null(\"fqid_None\"),\n    pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n]","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:42.959829Z","iopub.execute_input":"2023-07-18T16:45:42.960788Z","iopub.status.idle":"2023-07-18T16:45:42.985117Z","shell.execute_reply.started":"2023-07-18T16:45:42.960749Z","shell.execute_reply":"2023-07-18T16:45:42.983757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = [\n    'event_name',\n    'name',\n    'fqid',\n    'room_fqid',\n    'text_fqid'\n]\nNUMS = [\n    'room_coor_x',\n    'room_coor_y',\n    'screen_coor_x',\n    'screen_coor_y',\n    'et_diff',\n    'page'\n]","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:42.987222Z","iopub.execute_input":"2023-07-18T16:45:42.987744Z","iopub.status.idle":"2023-07-18T16:45:43.021483Z","shell.execute_reply.started":"2023-07-18T16:45:42.987698Z","shell.execute_reply":"2023-07-18T16:45:43.020495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_common = pl.read_parquet(\"/kaggle/input/torch-models/most_common.parquet\").to_series()\n\nnorms = pl.read_parquet(\"/kaggle/input/torch-models/norms.parquet\")\n\nf_read = open('/kaggle/input/torch-models/point_ixs.pkl', 'rb')\npoint_ixs = pickle.load(f_read)\nf_read.close()\n\nf_read = open('/kaggle/input/group-models/grp_cat_vals.pkl', 'rb')\ngrp_cat_vals = pickle.load(f_read)\nf_read.close()\n\npoint_pos = pl.read_parquet(\"/kaggle/input/group-models/point_pos.parquet\")\n\nf_read = open('/kaggle/input/group-models/feature_dict.pkl', 'rb')\nfeature_dict = pickle.load(f_read)\nf_read.close()","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:43.024152Z","iopub.execute_input":"2023-07-18T16:45:43.024840Z","iopub.status.idle":"2023-07-18T16:45:43.252180Z","shell.execute_reply.started":"2023-07-18T16:45:43.024807Z","shell.execute_reply":"2023-07-18T16:45:43.251185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineer_base(df):\n    \"\"\"\n    Create initial features, distance (cumulative distance moved) and et_diff,\n    that are used to create other features\n    in feature_engineer_xgb and feature_engineer_tr\n    \"\"\"\n    cols = [\n        (\n            (((pl.col(\"screen_coor_y\")-pl.col(\"screen_coor_y\").shift(1)).fill_null(0) ** 2\n            + (pl.col(\"screen_coor_x\")-pl.col(\"screen_coor_x\").shift(1)).fill_null(0) ** 2) ** (1/2))\n            .cumsum()\n            .fill_null(0)\n            .over([\"session_id\"])\n            .alias(\"distance\")\n        ),\n        (\n            (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n            .fill_null(0)\n            .clip(0, 1e9)\n            .over([\"session_id\"])\n            .alias(\"et_diff\")\n        ),\n    ]\n    return df.with_columns(cols)\n\n\ndef feature_engineer_xgb(df, grp):\n    \"\"\"\n    Create features that only xgboost and catboost uses\n    \"\"\"\n\n    aggs = [\n\n        # fqid\n        # count of each fqid value\n        *[((pl.col(\"fqid\") == c).sum()).alias(f\"{c}_num\") for c in grp_cat_vals[(grp, \"fqid\")]],\n        # mean elapsed time difference over each fqid value\n        *[pl.col(\"et_diff\").filter(pl.col(\"fqid\")==c).mean().alias(f\"{c}_et_mean\") for c in grp_cat_vals[(grp, \"fqid\")]],\n        # max elapsed time difference over each fqid value\n        *[pl.col(\"et_diff\").filter(pl.col(\"fqid\")==c).max().alias(f\"{c}_et_max\") for c in grp_cat_vals[(grp, \"fqid\")]],\n\n        # room_fqid\n        *[((pl.col(\"room_fqid\") == c).sum()).alias(f\"{c}_num\") for c in grp_cat_vals[(grp, \"room_fqid\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"room_fqid\")==c).mean().alias(f\"{c}_et_mean\") for c in grp_cat_vals[(grp, \"room_fqid\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"room_fqid\")==c).max().alias(f\"{c}_et_max\") for c in grp_cat_vals[(grp, \"room_fqid\")]],\n\n        # text_fqid\n        *[((pl.col(\"text_fqid\") == c).sum()).alias(f\"{c}_num\") for c in grp_cat_vals[(grp, \"text_fqid\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"text_fqid\")==c).mean().alias(f\"{c}_et_mean\") for c in grp_cat_vals[(grp, \"text_fqid\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"text_fqid\")==c).max().alias(f\"{c}_et_max\") for c in grp_cat_vals[(grp, \"text_fqid\")]],\n\n        # event_name\n        *[((pl.col(\"event_name\") == c).sum()).alias(f\"{c}_num\") for c in grp_cat_vals[(grp, \"event_name\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"event_name\")==c).mean().alias(f\"{c}_et_mean\") for c in grp_cat_vals[(grp, \"event_name\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"event_name\")==c).max().alias(f\"{c}_et_max\") for c in grp_cat_vals[(grp, \"event_name\")]],\n\n        # name\n        *[((pl.col(\"name\") == c).sum()).alias(f\"{c}_num\") for c in grp_cat_vals[(grp, \"name\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"name\")==c).mean().alias(f\"{c}_et_mean\") for c in grp_cat_vals[(grp, \"name\")]],\n        *[pl.col(\"et_diff\").filter(pl.col(\"name\")==c).max().alias(f\"{c}_et_max\") for c in grp_cat_vals[(grp, \"name\")]],\n\n        # stats of the numericals\n        *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique\") for c in CATS],\n        *[pl.col(c).mean().alias(f\"{c}_mean\") for c in NUMS],\n        *[pl.col(c).max().alias(f\"{c}_max\") for c in NUMS],\n        *[pl.col(c).min().alias(f\"{c}_min\") for c in NUMS],\n        *[pl.col(c).std().alias(f\"{c}_std\") for c in NUMS],\n        *[pl.col(c).sum().alias(f\"{c}_sum\") for c in NUMS],\n\n        # whether or not hq, music, fullscreen are used\n        pl.col(\"hq\").apply(lambda x: x[0]),\n        pl.col(\"music\").apply(lambda x: x[0]),\n        pl.col(\"fullscreen\").apply(lambda x: x[0]),\n\n    ]\n\n    df = df.groupby([\"session_id\"], maintain_order=True).agg(aggs).sort(\"session_id\")\n\n    # weekday and hour\n    df = df.with_columns([\n        pl.col(\"session_id\").apply(lambda x: int(str(x)[4:6])).alias(\"weekday\"),\n        pl.col(\"session_id\").apply(lambda x: int(str(x)[6:8])).alias(\"hour\"),\n    ])\n\n    return df\n\n\ndef point_filter_tr(df):\n    \"\"\"\n    Filter out the points that exists in more than 0.999 of sessions\n    (that are in most_common)\n    \"\"\"\n    df = df.with_columns(pl.col(\"index\").cumcount().over([\"session_id\", \"event_name\", \"level\", \"name\", \"page\", \"fqid\", \"room_fqid\", \"text_fqid\"]).cast(pl.Utf8).alias(\"dup_count\"))\n    df = df.with_columns((pl.col(\"event_name\") + pl.col(\"level\") + pl.col(\"name\") + pl.col(\"page\") + pl.col(\"fqid\") + pl.col(\"room_fqid\") + pl.col(\"text_fqid\") + pl.col(\"dup_count\")).alias(\"point\"))\n    df = df.filter(pl.col(\"point\").is_in(most_common))\n    return df\n\n\ndef pad_tr(df, grp):\n    \"\"\"\n    Makes sure that the transformer features are of constant length\n    and in the same order when used by the gbdt models.\n    \"\"\"\n    # the level group predicted mapped to the level groups seen/used\n    point_lgs = {\"0-4\": [\"0-4\"], \"5-12\": [\"0-4\", \"5-12\"], \"13-22\": [\"0-4\", \"5-12\", \"13-22\"]}[grp]\n    # get order of occurance of the points used\n    point_pos_lgs = point_pos.filter(pl.col(\"level_group\").is_in(point_lgs))\n    seq_len = min(256, point_pos_lgs.shape[0])\n    order = point_pos_lgs[-seq_len:]\n\n    def sort_points(df_sess):\n        \"\"\"\n        sorts the points in a df containing one session,\n        in order specified by order\n        \"\"\"\n        sid = df_sess[\"session_id\"][0]\n        p = order.with_columns(pl.lit(sid).alias(\"session_id\"))\n        return p.join(df_sess, on=\"point\", how=\"left\")\n\n    df = df.groupby(\"session_id\", maintain_order=True).apply(sort_points)\n    df = df.fill_null(0)\n    return df\n\n\ndef feature_engineer_tr(df):\n    \"\"\"\n    Generate the numerical transformer features:\n    time difference,\n    index difference,\n    distance (cumulative distance moved, calculated from screen_coor's) difference,\n    room_coor_x,\n    room_coor_y\n    \"\"\"\n    return df.with_columns([\n        (pl.col(\"elapsed_time\").clip(0, 1e9) - pl.col(\"elapsed_time\").clip(0, 1e9).shift(1)).fill_null(0).over(\"session_id\").alias(\"et_diff\"),\n        (pl.col(\"index\") - pl.col(\"index\").shift(1)).fill_null(0).over(\"session_id\").alias(\"ix_diff\"),\n        (pl.col(\"distance\") - pl.col(\"distance\").shift(1)).fill_null(0).over(\"session_id\").alias(\"dist_diff\"),\n        pl.col(\"room_coor_x\").fill_null(0),\n        pl.col(\"room_coor_y\").fill_null(0),\n    ])\n\n\ndef flatten_tr(df):\n    \"\"\"\n    Flatten the transformer features so they can be used by the GBDT models\n    \"\"\"\n    df = df.select([\"session_id\", \"et_diff\", \"ix_diff\", \"dist_diff\", \"room_coor_x\", \"room_coor_y\"])\n    df = df.groupby(\"session_id\").agg([pl.col(c) for c in df.columns[1:]])\n    vs = {}\n    for r in df.iter_rows():\n        ls = []\n        for l in r[1:]:\n            ls.extend(l)\n        vs[str(r[0])] = ls\n    df = pl.DataFrame(vs).transpose(include_header=True)\n    df = df.with_columns(pl.col(\"column\").cast(pl.Int64)).rename({\"column\": \"session_id\"})\n    return df\n\n\ndef xgb_data_pipe(df, grp):\n    \"\"\"\n    Generate all features for training and infering with the GBDT models\n    \"\"\"\n    df = feature_engineer_base(df)\n    # convert to string for combining the categoricals when creating points\n    df_tr = df.with_columns([\n        pl.col(\"page\").cast(pl.Utf8),\n        pl.col(\"level\").cast(pl.Utf8),\n    ])\n    df_tr = point_filter_tr(df_tr)\n    df_tr = feature_engineer_tr(df_tr)\n    df_tr = pad_tr(df_tr, grp)\n    df_tr = flatten_tr(df_tr)\n    df_xgb = feature_engineer_xgb(df, grp)\n    df = df_xgb.join(df_tr, on=\"session_id\", how=\"inner\")\n    return df\n\n\ndef tr_data_pipe(df):\n    \"\"\"\n    Generate all features for training and infering with the transformer\n    \"\"\"\n    df = feature_engineer_base(df)\n    # convert to string for combining the categoricals when creating points\n    df = df.with_columns([\n        pl.col(\"page\").cast(pl.Utf8),\n        pl.col(\"level\").cast(pl.Utf8),\n    ])\n    df = point_filter_tr(df)\n    df = feature_engineer_tr(df)\n    df = df.with_columns([\n        np.sign(pl.col(f)) * np.log1p(np.absolute(pl.col(f))) for f in [\"et_diff\", \"ix_diff\", \"dist_diff\"]\n    ])\n    return df.select([\"session_id\", \"level_group\", \"et_diff\", \"ix_diff\",  \"dist_diff\", \"room_coor_x\", \"room_coor_y\", \"point\"])","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:43.253575Z","iopub.execute_input":"2023-07-18T16:45:43.254093Z","iopub.status.idle":"2023-07-18T16:45:43.303434Z","shell.execute_reply.started":"2023-07-18T16:45:43.254064Z","shell.execute_reply":"2023-07-18T16:45:43.302316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nseq_lens = [76, 249, 452]\nmax_seq_len = 256\n\n\nclass NN(nn.Module):\n    \"\"\"\n    Lightweight transformer model\n    \"\"\"\n\n    def __init__(self, num_cont_cols, embed_dim, num_layers, num_heads, max_seq_len):\n        super(NN, self).__init__()\n        self.emb_cont = nn.Sequential(\n            nn.Linear(num_cont_cols, embed_dim//2),\n            nn.LayerNorm(embed_dim//2)\n        )\n        self.emb_cats = nn.Sequential(\n            nn.Embedding(max_seq_len + 1, embed_dim//2),\n            nn.LayerNorm(embed_dim//2)\n        )\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=embed_dim,\n            nhead=num_heads,\n            dim_feedforward=embed_dim,\n            dropout=0.1,\n            batch_first=True,\n            activation=\"relu\",\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        # one head for each level group\n        self.clf_heads = nn.ModuleList([\n            nn.Linear(embed_dim, out_dim) for out_dim in [3, 10, 5]\n        ])\n\n    def forward(self, x, grp):\n        emb_conts = self.emb_cont(x[:, :, :-1])\n        emb_cats = self.emb_cats(x[:, :, -1].type(torch.int32))\n        x = torch.cat([emb_conts, emb_cats], dim=2)\n        x = self.encoder(x)\n        x = x.mean(dim=1)\n        x = self.clf_heads[[\"0-4\", \"5-12\", \"13-22\"].index(grp)](x)\n        return x.unsqueeze(2)\n  ","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:43.304963Z","iopub.execute_input":"2023-07-18T16:45:43.305675Z","iopub.status.idle":"2023-07-18T16:45:43.323678Z","shell.execute_reply.started":"2023-07-18T16:45:43.305625Z","shell.execute_reply":"2023-07-18T16:45:43.322342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the models\n\nlimits = {'0-4': range(1, 4), '5-12': range(4, 14), '13-22': range(14, 19)}\nnum_seeds = 3\nnum_folds = 5\nmodels_tr = {}\nfor f in range(num_folds):  \n    for s in range(num_seeds):\n\n        model = NN(\n            num_cont_cols=5,\n            embed_dim=64,\n            num_layers=1,\n            num_heads=8,\n            max_seq_len=seq_lens[-1],\n        )\n        shutil.make_archive(\"model\", 'zip', f\"/kaggle/input/torch-models/model_{f}_s{s}\")\n        model.load_state_dict(torch.load(f\"/kaggle/working/model.zip\", map_location=torch.device('cpu')))\n        model.eval()\n        models_tr[(f, s)] = model\n\n        \nmodels_xgb = {}\nmodels_cat = {}\nfor grp in [\"0-4\", \"5-12\", \"13-22\"]:\n    for f in range(num_folds):\n        for s in range(num_seeds):\n                clf = XGBClassifier()\n                clf.load_model(f\"/kaggle/input/group-models/model_{f}_grp_{grp}_s{s}.xgb\")\n                models_xgb[(grp, f, s)] = clf\n\n                clf = CatBoostClassifier()\n                clf.load_model(f\"/kaggle/input/cat-models/model_{f}_grp_{grp}_s{s}.cbm\")\n                models_cat[(grp, f, s)] = clf\n                \nmodels_lin_reg = {}                \nfor f in range(num_folds):\n    for q in range(1, 19):\n        lin_reg = pickle.load(open(f\"/kaggle/input/meta-models/lin_reg_{f}_q{q}.sav\", 'rb'))\n        models_lin_reg[(f, q)] = lin_reg","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:43.325249Z","iopub.execute_input":"2023-07-18T16:45:43.325761Z","iopub.status.idle":"2023-07-18T16:45:59.302528Z","shell.execute_reply.started":"2023-07-18T16:45:43.325725Z","shell.execute_reply":"2023-07-18T16:45:59.301199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jo_wilder_310.make_env.__called__ = False\n# type(env)._state = type(type(env)._state).__dict__['INIT']","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:45:59.304124Z","iopub.execute_input":"2023-07-18T16:45:59.304710Z","iopub.status.idle":"2023-07-18T16:45:59.310353Z","shell.execute_reply.started":"2023-07-18T16:45:59.304646Z","shell.execute_reply":"2023-07-18T16:45:59.309136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMPORT KAGGLE API\nimport jo_wilder_310\nenv = jo_wilder_310.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:46:13.382284Z","iopub.execute_input":"2023-07-18T16:46:13.382742Z","iopub.status.idle":"2023-07-18T16:46:13.413067Z","shell.execute_reply.started":"2023-07-18T16:46:13.382708Z","shell.execute_reply":"2023-07-18T16:46:13.412026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {'0-4':(1, 4), '5-12':(4, 14), '13-22':(14, 19)}\npad_lens = {'0-4': 76, '5-12': 249, '13-22': 256}\n\ndfs = {}\nprev_probas = {}\n\nfor (test, sample_submission) in iter_test:\n    test = test.sort_values('index').reset_index(drop=True)\n    grp = test.level_group.values[0]\n    sess_id = test.session_id.values[0]\n    \n    df = pl.from_pandas(test).with_columns(dtypes).with_columns(fills)\n    \n    # use data from previous level groups\n    if grp == \"0-4\":\n        dfs[sess_id] = df\n    elif grp == \"5-12\":\n        df = pl.concat([dfs[sess_id], df])\n        dfs[sess_id] = df\n    else:\n        df = pl.concat([dfs[sess_id], df])\n                \n    # drop hovers and create new index for each session, from 0 to len(session)\n    df = df.filter(pl.col(\"hover_duration\").is_null())\n    df = df.with_columns(pl.arange(0, pl.col(\"index\").count()).over([\"session_id\"]).alias(\"new_index\"))\n    df = df.drop(\"index\").rename({\"new_index\": \"index\"})\n       \n    # gbdt feature engineering\n    df_xgb = xgb_data_pipe(df, grp)\n    \n    df_xgb = df_xgb.to_pandas().set_index('session_id')\n    df_xgb = df_xgb[feature_dict[grp]]\n\n    # use question number as a input feature since one model is trained per level group\n    df_qs = [] \n    a, b = limits[grp]\n    for q in range(a, b):\n        df_xgb[\"q\"] = q\n        df_qs.append(df_xgb.copy())\n    df_xgb = pd.concat(df_qs)\n    \n    # transformer feature engineering\n    df_tr = tr_data_pipe(df)\n    \n    # normalize the input\n    df_tr = df_tr.join(norms, on=\"point\", how=\"left\")\n    df_tr = df_tr.with_columns([\n        (pl.col(\"et_diff\") - pl.col(\"et_mean\")) / pl.col(\"et_std\"),\n        (pl.col(\"ix_diff\") - pl.col(\"ix_mean\")) / pl.col(\"ix_std\"),\n        (pl.col(\"dist_diff\") - pl.col(\"dist_mean\")) / pl.col(\"dist_std\"),\n        (pl.col(\"room_coor_x\") - pl.col(\"room_coor_x_mean\")) / pl.col(\"room_coor_x_std\"),\n        (pl.col(\"room_coor_y\") - pl.col(\"room_coor_y_mean\")) / pl.col(\"room_coor_y_std\"),\n    ]).fill_nan(0)\n    # replace some outlier values with 0\n    df_tr = df_tr.with_columns([pl.when(pl.col(\"ix_diff\").abs() > 7.5).then(0).otherwise(pl.col(\"ix_diff\")).keep_name()])\n    # map point categoricals to integer index, so transformers can handle it\n    df_tr = df_tr.with_columns([pl.col(\"point\").map_dict(point_ixs)])\n    \n    input = df_tr.select([\"et_diff\", \"ix_diff\", \"dist_diff\",\"room_coor_x\", \"room_coor_y\", \"point\"]).to_numpy()\n\n    pad_len = pad_lens[grp]\n    seq_len = input.shape[0]\n    if seq_len < pad_len:\n        input = np.pad(input, ((pad_len - seq_len, 0), (0, 0)), 'constant')\n    if seq_len > pad_len:\n        input = input[-pad_len:]\n    \n    input = torch.tensor(input, dtype=torch.float).unsqueeze(0)\n    \n    # predict\n    \n    # folds average\n    probas = np.zeros(b - a)\n    for f in range(num_folds):\n        \n        # seeds average\n        probas_seeds_avg_xgb = 0\n        probas_seeds_avg_cat = 0\n        probas_seeds_avg_tr = 0\n        for s in range(num_seeds):\n            probas_xgb = models_xgb[(grp, f, s)].predict_proba(df_xgb)[:, 1]\n            probas_cat = models_cat[(grp, f, s)].predict_proba(df_xgb)[:, 1]\n            out_tr = models_tr[(f, s)](input, grp)[0, :, 0]\n            probas_tr = torch.sigmoid(out_tr).detach().numpy()\n            probas_seeds_avg_xgb += probas_xgb / num_seeds\n            probas_seeds_avg_cat += probas_cat / num_seeds\n            probas_seeds_avg_tr += probas_tr / num_seeds\n        \n        # get previous predictions\n        if grp == \"0-4\":\n            prev_probas[(sess_id, f, \"xgb\")] = probas_seeds_avg_xgb\n            prev_probas[(sess_id, f, \"cat\")] = probas_seeds_avg_cat\n            prev_probas[(sess_id, f, \"tr\")] = probas_seeds_avg_tr\n        elif grp == \"5-12\":\n            probas_seeds_avg_xgb = np.concatenate([prev_probas[(sess_id, f, \"xgb\")], probas_seeds_avg_xgb])\n            probas_seeds_avg_cat = np.concatenate([prev_probas[(sess_id, f, \"cat\")], probas_seeds_avg_cat])\n            probas_seeds_avg_tr = np.concatenate([prev_probas[(sess_id, f, \"tr\")], probas_seeds_avg_tr])\n            prev_probas[(sess_id, f, \"xgb\")] = probas_seeds_avg_xgb\n            prev_probas[(sess_id, f, \"cat\")] = probas_seeds_avg_cat\n            prev_probas[(sess_id, f, \"tr\")] = probas_seeds_avg_tr\n        else:\n            probas_seeds_avg_xgb = np.concatenate([prev_probas[(sess_id, f, \"xgb\")], probas_seeds_avg_xgb])\n            probas_seeds_avg_cat = np.concatenate([prev_probas[(sess_id, f, \"cat\")], probas_seeds_avg_cat])\n            probas_seeds_avg_tr = np.concatenate([prev_probas[(sess_id, f, \"tr\")], probas_seeds_avg_tr])\n\n        probas_seeds_avg = np.concatenate([probas_seeds_avg_xgb, probas_seeds_avg_cat, probas_seeds_avg_tr])\n        probas_seeds_avg = probas_seeds_avg.reshape(1, -1)\n        \n        # linear regression prediction\n        for q in range(a, b):\n            for f1 in range(num_folds):\n                probas[q-a] += models_lin_reg[(f1, q)].predict(probas_seeds_avg) / num_folds / num_folds\n            \n    \n    preds = (probas > 0.62).astype(np.int32)\n    for q in range(a, b):\n        mask = sample_submission.session_id.str.contains(f'q{q}')         \n        sample_submission.loc[mask,'correct'] = preds[q-a]\n        \n\n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:46:21.929000Z","iopub.execute_input":"2023-07-18T16:46:21.929429Z","iopub.status.idle":"2023-07-18T16:46:41.187426Z","shell.execute_reply.started":"2023-07-18T16:46:21.929397Z","shell.execute_reply":"2023-07-18T16:46:41.185972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape)\nsub.head(40)","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:46:41.189421Z","iopub.execute_input":"2023-07-18T16:46:41.189841Z","iopub.status.idle":"2023-07-18T16:46:41.222196Z","shell.execute_reply.started":"2023-07-18T16:46:41.189803Z","shell.execute_reply":"2023-07-18T16:46:41.221080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sub.correct.mean())","metadata":{"execution":{"iopub.status.busy":"2023-07-18T16:46:41.223622Z","iopub.execute_input":"2023-07-18T16:46:41.224674Z","iopub.status.idle":"2023-07-18T16:46:41.230790Z","shell.execute_reply.started":"2023-07-18T16:46:41.224622Z","shell.execute_reply":"2023-07-18T16:46:41.229775Z"},"trusted":true},"execution_count":null,"outputs":[]}]}