{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Temporal Convolution with Only 4 Features (Inference Part)\n\n## Introduction\nBefore diving into the manual feature engineering, I try to see how far a DL-based model can go with a small set of features, including only:\n1. Difference of `elapsed_time`\n2. `event_name`\n3. `name`\n4. `room_fqid`\n\nFirst, `event_name` is combined with `name` to form a new feature `event_comb` (*i.e.*, event combination). Then, I feed these three processed features into the model to learn **event-aware temporal patterns**. Finally, this model can achieve the performance summarized as follows:\n\n| CV (GroupKFold with k=5) | Holdout (Released Old Test Set) | LB    |\n| ------------------------ | ------------------------------- | ----- |\n| 0.6914                   | 0.6911                          | 0.694 |\n\n\n## About this Notebook\nIn this kernel, I run the inference process with 5-fold model blending (equally-weighted). The training part is still a work in progress and will be published soon.\n\n\n## Acknowledgements\nSpecial thanks to [@chrisqiu](https://www.kaggle.com/chrisqiu)'s sharing [here](https://www.kaggle.com/code/chrisqiu/pytorch-using-only-1-column-submission), which provides the inspiration for model building.\n\n## Import Packages","metadata":{"execution":{"iopub.status.busy":"2023-03-30T13:29:47.489282Z","iopub.execute_input":"2023-03-30T13:29:47.489756Z","iopub.status.idle":"2023-03-30T13:29:50.289644Z","shell.execute_reply.started":"2023-03-30T13:29:47.489717Z","shell.execute_reply":"2023-03-30T13:29:50.288355Z"}}},{"cell_type":"code","source":"import gc\nimport os\nfrom typing import Any, Dict, List, Optional, Tuple\nimport pickle\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport yaml\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import Tensor","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-30T16:36:46.591264Z","iopub.execute_input":"2023-03-30T16:36:46.591989Z","iopub.status.idle":"2023-03-30T16:36:49.427948Z","shell.execute_reply.started":"2023-03-30T16:36:46.591947Z","shell.execute_reply":"2023-03-30T16:36:49.426774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Data Path","metadata":{}},{"cell_type":"code","source":"exp_dump_path = \"/kaggle/input/0330-16-53-13/0330-16_53_13\"\ncat2code_path = \"/kaggle/input/cat2code-v2/cat2code\"","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:36:49.430015Z","iopub.execute_input":"2023-03-30T16:36:49.430944Z","iopub.status.idle":"2023-03-30T16:36:49.437500Z","shell.execute_reply.started":"2023-03-30T16:36:49.430898Z","shell.execute_reply":"2023-03-30T16:36:49.436388Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evoke Experiment Configuration","metadata":{}},{"cell_type":"code","source":"CAT_FEATS = [\"event_comb_code\", \"room_fqid_code\"]\nCAT_FEAT_SIZE = {\n    \"event_comb_code\": 19,\n    \"room_fqid_code\": 19,\n}\nDEVICVE = torch.device(\"cpu\")\n\n# Load model configuration\nwith open(os.path.join(exp_dump_path, \"config/cfg.yaml\"), \"r\") as f:\n    cfg = yaml.full_load(f)\nmodel_cfg = cfg[\"model\"]\n\nbest_thres = 0.63\nt_window = 1000","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-03-30T16:36:49.439381Z","iopub.execute_input":"2023-03-30T16:36:49.440218Z","iopub.status.idle":"2023-03-30T16:36:49.475563Z","shell.execute_reply.started":"2023-03-30T16:36:49.440175Z","shell.execute_reply":"2023-03-30T16:36:49.474385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Model","metadata":{}},{"cell_type":"code","source":"class TConvLayer(nn.Module):\n    \"\"\"Dilated temporal convolution layer.\"\"\"\n    \n    def __init__(\n        self,\n        in_dim: int,\n        out_dim: int,\n        kernel_size: int,\n        dilation: int,\n        bias: bool = True,\n        act: str = \"relu\",\n        dropout: float = 0.1,\n    ):\n        super(TConvLayer, self).__init__()\n        \n        # Network parameters\n        self.in_dim = in_dim\n        self.out_dim = out_dim\n        self.kernel_size = kernel_size\n        self.dilation = dilation\n        self.bias = bias\n        \n        # Model blocks\n        self.conv = nn.utils.weight_norm(\n            nn.Conv1d(in_dim, out_dim, kernel_size, dilation=dilation, bias=bias),\n            dim=None\n        )\n        self.bn = nn.BatchNorm1d(out_dim)\n        if act == \"relu\":\n            self.act = nn.ReLU()\n        if dropout == 0:\n            self.dropout = None\n        else:\n            self.dropout = nn.Dropout(dropout)\n            \n    def forward(self, x: Tensor) -> Tensor:\n        \"\"\"Forward pass.\n        \n        Shape:\n            x: (B, C, P)\n        \"\"\"\n        x = self.conv(x)\n        x = self.bn(x)\n        x = self.act(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n            \n        return x\n    \n\nclass EventAwareEncoder(nn.Module):\n    \n    def __init__(\n        self,\n        h_dim: int = 128,\n        out_dim: int = 128,\n        readout: bool = True,\n        cat_feats: List[str] = [\"event_comb_code\", \"room_fqid_code\"]\n    ):\n        super(EventAwareEncoder, self).__init__()\n\n        # Network parameters\n        self.h_dim = h_dim\n        self.out_dim = out_dim\n        self.cat_feats = cat_feats\n\n        # Model blocks\n        # Categorical embeddings\n        self.embs = nn.ModuleList()\n        for cat_feat in cat_feats:\n            self.embs.append(nn.Embedding(CAT_FEAT_SIZE[cat_feat] + 1, 32, padding_idx=0))\n        self.emb_enc = nn.Sequential(\n            nn.Linear(64, 128),\n            nn.ReLU(),\n            nn.Linear(128, 64),\n            nn.ReLU(),\n        )\n        self.dropout = nn.Dropout(0.2)\n        # Feature extractor\n        self.convs = nn.ModuleList()\n        for l, (dilation, kernel_size) in enumerate(zip([2**i for i in range(3)], [7, 7, 5])):\n            self.convs.append(TConvLayer(64, h_dim, kernel_size, dilation=1))   # No dilation\n        # Readout layer\n        if readout:\n            self.readout = nn.Sequential(\n                nn.Linear(2 * (h_dim // 2), out_dim),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n            )\n        else:\n            self.readout = None\n\n    def forward(self, x: Tensor, x_cat: Tensor) -> Tensor:\n        \"\"\"Forward pass.\n\n        Shape:\n            x: (B, P, C)\n            x_cat: (B, P, #cat_feats)\n        \"\"\"\n\n        # Categorical embeddings\n        x_cat = x_cat + 1\n        x_emb = []\n        for i in range(len(self.cat_feats)):\n            x_emb.append(self.embs[i](x_cat[..., i]))  # (B, P, emb_dim)\n        x_emb = torch.cat(x_emb, dim=-1)  # (B, P, emb_dim_cat)\n        x_emb = self.emb_enc(x_emb) + x_emb  # (B, P, emb_dim_cat)\n        x = x * x_emb  # (B, P, emb_dim_cat)\n        x = self.dropout(x)\n        \n        # Feature extractor\n        x = x.transpose(1, 2)  # (B, C, P)\n        x_skip = []\n        for l in range(3):\n            x_conv = self.convs[l](x)   # (B, C * 2, P')\n            x_filter, x_gate = torch.split(x_conv, x_conv.size(1) // 2, dim=1)\n            x_conv = F.tanh(x_filter) * F.sigmoid(x_gate)   # (B, C, P')\n            \n            x_conv = self.dropout(x_conv)\n            \n            # Skip connection\n            x_skip.append(x_conv.unsqueeze(dim=1))  # (B, L (1), C, P')\n            \n            # Residual\n            x = x_conv\n            \n        # Process skipped latent representation\n        for l in range(3-1):\n            x_skip[l] = x_skip[l][..., -x_skip[-1].size(3) : ]\n        x_skip = torch.cat(x_skip, dim=1)   # (B, L, C, P)\n        x_skip = torch.sum(x_skip, dim=1)   # (B, C, P')\n\n        # Readout layer\n        if self.readout is not None:\n            x_std = torch.std(x_skip, dim=-1)  # Std pooling\n            x_mean = torch.mean(x_skip, dim=-1)  # Mean pooling\n            x = torch.cat([x_std, x_mean], dim=1)\n            x = self.readout(x)  # (B, out_dim)\n\n        return x\n\n\nclass EventConvSimple(nn.Module):\n\n    def __init__(self, n_lvs: int, out_dim: int, **model_cfg: Any):\n        self.name = self.__class__.__name__\n        super(EventConvSimple, self).__init__()\n\n        enc_out_dim = 128\n\n        # Network parameters\n        self.n_lvs = n_lvs\n        self.out_dim = out_dim\n        self.cat_feats = model_cfg[\"cat_feats\"]\n        \n        self.encoder = EventAwareEncoder(h_dim=128, out_dim=enc_out_dim, cat_feats=self.cat_feats)\n        self.clf = nn.Sequential(\n            nn.Linear(enc_out_dim, enc_out_dim // 2),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(enc_out_dim // 2, out_dim),\n        )\n\n    def forward(self, x: Tensor, x_cat: Tensor) -> Tensor:\n        \"\"\"Forward pass.\n\n        Shape:\n            x: (B, P, C)\n            x_cat: (B, P, 3)\n        \"\"\"\n        x = self.encoder(x, x_cat)\n        x = self.clf(x)\n        \n        x = F.sigmoid(x)\n\n        return x","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-30T16:36:49.481473Z","iopub.execute_input":"2023-03-30T16:36:49.484202Z","iopub.status.idle":"2023-03-30T16:36:49.516877Z","shell.execute_reply.started":"2023-03-30T16:36:49.484148Z","shell.execute_reply":"2023-03-30T16:36:49.515762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = os.path.join(exp_dump_path, \"models\")\n\nmodels = {\"0-4\": [], \"5-12\": [], \"13-22\": []}\nfor model_file in os.listdir(model_path):\n    if \"encoder\" in model_file: continue\n    if \"0-4\" in model_file:\n        model = EventConvSimple(5, 3, **model_cfg)\n        model.load_state_dict(\n            torch.load(\n                os.path.join(model_path, model_file),\n                map_location=torch.device('cpu')\n            )\n        )\n        models[\"0-4\"].append(model)\n    elif \"5-12\" in model_file:\n        model = EventConvSimple(8, 10, **model_cfg)\n        model.load_state_dict(\n            torch.load(\n                os.path.join(model_path, model_file),\n                map_location=torch.device('cpu')\n            )\n        )\n        models[\"5-12\"].append(model)\n    elif \"13-22\" in model_file:\n        model = EventConvSimple(10, 5, **model_cfg)\n        model.load_state_dict(\n            torch.load(\n                os.path.join(model_path, model_file),\n                map_location=torch.device('cpu')\n            )\n        )\n        models[\"13-22\"].append(model)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-30T16:36:49.522358Z","iopub.execute_input":"2023-03-30T16:36:49.524942Z","iopub.status.idle":"2023-03-30T16:36:50.191628Z","shell.execute_reply.started":"2023-03-30T16:36:49.524892Z","shell.execute_reply":"2023-03-30T16:36:50.190493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Inference","metadata":{}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:36:50.192897Z","iopub.execute_input":"2023-03-30T16:36:50.193231Z","iopub.status.idle":"2023-03-30T16:36:50.216643Z","shell.execute_reply.started":"2023-03-30T16:36:50.193199Z","shell.execute_reply":"2023-03-30T16:36:50.215380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat2code_ = {}\nfor cat_feat in CAT_FEATS:\n    with open(os.path.join(cat2code_path, f\"{cat_feat[:-5]}2code.pkl\"), \"rb\") as f:\n        cat2code_[cat_feat] = pickle.load(f)\n        \net_diff_upper_bound = 3.6e6","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:36:50.217977Z","iopub.execute_input":"2023-03-30T16:36:50.218344Z","iopub.status.idle":"2023-03-30T16:36:50.234676Z","shell.execute_reply.started":"2023-03-30T16:36:50.218290Z","shell.execute_reply":"2023-03-30T16:36:50.233442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_multi_game_naive(df: pd.DataFrame, local: bool = True) -> pd.DataFrame:\n    \"\"\"Drop events not occurring at the first game play.\n    \n    Parameters:\n        df: input DataFrame\n    \n    Return:\n        df: DataFrame with events occurring at the first game play only\n    \"\"\"\n    if local:\n        df[\"lv_diff\"] = df.groupby(\"session_id\").apply(lambda x: x[\"level\"].diff().fillna(0)).values\n    else:\n        df[\"lv_diff\"] = df[\"level\"].diff().fillna(0)\n    reversed_lv_pts = df[\"lv_diff\"] < 0\n    df.loc[~reversed_lv_pts, \"lv_diff\"] = 0\n    if local:\n        df[\"multi_game_flag\"] = df.groupby(\"session_id\")[\"lv_diff\"].cumsum()\n    else:\n        df[\"multi_game_flag\"] = df[\"lv_diff\"].cumsum()\n    multi_game_mask = df[\"multi_game_flag\"] < 0\n    multi_game_rows = df[multi_game_mask].index\n    df = df.drop(multi_game_rows).reset_index(drop=True)\n    \n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-30T16:36:50.235700Z","iopub.execute_input":"2023-03-30T16:36:50.236022Z","iopub.status.idle":"2023-03-30T16:36:50.246045Z","shell.execute_reply.started":"2023-03-30T16:36:50.235991Z","shell.execute_reply":"2023-03-30T16:36:50.245106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef quick_infer(X: Tuple[Tensor, Optional[Tensor]], models: List[nn.Module]) -> Tensor:\n    \n    x, x_cat = X\n    n_models = len(models)\n    \n    for i, model in enumerate(models):\n        model.eval()\n        if i == 0:\n            y_pred = model(x, x_cat) / n_models   # (1, n_qns (out_dim))\n        else:\n            y_pred += model(x, x_cat) / n_models   # (1, n_qns (out_dim))\n    \n    y_pred = y_pred.reshape((-1, 1)).detach().numpy()\n    \n    \n    return y_pred","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-30T16:36:50.247384Z","iopub.execute_input":"2023-03-30T16:36:50.247940Z","iopub.status.idle":"2023-03-30T16:36:50.268373Z","shell.execute_reply.started":"2023-03-30T16:36:50.247902Z","shell.execute_reply":"2023-03-30T16:36:50.266901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test, sample_submission) in iter_test:\n    cur_lv_gp = test[\"level_group\"].unique()[0]\n    test = drop_multi_game_naive(test, local=False)\n    test[\"et_diff\"] = test[\"elapsed_time\"].diff().fillna(0).clip(0, et_diff_upper_bound)\n    test[\"event_comb\"] = test[\"event_name\"] + \"_\" + test[\"name\"]\n    for cat_feat in CAT_FEATS:\n        test[cat_feat] = test[cat_feat[:-5]].map(cat2code_[cat_feat]).astype(np.int32)\n    \n    x = test[\"et_diff\"].values[-t_window:]\n    x_cat = test[CAT_FEATS].values[-t_window:]\n    if len(x) < t_window:\n        pad_len = t_window - len(x)\n        x = np.pad(x, (pad_len, 0), \"constant\")\n        x_cat = np.pad(x_cat, ((pad_len, 0), (0, 0)), \"constant\", constant_values=-1)  # (P, #cat_feats)\n    x = torch.tensor(x, dtype=torch.float32).unsqueeze(dim=0).unsqueeze(dim=-1)   # Add B, C dim, (1, P, 1)\n    x_cat = torch.tensor(x_cat, dtype=torch.int32).unsqueeze(dim=0)   # (1, P, 3)\n    \n    try:\n        # Run quick inference\n        y_pred = quick_infer((x, x_cat), models[cur_lv_gp])   # (1, n_qns)\n        y_pred = (y_pred > best_thres).astype(np.int)\n\n        sample_submission.loc[:, \"correct\"] = y_pred\n    except:\n        sample_submission.loc[:, \"correct\"] = 0\n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:36:50.273182Z","iopub.execute_input":"2023-03-30T16:36:50.276024Z","iopub.status.idle":"2023-03-30T16:36:51.868035Z","shell.execute_reply.started":"2023-03-30T16:36:50.275977Z","shell.execute_reply":"2023-03-30T16:36:51.866292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}