{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Config / Library","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input\")","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:26:46.268223Z","iopub.execute_input":"2023-06-24T23:26:46.268651Z","iopub.status.idle":"2023-06-24T23:26:46.273158Z","shell.execute_reply.started":"2023-06-24T23:26:46.268609Z","shell.execute_reply":"2023-06-24T23:26:46.272163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport pdb\nimport copy\nimport pickle\nimport random\nimport warnings\nimport functools\nimport itertools\nfrom pathlib import Path\nfrom typing import *\nfrom typing_extensions import Literal\nfrom tqdm.auto import tqdm\nfrom collections import defaultdict\n\nimport numba\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier, Pool\nfrom xgboost import XGBClassifier, XGBRegressor\n\nimport kagglib\n\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:26:46.274891Z","iopub.execute_input":"2023-06-24T23:26:46.275482Z","iopub.status.idle":"2023-06-24T23:27:04.006238Z","shell.execute_reply.started":"2023-06-24T23:26:46.275453Z","shell.execute_reply":"2023-06-24T23:27:04.004857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ryota & tereka config\nclass CFG:\n    input_path = Path('/kaggle/input/predict-student-performance-from-game-play')\n    ryota_gbdt_path = Path('/kaggle/input/psp-exp-067')\n    ryota_nn_path = Path('/kaggle/input/psp-exp-165')\n    tereka_gbdt_path = Path('/kaggle/input/psp-exp-601') # 603\n    \n    device = 'cpu'\n    seed = 10\n\n    q_range = {0: range(1, 4), 1: range(4, 14), 2: range(14, 19)}\n    limits = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)} \n    \n    max_len = {0: 384, 1: 768, 2: 1024}\n    cat_emb_dim = [8, 32, 8]\n    q_dim = 18\n    dense_dim = 128\n    nhead = 8\n    dim_ff = 512\n    tfm_drop_out = 0.1\n    num_layers = 1","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.007583Z","iopub.execute_input":"2023-06-24T23:27:04.007856Z","iopub.status.idle":"2023-06-24T23:27:04.017257Z","shell.execute_reply.started":"2023-06-24T23:27:04.00783Z","shell.execute_reply":"2023-06-24T23:27:04.016088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shu config\nclass Config:\n    AUTHOR = \"shu421\"\n\n    EXP = \"exp108\"\n    COMPETITION = \"predict-student-performance-from-game-play\"\n    \n    EXP_MODEL = f\"../input/psp-{EXP}/model\"\n    FEAT_PATH = f\"../input/psp-{EXP}/feat\"\n\n    seed = 42\n    n_folds = 5\n    \n    thr = pickle.load(open(f\"{FEAT_PATH}/all_thresholds.pkl\", \"rb\"))\n\n    folds = [0, 1, 2, 3, 4]\n\ndef setup(cfg):    \n    seed_everything(cfg.seed)\n    return cfg\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    \nshu_cfg = setup(Config)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.019631Z","iopub.execute_input":"2023-06-24T23:27:04.01997Z","iopub.status.idle":"2023-06-24T23:27:04.044394Z","shell.execute_reply.started":"2023-06-24T23:27:04.019945Z","shell.execute_reply":"2023-06-24T23:27:04.043436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ryota Part","metadata":{}},{"cell_type":"markdown","source":"### Utils","metadata":{}},{"cell_type":"code","source":"def reduce_memory(df: pl.DataFrame):\n#     df = df.drop(['fullscreen', 'hq', 'music'])\n    df = df.with_columns([\n        pl.col('index').cast(pl.Int32),\n        pl.col('level').cast(pl.Int8),\n        pl.col('page').fill_null('-1.0').apply(lambda x: x.split('.')[0]).cast(pl.Int8),\n        pl.col('room_coor_x').cast(pl.Float32),\n        pl.col('room_coor_y').cast(pl.Float32),\n        pl.col('screen_coor_x').cast(pl.Float32),\n        pl.col('screen_coor_y').cast(pl.Float32),\n    ])\n    return df\n\n\ndef compare_func(item1, item2):\n    if item1['level'] == item2['level']:\n        if abs(item1['index'] - item2['index']) > 50:\n            if item1['elapsed_time'] < item2['elapsed_time']:\n                return -1\n            elif item1['elapsed_time'] > item2['elapsed_time']:\n                return 1\n            else:\n                return 0\n        else:\n            if item1['index'] < item2['index']:\n                return -1\n            elif item1['index'] > item2['index']:\n                return 1\n            else:\n                return 0\n    else:\n        return item1['level'] - item2['level']\n\n    \ndef sort_frame(df):\n    v = [{'idx': idx, 'index':row['index'], 'elapsed_time': row['elapsed_time'], 'level': row['level']} for idx, row in df.iterrows()]\n    v.sort(key=functools.cmp_to_key(compare_func))\n    mdf = pd.DataFrame(v)\n    assert len(mdf) == len(df)\n    return df.loc[mdf.idx]\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.047665Z","iopub.execute_input":"2023-06-24T23:27:04.048003Z","iopub.status.idle":"2023-06-24T23:27:04.054499Z","shell.execute_reply.started":"2023-06-24T23:27:04.047977Z","shell.execute_reply":"2023-06-24T23:27:04.053396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GBDT","metadata":{}},{"cell_type":"code","source":"def ryota_stream_text_fill_null_forward(df: pl.DataFrame, text_col: str):\n    texts = df[text_col].to_numpy()\n    texts_shift = np.concatenate([[None], texts[:-1]])\n    flip_idx = np.where(texts != texts_shift, True, False)\n    null_idx = np.where(texts == None, True, False)\n    texts[flip_idx & null_idx] = [f'{val}_NONE' for val in texts_shift[flip_idx & null_idx]]\n    \n    df = df.with_columns([\n        pl.Series(list(texts)).fill_null(strategy='forward').alias(text_col)\n    ])\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.055935Z","iopub.execute_input":"2023-06-24T23:27:04.056202Z","iopub.status.idle":"2023-06-24T23:27:04.073019Z","shell.execute_reply.started":"2023-06-24T23:27:04.056173Z","shell.execute_reply":"2023-06-24T23:27:04.071376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ryota_stream_bingo_feature(features: dict, df: pl.DataFrame, level_group: int):\n    if level_group == 1:\n        bingo1 = (\n            df.filter((pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n            .select(\n                logbook_bingo_duration = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                logbook_bingo_indexCount = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo2 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (pl.col(\"fqid\") == \"reader.paper2.bingo\"))\n            .select(\n                reader_bingo_duration = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                reader_bingo_indexCount = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo3 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (pl.col(\"fqid\") == \"journals.pic_2.bingo\"))\n            .select(\n                journals_bingo_duration = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                journals_bingo_indexCount = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo = pl.concat([bingo1, bingo2, bingo3], how='horizontal')\n        bingo = dict(zip(bingo.columns, bingo.to_numpy()[0]))\n        features.update(bingo)\n\n    elif level_group == 2:\n        bingo1 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\"))\n            .select(\n                reader_flag_duration = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                reader_flag_indexCount = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo2 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\"))\n            .select(\n                journalsFlag_bingo_duration = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                journalsFlag_bingo_indexCount = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo = pl.concat([bingo1, bingo2], how='horizontal')\n        bingo = dict(zip(bingo.columns, bingo.to_numpy()[0]))\n        features.update(bingo)\n        \n    return features","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.074409Z","iopub.execute_input":"2023-06-24T23:27:04.07497Z","iopub.status.idle":"2023-06-24T23:27:04.088237Z","shell.execute_reply.started":"2023-06-24T23:27:04.074942Z","shell.execute_reply":"2023-06-24T23:27:04.087523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ryota_stream_feature_engineering(df: pl.DataFrame, level_group: int, use_bingo: bool = True):\n    ### preprocess\n    df = df.with_columns(\n        elapsed_time_diff = pl.col('elapsed_time').diff().shift(-1).over('session_id'),\n        room_coor_move = (pl.col('room_coor_x').diff().shift(-1).over('session_id') ** 2 + pl.col('room_coor_y').diff().shift(-1).over('session_id') ** 2).sqrt().fill_null(0),\n        screen_coor_move = (pl.col('screen_coor_x').diff().shift(-1).over('session_id') ** 2 + pl.col('screen_coor_y').diff().shift(-1).over('session_id') ** 2).sqrt().fill_null(0),\n        text_fqid_null_flag = pl.col('text_fqid').is_null().cast(pl.Int8)\n    )\n    df = df.with_columns([\n        (pl.col('event_name').fill_null('None') + '_&_' + pl.col('name').fill_null('None')).alias('event_name_&_name'),\n        (pl.col('room_fqid').fill_null('None') + '_&_' + pl.col('level').cast(pl.Utf8)).alias('room_fqid_&_level'),\n        (pl.col('room_fqid').fill_null('None') + '_&_' + pl.col('fqid').fill_null('None')).alias('room_fqid_&_fqid'),\n        (pl.col('room_fqid').fill_null('None') + '_&_' + pl.col('fqid').fill_null('None') + '_&_' + pl.col('level').cast(pl.Utf8)).alias('room_fqid_&_fqid_&_level'),\n        (pl.col('room_fqid').fill_null('None') + '_&_' + pl.col('event_name').fill_null('None')).alias('room_fqid_&_event_name'),\n        (pl.col('fqid').fill_null('None') + '_&_' + pl.col('event_name').fill_null('None')).alias('fqid_&_event_name'),\n        (pl.col('event_name').fill_null('None') + '_&_' + pl.col('room_fqid').fill_null('None') + '_&_' + pl.col('text_fqid').fill_null('None')).alias('event_name_&_room_fqid_&_text_fqid'),\n    ])\n\n    ### text_fqid None → イベンごとに識別\n    df = ryota_stream_text_fill_null_forward(df, 'text_fqid')\n\n    ### features作成\n    features = {}\n    features['session_id'] = df['session_id'][0]\n    level_group = df['level_group'][0]\n    features['level_group'] = level_group\n    \n    ### 基本的な特徴量\n    features['event_count'] = len(df)\n    features['elapsed_time_duration'] = df['elapsed_time'].max() - df['elapsed_time'].min()\n    features['text_unique_rate'] = df['text'].n_unique() / len(df)\n    features['elapsed_time_per_event'] = features['elapsed_time_duration'] / features['event_count']\n    \n    ### 基本的な特徴量2\n    cat_cols = ['event_name', 'name', 'fqid', 'room_fqid', 'text_fqid', 'text']\n    num_cols = ['elapsed_time_diff', 'level', 'page', 'hover_duration', 'room_coor_x',\n                'room_coor_y', 'screen_coor_x', 'screen_coor_y', 'room_coor_move', 'screen_coor_move']\n\n    for col in cat_cols:\n        features[f'{col}_nunique'] = df[col].n_unique()\n        features[f'{col}_duplicate'] = sum(df[col].shift() == df[col])\n\n    for col in num_cols:\n        features[f'{col}_mean'] = df[col].mean()\n        features[f'{col}_std'] = df[col].std()\n        features[f'{col}_sum'] = df[col].sum()\n        features[f'{col}_max'] = df[col].max()\n        \n    ### Aggregation (polars処理の方が速い)\n    agg_cols = ['level', 'event_name', 'room_fqid', 'fqid', 'room_fqid_&_level', 'room_fqid_&_event_name']\n    if level_group >= 1:\n        agg_cols.append('level_group')\n\n    for col in agg_cols:\n        agg_feat = df.groupby(col).agg([\n            pl.col('index').count().alias('index_count'),\n            pl.col('elapsed_time_diff').mean().alias('elapsed_time_diff_mean'),\n            pl.col('elapsed_time_diff').std().alias('elapsed_time_diff_std'),\n            pl.col('elapsed_time_diff').sum().alias('elapsed_time_diff_sum'),\n            pl.col('room_coor_move').mean().alias('room_coor_move_mean'),\n            pl.col('room_coor_move').std().alias('room_coor_move_std'),\n            pl.col('room_coor_move').sum().alias('room_coor_move_sum'),\n            pl.col('hover_duration').sum().alias('hover_duration_sum'),\n        ])\n        agg_feat = agg_feat.with_columns(\n            elapsed_time_diff_rate = pl.col('elapsed_time_diff_sum') / features['elapsed_time_duration']\n        )\n        agg_feat = agg_feat.melt(id_vars=[col], value_vars=agg_feat.columns[1:])\n        agg_feat = agg_feat.with_columns(\n            col_name = f'{col}_' + pl.col(col).cast(pl.Utf8) + '_' + pl.col('variable')\n        )\n        agg_feat = dict(agg_feat[['col_name', 'value']].to_numpy())\n        features.update(agg_feat)\n\n    ### Aggregation2\n    agg_cols2 = ['text_fqid', 'text', 'room_fqid_&_fqid', 'event_name_&_room_fqid_&_text_fqid']\n    for col in agg_cols2:\n        agg_feat = df.groupby(col).agg([\n            pl.col('index').count().alias('index_count'),\n            pl.col('elapsed_time_diff').sum().alias('elapsed_time_diff_sum'),\n            pl.col('room_coor_move').sum().alias('room_coor_move_sum'),\n            pl.col('hover_duration').sum().alias('hover_duration_sum'),\n            pl.col('text_fqid_null_flag').first().alias('text_fqid_null_flag')\n            \n        ])\n        if col in ['event_name_&_room_fqid_&_text_fqid']:\n            agg_feat = agg_feat.filter(pl.col('text_fqid_null_flag')==1)\n            \n        agg_feat = agg_feat.drop('text_fqid_null_flag')\n        agg_feat = agg_feat.with_columns(\n            elapsed_time_diff_rate = pl.col('elapsed_time_diff_sum') / features['elapsed_time_duration']\n        )\n        agg_feat = agg_feat.melt(id_vars=[col], value_vars=agg_feat.columns[1:])\n        agg_feat = agg_feat.with_columns(\n            col_name = f'{col}_' + pl.col(col).cast(pl.Utf8) + '_' + pl.col('variable')\n        )\n        agg_feat = dict(agg_feat[['col_name', 'value']].to_numpy())\n        features.update(agg_feat)\n\n    ### bingo feature\n    if use_bingo and (level_group >= 1):\n        print('use_bingo')\n        features = batch_bingo_feature(features, df, level_group)\n        \n    return features","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.089203Z","iopub.execute_input":"2023-06-24T23:27:04.089973Z","iopub.status.idle":"2023-06-24T23:27:04.114864Z","shell.execute_reply.started":"2023-06-24T23:27:04.089948Z","shell.execute_reply":"2023-06-24T23:27:04.113256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformer","metadata":{}},{"cell_type":"code","source":"def ryota_test_get_cat_map(df: pl.DataFrame, level_group: int, use_objects: dict):\n    for col in ['event_name', 'text_fqid', 'room_fqid']:\n        cat_map = use_objects['cat_map'][col]\n        df = df.with_columns([\n            pl.col(col).map_dict(cat_map, default=1)\n        ])\n    return df\n\ndef ryota_test_get_num_scaler(df: pl.DataFrame, level_group: int, use_objects: dict):\n    scaler = use_objects['num_scaler']\n    scaler_values = scaler.transform(df[['elapsed_time_diff']].to_numpy())\n    df = df.with_columns([\n        pl.Series(scaler_values[:, 0]).alias('elapsed_time_diff'),\n    ])\n    return df\n\ndef ryota_train_preprocess(config, df: pl.DataFrame, level_group: int, use_objects: dict):\n    df = df.with_columns([\n        pl.col('elapsed_time').diff().shift(-1).over(['session_id']).fill_null(0).clip(0, 10 * 60 * 1000).alias('elapsed_time_diff'),\n        (pl.col('event_name') + '-' + pl.col('name')).alias('event_name')\n    ])\n    # 前処理\n    df = ryota_stream_text_fill_null_forward(df, text_col='text_fqid')\n    df = ryota_test_get_cat_map(df, level_group, use_objects)\n    df = ryota_test_get_num_scaler(df, level_group, use_objects)\n    \n    # convert to torch.Tensor\n    max_len = config.max_len[level_group]\n    use_cols = ['event_name', 'text_fqid', 'room_fqid', 'elapsed_time_diff']\n    feat_array = df[use_cols].to_numpy()\n    \n    # truncation\n    if feat_array.shape[0] > max_len:\n        feat_array = feat_array[-max_len:, :]\n        \n    mask = np.ones(feat_array.shape[0])\n    array_len = feat_array.shape[0]\n    \n    # padding\n    if feat_array.shape[0] < max_len:\n        shortage = max_len - feat_array.shape[0]\n        feat_array = np.concatenate((feat_array, np.zeros((shortage, feat_array.shape[1]))), axis=0)\n        mask = np.append(mask, np.zeros(shortage))\n\n    length = torch.tensor([array_len], dtype=torch.long, device=config.device)\n    cat_feat = torch.tensor(feat_array[:, :3], dtype=torch.long).unsqueeze(dim=0) # (1, seq, cat_dim)\n    num_feat = torch.tensor(feat_array[:, 3], dtype=torch.float).unsqueeze(dim=0) # (1, seq, num_dim)\n    mask = torch.tensor(mask, dtype=torch.float).unsqueeze(dim=0)\n    questions = torch.tensor(range(18), dtype=torch.long).unsqueeze(dim=0) # (1, q_num)\n\n    return cat_feat, num_feat, mask, questions, length","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.116476Z","iopub.execute_input":"2023-06-24T23:27:04.116757Z","iopub.status.idle":"2023-06-24T23:27:04.136521Z","shell.execute_reply.started":"2023-06-24T23:27:04.116733Z","shell.execute_reply":"2023-06-24T23:27:04.135119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SequenceDropout(nn.Module):\n    def __init__(self, p: float = 0.50):\n        super().__init__()\n        if p < 0 or p > 1:\n            raise ValueError(\"dropout probability has to be between 0 and 1, \" \"but got {}\".format(p))\n        self.p = p\n        \n    def forward(self, x):\n        if self.training:\n            binary_mask = (torch.rand(x.shape[1], device=x.device) > self.p).to(torch.float) / (1 - self.p) # inversed dropout\n            binary_mask = binary_mask[None, ..., None]\n            x = x * binary_mask\n        return x\n\nclass PSPTransformerModel(nn.Module):\n    def __init__(\n        self,\n        cat_inp_dim: int,\n        cat_emb_dim: int,\n        q_dim: int,\n        dense_dim: int = 128,\n        nhead: int = 8,\n        dim_ff: int = 512,\n        dropout: float = 0.1,\n        num_layers: int = 1\n    ):\n        super().__init__()\n        \n        self.dense_dim = dense_dim\n        cat_dim1, cat_dim2, cat_dim3 = cat_inp_dim\n        cat_emb1, cat_emb2, cat_emb3 = cat_emb_dim\n        self.emb1 = nn.Embedding(cat_dim1, cat_emb1, padding_idx=0)\n        self.emb2 = nn.Embedding(cat_dim2, cat_emb2, padding_idx=0)\n        self.emb3 = nn.Embedding(cat_dim3, cat_emb3, padding_idx=0)\n        self.q_emb = nn.Embedding(q_dim, dense_dim)\n        \n        inp_dim = cat_emb1 + cat_emb2 + cat_emb3\n        self.linear = nn.Linear(inp_dim, dense_dim//2)\n        self.linear2 = nn.Linear(1, dense_dim//2)\n        self.linear3 = nn.Linear(dense_dim, dense_dim)\n        \n        self.lstm1 = nn.LSTM(input_size=dense_dim, hidden_size=dense_dim, num_layers=1, batch_first=True, bidirectional=False)\n        self.lstm2 = nn.LSTM(input_size=dense_dim, hidden_size=dense_dim, num_layers=1, batch_first=True, bidirectional=False)\n        encoder_layer = nn.TransformerEncoderLayer(d_model=dense_dim, nhead=nhead, dim_feedforward=dim_ff, dropout=dropout, batch_first=True)\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        \n        self.head = nn.Linear(dense_dim * 2, q_dim)\n        self.mean_prehead = nn.Sequential(\n            nn.Linear(dense_dim, dense_dim),\n            nn.CELU()\n        )\n        self.attention_prehead = nn.Sequential(\n            nn.Linear(dense_dim * q_dim, dense_dim),\n            nn.CELU()\n        )\n        self.seq_dropout = SequenceDropout(p=0.10)\n        self.dropout1 = nn.Dropout(p=0.25)\n        self.dropout2 = nn.Dropout(p=0.10)\n        self.relu = nn.ReLU()\n        self.layernorm1 = nn.LayerNorm(dense_dim)\n        self.layernorm2 = nn.LayerNorm(dense_dim)\n        self.layernorm3 = nn.LayerNorm(dense_dim * 2)\n\n    def forward(self, cat_feat, num_feat, mask, questions, length):\n        ### Input ###\n        # cat input\n        cat_x1 = self.emb1(cat_feat[:, :, 0])\n        cat_x2 = self.emb2(cat_feat[:, :, 1])\n        cat_x3 = self.emb3(cat_feat[:, :, 2])\n        cat_x = torch.cat([cat_x1, cat_x2, cat_x3], dim=-1)\n        cat_x = self.linear(cat_x)\n        # num input\n        num_x = num_feat.unsqueeze(dim=-1)\n        num_x = self.linear2(num_x)\n        # input\n        base_x = torch.cat([cat_x, num_x], dim=-1)\n        x = self.seq_dropout(base_x)\n\n        ### LSTM ###\n        x, _ = self.lstm1(x)\n        # skip connection\n        x = x + base_x\n        x = self.layernorm1(x)\n        \n        ### Transformer ###\n        x = self.encoder(x)\n\n        # Pooling\n        # mean pooling\n        mask_ = mask.unsqueeze(dim=-1)\n        mean_out = (x * mask_).sum(dim=1) / mask_.sum(dim=1) # (batch, dim)\n        mean_out = self.mean_prehead(mean_out)\n\n        # attention pooling\n        q = self.q_emb(questions) # (batch, q_num, dim)\n        k = x.transpose(1, 2) # (batch, dim, q_num)\n        qk = torch.matmul(q, k) / torch.sqrt(torch.tensor(self.dense_dim, device=x.device))\n        qk = F.softmax(qk + torch.where(mask == 1, 0., float('-inf')).unsqueeze(dim=1), dim=-1).unsqueeze(dim=2)\n        v = k.unsqueeze(dim=1) # (batch, 1, dim, q_num)\n        attention_out = (qk * v).sum(dim=-1) # (batch, q_num, dim)\n        \n        q_num, dim = attention_out.shape[1], attention_out.shape[2]\n        attention_out = attention_out.view(-1, q_num * dim)\n        attention_out = self.attention_prehead(attention_out) # (batch, dense_dim)\n\n        # all output concat\n        concat_out = torch.cat([mean_out, attention_out], dim=-1)\n        concat_out = self.layernorm3(concat_out)\n        out = self.head(concat_out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.140186Z","iopub.execute_input":"2023-06-24T23:27:04.140499Z","iopub.status.idle":"2023-06-24T23:27:04.163287Z","shell.execute_reply.started":"2023-06-24T23:27:04.140474Z","shell.execute_reply":"2023-06-24T23:27:04.16246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ryota_nn_model_load(config, level_group: int):\n    \n    models = []\n    n_folds = len(list((config.ryota_nn_path).glob(f'model_fold*_lg{level_group}.pth')))\n    for fold in range(n_folds):\n        cat_inp_dim = pickle.load(open(config.ryota_nn_path / f'cat_inp_dim_lg{level_group}.pkl', 'rb'))\n        model = PSPTransformerModel(\n            cat_inp_dim=cat_inp_dim,\n            cat_emb_dim=config.cat_emb_dim,\n            q_dim=config.q_dim,\n            dense_dim=config.dense_dim,\n            nhead=config.nhead,\n            dim_ff=config.dim_ff,\n            dropout=config.tfm_drop_out,\n            num_layers=config.num_layers\n        )\n        \n        weight_path = config.ryota_nn_path / f'model_fold{fold}_lg{level_group}.pth'\n        model.load_state_dict(torch.load(\n            weight_path,\n            map_location=torch.device(config.device)\n        ))\n        model.eval()\n        models.append(model)\n        \n    return models","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.164465Z","iopub.execute_input":"2023-06-24T23:27:04.164876Z","iopub.status.idle":"2023-06-24T23:27:04.179762Z","shell.execute_reply.started":"2023-06-24T23:27:04.164852Z","shell.execute_reply":"2023-06-24T23:27:04.178632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## shu421 Part","metadata":{}},{"cell_type":"code","source":"SHU_KEY_COLS = [\"session_id\", \"level_group\"]\n\nSHU_CAT_COLS = [\n    \"event_name\", \n    \"name\", \n    \"fqid\", \n    \"room_fqid\", \n    ]\n\nSHU_NUMERIC_COLS = [\n    \"elapsed_time\",\n    \"level\",\n    \"page\",\n    \"room_coor_x\",\n    \"room_coor_y\",\n    \"screen_coor_x\",\n    \"screen_coor_y\",\n    \"hover_duration\",\n]\nSHU_DIFF_COLS = [\n    \"elapsed_time_diff\",\n    \"room_coor_x_diff\",\n    \"room_coor_y_diff\",\n    \"screen_coor_x_diff\",\n    \"screen_coor_y_diff\",\n    \"room_coor_move\",\n]\nSHU_NUMERIC_COLS += SHU_DIFF_COLS","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.182225Z","iopub.execute_input":"2023-06-24T23:27:04.182858Z","iopub.status.idle":"2023-06-24T23:27:04.192739Z","shell.execute_reply.started":"2023-06-24T23:27:04.182826Z","shell.execute_reply":"2023-06-24T23:27:04.192062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@numba.jit('Tuple((f8[:], f8[:]))(i8[:], i8[:], i8[:], i8[:], i8[:], i8[:], i8[:], i8[:], i8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], i8[:], i8[:], i8[:], i8[:], i8[:], i8[:], i8[:], i8[:])', nopython=True)\ndef shu_feature_engineering(\n    event_name,\n    name,\n    fqid,\n    room_fqid,\n    text,\n    text_fqid,\n    elapsed_time,\n    level,\n    level_group,\n    page,\n    room_coor_x,\n    room_coor_y,\n    screen_coor_x,\n    screen_coor_y,\n    hover_duration,\n    elapsed_time_diff,\n    room_coor_x_diff,\n    room_coor_y_diff,\n    screen_coor_x_diff,\n    screen_coor_y_diff,\n    room_coor_move,\n    event_name_cols,\n    name_cols,\n    text_cols,\n    fqid_cols,\n    room_fqid_cols,\n    text_fqid_cols,\n    level_cols,\n    level_group_cols,\n):\n    feat = [\n        np.nanquantile(elapsed_time, 0.1),\n        np.nanquantile(elapsed_time, 0.2),\n        np.nanquantile(elapsed_time, 0.4),\n        np.nanquantile(elapsed_time, 0.6),\n        np.nanquantile(elapsed_time, 0.8),\n        np.nanquantile(elapsed_time, 0.9),\n        np.nanquantile(level, 0.1),\n        np.nanquantile(level, 0.2),\n        np.nanquantile(level, 0.4),\n        np.nanquantile(level, 0.6),\n        np.nanquantile(level, 0.8),\n        np.nanquantile(level, 0.9),\n        np.nanquantile(page, 0.1),\n        np.nanquantile(page, 0.2),\n        np.nanquantile(page, 0.4),\n        np.nanquantile(page, 0.6),\n        np.nanquantile(page, 0.8),\n        np.nanquantile(page, 0.9),\n        np.nanquantile(room_coor_x, 0.1),\n        np.nanquantile(room_coor_x, 0.2),\n        np.nanquantile(room_coor_x, 0.4),\n        np.nanquantile(room_coor_x, 0.6),\n        np.nanquantile(room_coor_x, 0.8),\n        np.nanquantile(room_coor_x, 0.9),\n        np.nanquantile(room_coor_y, 0.1),\n        np.nanquantile(room_coor_y, 0.2),\n        np.nanquantile(room_coor_y, 0.4),\n        np.nanquantile(room_coor_y, 0.6),\n        np.nanquantile(room_coor_y, 0.8),\n        np.nanquantile(room_coor_y, 0.9),\n        np.nanquantile(screen_coor_x, 0.1),\n        np.nanquantile(screen_coor_x, 0.2),\n        np.nanquantile(screen_coor_x, 0.4),\n        np.nanquantile(screen_coor_x, 0.6),\n        np.nanquantile(screen_coor_x, 0.8),\n        np.nanquantile(screen_coor_x, 0.9),\n        np.nanquantile(screen_coor_y, 0.1),\n        np.nanquantile(screen_coor_y, 0.2),\n        np.nanquantile(screen_coor_y, 0.4),\n        np.nanquantile(screen_coor_y, 0.6),\n        np.nanquantile(screen_coor_y, 0.8),\n        np.nanquantile(screen_coor_y, 0.9),\n        np.nanquantile(hover_duration, 0.1),\n        np.nanquantile(hover_duration, 0.2),\n        np.nanquantile(hover_duration, 0.4),\n        np.nanquantile(hover_duration, 0.6),\n        np.nanquantile(hover_duration, 0.8),\n        np.nanquantile(hover_duration, 0.9),\n        np.nanquantile(elapsed_time_diff, 0.1),\n        np.nanquantile(elapsed_time_diff, 0.2),\n        np.nanquantile(elapsed_time_diff, 0.4),\n        np.nanquantile(elapsed_time_diff, 0.6),\n        np.nanquantile(elapsed_time_diff, 0.8),\n        np.nanquantile(elapsed_time_diff, 0.9),\n        np.nanquantile(room_coor_x_diff, 0.1),\n        np.nanquantile(room_coor_x_diff, 0.2),\n        np.nanquantile(room_coor_x_diff, 0.4),\n        np.nanquantile(room_coor_x_diff, 0.6),\n        np.nanquantile(room_coor_x_diff, 0.8),\n        np.nanquantile(room_coor_x_diff, 0.9),\n        np.nanquantile(room_coor_y_diff, 0.1),\n        np.nanquantile(room_coor_y_diff, 0.2),\n        np.nanquantile(room_coor_y_diff, 0.4),\n        np.nanquantile(room_coor_y_diff, 0.6),\n        np.nanquantile(room_coor_y_diff, 0.8),\n        np.nanquantile(room_coor_y_diff, 0.9),\n        np.nanquantile(screen_coor_x_diff, 0.1),\n        np.nanquantile(screen_coor_x_diff, 0.2),\n        np.nanquantile(screen_coor_x_diff, 0.4),\n        np.nanquantile(screen_coor_x_diff, 0.6),\n        np.nanquantile(screen_coor_x_diff, 0.8),\n        np.nanquantile(screen_coor_x_diff, 0.9),\n        np.nanquantile(screen_coor_y_diff, 0.1),\n        np.nanquantile(screen_coor_y_diff, 0.2),\n        np.nanquantile(screen_coor_y_diff, 0.4),\n        np.nanquantile(screen_coor_y_diff, 0.6),\n        np.nanquantile(screen_coor_y_diff, 0.8),\n        np.nanquantile(screen_coor_y_diff, 0.9),\n        np.nanquantile(room_coor_move, 0.1),\n        np.nanquantile(room_coor_move, 0.2),\n        np.nanquantile(room_coor_move, 0.4),\n        np.nanquantile(room_coor_move, 0.6),\n        np.nanquantile(room_coor_move, 0.8),\n        np.nanquantile(room_coor_move, 0.9),\n        len(np.unique(event_name)),\n        len(np.unique(name)),\n        len(np.unique(fqid)),\n        len(np.unique(room_fqid)),\n    ]\n    save_feat = [\n        np.nansum(elapsed_time),\n        np.nansum(level),\n        np.nansum(page),\n        np.nansum(room_coor_x),\n        np.nansum(room_coor_y),\n        np.nansum(screen_coor_x),\n        np.nansum(screen_coor_y),\n        np.nansum(hover_duration),\n        np.nansum(elapsed_time_diff),\n        np.nansum(room_coor_x_diff),\n        np.nansum(room_coor_y_diff),\n        np.nansum(screen_coor_x_diff),\n        np.nansum(screen_coor_y_diff),\n        np.nansum(room_coor_move),\n        np.nanstd(elapsed_time),\n        np.nanstd(level),\n        np.nanstd(page),\n        np.nanstd(room_coor_x),\n        np.nanstd(room_coor_y),\n        np.nanstd(screen_coor_x),\n        np.nanstd(screen_coor_y),\n        np.nanstd(hover_duration),\n        np.nanstd(elapsed_time_diff),\n        np.nanstd(room_coor_x_diff),\n        np.nanstd(room_coor_y_diff),\n        np.nanstd(screen_coor_x_diff),\n        np.nanstd(screen_coor_y_diff),\n        np.nanstd(room_coor_move),\n    ]\n\n    feat += save_feat\n\n    ############################ elapsed_time_diff, hover_duration ############################\n    # event_name\n    for c in event_name_cols:\n        idx = event_name == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(12)]\n            save_feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanquantile(elapsed_time_diff[idx], 0.1),\n                np.nanquantile(elapsed_time_diff[idx], 0.2),\n                np.nanquantile(elapsed_time_diff[idx], 0.4),\n                np.nanquantile(elapsed_time_diff[idx], 0.6),\n                np.nanquantile(elapsed_time_diff[idx], 0.8),\n                np.nanquantile(elapsed_time_diff[idx], 0.9),\n                np.nanmean(hover_duration[idx]),\n            ]\n            _save_feat = [\n                np.nansum(elapsed_time_diff[idx]),\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nansum(hover_duration[idx]),\n                np.nanstd(hover_duration[idx]),\n            ]\n            feat += _save_feat\n            save_feat += _save_feat\n\n    # name\n    for c in name_cols:\n        idx = name == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(12)]\n            save_feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanmean(hover_duration[idx]),\n                np.nanquantile(elapsed_time_diff[idx], 0.1),\n                np.nanquantile(elapsed_time_diff[idx], 0.2),\n                np.nanquantile(elapsed_time_diff[idx], 0.4),\n                np.nanquantile(elapsed_time_diff[idx], 0.6),\n                np.nanquantile(elapsed_time_diff[idx], 0.8),\n                np.nanquantile(elapsed_time_diff[idx], 0.9),\n            ]\n            _save_feat = [\n                np.nansum(elapsed_time_diff[idx]),\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nansum(hover_duration[idx]),\n                np.nanstd(hover_duration[idx]),\n            ]\n            feat += _save_feat\n            save_feat += _save_feat\n\n    # fqid\n    for c in fqid_cols:\n        idx = fqid == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(12)]\n            save_feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanmean(hover_duration[idx]),\n                np.nanquantile(elapsed_time_diff[idx], 0.1),\n                np.nanquantile(elapsed_time_diff[idx], 0.2),\n                np.nanquantile(elapsed_time_diff[idx], 0.4),\n                np.nanquantile(elapsed_time_diff[idx], 0.6),\n                np.nanquantile(elapsed_time_diff[idx], 0.8),\n                np.nanquantile(elapsed_time_diff[idx], 0.9),\n            ]\n            _save_feat = [\n                np.nansum(elapsed_time_diff[idx]),\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nansum(hover_duration[idx]),\n                np.nanstd(hover_duration[idx]),\n            ]\n            feat += _save_feat\n            save_feat += _save_feat\n\n    # room_fqid\n    for c in room_fqid_cols:\n        idx = room_fqid == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(12)]\n            save_feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanmean(hover_duration[idx]),\n                np.nanquantile(elapsed_time_diff[idx], 0.1),\n                np.nanquantile(elapsed_time_diff[idx], 0.2),\n                np.nanquantile(elapsed_time_diff[idx], 0.4),\n                np.nanquantile(elapsed_time_diff[idx], 0.6),\n                np.nanquantile(elapsed_time_diff[idx], 0.8),\n                np.nanquantile(elapsed_time_diff[idx], 0.9),\n            ]\n            _save_feat = [\n                np.nansum(elapsed_time_diff[idx]),\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nansum(hover_duration[idx]),\n                np.nanstd(hover_duration[idx]),\n            ]\n            feat += _save_feat\n            save_feat += _save_feat\n\n    # text\n    for c in text_cols:\n        idx = text == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanstd(hover_duration[idx]),\n                np.nanmean(hover_duration[idx]),\n            ]\n\n    # text_fqid\n    for c in text_fqid_cols:\n        idx = text_fqid == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanstd(hover_duration[idx]),\n                np.nanmean(hover_duration[idx]),\n            ]\n\n    # level\n    for c in level_cols:\n        idx = level == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(4)]\n        else:\n            feat += [\n                np.nanstd(elapsed_time_diff[idx]),\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanstd(hover_duration[idx]),\n                np.nanmean(hover_duration[idx]),\n            ]\n\n    # level_group\n    for c in level_group_cols:\n        idx = level_group == c\n        count = np.sum(idx)\n        feat.append(count)\n        if count == 0:\n            feat += [0.0 for k in range(2)]\n        else:\n            feat += [\n                np.nanmean(elapsed_time_diff[idx]),\n                np.nanmean(hover_duration[idx]),\n            ]\n\n\n    ############################ categorical features ############################\n    for c in level_cols:\n        for d in room_fqid_cols:\n            idx = (c == level) & (d == room_fqid)\n            count = np.sum(idx)\n            feat.append(count)\n            if count == 0:\n                feat += [0.0]\n            else:\n                feat += [\n                    len(np.unique(room_fqid[idx])),\n                    ]\n\n    feat = np.array(feat, dtype=np.float64)\n    save_feat = np.array(save_feat, dtype=np.float64)\n    return feat, save_feat","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:04.194009Z","iopub.execute_input":"2023-06-24T23:27:04.194463Z","iopub.status.idle":"2023-06-24T23:27:29.558829Z","shell.execute_reply.started":"2023-06-24T23:27:04.19444Z","shell.execute_reply":"2023-06-24T23:27:29.557811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_date_feat(session_id: str):\n    date_feat_arr = []\n    date_feat_arr.append(int(session_id[:2])) # year\n    date_feat_arr.append(int(session_id[2:4])) # month\n    date_feat_arr.append(int(session_id[4:6])) # day\n    date_feat_arr.append(int(session_id[6:8])) # hour\n    return np.array(date_feat_arr)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.560223Z","iopub.execute_input":"2023-06-24T23:27:29.560508Z","iopub.status.idle":"2023-06-24T23:27:29.566526Z","shell.execute_reply.started":"2023-06-24T23:27:29.560484Z","shell.execute_reply":"2023-06-24T23:27:29.565437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_previous_features(train_feat_arr, grp, prev_feat_dict):\n    if grp == \"5-12\":\n        train_feat_arr = np.concatenate([train_feat_arr, prev_feat_dict[\"0-4\"]], axis=1)\n        train_feat_arr = np.concatenate([train_feat_arr, prev_feat_dict[\"5-12_first_et\"]-prev_feat_dict[\"0-4_last_et\"]], axis=1) # grp間の時間差\n    elif grp == \"13-22\":\n        train_feat_arr = np.concatenate([train_feat_arr, prev_feat_dict[\"0-4\"], prev_feat_dict[\"5-12\"]], axis=1)\n        train_feat_arr = np.concatenate([train_feat_arr, prev_feat_dict[\"13-22_first_et\"]-prev_feat_dict[\"5-12_last_et\"]], axis=1) # grp間の平均経過時間の差\n    return train_feat_arr\n\ndef add_previous_predictions(train_feat_arr: np.array, grp, oof):\n    if not grp==\"0-4\":\n        if grp==\"5-12\":\n            pre_limits = (1, 4)\n        elif grp==\"13-22\":\n            pre_limits = (1, 14)\n        for pre_q in range(*pre_limits):\n            train_feat_arr = np.concatenate([train_feat_arr, oof.loc[:, pre_q-1].to_numpy().reshape(-1, 1)], axis=1)\n    return train_feat_arr","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.567977Z","iopub.execute_input":"2023-06-24T23:27:29.568406Z","iopub.status.idle":"2023-06-24T23:27:29.593042Z","shell.execute_reply.started":"2023-06-24T23:27:29.568381Z","shell.execute_reply":"2023-06-24T23:27:29.591451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def shu_load_model(filepath):\n    with open(filepath, \"rb\") as f:\n        model = pickle.load(f)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.594376Z","iopub.execute_input":"2023-06-24T23:27:29.594698Z","iopub.status.idle":"2023-06-24T23:27:29.605245Z","shell.execute_reply.started":"2023-06-24T23:27:29.594674Z","shell.execute_reply":"2023-06-24T23:27:29.604055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def shu_get_cols(grp, event_name2label, name2label, fqid2label, room_fqid2label, text2label, text_fqid2label, level_group2label):\n    event_name_cols = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/event_name_cols_{grp}.pkl\", \"rb\"))\n    name_cols = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/name_cols_{grp}.pkl\", \"rb\"))\n    text_cols = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/text_cols_{grp}.pkl\", \"rb\"))\n    fqid_cols = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/fqid_cols_{grp}.pkl\", \"rb\"))\n    room_fqid_cols = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/room_fqid_cols_{grp}.pkl\", \"rb\"))\n    text_fqid_cols = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/text_fqid_cols_{grp}.pkl\", \"rb\"))\n\n    if grp==\"0-4\":\n        level_cols = [1, 2, 3, 4]\n    elif grp==\"5-12\":\n        level_cols = [5, 6, 7, 8, 9, 10, 11, 12]\n    elif grp==\"13-22\":\n        level_cols = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22]\n        \n    ############### encoding ###############\n    event_name_cols = np.vectorize(event_name2label.get)(event_name_cols).astype(np.int64)\n    name_cols = np.vectorize(name2label.get)(name_cols).astype(np.int64)\n    text_cols = np.vectorize(text2label.get)(text_cols).astype(np.int64)\n    fqid_cols = np.vectorize(fqid2label.get)(fqid_cols).astype(np.int64)\n    room_fqid_cols = np.vectorize(room_fqid2label.get)(room_fqid_cols).astype(np.int64)\n    text_fqid_cols = np.vectorize(text_fqid2label.get)(text_fqid_cols).astype(np.int64)\n    level_cols = np.vectorize(np.int64)(level_cols)\n    level_group_cols = np.array([level_group2label[grp]]).astype(np.int64)\n\n    return event_name_cols, name_cols, text_cols, fqid_cols, room_fqid_cols, text_fqid_cols, level_cols, level_group_cols","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.6068Z","iopub.execute_input":"2023-06-24T23:27:29.6071Z","iopub.status.idle":"2023-06-24T23:27:29.619479Z","shell.execute_reply.started":"2023-06-24T23:27:29.607074Z","shell.execute_reply":"2023-06-24T23:27:29.618093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## tereka Part","metadata":{}},{"cell_type":"code","source":"CATS = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMS = [\n    'page', 'room_coor_x', 'room_coor_y', 'screen_coor_x', \n    'screen_coor_y', 'hover_duration', 'elapsed_time_diff'\n]\nDIALOGS = [\n    'that', 'this', 'it', 'you','find','found',\n    'Found','notebook','Wells','wells','help',\n    'need', 'Oh','Ooh','Jo', 'flag', 'can','and',\n    'is','the','to'\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.62092Z","iopub.execute_input":"2023-06-24T23:27:29.621233Z","iopub.status.idle":"2023-06-24T23:27:29.637879Z","shell.execute_reply.started":"2023-06-24T23:27:29.621206Z","shell.execute_reply":"2023-06-24T23:27:29.636327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tereka_make_base_feat(df: pl.DataFrame):\n    columns = [\n        pl.col(\"page\").cast(pl.Float32),\n        (\n            (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1)) # time used for each action\n             .fill_null(0)\n             .clip(0, 1e9)\n             .over([\"session_id\", \"level_group\"])\n             .alias(\"elapsed_time_diff\")\n        ),\n        (\n            (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(2)) # time used for each action\n             .fill_null(0)\n             .clip(0, 1e9)\n             .over([\"session_id\", \"level_group\"])\n             .alias(\"elapsed_time_diff_s2\")\n        ),\n        (\n            (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(3)) # time used for each action\n             .fill_null(0)\n             .clip(0, 1e9)\n             .over([\"session_id\", \"level_group\"])\n             .alias(\"elapsed_time_diff_s3\")\n        ),\n        (\n            (pl.col(\"screen_coor_x\") - pl.col(\"screen_coor_x\").shift(1)) # location x changed for click \n             .abs()\n             .over([\"session_id\", \"level_group\"])\n        ),\n        (\n            (pl.col(\"screen_coor_y\") - pl.col(\"screen_coor_y\").shift(1)) # location y changed for click \n             .abs()\n             .over([\"session_id\", \"level_group\"])\n        ),\n\n        pl.col(\"fqid\").fill_null(\"fqid_None\"),\n        pl.col(\"text_fqid\").fill_null(\"text_fqid_None\")\n    ]\n    df = df.with_columns(columns)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.639586Z","iopub.execute_input":"2023-06-24T23:27:29.639934Z","iopub.status.idle":"2023-06-24T23:27:29.656677Z","shell.execute_reply.started":"2023-06-24T23:27:29.639906Z","shell.execute_reply":"2023-06-24T23:27:29.65526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tereka_stream_bingo_feature(features: dict, df: pl.DataFrame, level_group: int):\n    if level_group == 1:\n        bingo1 = (\n            df.filter((pl.col(\"text\") == \"Here's the log book.\") | (pl.col(\"fqid\") == 'logbook.page.bingo'))\n            .select(\n                logbook_bingo_duration1 = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                logbook_bingo_indexCount1 = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo2 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader')) | (pl.col(\"fqid\") == \"reader.paper2.bingo\"))\n            .select(\n                reader_bingo_duration1 = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                reader_bingo_indexCount1 = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo3 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals')) | (pl.col(\"fqid\") == \"journals.pic_2.bingo\"))\n            .select(\n                journals_bingo_duration2 = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                journals_bingo_indexCount2 = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo = pl.concat([bingo1, bingo2, bingo3], how='horizontal')\n        bingo = dict(zip(bingo.columns, bingo.to_numpy()[0]))\n        features.update(bingo)\n\n    elif level_group == 2:\n        bingo1 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'reader_flag')) | (pl.col(\"fqid\") == \"tunic.library.microfiche.reader_flag.paper2.bingo\"))\n            .select(\n                reader_flag_duration2 = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                reader_flag_indexCount2 = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo2 = (\n            df.filter(((pl.col(\"event_name\") == 'navigate_click') & (pl.col(\"fqid\") == 'journals_flag')) | (pl.col(\"fqid\") == \"journals_flag.pic_0.bingo\"))\n            .select(\n                journalsFlag_bingo_duration2 = pl.col('elapsed_time').max() - pl.col('elapsed_time').min(),\n                journalsFlag_bingo_indexCount2 = pl.col('index').max() - pl.col('index').min()\n            )\n        )\n        bingo = pl.concat([bingo1, bingo2], how='horizontal')\n        bingo = dict(zip(bingo.columns, bingo.to_numpy()[0]))\n        features.update(bingo)\n        \n    return features","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.658431Z","iopub.execute_input":"2023-06-24T23:27:29.658741Z","iopub.status.idle":"2023-06-24T23:27:29.67296Z","shell.execute_reply.started":"2023-06-24T23:27:29.658712Z","shell.execute_reply":"2023-06-24T23:27:29.672008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tereka_feature_engineering(df: pl.DataFrame, grp: str, use_extra: bool, feature_suffix: str):\n    ### PREPROCESS\n    df = tereka_make_base_feat(df)\n    \n    ### BASIC\n    features = {}\n    features['session_id'] = df['session_id'][0]\n    # features['level_group'] = df['level_group'][0]\n    features[f'session_number_{feature_suffix}'] = len(df)\n    features[f'elapsed_time_diff_all_sum_{feature_suffix}'] = df['elapsed_time_diff'].sum()\n    \n    ### CATS\n    for col in CATS:\n        features[f'{col}_unique_{feature_suffix}'] = df[col].n_unique()\n        features[f'{col}_cnt_{feature_suffix}'] = len(df[col].drop_nulls())\n       \n    ### NUMS\n    for col in NUMS:\n        features[f'{col}_mean_{feature_suffix}'] = df[col].mean()\n        features[f'{col}_min_{feature_suffix}'] = df[col].min()\n        features[f'{col}_max_{feature_suffix}'] = df[col].max()\n        features[f'{col}_median_{feature_suffix}'] = df[col].median()\n        features[f'{col}_std_{feature_suffix}'] = df[col].std()\n    \n    ### DIALOGS\n    aggs = (\n        df.with_columns(\n            c = pl.col('text')\n            .apply(lambda x: [c for c in DIALOGS if c in x])\n            .apply(lambda x: None if len(x) == 0 else x[0])\n        )\n        .filter(pl.col('c').is_not_null())\n        .groupby('c')\n        .agg([\n            pl.col('elapsed_time_diff').mean().alias('word_mean'),\n            pl.col('elapsed_time_diff').std().alias('word_std'),\n            pl.col('elapsed_time_diff').max().alias('word_max'),\n            pl.col('elapsed_time_diff').sum().alias('word_sum'),\n            pl.col('elapsed_time_diff').median().alias('word_median'),\n        ])\n    )\n    aggs = (\n        aggs.melt(id_vars='c', value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col('variable') + '_' + pl.col('c') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n    \n    ### EVENT NAME\n    key_column = 'event_name'\n    aggs = (\n    df.groupby(key_column)\n        .agg([\n            pl.col('index').count().alias('cnt'),  \n            pl.col('elapsed_time_diff').median().alias('ET_median'),\n            pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n            pl.col('elapsed_time_diff').max().alias('ET_max'),\n            pl.col('elapsed_time_diff').min().alias('ET_min'),\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n            pl.col('elapsed_time_diff').std().alias('ET_std'),\n            pl.col('elapsed_time_diff').quantile(0.1, 'nearest').alias('ET_quantile1'),\n            pl.col('elapsed_time_diff').quantile(0.2, 'nearest').alias('ET_quantile2'),\n            pl.col('elapsed_time_diff').quantile(0.4, 'nearest').alias('ET_quantile4'),\n            pl.col('elapsed_time_diff').quantile(0.6, 'nearest').alias('ET_quantile6'),\n            pl.col('elapsed_time_diff').quantile(0.8, 'nearest').alias('ET_quantile8'),\n            pl.col('elapsed_time_diff').quantile(0.9, 'nearest').alias('ET_quantile9'),\n            pl.col('hover_duration').median().alias('HD_median'),\n            pl.col('hover_duration').mean().alias('HD_mean'),\n            pl.col('hover_duration').max().alias('HD_max'),\n            pl.col('hover_duration').min().alias('HD_min'),\n            pl.col('hover_duration').sum().alias('HD_sum'),\n            pl.col('hover_duration').std().alias('HD_std'),\n        ])\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col(key_column) + '_' + pl.col('variable') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n        \n    ### NAME\n    key_column = 'name'\n    aggs = (\n        df.groupby(key_column)\n        .agg([\n            pl.col('elapsed_time_diff').median().alias('ET_median'),\n            pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n            pl.col('elapsed_time_diff').max().alias('ET_max'),\n            pl.col('elapsed_time_diff').min().alias('ET_min'),\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n            pl.col('elapsed_time_diff').std().alias('ET_std'),\n        ])\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col(key_column) + '_' + pl.col('variable') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n    \n    ### FQID\n    key_column = 'fqid'\n    aggs = (\n        df.groupby(key_column)\n        .agg([\n            pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n            pl.col('elapsed_time_diff').max().alias('ET_max'),\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n            pl.col('elapsed_time_diff').std().alias('ET_std'),\n        ])\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col(key_column) + '_' + pl.col('variable') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n    \n    ### TEXT\n    key_column = 'text'\n    aggs = (\n        df.filter(pl.col('text').is_not_null())\n        .groupby(key_column)\n        .agg(\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n        )\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = 'text_sum_' + pl.col('text') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n        \n    ### ROOM_FQID\n    key_column = 'room_fqid'\n    aggs = (\n        df.groupby(key_column)\n        .agg(\n            pl.col('index').count().alias('cnt'),\n            pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n            pl.col('elapsed_time_diff').max().alias('ET_max'),\n            pl.col('elapsed_time_diff').min().alias('ET_min'),\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n            pl.col('elapsed_time_diff').std().alias('ET_std'),\n        )\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col(key_column) + '_' + pl.col('variable') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n    \n    ### TEXT FQIDS\n    key_column = 'text_fqid'\n    aggs = (\n        df.filter(pl.col(key_column).is_not_null())\n        .groupby(key_column)\n        .agg(\n            pl.col('index').count().alias('cnt'),\n            pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n            pl.col('elapsed_time_diff').max().alias('ET_max'),\n            pl.col('elapsed_time_diff').std().alias('ET_std'),\n            pl.col('hover_duration').mean().alias('HD_mean'),\n            pl.col('hover_duration').sum().alias('HD_sum'),\n            pl.col('hover_duration').max().alias('HD_max'),\n            pl.col('hover_duration').std().alias('HD_std'),\n        )\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col(key_column) + '_' + pl.col('variable') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n    \n    ### LEVEL\n    key_column = 'level'\n    aggs = (\n        df.groupby(key_column)\n        .agg(\n            pl.col('index').count().alias('cnt'),\n            pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n            pl.col('elapsed_time_diff').max().alias('ET_max'),\n            pl.col('elapsed_time_diff').min().alias('ET_min'),\n            pl.col('elapsed_time_diff').std().alias('ET_std'),\n            pl.col('elapsed_time_diff').sum().alias('ET_sum'),\n            pl.col('hover_duration').mean().alias('HD_mean'),\n            pl.col('hover_duration').max().alias('HD_max'),\n            pl.col('hover_duration').min().alias('HD_min'),\n            pl.col('hover_duration').std().alias('HD_std'),\n        )\n    )\n    aggs = (\n        aggs.melt(id_vars=key_column, value_vars=aggs.columns[1:])\n        .with_columns(col_name = pl.col(key_column).cast(pl.Utf8) + '_' + pl.col('variable') + '_' + feature_suffix)\n    )\n    aggs = dict(aggs[['col_name', 'value']].to_numpy())\n    features.update(aggs)\n        \n    ### COMBINATIONS1\n    for col1, col2 in [['text', 'room_fqid'], ['text', 'fqid']]:\n        key_column = [col1, col2]\n        aggs = (\n            df.filter(pl.col('text').is_not_null())\n            .groupby(key_column)\n            .agg(\n                pl.col('elapsed_time_diff').sum().alias('ET_sum')\n            )\n        )\n        aggs = (\n            aggs.melt(id_vars=key_column, value_vars=aggs.columns[len(key_column):])\n            .with_columns(col_name = 'text_sum_' + pl.col(col1) + '_' + pl.col(col2) + '_' + feature_suffix)\n        )\n        aggs = dict(aggs[['col_name', 'value']].to_numpy())\n        features.update(aggs)\n    \n    ### COMBINATIONS2\n    for col1, col2 in [\n        ['event_name', 'name'],\n        ['event_name', 'level'],\n        ['room_fqid', 'level'],\n        ['room_fqid', 'event_name'],\n        ['room_fqid', 'fqid'],\n        ['text_fqid', 'fqid']\n    ]:\n        key_column = [col1, col2]\n        aggs = (\n            df.groupby(key_column)\n            .agg(\n                pl.col('elapsed_time_diff').mean().alias('ET_mean'),\n                pl.col('elapsed_time_diff').sum().alias('ET_sum')\n            )\n        )\n        aggs = (\n            aggs.melt(id_vars=key_column, value_vars=aggs.columns[len(key_column):])\n            .with_columns(col_name = pl.col(col1).cast(pl.Utf8) + '_' + pl.col(col2).cast(pl.Utf8) + '_' + pl.col('variable') + '_' + feature_suffix)\n        )\n        aggs = dict(aggs[['col_name', 'value']].to_numpy())\n        features.update(aggs)\n    \n    ### EXTRA FEATURES (BINGO)\n    if (grp in ['5-12', '13-22']) and use_extra:\n        lg = {'5-12': 1, '13-22': 2}[grp]\n        features = tereka_stream_bingo_feature(features, df, lg)\n\n    return features","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.674254Z","iopub.execute_input":"2023-06-24T23:27:29.674631Z","iopub.status.idle":"2023-06-24T23:27:29.715034Z","shell.execute_reply.started":"2023-06-24T23:27:29.67461Z","shell.execute_reply":"2023-06-24T23:27:29.713762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tereka_use_features_load(feat_file: Path):\n    f_read = open(feat_file, 'rb')\n    features_list = pickle.load(f_read)\n    \n    features_zero = list(set(features_list[0] + features_list[1] + features_list[2]))\n    features_one = list(set(\n        features_list[3] + features_list[4] + features_list[5] +\n        features_list[6] + features_list[7] + features_list[8] +\n        features_list[9] + features_list[10] + features_list[11] +\n        features_list[12]\n    ))\n    features_two = list(set(features_list[13] + features_list[14] + features_list[15] + features_list[16] + features_list[17]))\n\n    features_zero = list(set(features_zero + [column for column in features_two if \"_grp0\" in column] + [column for column in features_one if \"_grp0\" in column]))\n    features_one = list(set(features_one + [column for column in features_two if \"_grp1\" in column] + features_zero))\n\n    return features_list, features_zero, features_one, features_two","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.716963Z","iopub.execute_input":"2023-06-24T23:27:29.717339Z","iopub.status.idle":"2023-06-24T23:27:29.73601Z","shell.execute_reply.started":"2023-06-24T23:27:29.717291Z","shell.execute_reply":"2023-06-24T23:27:29.734369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# theta Part","metadata":{}},{"cell_type":"code","source":"with open('/kaggle/input/pickle/FQID_v2.pickle', 'rb') as f:\n    FQID_V2 = pickle.load(f)\nwith open('/kaggle/input/pickle/ROOM_FQID_v2.pickle', 'rb') as f:\n    ROOM_FQID_V2 = pickle.load(f)\nwith open('/kaggle/input/pickle/TEXT_FQID_v2.pickle', 'rb') as f:\n    TEXT_FQID_V2 = pickle.load(f)\nwith open('/kaggle/input/pickle/TEXT_v2.pickle', 'rb') as f:\n    TEXT_V2 = pickle.load(f)\n\nFQID2LABEL_V2 = defaultdict(lambda :len(FQID2LABEL_V2))\nROOM_FQID2LABEL_V2 = defaultdict(lambda :len(ROOM_FQID2LABEL_V2))\nTEXT_FQID2LABEL_V2 = defaultdict(lambda :len(TEXT_FQID2LABEL_V2))\nTEXT2LABEL_V2 = defaultdict(lambda :len(TEXT2LABEL_V2))\nEVENT_NAME2LABEL_V2 = defaultdict(lambda :len(EVENT_NAME2LABEL_V2))\n\nGRP2LABEL_V2 = {'0-4':0, '5-12':1, '13-22':2}\nNAME2LABEL_V2 = {'basic':0, 'open':1, 'close':2, 'next':3, 'prev':4, 'undefined':5}\n\nfor grp in ['0-4', '5-12', '13-22']:\n    FQID_V2[grp] = np.vectorize(lambda x:FQID2LABEL_V2[x])(FQID_V2[grp])\n    ROOM_FQID_V2[grp] = np.vectorize(lambda x:ROOM_FQID2LABEL_V2[x])(ROOM_FQID_V2[grp])\n    TEXT_FQID_V2[grp] = np.vectorize(lambda x:TEXT_FQID2LABEL_V2[x])(TEXT_FQID_V2[grp])\n    TEXT_V2[grp] = np.vectorize(lambda x:TEXT2LABEL_V2[x])(TEXT_V2[grp])\n\nEVENT_NAME_V2 = np.array(['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'notebook_click', 'object_hover', 'checkpoint'])\nEVENT_NAME_V2 = np.vectorize(lambda x:EVENT_NAME2LABEL_V2[x])(EVENT_NAME_V2)\nNAME_V2 = np.array(['basic', 'open', 'close', 'next', 'prev', 'undefined'])\nNAME_V2 = np.vectorize(lambda x:NAME2LABEL_V2[x])(NAME_V2)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.737747Z","iopub.execute_input":"2023-06-24T23:27:29.73811Z","iopub.status.idle":"2023-06-24T23:27:29.801374Z","shell.execute_reply.started":"2023-06-24T23:27:29.738044Z","shell.execute_reply":"2023-06-24T23:27:29.800216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/pickle/FQID_v3.pickle', 'rb') as f:\n    FQID = pickle.load(f)\nwith open('/kaggle/input/pickle/ROOM_FQID_v3.pickle', 'rb') as f:\n    ROOM_FQID = pickle.load(f)\nwith open('/kaggle/input/pickle/TEXT_FQID_v3.pickle', 'rb') as f:\n    TEXT_FQID = pickle.load(f)\nwith open('/kaggle/input/pickle/TEXT_v3.pickle', 'rb') as f:\n    TEXT = pickle.load(f)\n\nFQID2LABEL = defaultdict(lambda :len(FQID2LABEL))\nROOM_FQID2LABEL = defaultdict(lambda :len(ROOM_FQID2LABEL))\nTEXT_FQID2LABEL = defaultdict(lambda :len(TEXT_FQID2LABEL))\nTEXT2LABEL = defaultdict(lambda :len(TEXT2LABEL))\nEVENT_NAME2LABEL = defaultdict(lambda :len(EVENT_NAME2LABEL))\n\nGRP2LABEL = {'0-4':0, '5-12':1, '13-22':2}\nNAME2LABEL = {'basic':0, 'open':1, 'close':2, 'next':3, 'prev':4, 'undefined':5}","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.80323Z","iopub.execute_input":"2023-06-24T23:27:29.803611Z","iopub.status.idle":"2023-06-24T23:27:29.843454Z","shell.execute_reply.started":"2023-06-24T23:27:29.803582Z","shell.execute_reply":"2023-06-24T23:27:29.841718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FQID_NAN_LABEL = FQID2LABEL[np.nan]\nTEXT_NAN_LABEL = TEXT2LABEL[np.nan]\nTEXT_FQID_NAN_LABEL = TEXT_FQID2LABEL[np.nan]","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.845149Z","iopub.execute_input":"2023-06-24T23:27:29.845699Z","iopub.status.idle":"2023-06-24T23:27:29.850155Z","shell.execute_reply.started":"2023-06-24T23:27:29.845672Z","shell.execute_reply":"2023-06-24T23:27:29.848878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BUSINESSCARDS_BINGO_TEXT_FQID_LABEL = TEXT_FQID2LABEL[\"tunic.humanecology.frontdesk.businesscards.card_bingo.bingo\"]\nBUSINESSCARDS_BINGO_FQID_LABEL = FQID2LABEL[\"businesscards\"]\nLOGBOOK_BINGO_TEXT_FQID_LABEL = TEXT_FQID2LABEL[\"tunic.drycleaner.frontdesk.logbook.page.bingo\"]\nLOGBOOK_BINGO_FQID_LABEL = FQID2LABEL[\"logbook\"]\nREADER_BINGO_TEXT_FQID_LABEL = TEXT_FQID2LABEL[\"tunic.library.microfiche.reader.paper2.bingo\"]\nREADER_BINGO_FQID_LABEL = FQID2LABEL[\"reader\"]\nJOURNALS_BINGO_TEXT_FQID_LABEL = TEXT_FQID2LABEL[\"tunic.historicalsociety.stacks.journals.pic_2.bingo\"]\nJOURNALS_BINGO_FQID_LABEL = FQID2LABEL[\"journals\"]\nREADER_FLAG_BINGO_TEXT_FQID_LABEL = TEXT_FQID2LABEL[\"tunic.library.microfiche.reader_flag.paper2.bingo\"]\nREADER_FLAG_BINGO_FQID_LABEL = FQID2LABEL[\"reader_flag\"]\nJOURNALS_FLAG_BINGO_TEXT_FQID_LABEL = np.array([TEXT_FQID2LABEL[f\"tunic.historicalsociety.stacks.journals_flag.pic_{i}.bingo\"] for i in range(3)])\nJOURNALS_FLAG_BINGO_FQID_LABEL = FQID2LABEL[\"journals_flag\"]","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.851923Z","iopub.execute_input":"2023-06-24T23:27:29.852282Z","iopub.status.idle":"2023-06-24T23:27:29.866719Z","shell.execute_reply.started":"2023-06-24T23:27:29.852255Z","shell.execute_reply":"2023-06-24T23:27:29.865349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for grp in ['0-4', '5-12', '13-22']:\n    FQID[grp] = np.vectorize(lambda x:FQID2LABEL[x])(FQID[grp])\n    ROOM_FQID[grp] = np.vectorize(lambda x:ROOM_FQID2LABEL[x])(ROOM_FQID[grp])\n    TEXT_FQID[grp] = np.vectorize(lambda x:TEXT_FQID2LABEL[x])(TEXT_FQID[grp])\n    TEXT[grp] = np.vectorize(lambda x:TEXT2LABEL[x])(TEXT[grp])\n\nEVENT_NAME = np.array(['navigate_click','person_click','cutscene_click','object_click',\n          'map_hover','notification_click','map_click','observation_click',\n          'notebook_click', 'object_hover', 'checkpoint'])\nEVENT_NAME = np.vectorize(lambda x:EVENT_NAME2LABEL[x])(EVENT_NAME)\nNAME = np.array(['basic', 'open', 'close', 'next', 'prev', 'undefined'])\nNAME = np.vectorize(lambda x:NAME2LABEL[x])(NAME)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.870263Z","iopub.execute_input":"2023-06-24T23:27:29.870629Z","iopub.status.idle":"2023-06-24T23:27:29.883442Z","shell.execute_reply.started":"2023-06-24T23:27:29.870601Z","shell.execute_reply":"2023-06-24T23:27:29.881946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgb\n@numba.jit('Tuple((float32[:], float32[:]))(i8, i8[:], float32[:], i8[:], i8[:], i8[:], float32[:], float32[:], float32[:], float32[:], float32[:], float32[:], float32[:], i8[:], i8[:], i8[:], i8[:], i8, i8, i8[:], i8[:], i8[:], i8[:], i8[:], i8[:])', nopython=True)\ndef theta_xgb_feature_engineer(grp, elapsed_time, diff_elapsed_time, event_name, name, level, page, room_coor_x, room_coor_y, \n                    screen_coor_x, screen_coor_y, diff_screen_x, hover_duration, text, fqid, room_fqid, text_fqid, LV_MIN, \n                     LV_MAX, EVENT_NAME, NAME, FQID, ROOM_FQID, TEXT_FQID, TEXT):\n    \n    df = [\n        float(elapsed_time.shape[0]),\n        np.unique(event_name).shape[0],\n        np.unique(name).shape[0],\n        np.unique(fqid).shape[0],\n        np.unique(room_fqid).shape[0],\n        np.unique(text_fqid).shape[0]\n    ]\n    \n    tmp = [\n        np.nanquantile(page, 0.1),\n        np.nanquantile(page, 0.2),\n        np.nanquantile(page, 0.4),\n        np.nanquantile(page, 0.6),\n        np.nanquantile(page, 0.8),\n        np.nanquantile(page, 0.9),\n        np.nanmean(page),\n        np.nanmin(page),\n        np.nanmax(page),\n        np.nanstd(page),\n        np.nanquantile(room_coor_x, 0.1),\n        np.nanquantile(room_coor_x, 0.2),\n        np.nanquantile(room_coor_x, 0.4),\n        np.nanquantile(room_coor_x, 0.6),\n        np.nanquantile(room_coor_x, 0.8),\n        np.nanquantile(room_coor_x, 0.9),\n        np.nanmean(room_coor_x),\n        np.nanmin(room_coor_x),\n        np.nanmax(room_coor_x),\n        np.nanstd(room_coor_x),\n        np.nanquantile(room_coor_y, 0.1),\n        np.nanquantile(room_coor_y, 0.2),\n        np.nanquantile(room_coor_y, 0.4),\n        np.nanquantile(room_coor_y, 0.6),\n        np.nanquantile(room_coor_y, 0.8),\n        np.nanquantile(room_coor_y, 0.9),\n        np.nanmean(room_coor_y),\n        np.nanmin(room_coor_y),\n        np.nanmax(room_coor_y),\n        np.nanstd(room_coor_y),\n        np.nanquantile(screen_coor_x, 0.1),\n        np.nanquantile(screen_coor_x, 0.2),\n        np.nanquantile(screen_coor_x, 0.4),\n        np.nanquantile(screen_coor_x, 0.6),\n        np.nanquantile(screen_coor_x, 0.8),\n        np.nanquantile(screen_coor_x, 0.9),\n        np.nanmean(screen_coor_x),\n        np.nanmin(screen_coor_x),\n        np.nanmax(screen_coor_x),\n        np.nanstd(screen_coor_x),\n        np.nanquantile(screen_coor_y, 0.1),\n        np.nanquantile(screen_coor_y, 0.2),\n        np.nanquantile(screen_coor_y, 0.4),\n        np.nanquantile(screen_coor_y, 0.6),\n        np.nanquantile(screen_coor_y, 0.8),\n        np.nanquantile(screen_coor_y, 0.9),\n        np.nanmean(screen_coor_y),\n        np.nanmin(screen_coor_y),\n        np.nanmax(screen_coor_y),\n        np.nanstd(screen_coor_y),\n        np.nanquantile(hover_duration, 0.1),\n        np.nanquantile(hover_duration, 0.2),\n        np.nanquantile(hover_duration, 0.4),\n        np.nanquantile(hover_duration, 0.6),\n        np.nanquantile(hover_duration, 0.8),\n        np.nanquantile(hover_duration, 0.9),\n        np.nanmean(hover_duration),\n        np.nanmin(hover_duration),\n        np.nanmax(hover_duration),\n        np.nanstd(hover_duration),\n        np.nanquantile(diff_elapsed_time, 0.1),\n        np.nanquantile(diff_elapsed_time, 0.2),\n        np.nanquantile(diff_elapsed_time, 0.4),\n        np.nanquantile(diff_elapsed_time, 0.6),\n        np.nanquantile(diff_elapsed_time, 0.8),\n        np.nanquantile(diff_elapsed_time, 0.9),\n        np.nanmean(diff_elapsed_time),\n        np.nanmin(diff_elapsed_time),\n        np.nanmax(diff_elapsed_time),\n        np.nanstd(diff_elapsed_time)\n    ]\n    \n    df += tmp\n    df_save = tmp[:]\n    \n    for i in range(EVENT_NAME.shape[0]):\n        c = EVENT_NAME[i]\n        idx = (event_name == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(14)]\n            df_save += [0 for k in range(10)]\n        else:\n            tmp = [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]\n            df += tmp\n            df += [\n                np.nanmean(diff_screen_x[idx]),\n                np.nanmin(diff_screen_x[idx]),\n                np.nanmax(diff_screen_x[idx]),\n                np.nanstd(diff_screen_x[idx])\n            ]\n            df_save += tmp\n\n    for i in range(NAME.shape[0]):\n        c = NAME[i]\n        idx = (name == c)\n        count = np.sum(idx)\n        df.append(count)\n        if count == 0:\n            df += [0, 0, 0, 0]\n        else:\n            df += [\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]        \n\n    for i in range(FQID.shape[0]):\n        c = FQID[i]\n        idx = (fqid == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(11)]\n            df_save += [0 for k in range(5)]\n        else:\n            df += [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9)\n            ]\n            tmp = [\n                np.nansum(diff_elapsed_time[idx]),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]            \n            df += tmp\n            df_save += tmp\n            \n    for i in range(ROOM_FQID.shape[0]):\n        c = ROOM_FQID[i]\n        idx = (room_fqid == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(11)]\n            df_save += [0 for k in range(5)]\n        else:\n            df += [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9)\n            ]\n            tmp = [\n                np.nansum(diff_elapsed_time[idx]),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]            \n            df += tmp\n            df_save += tmp\n            \n    for i in range(TEXT_FQID.shape[0]):\n        c = TEXT_FQID[i]\n        idx = (text_fqid == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(11)]\n            df_save += [0 for k in range(5)]\n        else:\n            df += [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9)\n            ]\n            tmp = [\n                np.nansum(diff_elapsed_time[idx]),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]            \n            df += tmp\n            df_save += tmp\n    for i in range(TEXT.shape[0]):\n        c = TEXT[i]\n        idx = (text == c)\n        if np.sum(idx) == 0:\n            df.append(0)\n        else:\n            df.append(np.nansum(diff_elapsed_time[idx]))\n    \n    for lv in range(LV_MIN, LV_MAX+1):\n        for i in range(EVENT_NAME.shape[0]):\n            c = EVENT_NAME[i]\n            idx = (level == lv) & (event_name == c)\n            count = np.sum(idx)\n            df.append(count)\n            if count == 0:\n                df += [0, 0, 0, 0]\n            else:\n                df += [\n                    np.nansum(diff_elapsed_time[idx]),\n                    np.nanmean(diff_elapsed_time[idx]),\n                    np.nanmin(diff_elapsed_time[idx]),\n                    np.nanmax(diff_elapsed_time[idx])\n                ]\n\n    if grp == 1: \n        mask1 = (text_fqid == BUSINESSCARDS_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == BUSINESSCARDS_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)\n        mask1 = (text_fqid == LOGBOOK_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == LOGBOOK_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)        \n        mask1 = (text_fqid == READER_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == READER_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)         \n        mask1 = (text_fqid == JOURNALS_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == JOURNALS_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)         \n    elif grp == 2:\n        mask1 = (text_fqid == READER_FLAG_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == READER_FLAG_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)   \n        mask1 = (text_fqid == JOURNALS_FLAG_BINGO_TEXT_FQID_LABEL[0]) | (text_fqid == JOURNALS_FLAG_BINGO_TEXT_FQID_LABEL[1]) | (text_fqid == JOURNALS_FLAG_BINGO_TEXT_FQID_LABEL[2])\n        mask2 = (fqid == JOURNALS_FLAG_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)   \n            \n    df = np.array(df, dtype=np.float32)\n    df[np.isnan(df)] = 0\n    df_save = np.array(df_save, dtype=np.float32)\n    df_save[np.isnan(df_save)] = 0\n    return df, df_save","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:29.888398Z","iopub.execute_input":"2023-06-24T23:27:29.888703Z","iopub.status.idle":"2023-06-24T23:27:48.858009Z","shell.execute_reply.started":"2023-06-24T23:27:29.888679Z","shell.execute_reply":"2023-06-24T23:27:48.85645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb\n@numba.jit('Tuple((f8[:], f8[:]))(i8, i8[:], f8[:], i8[:], i8[:], i8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], f8[:], i8[:], i8[:], i8[:], i8[:], i8, i8, i8[:], i8[:], i8[:], i8[:], i8[:], i8[:])', nopython=True)\ndef theta_lgb_feature_engineer(grp, elapsed_time, diff_elapsed_time, event_name, name, level, page, room_coor_x, room_coor_y, \n                    screen_coor_x, screen_coor_y, diff_screen_x, hover_duration, text, fqid, room_fqid, text_fqid, LV_MIN, \n                     LV_MAX, EVENT_NAME, NAME, FQID, ROOM_FQID, TEXT_FQID, TEXT):\n    \n    df = [\n        float(elapsed_time.shape[0]),\n        np.unique(event_name).shape[0],\n        np.unique(name).shape[0],\n        np.unique(fqid[fqid!=FQID_NAN_LABEL]).shape[0],\n        np.unique(room_fqid).shape[0],\n        np.unique(text_fqid[text_fqid!=TEXT_FQID_NAN_LABEL]).shape[0]\n    ]\n    \n    tmp = [\n        np.nanquantile(page, 0.1),\n        np.nanquantile(page, 0.2),\n        np.nanquantile(page, 0.4),\n        np.nanquantile(page, 0.6),\n        np.nanquantile(page, 0.8),\n        np.nanquantile(page, 0.9),\n        np.nanmean(page),\n        np.nanmin(page),\n        np.nanmax(page),\n        np.nanstd(page),\n        np.nanquantile(room_coor_x, 0.1),\n        np.nanquantile(room_coor_x, 0.2),\n        np.nanquantile(room_coor_x, 0.4),\n        np.nanquantile(room_coor_x, 0.6),\n        np.nanquantile(room_coor_x, 0.8),\n        np.nanquantile(room_coor_x, 0.9),\n        np.nanmean(room_coor_x),\n        np.nanmin(room_coor_x),\n        np.nanmax(room_coor_x),\n        np.nanstd(room_coor_x),\n        np.nanquantile(room_coor_y, 0.1),\n        np.nanquantile(room_coor_y, 0.2),\n        np.nanquantile(room_coor_y, 0.4),\n        np.nanquantile(room_coor_y, 0.6),\n        np.nanquantile(room_coor_y, 0.8),\n        np.nanquantile(room_coor_y, 0.9),\n        np.nanmean(room_coor_y),\n        np.nanmin(room_coor_y),\n        np.nanmax(room_coor_y),\n        np.nanstd(room_coor_y),\n        np.nanquantile(screen_coor_x, 0.1),\n        np.nanquantile(screen_coor_x, 0.2),\n        np.nanquantile(screen_coor_x, 0.4),\n        np.nanquantile(screen_coor_x, 0.6),\n        np.nanquantile(screen_coor_x, 0.8),\n        np.nanquantile(screen_coor_x, 0.9),\n        np.nanmean(screen_coor_x),\n        np.nanmin(screen_coor_x),\n        np.nanmax(screen_coor_x),\n        np.nanstd(screen_coor_x),\n        np.nanquantile(screen_coor_y, 0.1),\n        np.nanquantile(screen_coor_y, 0.2),\n        np.nanquantile(screen_coor_y, 0.4),\n        np.nanquantile(screen_coor_y, 0.6),\n        np.nanquantile(screen_coor_y, 0.8),\n        np.nanquantile(screen_coor_y, 0.9),\n        np.nanmean(screen_coor_y),\n        np.nanmin(screen_coor_y),\n        np.nanmax(screen_coor_y),\n        np.nanstd(screen_coor_y),\n        np.nanquantile(hover_duration, 0.1),\n        np.nanquantile(hover_duration, 0.2),\n        np.nanquantile(hover_duration, 0.4),\n        np.nanquantile(hover_duration, 0.6),\n        np.nanquantile(hover_duration, 0.8),\n        np.nanquantile(hover_duration, 0.9),\n        np.nanmean(hover_duration),\n        np.nanmin(hover_duration),\n        np.nanmax(hover_duration),\n        np.nanstd(hover_duration),\n        np.nanquantile(diff_elapsed_time, 0.1),\n        np.nanquantile(diff_elapsed_time, 0.2),\n        np.nanquantile(diff_elapsed_time, 0.4),\n        np.nanquantile(diff_elapsed_time, 0.6),\n        np.nanquantile(diff_elapsed_time, 0.8),\n        np.nanquantile(diff_elapsed_time, 0.9),\n        np.nanmean(diff_elapsed_time),\n        np.nanmin(diff_elapsed_time),\n        np.nanmax(diff_elapsed_time),\n        np.nanstd(diff_elapsed_time)\n    ]\n    \n    df += tmp\n    df_save = tmp[:]\n    \n    for i in range(EVENT_NAME.shape[0]):\n        c = EVENT_NAME[i]\n        idx = (event_name == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(14)]\n            df_save += [0 for k in range(10)]\n        else:\n            tmp = [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]\n            df += tmp\n            df += [\n                np.nanmean(diff_screen_x[idx]),\n                np.nanmin(diff_screen_x[idx]),\n                np.nanmax(diff_screen_x[idx]),\n                np.nanstd(diff_screen_x[idx])\n            ]\n            df_save += tmp\n\n    for i in range(NAME.shape[0]):\n        c = NAME[i]\n        idx = (name == c)\n        count = np.sum(idx)\n        df.append(count)\n        if count == 0:\n            df += [0, 0, 0, 0]\n        else:\n            df += [\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]        \n\n    for i in range(FQID.shape[0]):\n        c = FQID[i]\n        idx = (fqid == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(11)]\n            df_save += [0 for k in range(5)]\n        else:\n            df += [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9)\n            ]\n            tmp = [\n                np.nansum(diff_elapsed_time[idx]),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]            \n            df += tmp\n            df_save += tmp\n            \n    for i in range(ROOM_FQID.shape[0]):\n        c = ROOM_FQID[i]\n        idx = (room_fqid == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(11)]\n            df_save += [0 for k in range(5)]\n        else:\n            df += [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9)\n            ]\n            tmp = [\n                np.nansum(diff_elapsed_time[idx]),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]            \n            df += tmp\n            df_save += tmp\n            \n    for i in range(TEXT_FQID.shape[0]):\n        c = TEXT_FQID[i]\n        idx = (text_fqid == c)\n        count = np.sum(idx)\n        df.append(count)\n        df_save.append(count)\n        if count == 0:\n            df += [0 for k in range(11)]\n            df_save += [0 for k in range(5)]\n        else:\n            df += [\n                np.nanquantile(diff_elapsed_time[idx], 0.1),\n                np.nanquantile(diff_elapsed_time[idx], 0.2),\n                np.nanquantile(diff_elapsed_time[idx], 0.4),\n                np.nanquantile(diff_elapsed_time[idx], 0.6),\n                np.nanquantile(diff_elapsed_time[idx], 0.8),\n                np.nanquantile(diff_elapsed_time[idx], 0.9)\n            ]\n            tmp = [\n                np.nansum(diff_elapsed_time[idx]),\n                np.nanmean(diff_elapsed_time[idx]),\n                np.nanmin(diff_elapsed_time[idx]),\n                np.nanmax(diff_elapsed_time[idx]),\n                np.nanstd(diff_elapsed_time[idx])\n            ]            \n            df += tmp\n            df_save += tmp\n    for i in range(TEXT.shape[0]):\n        c = TEXT[i]\n        idx = (text == c)\n        if np.sum(idx) == 0:\n            df.append(0)\n        else:\n            df.append(np.nansum(diff_elapsed_time[idx]))\n    \n    for lv in range(LV_MIN, LV_MAX+1):\n        for i in range(EVENT_NAME.shape[0]):\n            c = EVENT_NAME[i]\n            idx = (level == lv) & (event_name == c)\n            count = np.sum(idx)\n            df.append(count)\n            if count == 0:\n                df += [0, 0, 0, 0]\n            else:\n                df += [\n                    np.nansum(diff_elapsed_time[idx]),\n                    np.nanmean(diff_elapsed_time[idx]),\n                    np.nanmin(diff_elapsed_time[idx]),\n                    np.nanmax(diff_elapsed_time[idx])\n                ]\n\n    if grp == 1: \n        mask1 = (text_fqid == BUSINESSCARDS_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == BUSINESSCARDS_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)\n        mask1 = (text_fqid == LOGBOOK_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == LOGBOOK_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)        \n        mask1 = (text_fqid == READER_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == READER_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)         \n        mask1 = (text_fqid == JOURNALS_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == JOURNALS_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)         \n    elif grp == 2:\n        mask1 = (text_fqid == READER_FLAG_BINGO_TEXT_FQID_LABEL)\n        mask2 = (fqid == READER_FLAG_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)   \n        mask1 = (text_fqid == JOURNALS_FLAG_BINGO_TEXT_FQID_LABEL[0]) | (text_fqid == JOURNALS_FLAG_BINGO_TEXT_FQID_LABEL[1]) | (text_fqid == JOURNALS_FLAG_BINGO_TEXT_FQID_LABEL[2])\n        mask2 = (fqid == JOURNALS_FLAG_BINGO_FQID_LABEL)\n        if len(elapsed_time[mask1]) != 0 and len(elapsed_time[mask2]) != 0:\n            end = elapsed_time[mask1][0]\n            start = elapsed_time[mask2][0]\n            df.append(end-start)\n        else:\n            df.append(0)   \n\n    df = np.array(df, dtype=np.float64)\n    df[np.isnan(df)] = 0\n    df_save = np.array(df_save, dtype=np.float64)\n    df_save[np.isnan(df_save)] = 0\n    return df, df_save","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:27:48.859504Z","iopub.execute_input":"2023-06-24T23:27:48.859817Z","iopub.status.idle":"2023-06-24T23:28:05.849847Z","shell.execute_reply.started":"2023-06-24T23:27:48.859792Z","shell.execute_reply":"2023-06-24T23:28:05.848716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stacking part","metadata":{}},{"cell_type":"code","source":"class MLP(nn.Module):\n    def __init__(self, input_size, hidden_size, output_size):\n        super().__init__()\n        self.fc1 = nn.Linear(input_size, hidden_size)\n        self.fc2 = nn.Linear(hidden_size, output_size)\n        self.dropout = nn.Dropout(0.2)\n\n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        x = self.fc2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:28:06.071486Z","iopub.execute_input":"2023-06-24T23:28:06.071928Z","iopub.status.idle":"2023-06-24T23:28:06.088651Z","shell.execute_reply.started":"2023-06-24T23:28:06.071898Z","shell.execute_reply":"2023-06-24T23:28:06.086953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"######################## 使用するオブジェクトをロード ########################\n\n############ ryota object ############\n\nryota_objects = {}\nfor lg in range(3):\n    ryota_objects[lg] = {\n        'use_features': pickle.load(open(CFG.ryota_gbdt_path / f'use_features_lg{lg}_with_fs.pkl', 'rb')),\n        'gbdt': pickle.load(open(CFG.ryota_gbdt_path / f'xgb_models_lg{lg}_with_fs.pkl', 'rb')),\n        'cat_map': {col: pickle.load(open(CFG.ryota_nn_path / f'{col}_map_lg{lg}.pkl', 'rb')) for col in ['event_name', 'text_fqid', 'room_fqid']},\n        'num_scaler': pickle.load(open(CFG.ryota_nn_path / f'numerical_scaler_lg{lg}.pkl', 'rb')),\n        'transformer': ryota_nn_model_load(CFG, lg),\n    }\n    \n\n############ shu object ############\nshu_cols = {\"0-4\": None, \"5-12\": None, \"13-22\": None}\nshu_feat_cols = defaultdict(dict)\nshu_models = defaultdict(dict)\n\nevent_name2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/event_name2label.pkl\", \"rb\"))\nname2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/name2label.pkl\", \"rb\"))\nfqid2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/fqid2label.pkl\", \"rb\"))\nroom_fqid2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/room_fqid2label.pkl\", \"rb\"))\ntext2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/text2label.pkl\", \"rb\"))\ntext_fqid2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/text_fqid2label.pkl\", \"rb\"))\nlevel_group2label = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/level_group2label.pkl\", \"rb\"))\n\nfor grp in shu_cols.keys():\n    shu_cols[grp] = shu_get_cols(grp, event_name2label, name2label, fqid2label, room_fqid2label, text2label, text_fqid2label, level_group2label)\n    for i_fold in shu_cfg.folds:\n        shu_feat_cols[i_fold][grp] = pickle.load(open(f\"{shu_cfg.FEAT_PATH}/feat_cols_fold{i_fold}_grp{grp}.pkl\", \"rb\"))\n        shu_models[i_fold][grp] = shu_load_model(Path(shu_cfg.EXP_MODEL) / f\"fold{i_fold}_grp{grp}.pkl\")\n        \n\n############ tereka object ############\ntereka_models = defaultdict(list)\nfor t in range(1, 19):\n    for fold in range(5):\n        model = XGBClassifier()\n        model.load_model(CFG.tereka_gbdt_path / f'XGB_question{t}_fold{fold}.xgb')\n        tereka_models[t].append(model)\n\ntereka_features_list, feat0, feat1, feat2 = tereka_use_features_load(CFG.tereka_gbdt_path / 'xgb_filtered_features.pkl')\ntereka_use_features = {\n    '0-4': feat0,\n    '5-12': feat1,\n    '13-22': feat2     \n}\n\n\n############ theta object ############  \ntheta_n_splits = 5\ntheta_xgb_model = {}\ntheta_lgb_model = {}\nfor i in range(theta_n_splits):\n    for t in range(1, 19):\n        clf = XGBClassifier()\n        clf.load_model(\n            f\"/kaggle/input/xgb-model-5folds-v5/model_for_q{t}_fold{i}.json\"\n        )\n        theta_xgb_model[(i, t)] = clf\n        clf = lgb.Booster(\n            model_file=f\"/kaggle/input/lgb-model-5folds-v4-max-depth5/model_for_q{t}_fold{i}.json\"\n        )\n        theta_lgb_model[(i, t)] = clf\n\n############ stacking object ############\nensemble_models = defaultdict(lambda: defaultdict(dict))\nfor q in range(1, 19):\n    for i_fold in range(5):\n        mlp_model = MLP(\n            input_size=6, \n            hidden_size=64, \n            output_size=1\n            )\n        mlp_model.load_state_dict(torch.load(f\"/kaggle/input/psp-exp-ensemble015/model/mlp/q{q}_fold{i_fold}.pth\", map_location=torch.device('cpu')))\n        mlp_model.eval()\n        ensemble_models[\"mlp\"][i_fold][q] = mlp_model\n        \n        ensemble_models[\"lr\"][i_fold][q] = pickle.load(open(f\"/kaggle/input/psp-exp-ensemble015/model/lr/q{q}_fold{i_fold}.pkl\", \"rb\"))\n\n        \n######################## データを保存するインスタンス ########################\n\n############ overall ############\ncnt = 0\n\n\n############ ryota store ############\nryota_data_store = defaultdict(pl.DataFrame)\nryota_pred_store = defaultdict(dict)\n\n\n############ shu store ############\nshu_prev_feat_dict = {\n    \"0-4\": None, \n    \"0-4_first_et\": None, \n    \"0-4_last_et\": None, \n    \"5-12\": None, \n    \"5-12_first_et\": None, \n    \"5-12_last_et\": None,\n    \"13-22\": None, \n    \"13-22_first_et\": None,\n    \"13-22_last_et\": None,\n    }\nshu_oof = pd.DataFrame(data=np.zeros((1, 18)))\n\n\n############ tereka store ############\ntereka_data_dict = {}\ntereka_meta_columns = []\ntereka_meta_dict = {}\n\n\n############ theta store ############\ntheta_level4_endpoint_xgb = {}\ntheta_level4_checkpoint_duration_xgb = {}\ntheta_level12_endpoint_xgb = {}\ntheta_xgb_prev_answer = {}\ntheta_df0_4_xgb = {}\ntheta_df5_12_xgb = {}\ntheta_level4_endpoint_lgb = {}\ntheta_level4_checkpoint_duration_lgb = {}\ntheta_level12_endpoint_lgb = {}\ntheta_lgb_prev_answer = {}\ntheta_df0_4_lgb = {}\ntheta_df5_12_lgb = {}","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:28:06.103345Z","iopub.execute_input":"2023-06-24T23:28:06.103632Z","iopub.status.idle":"2023-06-24T23:28:30.21155Z","shell.execute_reply.started":"2023-06-24T23:28:06.103602Z","shell.execute_reply":"2023-06-24T23:28:30.210454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_th = [\n    0.63223511, 0.66983917, 0.54309771, 0.63995665, 0.62244779,\n    0.62261213, 0.590253  , 0.62167946, 0.57819705, 0.62473673,\n    0.62468257, 0.61539037, 0.63009761, 0.61601066, 0.6369451 ,\n    0.61671963, 0.61570777, 0.23611416\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:28:30.214638Z","iopub.execute_input":"2023-06-24T23:28:30.214985Z","iopub.status.idle":"2023-06-24T23:28:30.222412Z","shell.execute_reply.started":"2023-06-24T23:28:30.21496Z","shell.execute_reply":"2023-06-24T23:28:30.221541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder_310\n\njo_wilder_310.make_env.__called__ = False\nenv = jo_wilder_310.make_env()\niter_test = env.iter_test()\n\nseed_everything(42)\nfor test_pd, sample_submission in iter_test:\n    ######################## common part ########################\n    test_pd = sort_frame(test_pd)\n    test_pd = test_pd.drop(columns=[\"fullscreen\", \"hq\", \"music\"])\n    \n    grp = test_pd.level_group.values[0]\n    session_id = test_pd.session_id.values[0]\n    \n    test_pl = pl.from_pandas(test_pd)\n    a,b = CFG.limits[grp]\n    \n    \n    ######################## ryota part ########################\n    session_df = test_pl.clone()\n    session_df = session_df.with_columns(\n        level_group = pl.col('level_group').map_dict({'0-4': 0, '5-12': 1, '13-22': 2}).cast(pl.Int8),\n        load_index = pl.col('index').cumcount().over('session_id')\n    )\n    level_group = session_df['level_group'][0]\n    \n    ### Transformer ####\n    session_nn_df = session_df.clone()\n    cat_feat, num_feat, mask, questions, length = ryota_train_preprocess(\n        CFG,\n        session_nn_df,\n        level_group,\n        ryota_objects[level_group]\n    )\n    \n    q_range = CFG.q_range[level_group]\n    models = ryota_objects[level_group]['transformer']\n    \n    preds = []\n    for model in models:\n        with torch.no_grad():\n            pred = model(cat_feat, num_feat, mask, questions, length)\n            pred = 1 / (1 + torch.exp(-1 * pred))\n            pred = pred.numpy()[:, np.array(q_range) - 1][0]\n            preds.append(pred)\n            \n    ryota_nn_preds = np.mean(np.array(preds), axis=0)\n        \n    ### GBDT ###\n    session_df = reduce_memory(session_df)\n\n    if level_group < 2:\n        ryota_data_store[session_id] = pl.concat([ryota_data_store[session_id], session_df])\n        session_df = ryota_data_store[session_id]\n    else:\n        session_df = pl.concat([ryota_data_store[session_id], session_df])\n        del ryota_data_store[session_id]\n\n    # feature engineering\n    features = ryota_stream_feature_engineering(session_df, level_group, use_bingo=True)\n    \n    # 過去pred_feature\n    if level_group >= 1:\n        qs = []\n        prev_q = range(1, 4) if level_group == 1 else range(1, 14)\n        for q in prev_q:\n            features[f'prev_pred_q{q}'] = ryota_pred_store[session_id][q]\n            qs.append(ryota_pred_store[session_id][q])\n            \n        qs = pl.Series(qs)\n        features['prev_pred_mean'] = qs.mean()\n        features['prev_pred_std'] = qs.std()\n        features['prev_pred_max'] = qs.max()\n        features['prev_pred_min'] = qs.min()\n        \n    # 不足しているfeatureを作成する\n    use_features = ryota_objects[level_group]['use_features']\n    for feature_name in set(use_features).difference(set(features.keys())):\n        features[feature_name] = 0\n    \n    q_range = range(1, 4) if level_group == 0 else range(4, 14) if level_group == 1 else range(14, 19)\n    features_ = []\n    for q in q_range:\n        features['q'] = q\n        features_.append(features.copy())\n    \n    features = pl.from_pandas(pd.DataFrame(features_)).fill_null(0).fill_nan(0)\n    features = features[use_features] # 必要なものだけ抽出\n    \n    preds = []\n    for model in ryota_objects[level_group]['gbdt']:\n        pred = model.predict(xgb.DMatrix(features[:, use_features].to_numpy(), feature_names=features.columns))\n        preds.append(pred)\n    \n    ryota_gbdt_preds = np.mean(np.array(preds), axis=0)\n    for q, p in zip(q_range, ryota_gbdt_preds):\n        ryota_pred_store[session_id][q] = p\n    \n\n    ######################## shu421 part ########################\n    q_list = list(range(a, b))\n    row_df = test_pd.copy()\n    row_df = row_df.sort_values(\"index\")\n    \n    _cat_cols = [\"event_name\", \"name\", \"fqid\", \"room_fqid\", \"text\", \"text_fqid\"]\n    row_df[_cat_cols] = row_df[_cat_cols].fillna(\"NAN\")\n    \n    event_name_cols, name_cols, text_cols, fqid_cols, room_fqid_cols, text_fqid_cols, level_cols, level_group_cols = shu_cols[grp]\n    ############### get features ###############\n    # categorical features\n    event_name = row_df[\"event_name\"].to_numpy()\n    name = row_df[\"name\"].to_numpy()\n    fqid = row_df[\"fqid\"].to_numpy()\n    room_fqid = row_df[\"room_fqid\"].to_numpy()\n    text = row_df[\"text\"].to_numpy()\n    text_fqid = row_df[\"text_fqid\"].to_numpy()\n    # numeric features\n    elapsed_time = row_df[\"elapsed_time\"].to_numpy().astype(np.int64)\n    level = row_df[\"level\"].to_numpy().astype(np.int64)\n    level_group = row_df[\"level_group\"].to_numpy()\n    page = row_df[\"page\"].to_numpy().astype(np.float64)\n    room_coor_x = row_df[\"room_coor_x\"].to_numpy().astype(np.float64)\n    room_coor_y = row_df[\"room_coor_y\"].to_numpy().astype(np.float64)\n    screen_coor_x = row_df[\"screen_coor_x\"].to_numpy().astype(np.float64)\n    screen_coor_y = row_df[\"screen_coor_y\"].to_numpy().astype(np.float64)\n    hover_duration = row_df[\"hover_duration\"].to_numpy().astype(np.float64)\n    # diff features\n    elapsed_time_diff = row_df[\"elapsed_time\"].diff().clip(0, None).fillna(0).to_numpy().astype(np.float64)\n    room_coor_x_diff = (row_df[\"room_coor_x\"] - row_df[\"room_coor_x\"].shift(1)).abs().to_numpy().astype(np.float64)\n    room_coor_y_diff = (row_df[\"room_coor_y\"] - row_df[\"room_coor_y\"].shift(1)).abs().to_numpy().astype(np.float64)\n    screen_coor_x_diff = (row_df[\"screen_coor_x\"] - row_df[\"screen_coor_x\"].shift(1)).abs().to_numpy().astype(np.float64)\n    screen_coor_y_diff = (row_df[\"screen_coor_y\"] - row_df[\"screen_coor_y\"].shift(1)).abs().to_numpy().astype(np.float64)\n    room_coor_move = ((room_coor_x_diff**2 + room_coor_y_diff**2)**0.5).astype(np.float64)\n\n    ############### encoding ###############\n    event_name = np.vectorize(lambda x: event_name2label[x] if x in event_name2label.keys() else len(event_name2label))(event_name).astype(np.int64)\n    name = np.vectorize(lambda x: name2label[x] if x in name2label.keys() else len(name2label))(name).astype(np.int64)\n    text = np.vectorize(lambda x: text2label[x] if x in text2label.keys() else len(text2label))(text).astype(np.int64)\n    fqid = np.vectorize(lambda x: fqid2label[x] if x in fqid2label.keys() else len(fqid2label))(fqid).astype(np.int64)\n    room_fqid = np.vectorize(lambda x: room_fqid2label[x] if x in room_fqid2label.keys() else len(room_fqid2label))(room_fqid).astype(np.int64)\n    text_fqid = np.vectorize(lambda x: text_fqid2label[x] if x in text_fqid2label.keys() else len(text_fqid2label))(text_fqid).astype(np.int64)\n    level_group = np.vectorize(lambda x: level_group2label[x] if x in level_group2label.keys() else len(level_group2label.keys()))(level_group).astype(np.int64)\n\n\n    ############### feature engineering ###############\n    feat_arr = []\n    feat_session_arr, save_feat_session_arr = shu_feature_engineering(\n        event_name,\n        name,\n        fqid,\n        room_fqid,\n        text,\n        text_fqid,\n        elapsed_time,\n        level,\n        level_group,\n        page,\n        room_coor_x,\n        room_coor_y,\n        screen_coor_x,\n        screen_coor_y,\n        hover_duration,\n        elapsed_time_diff,\n        room_coor_x_diff,\n        room_coor_y_diff,\n        screen_coor_x_diff,\n        screen_coor_y_diff,\n        room_coor_move,\n        event_name_cols, \n        name_cols, \n        text_cols, \n        fqid_cols, \n        room_fqid_cols, \n        text_fqid_cols, \n        level_cols, \n        level_group_cols,\n        )\n    # date features\n    date_feat_arr = get_date_feat(str(session_id))\n    feat_session_arr = np.concatenate([feat_session_arr, date_feat_arr], axis=0)\n    \n    # save\n    feat_arr.append(feat_session_arr)\n    shu_prev_feat_dict[grp] = [save_feat_session_arr]\n    shu_prev_feat_dict[grp+\"_first_et\"] = [elapsed_time[0]]\n    shu_prev_feat_dict[grp+\"_last_et\"] = [elapsed_time[-1]]\n    \n    # 学習時と次元合わせ\n    feat_arr = np.stack(feat_arr, axis=0)\n    shu_prev_feat_dict[grp] = np.stack(shu_prev_feat_dict[grp], axis=0)\n    shu_prev_feat_dict[grp+\"_first_et\"] = np.array(shu_prev_feat_dict[grp+\"_first_et\"]).reshape(-1, 1)\n    shu_prev_feat_dict[grp+\"_last_et\"] = np.array(shu_prev_feat_dict[grp+\"_last_et\"]).reshape(-1, 1)\n    \n    # add previous features\n    feat_arr = add_previous_features(feat_arr, grp, shu_prev_feat_dict)\n\n    # add previous prediction\n    feat_arr = add_previous_predictions(feat_arr, grp, shu_oof)\n    \n    ############### predict ###############\n    shu_gbdt_preds = []\n    for i_fold in shu_cfg.folds:\n        feat_cols = shu_feat_cols[i_fold][grp]\n        fold_feat_arr = feat_arr[:, feat_cols]\n        model = shu_models[i_fold][grp]\n        \n        _shu_gbdt_preds = model.predict(fold_feat_arr)\n        shu_gbdt_preds.append(_shu_gbdt_preds)\n\n    shu_gbdt_preds = np.mean(shu_gbdt_preds, axis=0)\n    shu_gbdt_preds = shu_gbdt_preds.flatten()\n\n    # keep current preds for next prediction\n    shu_oof.loc[:, np.array(q_list)-1] = shu_gbdt_preds\n    del model\n    \n    \n    ######################## tereka part ########################\n    feature_suffix = {'0-4': 'grp0', '5-12': 'grp1', '13-22': 'grp2'}[grp]\n    tereka_test = test_pl.clone()\n    tereka_test = tereka_feature_engineering(tereka_test, grp=grp, use_extra=True, feature_suffix=feature_suffix)\n    tereka_test = pl.DataFrame(tereka_test)\n    \n    # 過去データ保存\n    if grp =='0-4':\n        save_cols = [col for col in tereka_test.columns if col in tereka_use_features[grp]]\n        tereka_data_dict[session_id] = tereka_test[save_cols]\n    else:\n        prev_df = tereka_data_dict[session_id]\n        tereka_test = pl.concat([prev_df, tereka_test], how=\"horizontal\")\n        if grp == '5-12':\n            save_cols = [col for col in tereka_test.columns if col in tereka_use_features[grp]]\n            tereka_data_dict[session_id] = tereka_test[save_cols]\n        else:\n            del tereka_data_dict[session_id]\n            \n    # 足りないfeaturesをNoneで埋める\n    use_features = tereka_use_features[grp]\n    shortage_cols = list(set(use_features).difference(set(tereka_test.columns)))\n    tereka_test = tereka_test.with_columns([pl.lit(None).alias(col) for col in shortage_cols])\n    tereka_test = tereka_test[use_features]\n\n    # to pandas\n    tereka_test = tereka_test.to_pandas()\n    \n    # prediction\n    a,b = CFG.limits[grp]\n    tereka_gbdt_preds = []\n    for t in range(a, b):\n        FEATURES = copy.copy(tereka_features_list[t-1])\n        FEATURES += [f'meta_{t_}' for t_ in range(t-1)]\n        for key, value in tereka_meta_dict.items():\n            tereka_test[key] = value\n        \n        p = 0.0\n        models = tereka_models[t]\n        for model in models:\n            p += model.predict_proba(tereka_test[FEATURES].astype('float32'))[:,1]\n            \n        p /= len(models)\n        p = p.item()\n        tereka_meta_dict[f'meta_{t-1}'] = p\n        tereka_gbdt_preds.append(p)\n    \n    tereka_gbdt_preds = np.array(tereka_gbdt_preds)\n\n    if grp == '13-22':\n        tereka_meta_dict = {}\n        tereka_meta_columns = []\n        \n        \n    ######################## theta part ########################\n    # xgb\n    theta_test = test_pd.copy()\n\n    theta_test['fqid'] = theta_test['fqid'].fillna('no_fqid')\n    theta_test['text_fqid'] = theta_test['text_fqid'].fillna('no_text_fqid')\n    theta_test['text'] = theta_test['text'].fillna('no_text')\n    \n#     # FEATURE ENGINEER TEST DATA\n    \n    LV_MIN = int(grp.split('-')[0])\n    LV_MAX = int(grp.split('-')[1])\n\n    elapsed_time = theta_test['elapsed_time'].to_numpy()\n    diff_elapsed_time = theta_test['elapsed_time'].diff().clip(0, None).fillna(0).to_numpy().astype('float32')\n    event_name = np.vectorize(lambda x:EVENT_NAME2LABEL_V2[x])(theta_test['event_name'].to_numpy())\n    name = np.vectorize(lambda x:NAME2LABEL_V2[x])(theta_test['name'].to_numpy())\n    level = theta_test['level'].to_numpy()\n    page = theta_test['page'].to_numpy().astype('float32')\n    room_coor_x = theta_test['room_coor_x'].to_numpy().astype('float32')\n    room_coor_y = theta_test['room_coor_y'].to_numpy().astype('float32')\n    screen_coor_x = theta_test['screen_coor_x'].to_numpy().astype('float32')\n    screen_coor_y = theta_test['screen_coor_y'].to_numpy().astype('float32')\n    diff_screen_x = (theta_test['screen_coor_x'] - theta_test['screen_coor_x'].shift(1)).abs().to_numpy().astype('float32')\n    hover_duration = theta_test['hover_duration'].to_numpy().astype('float32')\n    text = np.vectorize(lambda x:TEXT2LABEL_V2[x])(theta_test['text'].to_numpy())\n    fqid = np.vectorize(lambda x:FQID2LABEL_V2[x])(theta_test['fqid'].to_numpy())\n    room_fqid = np.vectorize(lambda x:ROOM_FQID2LABEL_V2[x])(theta_test['room_fqid'].to_numpy())\n    text_fqid = np.vectorize(lambda x:TEXT_FQID2LABEL_V2[x])(theta_test['text_fqid'].to_numpy())\n    \n    df, df_save = theta_xgb_feature_engineer(GRP2LABEL_V2[grp], elapsed_time, diff_elapsed_time, event_name, name, \n                                   level, page, room_coor_x, room_coor_y, screen_coor_x, screen_coor_y, \n                                   diff_screen_x, hover_duration, text, fqid, room_fqid, text_fqid, LV_MIN, LV_MAX, \n                                   EVENT_NAME_V2, NAME_V2, FQID_V2[grp], ROOM_FQID_V2[grp], TEXT_FQID_V2[grp], TEXT_V2[grp])\n    \n    if grp == '0-4':\n        theta_df0_4_xgb[session_id] = df_save\n        theta_level4_endpoint_xgb[session_id] = elapsed_time[-1]\n    elif grp == '5-12':\n        theta_df5_12_xgb[session_id] = df_save\n        level5_startpoint = elapsed_time[0]\n        theta_level12_endpoint_xgb[session_id] = elapsed_time[-1]\n        theta_level4_checkpoint_duration_xgb[session_id] = level5_startpoint - theta_level4_endpoint_xgb[session_id]\n        if theta_level4_checkpoint_duration_xgb[session_id] < 0: theta_level4_checkpoint_duration_xgb[session_id] = 0\n        df = np.hstack([\n            df, \n            theta_df0_4_xgb[session_id],\n            np.array([theta_level4_checkpoint_duration_xgb[session_id]])\n        ])\n    elif grp == '13-22':\n        level13_startpoint = elapsed_time[0]\n        level12_checkpoint_duration = level13_startpoint - theta_level12_endpoint_xgb[session_id]\n        if level12_checkpoint_duration < 0:level12_checkpoint_duration = 0\n        df = np.hstack([\n            df, \n            theta_df5_12_xgb[session_id],\n            np.array([level12_checkpoint_duration])\n        ])\n\n    # INFER TEST DATA\n    a, b = CFG.limits[grp]\n    \n    theta_xgb_preds = []\n    for t in range(a,b):\n        p_xgb_mean = np.zeros(1)\n        for i in range(theta_n_splits):\n\n            clf = theta_xgb_model[(i, t)]\n            df_feature = np.hstack([df]+[theta_xgb_prev_answer[(session_id, i, k)] for k in range(1, t)])\n            df_feature = np.array([df_feature])\n            p = clf.predict_proba(df_feature.astype('float32'))[:,1]\n            theta_xgb_prev_answer[(session_id, i, t)] = p\n            p_xgb_mean += p\n            \n        p_xgb_mean /= float(theta_n_splits)\n        theta_xgb_preds.append(p_xgb_mean)\n    theta_xgb_preds = np.array(theta_xgb_preds).flatten()\n    \n    \n\n    # lgb\n    theta_test = test_pd.copy()\n\n    theta_test['fqid'] = theta_test['fqid'].fillna('no_fqid')\n    theta_test['text_fqid'] = theta_test['text_fqid'].fillna('no_text_fqid')\n    theta_test['text'] = theta_test['text'].fillna('no_text')\n    \n#     # FEATURE ENGINEER TEST DATA\n    \n    LV_MIN = int(grp.split('-')[0])\n    LV_MAX = int(grp.split('-')[1])\n\n    elapsed_time = theta_test['elapsed_time'].to_numpy()\n    diff_elapsed_time = theta_test['elapsed_time'].diff().clip(0, None).fillna(0).to_numpy()\n    event_name = np.vectorize(lambda x:EVENT_NAME2LABEL[x])(theta_test['event_name'].to_numpy())\n    name = np.vectorize(lambda x:NAME2LABEL[x])(theta_test['name'].to_numpy())\n    level = theta_test['level'].to_numpy()\n    page = theta_test['page'].to_numpy()\n    room_coor_x = theta_test['room_coor_x'].to_numpy()\n    room_coor_y = theta_test['room_coor_y'].to_numpy()\n    screen_coor_x = theta_test['screen_coor_x'].to_numpy()\n    screen_coor_y = theta_test['screen_coor_y'].to_numpy()\n    diff_screen_x = (theta_test['screen_coor_x'] - theta_test['screen_coor_x'].shift(1)).abs().to_numpy()\n    hover_duration = theta_test['hover_duration'].to_numpy()\n    text = np.vectorize(lambda x:TEXT2LABEL[x])(theta_test['text'].to_numpy())\n    fqid = np.vectorize(lambda x:FQID2LABEL[x])(theta_test['fqid'].to_numpy())\n    room_fqid = np.vectorize(lambda x:ROOM_FQID2LABEL[x])(theta_test['room_fqid'].to_numpy())\n    text_fqid = np.vectorize(lambda x:TEXT_FQID2LABEL[x])(theta_test['text_fqid'].to_numpy())\n    \n    df, df_save = theta_lgb_feature_engineer(GRP2LABEL[grp], elapsed_time, diff_elapsed_time, event_name, name, \n                                   level, page, room_coor_x, room_coor_y, screen_coor_x, screen_coor_y, \n                                   diff_screen_x, hover_duration, text, fqid, room_fqid, text_fqid, LV_MIN, LV_MAX, \n                                   EVENT_NAME, NAME, FQID[grp], ROOM_FQID[grp], TEXT_FQID[grp], TEXT[grp])\n    \n    if grp == '0-4':\n        theta_df0_4_lgb[session_id] = df_save\n        theta_level4_endpoint_lgb[session_id] = elapsed_time[-1]\n    elif grp == '5-12':\n        theta_df5_12_lgb[session_id] = df_save\n        level5_startpoint_lgb = elapsed_time[0]\n        theta_level12_endpoint_lgb[session_id] = elapsed_time[-1]\n        theta_level4_checkpoint_duration_lgb[session_id] = level5_startpoint_lgb - theta_level4_endpoint_lgb[session_id]\n        if theta_level4_checkpoint_duration_lgb[session_id] < 0: theta_level4_checkpoint_duration_lgb[session_id] = 0\n        df = np.hstack([\n            df, \n            theta_df0_4_lgb[session_id],\n            np.array([theta_level4_checkpoint_duration_lgb[session_id]])\n        ])\n    elif grp == '13-22':\n        level13_startpoint = elapsed_time[0]\n        level12_checkpoint_duration = level13_startpoint - theta_level12_endpoint_lgb[session_id]\n        if level12_checkpoint_duration < 0:level12_checkpoint_duration = 0\n        df = np.hstack([\n            df, \n            theta_df5_12_lgb[session_id],\n            np.array([level12_checkpoint_duration])\n        ])\n        \n    \n    \n    theta_lgb_preds = []\n    for t in range(a,b):\n        p_lgb_mean = np.zeros(1)\n        for i in range(theta_n_splits):\n\n            clf = theta_lgb_model[(i, t)]\n            df_feature = np.hstack([df]+[theta_lgb_prev_answer[(session_id, i, k)] for k in range(1, t)])\n            df_feature = np.array([df_feature])\n            p = clf.predict(df_feature.astype('float32'))\n            theta_lgb_prev_answer[(session_id, i, t)] = p\n            p_lgb_mean += p\n            \n        p_lgb_mean /= float(theta_n_splits)\n        theta_lgb_preds.append(p_lgb_mean)\n    theta_lgb_preds = np.array(theta_lgb_preds).flatten()\n    \n    \n    \n    ######################## stacking part ########################\n    ensemble_input = np.stack([\n        ryota_gbdt_preds,\n        ryota_nn_preds,\n        shu_gbdt_preds,\n        tereka_gbdt_preds,\n        theta_xgb_preds,\n        theta_lgb_preds,\n    ], axis=1)\n    \n    final_preds = []\n    for idx, q in enumerate(range(a, b)):\n        ensemble_q_input = ensemble_input[idx, :].reshape(1, -1)\n        ensemble_pred = []\n        for i_fold in range(5):\n            \n            # mlp\n            mlp_model = ensemble_models[\"mlp\"][i_fold][q] # load時にmodel.eval()済み\n            with torch.no_grad():\n                mlp_pred = mlp_model(torch.tensor(ensemble_q_input, dtype=torch.float32))\n                mlp_pred = torch.sigmoid(mlp_pred)\n            \n            # logistic regression\n            lr_model = ensemble_models[\"lr\"][i_fold][q]\n            lr_pred = lr_model.predict_proba(ensemble_q_input)\n            lr_pred = lr_pred[:, 1]\n            \n            ensemble_fold_pred = (mlp_pred.item() + lr_pred.item()) / 2\n            ensemble_pred.append(ensemble_fold_pred)\n\n        ensemble_pred = np.array(ensemble_pred).mean()\n        final_preds.append(ensemble_pred)\n    final_preds = np.array(final_preds).flatten()\n    \n    \n    # submission\n    for q, p in zip(q_range, final_preds):\n        mask = sample_submission['session_id'].str.contains(f'q{q}')\n        sample_submission.loc[mask, 'correct'] = int(p > best_th[q-1]) # 変更\n    \n    cnt += 1\n    if (cnt % 1000) == 0:\n        gc.collect()\n\n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:28:54.936947Z","iopub.execute_input":"2023-06-24T23:28:54.937317Z","iopub.status.idle":"2023-06-24T23:29:10.607859Z","shell.execute_reply.started":"2023-06-24T23:28:54.937273Z","shell.execute_reply":"2023-06-24T23:29:10.607085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub[\"correct\"].mean())\nsub","metadata":{"execution":{"iopub.status.busy":"2023-06-24T23:29:10.611409Z","iopub.execute_input":"2023-06-24T23:29:10.612934Z","iopub.status.idle":"2023-06-24T23:29:10.641823Z","shell.execute_reply.started":"2023-06-24T23:29:10.612911Z","shell.execute_reply":"2023-06-24T23:29:10.641134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}