{"metadata":{"kernelspec":{"display_name":"gpu","language":"python","name":"gpu"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[An essay for a PSPFGP comepetition.](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/425798)\n\n1. [PSPFGP_NN_dataset](https://www.kaggle.com/code/dongyk/pspfgp-nn-dataset)\n2. [PSPFGP_NN_Pretrain](https://www.kaggle.com/code/dongyk/pspfgp-nn-pretrain)\n3. [PSPFGP_NN_Train](https://www.kaggle.com/code/dongyk/pspfgp-nn-train)\n4. [PSPFGP_NN_Inference](https://www.kaggle.com/code/dongyk/pspfgp-nn-inference)","metadata":{}},{"cell_type":"code","source":"VERSION = 4\nDATASET_VERSION = f'PSPFGP_NN_dataset_{VERSION}'\nCODE_VERSION = f'PSPFGP_NN_code_{VERSION}'\nprint(DATASET_VERSION)\nprint(CODE_VERSION)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport pickle\n\n%load_ext memory_profiler","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option(\"display.max_columns\", 50)\npd.set_option(\"display.max_rows\", 100)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\nSEED = 0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CAT_FEATURES = ['coded_event_name_name', 'coded_room_fqid', 'coded_text', 'coded_fqid']\nNUM_FEATURES = ['elapsed_time_diff']\nFEATURES = NUM_FEATURES + CAT_FEATURES\n\nLEVEL_GROUPS = ['0-4', '5-12', '13-22']\nLENGTHS = {'0-4':600, '5-12':1400, '13-22':2000}\nQNS = {'0-4':list(range(1, 4)), '5-12':list(range(4, 14)), '13-22':list(range(14, 19))}\nLVGP_ORDER = {'0-4': 0, '5-12': 1, '13-22': 2}\n\nN_FOLDS = 5","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = [\n    (\n        (pl.col('event_name') + '_' + pl.col('name'))\n        .alias('event_name_name')\n    ),\n    (\n        ( - pl.col('elapsed_time').diff(-1))\n        .over(['session_id', 'level_group'])\n        .fill_null(0)\n        .abs()\n        .clip(0, 60000)\n        .alias('elapsed_time_diff')\n        / 60000\n    ),\n    (\n        pl.col('level_group')\n        .map_dict(LVGP_ORDER)\n        .alias('level_group_order')\n    )\n]\ncat_columns = [*[pl.col(cat[6:]).fill_null('None') for cat in CAT_FEATURES]]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\ndf = (\n    pl.read_parquet('../train.parquet')\n    .drop(['fullscreen', 'hq', 'music', 'page', 'hover_duration'])\n    .with_columns(columns)\n    .with_columns(cat_columns)\n)\ntarget = pd.read_csv('../train_labels.csv')\ntarget['q'] = target['session_id'].apply(lambda x: x.split('_q')[1]).astype('int8')\ntarget['session_id'] = target['session_id'].apply(lambda x: x.split('_q')[0]).astype('int64')\n\ntarget_all = target.pivot(index=['session_id'], columns=['q'], values=['correct'])['correct'].reset_index()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_factorize_map(values: pd.Series):\n    vals_unique = values.unique()\n    # + 1 is needed, because we add zero padding.\n    val_code = {val: code + 1 for code, val in enumerate(vals_unique)}\n    \n    return val_code","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_code_dict(x: pl.DataFrame):\n\n    code = {}\n    code_len = {}\n    \n    for lvgp in LEVEL_GROUPS:\n    \n        code[lvgp] = {}\n        code_len[lvgp] = {}\n    \n        for cat in CAT_FEATURES:\n            \n            code[lvgp][cat] = get_factorize_map(x.filter(pl.col('level_group')==lvgp)[cat[6:]])\n            # + 1 is needed, because we add zero padding.\n            code_len[lvgp][cat] = len(code[lvgp][cat]) + 1\n            \n        with open(f\"{CODE_VERSION}.pkl\", \"wb\") as f:\n            pickle.dump((code, code_len), f)\n    \n    display(code_len)\n\n    return code, code_len","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mapping_code(x: pl.DataFrame):\n\n    code, code_len = get_code_dict(x)\n    \n    col = [\n        *[\n            pl.when(pl.col('level_group')=='0-4')\n            .then(pl.col(cat[6:]).map_dict(code['0-4'][cat]))\n            .when(pl.col('level_group')=='5-12')\n            .then(pl.col(cat[6:]).map_dict(code['5-12'][cat]))\n            .when(pl.col('level_group')=='13-22')\n            .then(pl.col(cat[6:]).map_dict(code['13-22'][cat]))\n            .alias(cat)\n            for cat in CAT_FEATURES\n        ]\n    ]\n\n    df = x.with_columns(col)\n    \n    return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineering(x: pl.DataFrame):\n\n    x = mapping_code(x)\n    col = pl.lit(1).alias('1')\n    x = x.with_columns(col)\n    col = (\n        pl.col('1').cumsum()\n        .over(['session_id', 'level_group'])\n        .alias('idx')\n    )\n    x = x.with_columns(col)\n\n    col = ['session_id', 'idx', 'level_group', 'level_group_order', 'level', 'text'] + FEATURES\n    df = x.select(pl.col(col))\n    \n    return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\ndf = feature_engineering(df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def CreateDataset(train_df, target_df):\n\n    dataset = {}\n    \n    session = train_df['session_id'].unique().to_list()\n    random.Random(SEED).shuffle(session)\n    col = [\n        (pl.col('level').diff()\n         .fill_null(0)\n         .over('session_id')\n         .alias('lv_diff')),\n        (pl.col('level_group_order').diff()\n         .fill_null(0)\n         .over('session_id')\n         .alias('lvgp_order_diff'))\n    ]\n    train_df = train_df.with_columns(col)\n    \n    abnormal_session = train_df.filter((train_df['lv_diff']<0) | (train_df['lvgp_order_diff']<0))['session_id'].unique().to_list()\n    chunk = len(session) // N_FOLDS\n    \n    for i in range(N_FOLDS):\n\n        X_train = {}\n        X_valid = {}\n        y_train = {}\n        y_valid = {}\n        \n        if i == N_FOLDS - 1:\n            valid_session = session[i * chunk: ]\n        else:\n            valid_session = session[i * chunk: (i+1) * chunk]\n        train_session = list(set(session) - set(valid_session) - set(abnormal_session))\n        \n        for lvgp, length in LENGTHS.items():\n            \n            X_train[lvgp] = {}\n            X_valid[lvgp] = {}\n        \n            for feat in FEATURES:\n        \n                tmp = train_df.filter(train_df['level_group']==lvgp)\\\n                .pivot(index=['session_id'], columns=['idx'], values=[feat])\\\n                [:, :length+1]\\\n                .fill_null(0)\n                \n                X_valid[lvgp][feat] = tmp.filter(tmp['session_id'].is_in(valid_session)).sort('session_id')[:, 1:].to_numpy()\n                X_train[lvgp][feat] = tmp.filter(tmp['session_id'].is_in(train_session)).sort('session_id')[:, 1:].to_numpy()\n                \n                print(i, lvgp, feat, 'valid', len(X_valid[lvgp][feat]))\n                print(i, lvgp, feat, 'train', len(X_train[lvgp][feat]))\n    \n            y_valid[lvgp] = target_df.loc[target_df['session_id'].isin(valid_session)].sort_values('session_id')[QNS[lvgp]].values\n            y_train[lvgp] = target_df.loc[target_df['session_id'].isin(train_session)].sort_values('session_id')[QNS[lvgp]].values\n\n            print(i, lvgp, 'valid', len(y_valid[lvgp]))\n            print(i, lvgp, 'train', len(y_train[lvgp]))\n\n        dataset[f'f{i}'] = (X_train, y_train), (X_valid, y_valid)\n        \n    return dataset","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\ndataset = CreateDataset(df, target_all)\nwith open(f\"{DATASET_VERSION}.pkl\", \"wb\") as f:\n    pickle.dump(dataset, f)","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]}]}