{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nimport xgboost as xgb\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import KFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-21T18:25:56.224058Z","iopub.execute_input":"2023-03-21T18:25:56.224477Z","iopub.status.idle":"2023-03-21T18:25:59.551686Z","shell.execute_reply.started":"2023-03-21T18:25:56.224441Z","shell.execute_reply":"2023-03-21T18:25:59.549773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config:\n    BATCH_SIZE=1024\n    N_epoch=5\n    device=torch.device(\"cuda\" if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:25:59.557204Z","iopub.execute_input":"2023-03-21T18:25:59.558012Z","iopub.status.idle":"2023-03-21T18:25:59.562395Z","shell.execute_reply.started":"2023-03-21T18:25:59.557976Z","shell.execute_reply":"2023-03-21T18:25:59.561669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train.csv\",\n                       usecols=['session_id', 'elapsed_time','text_fqid', 'level_group', 'page'],\n                       dtype={\n                           'elapsed_time': np.float32,\n                           'page': np.float16\n                       })\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:25:59.563857Z","iopub.execute_input":"2023-03-21T18:25:59.564352Z","iopub.status.idle":"2023-03-21T18:27:33.262459Z","shell.execute_reply.started":"2023-03-21T18:25:59.564323Z","shell.execute_reply":"2023-03-21T18:27:33.261280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:33.265165Z","iopub.execute_input":"2023-03-21T18:27:33.265535Z","iopub.status.idle":"2023-03-21T18:27:33.290362Z","shell.execute_reply.started":"2023-03-21T18:27:33.265502Z","shell.execute_reply":"2023-03-21T18:27:33.288253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.page.fillna(-1, inplace=True)\ntrain_df.page = train_df.page.astype(np.int8)\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:33.291824Z","iopub.execute_input":"2023-03-21T18:27:33.292173Z","iopub.status.idle":"2023-03-21T18:27:33.530161Z","shell.execute_reply.started":"2023-03-21T18:27:33.292138Z","shell.execute_reply":"2023-03-21T18:27:33.529421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.level_group.unique()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:33.531137Z","iopub.execute_input":"2023-03-21T18:27:33.531845Z","iopub.status.idle":"2023-03-21T18:27:35.126790Z","shell.execute_reply.started":"2023-03-21T18:27:33.531815Z","shell.execute_reply":"2023-03-21T18:27:35.125053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level0_df = train_df[train_df.level_group == '0-4']\nlevel1_df = train_df[train_df.level_group == '5-12']\nlevel2_df = train_df[train_df.level_group == '13-22']\n\nlevel0_df = level0_df[(~level0_df.text_fqid.isna()) | (level0_df.page!=-1)]\nlevel1_df = level1_df[(~level1_df.text_fqid.isna()) | (level1_df.page!=-1)]\nlevel2_df = level2_df[(~level2_df.text_fqid.isna()) | (level2_df.page!=-1)]\n\nlevel0_df.drop(columns=['level_group'], inplace=True)\nlevel1_df.drop(columns=['level_group'], inplace=True)\nlevel2_df.drop(columns=['level_group'], inplace=True)\n\nlevel0_df['log_elapsed_time'] = np.log(1+(level0_df.elapsed_time/1000/60))\nlevel1_df['log_elapsed_time'] = np.log(1+(level1_df.elapsed_time/1000/60))\nlevel2_df['log_elapsed_time'] = np.log(1+(level2_df.elapsed_time/1000/60))\n\nprint(level0_df.shape, level1_df.shape, level2_df.shape)\n\nlevel1_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:35.128760Z","iopub.execute_input":"2023-03-21T18:27:35.129125Z","iopub.status.idle":"2023-03-21T18:27:42.837192Z","shell.execute_reply.started":"2023-03-21T18:27:35.129090Z","shell.execute_reply":"2023-03-21T18:27:42.836337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"log elapsed time(5-12)\")\nplt.hist(level1_df.log_elapsed_time, bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:42.838664Z","iopub.execute_input":"2023-03-21T18:27:42.839167Z","iopub.status.idle":"2023-03-21T18:27:43.159158Z","shell.execute_reply.started":"2023-03-21T18:27:42.839132Z","shell.execute_reply":"2023-03-21T18:27:43.157967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"log elapsed time(13-22)\")\nplt.hist(level2_df.log_elapsed_time, bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:43.160382Z","iopub.execute_input":"2023-03-21T18:27:43.160794Z","iopub.status.idle":"2023-03-21T18:27:43.462805Z","shell.execute_reply.started":"2023-03-21T18:27:43.160767Z","shell.execute_reply":"2023-03-21T18:27:43.462016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(level0_df.log_elapsed_time.mean())\nprint(level1_df.log_elapsed_time.mean())\nprint(level2_df.log_elapsed_time.mean())","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:43.466147Z","iopub.execute_input":"2023-03-21T18:27:43.466633Z","iopub.status.idle":"2023-03-21T18:27:43.486049Z","shell.execute_reply.started":"2023-03-21T18:27:43.466605Z","shell.execute_reply":"2023-03-21T18:27:43.485320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pre process elapsed time","metadata":{}},{"cell_type":"code","source":"def get_page_features(df):\n    log_elapsed_mean = df.log_elapsed_time.mean()\n    page_feat_df = pd.pivot_table(\n        df[df.page!=-1],\n        index='session_id',\n        columns='page',\n        values=\"log_elapsed_time\",\n        aggfunc=np.max\n    )\n    \n    page_feat_df.fillna(0.0, inplace=True)\n    page_feat_df = page_feat_df-log_elapsed_mean\n    page_feat_df = page_feat_df.clip(-log_elapsed_mean, 5)\n    page_feat_df.columns=[\"page\"+str(c) for  c in page_feat_df.columns]\n    page_feat_df = page_feat_df.reset_index()\n    \n    return page_feat_df","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:43.487156Z","iopub.execute_input":"2023-03-21T18:27:43.487606Z","iopub.status.idle":"2023-03-21T18:27:43.494719Z","shell.execute_reply.started":"2023-03-21T18:27:43.487579Z","shell.execute_reply":"2023-03-21T18:27:43.493060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_text_features(df):\n    log_elapsed_mean = df.log_elapsed_time.mean()\n    feat_df = pd.pivot_table(\n        df[~df.text_fqid.isna()],\n        index='session_id',\n        columns='text_fqid',\n        values=\"log_elapsed_time\",\n        aggfunc=np.mean\n    )\n    \n    feat_df.fillna(0.0, inplace=True)\n    feat_df = feat_df-log_elapsed_mean\n    feat_df = feat_df.clip(-log_elapsed_mean, 5)\n    feat_df = feat_df.reset_index()\n    \n    return feat_df","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:43.496425Z","iopub.execute_input":"2023-03-21T18:27:43.496806Z","iopub.status.idle":"2023-03-21T18:27:43.507186Z","shell.execute_reply.started":"2023-03-21T18:27:43.496773Z","shell.execute_reply":"2023-03-21T18:27:43.506357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_features(df):\n    text_feat_df = get_text_features(df)\n    page_feat_df = get_page_features(df)\n    \n    feat_df = text_feat_df.merge(page_feat_df, how='left')\n    feat_df.fillna(0.0, inplace=True)\n    FEATURE_COLUMNS=[colname for colname in feat_df.columns if colname!='session_id']\n    return FEATURE_COLUMNS, feat_df","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:43.508522Z","iopub.execute_input":"2023-03-21T18:27:43.509023Z","iopub.status.idle":"2023-03-21T18:27:43.523255Z","shell.execute_reply.started":"2023-03-21T18:27:43.508993Z","shell.execute_reply":"2023-03-21T18:27:43.521167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level0_columns, level0_feat_df = build_features(level0_df)\nlevel1_columns, level1_feat_df = build_features(level1_df)\nlevel2_columns, level2_feat_df = build_features(level2_df)\n\nprint(\"number of level0 columns:\", len(level0_columns))\nprint(\"number of level1 columns:\", len(level1_columns))\nprint(\"number of level2 columns:\", len(level2_columns))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:43.524983Z","iopub.execute_input":"2023-03-21T18:27:43.525397Z","iopub.status.idle":"2023-03-21T18:27:46.848765Z","shell.execute_reply.started":"2023-03-21T18:27:43.525367Z","shell.execute_reply":"2023-03-21T18:27:46.846414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train labels","metadata":{}},{"cell_type":"code","source":"train_label = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\n\ntrain_label['qno'] = train_label.session_id.apply(lambda session_id: int(session_id.split(\"_\")[1][1:]))\ntrain_label['session_id']= train_label.session_id.apply(lambda session_id: int(session_id.split(\"_\")[0]))\ntrain_label.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:46.854472Z","iopub.execute_input":"2023-03-21T18:27:46.854998Z","iopub.status.idle":"2023-03-21T18:27:47.980336Z","shell.execute_reply.started":"2023-03-21T18:27:46.854948Z","shell.execute_reply":"2023-03-21T18:27:47.978832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train0_label = train_label[(train_label.qno<4)].copy()\ntrain1_label = train_label[(train_label.qno>=4) & (train_label.qno <=13)].copy()\ntrain2_label = train_label[(train_label.qno>=14)].copy()\n\ntrain0_label = train0_label.groupby(\"session_id\")[['qno', 'correct']].agg(list).reset_index()\ntrain1_label = train1_label.groupby(\"session_id\")[['qno', 'correct']].agg(list).reset_index()\ntrain2_label = train2_label.groupby(\"session_id\")[['qno', 'correct']].agg(list).reset_index()\n\ntrain1_label.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:47.981834Z","iopub.execute_input":"2023-03-21T18:27:47.982444Z","iopub.status.idle":"2023-03-21T18:27:49.436183Z","shell.execute_reply.started":"2023-03-21T18:27:47.982403Z","shell.execute_reply":"2023-03-21T18:27:49.434735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2_label.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.437711Z","iopub.execute_input":"2023-03-21T18:27:49.438242Z","iopub.status.idle":"2023-03-21T18:27:49.454222Z","shell.execute_reply.started":"2023-03-21T18:27:49.438170Z","shell.execute_reply":"2023-03-21T18:27:49.452907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level0_feat_df = level0_feat_df.merge(train0_label)\nlevel1_feat_df = level1_feat_df.merge(train1_label)\nlevel2_feat_df = level2_feat_df.merge(train2_label)\n\nlevel0_feat_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.455690Z","iopub.execute_input":"2023-03-21T18:27:49.456661Z","iopub.status.idle":"2023-03-21T18:27:49.518774Z","shell.execute_reply.started":"2023-03-21T18:27:49.456625Z","shell.execute_reply":"2023-03-21T18:27:49.516713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class PSPDataset(torch.utils.data.Dataset):\n    def __init__(self, X, qno_lst, correct_lst, min_qno):\n        self.X = X; \n        self.qno_lst=qno_lst\n        self.correct_lst = correct_lst\n        self.min_qno=min_qno\n        self.max_labels = qno_lst.shape[-1]\n        \n    def __getitem__(self, idx):\n        X = self.X[idx];\n        y = np.zeros(self.max_labels)\n        y[ self.qno_lst[idx]-self.min_qno ] = self.correct_lst[idx]\n        \n        X = torch.tensor(X, dtype=torch.float32)\n        X[X>0] = 0.1\n        y = torch.tensor(y, dtype=torch.float32)\n        return (X, y)\n    def __len__(self):\n        return len(self.X)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.521415Z","iopub.execute_input":"2023-03-21T18:27:49.521803Z","iopub.status.idle":"2023-03-21T18:27:49.530925Z","shell.execute_reply.started":"2023-03-21T18:27:49.521767Z","shell.execute_reply":"2023-03-21T18:27:49.529488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{}},{"cell_type":"code","source":"class PSPModel(nn.Module ):\n    def __init__(self, infeats, nlabels):\n        super().__init__()\n        self.mlp = nn.Sequential(\n            nn.Linear(infeats, 32),\n            nn.BatchNorm1d(32),\n            nn.LeakyReLU(),\n            nn.Dropout(0.5),\n            \n            nn.Linear(32, nlabels)\n        )\n    def forward(self, x):\n        return self.mlp(x)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.532840Z","iopub.execute_input":"2023-03-21T18:27:49.533177Z","iopub.status.idle":"2023-03-21T18:27:49.544196Z","shell.execute_reply.started":"2023-03-21T18:27:49.533149Z","shell.execute_reply":"2023-03-21T18:27:49.542094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# training model","metadata":{}},{"cell_type":"code","source":"def evaluate(model, val_dataloader):\n    model.eval()\n    ally=[]\n    allpred=[]\n    \n    for X,y in val_dataloader:\n        with torch.no_grad():\n            yhat = model(X)\n            \n            ally += list(y.flatten())\n            allpred += list((yhat>0.5).numpy().astype(int).flatten())\n    fscore = f1_score(ally, allpred)\n    return fscore","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.546734Z","iopub.execute_input":"2023-03-21T18:27:49.547899Z","iopub.status.idle":"2023-03-21T18:27:49.565562Z","shell.execute_reply.started":"2023-03-21T18:27:49.547834Z","shell.execute_reply":"2023-03-21T18:27:49.563719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(level1_feat_df.qno.values[0]))\nprint(len(level2_feat_df.qno.values[0]))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.567002Z","iopub.execute_input":"2023-03-21T18:27:49.567855Z","iopub.status.idle":"2023-03-21T18:27:49.578297Z","shell.execute_reply.started":"2023-03-21T18:27:49.567818Z","shell.execute_reply":"2023-03-21T18:27:49.576789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(level_id, foldnum, train_dataloader, val_dataloader):\n    if not os.path.exists(\"level{}_models\".format(level_id)):\n        os.mkdir( \"level{}_models\".format(level_id) )\n    \n    infeats = 0; outfeats=0;\n    if level_id==0:\n        infeats = len(level0_columns)\n        outfeats = 3\n    elif level_id==1:\n        infeats = len(level1_columns)\n        outfeats = 10\n    else:\n        infeats = len(level2_columns)\n        outfeats = 5\n    \n    model = PSPModel(infeats, outfeats).to(Config.device)\n    criteria=nn.BCEWithLogitsLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=5e-3, weight_decay=1.0)\n    schedular = torch.optim.lr_scheduler.CosineAnnealingLR(\n        optimizer,\n        T_max = len(train_dataloader) * Config.N_epoch, \n        eta_min=1e-6,\n    )\n    \n    best_eval=None\n    for e in range(Config.N_epoch):\n        epoch_loss=[]\n        model.train()\n        for (X, y) in train_dataloader:\n            X = X.to(Config.device)\n            y = y.to(Config.device)\n\n            optimizer.zero_grad()\n\n            yhat = model(X)\n            loss = criteria(yhat, y)\n\n            loss.backward()\n            optimizer.step()\n            schedular.step()\n            epoch_loss.append(loss.item())\n        \n        eval_metric = evaluate(model, val_dataloader)\n        if (best_eval is None) or (eval_metric>best_eval):\n            best_eval=eval_metric\n            torch.save(model, \"level{}_models/models_{}.pt\".format(level_id, foldnum))\n        print(\"epoch:{} | train loss:{:.4f}\".format(e, np.mean(epoch_loss)))\n        print(\"eval:{:.4f} | best eval:{:.4f}\".format(eval_metric, best_eval))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.580211Z","iopub.execute_input":"2023-03-21T18:27:49.580875Z","iopub.status.idle":"2023-03-21T18:27:49.597449Z","shell.execute_reply.started":"2023-03-21T18:27:49.580827Z","shell.execute_reply":"2023-03-21T18:27:49.595117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(level1_feat_df.session_id.nunique())\nprint(level2_feat_df.session_id.nunique())","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.599467Z","iopub.execute_input":"2023-03-21T18:27:49.599862Z","iopub.status.idle":"2023-03-21T18:27:49.620279Z","shell.execute_reply.started":"2023-03-21T18:27:49.599826Z","shell.execute_reply":"2023-03-21T18:27:49.618828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_ids = train_df.session_id.unique()\nkfold = KFold(n_splits=5, shuffle=True, random_state=44)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.622241Z","iopub.execute_input":"2023-03-21T18:27:49.622640Z","iopub.status.idle":"2023-03-21T18:27:49.732331Z","shell.execute_reply.started":"2023-03-21T18:27:49.622602Z","shell.execute_reply":"2023-03-21T18:27:49.730896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for level_id, train_feat_df in enumerate([level0_feat_df, level1_feat_df, level2_feat_df]):\n    if level_id == 0:\n        FEATURE_COLUMNS = level0_columns\n    elif level_id == 1:\n        FEATURE_COLUMNS = level1_columns\n    else:\n        FEATURE_COLUMNS = level2_columns\n    \n    for foldnum, (train_index, val_index) in enumerate(kfold.split(session_ids)):\n        train_sessions = session_ids[train_index]\n        val_sessions = session_ids[val_index]\n\n        fold_train_df = train_feat_df[train_feat_df.session_id.isin(train_sessions)]\n        fold_val_df = train_feat_df[train_feat_df.session_id.isin(val_sessions)]\n\n        print(\"===========LevelId:{}==============\".format(level_id))\n        print(\"==================Foldnum{}================\".format(foldnum))\n        \n        print(\"number of train sessions:\", len(train_sessions))\n        print(\"number of val sessions:\", len(val_sessions))\n\n        Xfold_train = fold_train_df[FEATURE_COLUMNS].values\n        qno_fold_train = np.stack(fold_train_df['qno'].values)\n        correct_fold_train = np.stack(fold_train_df['correct'].values)\n\n        Xfold_val = fold_val_df[FEATURE_COLUMNS].values\n        qno_fold_val = np.stack(fold_val_df['qno'].values)\n        correct_fold_val = np.stack(fold_val_df['correct'].values)\n\n        print(Xfold_train.shape, qno_fold_train.shape, correct_fold_train.shape)\n        print()\n        print()\n        \n        train_dataset = PSPDataset(Xfold_train, qno_fold_train, correct_fold_train, qno_fold_train.min())\n        val_dataset = PSPDataset(Xfold_val, qno_fold_val, correct_fold_val, qno_fold_train.min())\n\n        train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=Config.BATCH_SIZE, shuffle=True)\n        val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=Config.BATCH_SIZE, shuffle=False, drop_last=False)\n\n        train_model(level_id, foldnum, train_dataloader, val_dataloader)\n        print()\n        print()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:27:49.733930Z","iopub.execute_input":"2023-03-21T18:27:49.734210Z","iopub.status.idle":"2023-03-21T18:28:14.896861Z","shell.execute_reply.started":"2023-03-21T18:27:49.734182Z","shell.execute_reply":"2023-03-21T18:28:14.896035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"train_label = pd.read_csv(\"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\")\ntrain_label['q'] = train_label['session_id'].apply(lambda s: s.split(\"_\")[-1])\ntrain_label['session'] = train_label['session_id'].apply(lambda s: s.split(\"_\")[0])\n\nqprob_df = train_label.groupby(\"q\")[['correct']].mean().reset_index().sort_values(\"correct\", ascending=False)\nqprob_map={}\n\nfor _,row in qprob_df.iterrows():\n    q=row.q\n    p = row.correct\n    qprob_map[q]=p\nprint(len(qprob_map))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:14.897891Z","iopub.execute_input":"2023-03-21T18:28:14.898937Z","iopub.status.idle":"2023-03-21T18:28:15.497327Z","shell.execute_reply.started":"2023-03-21T18:28:14.898906Z","shell.execute_reply":"2023-03-21T18:28:15.496168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qprob_map","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:15.501687Z","iopub.execute_input":"2023-03-21T18:28:15.502562Z","iopub.status.idle":"2023-03-21T18:28:15.509946Z","shell.execute_reply.started":"2023-03-21T18:28:15.502525Z","shell.execute_reply":"2023-03-21T18:28:15.508758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_models(path):\n    models=[]\n    for modelname in os.listdir(path):\n        modelpath = os.path.join(path, modelname)\n        model = torch.load(modelpath)\n        models.append(model)\n    return models\n\nlevel0_models=load_models(\"/kaggle/working/level0_models\")\nlevel1_models=load_models(\"/kaggle/working/level1_models\")\nlevel2_models=load_models(\"/kaggle/working/level2_models\")\n\n\nprint(len(level0_models))\nprint(len(level1_models))\nprint(len(level2_models))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:15.511244Z","iopub.execute_input":"2023-03-21T18:28:15.511590Z","iopub.status.idle":"2023-03-21T18:28:15.530474Z","shell.execute_reply.started":"2023-03-21T18:28:15.511560Z","shell.execute_reply":"2023-03-21T18:28:15.528821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_random_model(submission):\n    q = submission.session_id.apply(lambda k: k.split(\"_\")[-1])\n    p = q.apply(lambda k: qprob_map[k])\n    r = np.random.uniform(0, 1, len(q))\n    correct = (r<=p).astype(int)\n    \n    submission['correct'] = correct\n    return submission","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:15.532222Z","iopub.execute_input":"2023-03-21T18:28:15.533374Z","iopub.status.idle":"2023-03-21T18:28:15.541983Z","shell.execute_reply.started":"2023-03-21T18:28:15.533255Z","shell.execute_reply":"2023-03-21T18:28:15.540297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_feats(level, submission, test):\n    log_elapsed_mean=0\n    \n    if level==0:\n        log_elapsed_mean = level0_df.log_elapsed_time.mean()\n        FEAT_COLUMNS = level0_columns\n        nlabel = 3\n    elif level==1:\n        log_elapsed_mean = level1_df.log_elapsed_time.mean()\n        FEAT_COLUMNS = level1_columns\n        nlabel = 10\n    else:\n        log_elapsed_mean = level2_df.log_elapsed_time.mean()\n        FEAT_COLUMNS = level2_columns\n        nlabel = 5\n    \n    test['log_elapsed_time'] = np.log(1+test.elapsed_time/1000/60)\n    _, test_feat_df = build_features(test)\n    unknown_columns = set(FEAT_COLUMNS ) - set(test_feat_df.columns)\n    for colname in unknown_columns:\n        test_feat_df[colname] = -log_elapsed_mean\n    \n    test_feat_df  = test_feat_df.reset_index()\n    return (test_feat_df,FEAT_COLUMNS, nlabel)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:15.544049Z","iopub.execute_input":"2023-03-21T18:28:15.544895Z","iopub.status.idle":"2023-03-21T18:28:15.559046Z","shell.execute_reply.started":"2023-03-21T18:28:15.544838Z","shell.execute_reply":"2023-03-21T18:28:15.556468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_model(models, test_feat_df, FEAT_COLS, nlabels, min_label):\n    X = test_feat_df[FEAT_COLS].values\n    X = torch.tensor(X, dtype=torch.float32)\n    X[X>0]=0.1\n    ypred = np.zeros( (len(X), nlabels) )\n    \n    for model in models:\n        model.eval()\n        with torch.no_grad():\n            yhat = model(X).numpy()\n            ypred += yhat\n    ypred = ypred/len(models)\n    \n    all_df = []\n    sessions = test_feat_df.session_id.values\n    for i in range(len(sessions)):\n        for j in range(nlabels):\n            session_id = str(sessions[i])+\"_q\"+str(min_label+j)\n            predi = (ypred[i][j]>0.5).astype(int)\n            all_df.append({\n                \"session_id\": session_id,\n                'correct': predi\n            })\n    all_df = pd.DataFrame.from_dict(all_df)\n    return all_df","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:15.561733Z","iopub.execute_input":"2023-03-21T18:28:15.562584Z","iopub.status.idle":"2023-03-21T18:28:15.577332Z","shell.execute_reply.started":"2023-03-21T18:28:15.562548Z","shell.execute_reply":"2023-03-21T18:28:15.575210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.level_group.unique()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:28:15.578546Z","iopub.execute_input":"2023-03-21T18:28:15.578860Z","iopub.status.idle":"2023-03-21T18:28:17.155146Z","shell.execute_reply.started":"2023-03-21T18:28:15.578831Z","shell.execute_reply":"2023-03-21T18:28:17.153819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\njo_wilder.make_env.__called__ = False\n\nenv = jo_wilder.make_env()\niter_test = iter(env.iter_test())\n\nfor (test, submission) in iter_test:\n    nlabels = 3\n    \n    test_feat_df, FEAT_COLUMNS, nlabels = get_test_feats(0, submission, test[test.level_group == '0-4'])\n    submission1 = predict_model(level0_models, test_feat_df, level0_columns, nlabels, 1)\n    \n    test_feat_df, FEAT_COLUMNS, nlabels = get_test_feats(1, submission, test[test.level_group == '5-12'])\n    submission2 = predict_model(level1_models, test_feat_df, level1_columns, nlabels, 4)\n    \n    test_feat_df, FEAT_COLUMNS, nlabels = get_test_feats(2, submission, test[test.level_group == '13-22'])\n    submission3 = predict_model(level2_models, test_feat_df, level2_columns, nlabels, 14)\n    \n    submission = pd.concat([submission1, submission2, submission3])\n    #submission = predict_random_model(submission)\n    env.predict(submission)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:36:26.563782Z","iopub.execute_input":"2023-03-21T18:36:26.564703Z","iopub.status.idle":"2023-03-21T18:36:26.717001Z","shell.execute_reply.started":"2023-03-21T18:36:26.564646Z","shell.execute_reply":"2023-03-21T18:36:26.715911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(\"submission.csv\")\nprint(sub_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:30:47.310759Z","iopub.execute_input":"2023-03-21T18:30:47.311147Z","iopub.status.idle":"2023-03-21T18:30:47.319360Z","shell.execute_reply.started":"2023-03-21T18:30:47.311112Z","shell.execute_reply":"2023-03-21T18:30:47.317713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df[sub_df.session_id.apply(lambda x: x.startswith(\"20090312143683264\"))]","metadata":{"execution":{"iopub.status.busy":"2023-03-21T18:30:54.339018Z","iopub.execute_input":"2023-03-21T18:30:54.339446Z","iopub.status.idle":"2023-03-21T18:30:54.353512Z","shell.execute_reply.started":"2023-03-21T18:30:54.339404Z","shell.execute_reply":"2023-03-21T18:30:54.352675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}