{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### I tried to construct SAINT model with ommiting some implementations from following links.<br>\n+ arxiv: https://arxiv.org/abs/2106.01342 <br>\n+ github: https://github.com/somepago/saint/tree/main","metadata":{}},{"cell_type":"markdown","source":"# 1. Preparation","metadata":{}},{"cell_type":"markdown","source":"## 1.1. Loading packages & Setting variables","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport warnings\nimport gc\nimport os\nimport random\nimport pickle\nfrom tqdm import tqdm\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport lightgbm as lgb\nimport plotly.express as px\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import train_test_split as tts\n#from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit\n\nwarnings.simplefilter(\"ignore\")\npd.options.display.max_columns=1000\n\nclass CFG:\n    # hyper parameters\n    EPOCHS = 1000\n    THRESHOLD = .5\n    BATCH_SIZE = 2**6\n    EARLY_STOPPING_ROUNDS = max(EPOCHS//20, 20)\n    WARM_UP = min(20, EPOCHS//10) # do not stop early\n    LEARNING_RATE = .01\n    SEED = 0\n    \n    # utils\n    DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    VERBOSE = True # or False\n    PREDICT_ALL = False # True or False\n    \n    SUBMISSION_MODE = True\n    CHECKPOINT = False\n    RESUME = True\n    PRETRAINING_OMIT = False\n    TRAINING_OMIT = False\n    if SUBMISSION_MODE:\n        CHECKPOINT = False\n        RESUME = False\n        PRETRAINING_OMIT = True\n        TRAINING_OMIT = True\n        VERBOSE = False\n    elif CHECKPOINT:\n        RESUME = False\n        PRETRAINING_OMIT = False\n        TRAINING_OMIT = False\n    else:\n        RESUME = True\n        if TRAINING_OMIT:\n            PRETRAINING_OMIT = True\n    INPUT = '/kaggle/input/predict-student-performance-from-game-play'\n    CHECKPOINT_PATH = '/kaggle/input/saint-pspfgp-iori'\n\nif CFG.SUBMISSION_MODE:\n    import jo_wilder_310\n    \nrandom.seed(CFG.SEED)\nnp.random.seed(CFG.SEED)\ntorch.manual_seed(CFG.SEED)\ntorch.cuda.manual_seed(CFG.SEED)\n\nlq_dict = {\n    \"0-4\":[\"q\"+str(i) for i in range(1,4)],\n    \"5-12\":[\"q\"+str(i) for i in range(4,14)],\n    \"13-22\":[\"q\"+str(i) for i in range(14,19)]\n}\nquestions = ['q'+str(i) for i in range(1,19)]\n\n%env CUDA_LAUNCH_BLOCKING=1\n%env PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-21T22:21:39.496474Z","iopub.execute_input":"2023-05-21T22:21:39.496823Z","iopub.status.idle":"2023-05-21T22:21:44.965081Z","shell.execute_reply.started":"2023-05-21T22:21:39.496794Z","shell.execute_reply":"2023-05-21T22:21:44.963938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2. Loading packages & Setting variables","metadata":{}},{"cell_type":"markdown","source":"### 1.2.1. Data Processing Class","metadata":{}},{"cell_type":"code","source":"class DataProcessing():\n    \n    def __init__(self):\n        self.agg_units = [\"session_id\", \"level_group\"]\n    \n    def read_table(self, usecols, inc_agg_units=True):\n        if inc_agg_units:\n            base_cols = self.agg_units + [\"level\"]\n            usecols += base_cols\n        if self.path:\n            data = pd.read_csv(self.path, usecols=usecols)\n        elif not self.data is None:\n            data = self.data[usecols]\n                \n        return data\n\n    def agg_by_level(self, data, agg_func):\n        data = data.copy()\n        if type(agg_func) == dict:\n            data = data.groupby(self.agg_units + [\"level\"]).agg(**agg_func)\n        elif type(agg_func) == list:\n            data = data.groupby(self.agg_units + [\"level\"]).agg(agg_func)\n            data.columns = [col[0]+\"_\"+col[1] for col in data.columns]\n        else:\n            data = data.groupby(self.agg_units + [\"level\"]).agg(agg_func)\n        \"\"\"    \n        tmp = data.diff().fillna(0)\n        tmp.columns = [col+\"_diff\" for col in data.columns]\n        data = pd.concat([data, tmp], axis=1)\n        del tmp\n        gc.collect()\n        \"\"\"\n        data = data.reset_index()\n        data = data.pivot(index=self.agg_units, columns=\"level\")\n        data.columns = [f\"{col[0]}_level{col[1]}\" for col in data.columns]\n\n        return data.sort_index()\n\n    def agg_by_level_group(self, data, agg_func):\n        data = data.copy()\n        if CFG.SUBMISSION_MODE:\n            if type(agg_func) == dict:\n                data = data.groupby(self.agg_units).agg(**agg_func)\n            elif type(agg_func) == list:\n                data = data.groupby(self.agg_units).agg(agg_func)\n                data.columns = [col[0]+\"_\"+col[1] for col in data.columns]\n            else:\n                data = data.groupby(self.agg_units).agg(agg_func)\n        else:        \n            data2 = data.query(\"level_group!='13-22'\")\n            data2.level_group = \"5-12\"\n            data1 = data.query(\"level_group=='0-4'\")\n            data1.level_group = \"0-4\"\n            data.level_group = \"13-22\"\n            if type(agg_func) == dict:\n                data = data.groupby(self.agg_units).agg(**agg_func)\n                data2 = data2.groupby(self.agg_units).agg(**agg_func)\n                data1 = data1.groupby(self.agg_units).agg(**agg_func)\n            elif type(agg_func) == list:\n                cols = [col for col in data.columns if col != \"level\"]\n                data = data[cols].groupby(self.agg_units).agg(agg_func)\n                data.columns = [col[0]+\"_\"+col[1] for col in data.columns]\n                data2 = data2[cols].groupby(self.agg_units).agg(agg_func)\n                data2.columns = [col[0]+\"_\"+col[1] for col in data2.columns]\n                data1 = data1[cols].groupby(self.agg_units).agg(agg_func)\n                data1.columns = [col[0]+\"_\"+col[1] for col in data1.columns]\n            else:        \n                cols = [col for col in data.columns if col != \"level\"]\n                data = data[cols].groupby(self.agg_units).agg(agg_func)\n                data2 = data2[cols].groupby(self.agg_units).agg(agg_func)\n                data1 = data1[cols].groupby(self.agg_units).agg(agg_func)\n\n            data = pd.concat([data1, data2, data], axis=0).fillna(float(\"inf\"))\n        \n            del data1, data2\n            gc.collect()\n        \n        return data.sort_index()\n    \n    def fit_transform(self, path=None, data=None):\n        self.fit = True\n        if path:\n            self.path = path\n            self.data = None\n        elif not data is None:\n            self.path = None\n            self.data = data\n        else:\n            assert (self.path is None and self.data is None, \"both path and data is None\")\n        \n        data = self.transform(path)\n        self.fit = False\n        \n        return data \n    \n    def transform(self, path=None, data=None, decided_level_group=None):\n        if CFG.SUBMISSION_MODE:\n            self.data = data\n            self.path = None\n            self.agg_units = [\"session_id\"]\n        else:\n            if path:\n                self.path = path\n                self.data = None\n            elif not data is None:\n                self.path = None\n                self.data = data\n            else:\n                assert (self.path is None and self.data is None, \"both path and data is None\")\n\n        self.level_group = decided_level_group\n\n        datas = []\n        \n        \"\"\"elapsed_time\"\"\"\n        datas.append(\n            self.col_processing(\n                [\"elapsed_time\"],\n                agg_func = dict(\n                    elapsed_time_max=(\"elapsed_time\", \"max\"),\n                    elapsed_time_mean=(\"elapsed_time\", \"mean\"),\n                    elapsed_time_median=(\"elapsed_time\", \"median\"),\n                    elapsed_time_75percentile=(\"elapsed_time\", lambda x: np.percentile(x, 75)),\n                    elapsed_time_25percentile=(\"elapsed_time\", lambda x: np.percentile(x, 25)),\n                    elapsed_time_std=(\"elapsed_time\", \"std\"),\n                    logs=(\"elapsed_time\", \"count\"),\n                )\n            )\n        )\n        if CFG.VERBOSE:\n            print(\"elapsed_time processing has finished!\")\n\n        \"\"\"fqids\"\"\"\n        datas.append(self.col_processing([\"fqid\"], \"nunique\"))\n        if CFG.VERBOSE:\n            print(\"fqid processing has finished!\")\n        \n        datas.append(self.col_processing([\"room_fqid\"], \"nunique\"))\n        if CFG.VERBOSE:\n            print(\"rfqid processing has finished!\")\n        \n        datas.append(self.col_processing([\"text_fqid\"], \"nunique\"))\n        if CFG.VERBOSE:\n            print(\"tfqid processing has finished!\")\n        \n        \"\"\"coor\"\"\"\n        datas.append(self.col_processing(\n            [\"screen_coor_x\", \"screen_coor_y\", \"room_coor_x\", \"room_coor_y\"],\n            [\"mean\", \"std\", \"max\", \"min\", \"median\"]))\n        if CFG.VERBOSE:\n            print(\"coor processing has finished!\")\n        \n        \"\"\"text\"\"\"\n        datas.append(self.text_processing())\n        if CFG.VERBOSE:\n            print(\"text processing has finished!\")\n        \n        \"\"\"name\"\"\"\n        datas.append(self.str_col_processing(\"name\"))\n        if CFG.VERBOSE:\n            print(\"name processing has finished!\")\n        \n        \"\"\"event_name\"\"\"\n        datas.append(self.str_col_processing(\"event_name\", \"_\"))\n        if CFG.VERBOSE:\n            print(\"event_name processing has finished!\")\n            \n        \"\"\"others\"\"\"\n        datas.append(\n            self.col_processing([\"page\", \"hover_duration\"],\n            dict(\n                page_max=(\"page\", \"max\"),\n                hover_duration_max=(\"hover_duration\", \"max\"),\n            )\n        ))\n        if CFG.VERBOSE:\n            print(\"others processing has finished!\")\n\n        \"\"\"concate data\"\"\"\n        datas = pd.concat(datas, axis=1)\n        \n        \"\"\"final process\"\"\"\n        datas = self.make_cols_same(datas)\n        datas = self.standardizing_and_masking(datas)\n        datas = self.feature_elimination(datas,pca_components=10)\n        if CFG.VERBOSE:\n            print(\"final processing has finished!\\n\")\n            \n        return datas\n    \n    def str_col_processing(self, col, split_str=None, agg_func=[\"sum\", \"mean\"]):\n        base_data = self.read_table([col])\n        data = pd.DataFrame({col: base_data[col].unique()})\n        \n        if split_str:\n            split = data[col].str.split(split_str, expand=True)\n            cols = [col+str(i+1) for i in range(split.shape[1])]\n            data[cols] = split\n            del split\n            gc.collect()\n            data = pd.get_dummies(data.set_index(col), sparse=False).reset_index()\n        else:\n            tmp = pd.get_dummies(data[col], sparse=False)\n            data = pd.concat([data, tmp], axis=1)\n        \n        data = pd.merge(base_data, data, how=\"left\", on=col)\n        \n        del base_data\n        gc.collect()\n        \n        data = self.agg_by_level_group(data, agg_func)\n        return data\n    \n    def col_processing(self, cols, agg_func=[\"sum\", \"mean\"], also_by_level=True):\n        data = self.read_table(cols)\n        if also_by_level:\n            datas = []\n            datas.append(self.agg_by_level_group(data, agg_func))\n            datas.append(self.agg_by_level(data, agg_func))\n            data = pd.concat(datas, axis=1)\n            del datas\n            gc.collect()\n        else:   \n            data = self.agg_by_level_group(data, agg_func)\n        return data\n    \n    def text_processing(self, also_by_level=True):\n        data = self.read_table([\"text\"])\n        data[\"text_byte\"] = data.text.str.len().tolist()\n        data[\"text_len\"] = data.text.str.split(\" \").apply(lambda x: len(x) if type(x)==list else 0).tolist()\n        data.text = data.text.notnull().astype(int).tolist()\n        \n        agg_func = dict(\n            text_len_max = (\"text_len\", \"max\"),\n            text_len_mean = (\"text_len\", \"mean\"),\n            text_len_std = (\"text_len\", \"std\"),\n            text_byte_max = (\"text_byte\", \"max\"),\n            text_byte_mean = (\"text_byte\", \"mean\"),\n            text_byte_std = (\"text_byte\", \"std\"),\n            text_count_mean = (\"text\", \"mean\"),\n            text_count_std = (\"text\", \"std\"),\n            text_count = (\"text\", \"sum\"),\n        )\n        \n        if also_by_level:\n            datas = []\n            datas.append(self.agg_by_level_group(data, agg_func))\n            datas.append(self.agg_by_level(data, agg_func))\n            data = pd.concat(datas, axis=1)\n            del datas\n            gc.collect()\n        else:\n            data = self.agg_by_level_group(data, agg_dict)\n        \n        return data\n    \n    def make_cols_same(self, data):\n        if not self.fit:\n            cols = self.columns\n        \n            if self.level_group:\n                lg = self.level_group\n                drop_cols = []\n                for col in data.columns:\n                    if not col in cols[lg]:\n                        drop_cols.append(col)\n                data = data.drop(drop_cols, axis=1)\n\n                for col in cols[lg]:\n                    if not col in data.columns:\n                        data[col] = 0\n\n                return data[cols[lg]]\n\n            else:\n                datas = {}\n                for lg in lq_dict.keys():\n                    dlg = data.query(f\"level_group=='{lg}'\")\n                    \n                    drop_cols = []\n                    for col in dlg.columns:\n                        if not col in cols[lg]:\n                            drop_cols.append(col)\n                    dlg = dlg.drop(drop_cols, axis=1)\n\n                    for col in cols[lg]:\n                        if not col in dlg.columns:\n                            dlg[col] = 0\n                            \n                    datas[lg] = dlg\n                return datas\n            \n        else:\n            self.columns = {}\n            datas = {}\n            useless_from_lgb = []\n\n            for lg in lq_dict.keys():\n                dlg = data.query(f\"level_group=='{lg}'\")\n                \n                all_one_value = dlg.nunique()\n                all_one_value = all_one_value[all_one_value<2].index\n                dlg = dlg.drop(all_one_value, axis=1)\n                dlg = dlg.drop([col for col in useless_from_lgb if col in dlg.columns], axis=1)\n                \n                print(f\"{lg}: all_one_value COLS ARE {all_one_value}\")\n                \n                datas[lg] = dlg\n                self.columns[lg] = dlg.columns\n            return datas\n \n    def feature_elimination(self, datas, pca_components=10):\n        top30_useful_lgb = {\n            \"0-4\":[\n                'basic_sum',\n                'elapsed_time_std_level4',\n                'elapsed_time_std_level2',\n                'logs',\n                'screen_coor_x_median_level4',\n                'event_name1_notification_mean',\n                'room_coor_y_median_level4',\n                'elapsed_time_std_level1',\n                'room_coor_y_max',\n                'room_coor_y_max_level3',\n                'screen_coor_x_median_level0',\n                'screen_coor_x_std_level2',\n                'room_coor_x_mean_level4',\n                'room_coor_x_min_level4',\n                'screen_coor_y_median_level3',\n                'text_len_mean_level3',\n                'room_coor_y_min_level0',\n                'room_coor_x_std',\n                'text_byte_std_level2',\n                'room_coor_y_std_level0',\n                'screen_coor_x_std_level1',\n                'room_coor_x_median_level3',\n                'room_coor_x_mean_level3',\n                'room_coor_y_std',\n                'room_coor_y_min_level3',\n                'text_len_std_level2',\n                'event_name2_click_mean',\n                'text_len_mean_level2',\n                'room_coor_x_mean_level1',\n                'screen_coor_y_max_level0'\n            ],\n            \"5-12\":[\n                'basic_sum',\n                'logs',\n                'logs_level11',\n                'event_name1_object_sum',\n                'elapsed_time_std_level8',\n                'elapsed_time_std_level12',\n                'logs_level9',\n                'event_name1_notification_mean',\n                'hover_duration_max_level11',\n                'event_name2_click_sum',\n                'text_count',\n                'text_byte_mean_level6',\n                'hover_duration_max_level9',\n                'elapsed_time_std_level10',\n                'text_len_std_level6',\n                'room_coor_x_min_level11',\n                'room_coor_y_max',\n                'room_coor_x_min_level9',\n                'room_coor_x_min',\n                'room_coor_x_max_level9',\n                'room_coor_x_max_level8',\n                'room_coor_x_max_level7',\n                'hover_duration_max_level8',\n                'elapsed_time_std_level5',\n                'hover_duration_max_level7',\n                'screen_coor_x_min_level7',\n                'room_coor_x_max_level5',\n                'room_coor_x_min_level10',\n                'elapsed_time_std_level11',\n                'room_coor_x_max_level6'\n            ],\n            \"13-22\":[\n                'event_name1_checkpoint_mean',\n                'room_coor_y_max_level15',\n                'logs',\n                'room_coor_y_mean_level22',\n                'text_byte_mean_level21',\n                'elapsed_time_std',\n                'screen_coor_x_median_level22',\n                'elapsed_time_std_level21',\n                'hover_duration_max_level20',\n                'screen_coor_x_min_level20',\n                'room_coor_x_max_level22',\n                'room_coor_x_max_level17',\n                'screen_coor_x_std_level17',\n                'screen_coor_x_mean_level21',\n                'elapsed_time_median',\n                'room_coor_x_max',\n                'elapsed_time_std_level22',\n                'text_byte_std',\n                'room_coor_y_min_level21',\n                'hover_duration_max_level15',\n                'screen_coor_x_max_level15',\n                'elapsed_time_std_level20',\n                'basic_sum',\n                'room_coor_y_std_level22',\n                'room_coor_x_std_level18',\n                'text_count_level20',\n                'room_coor_y_min_level18',\n                'screen_coor_x_median_level17',\n                'room_coor_x_min',\n                'elapsed_time_std_level19'\n            ]\n        }\n        \n        if self.level_group:\n            level_group = self.level_group\n            data, data_mask = datas\n            tmp = self.pca[level_group].transform(data)\n            tmp = pd.DataFrame(tmp, columns=[f\"pca_{i}\" for i in range(pca_components)])\n            data = data[top30_useful_lgb[level_group]]\n            data_mask = data_mask[top30_useful_lgb[level_group]]\n            for col in tmp.columns:\n                data[col] = tmp[col].values\n                data_mask[col] = 1\n            \n            del tmp\n            gc.collect()\n            \n            res = (data, data_mask)\n        \n        else:\n            res = {}\n\n            if self.fit:\n                self.pca = {}\n\n            for level_group in lq_dict.keys():\n                data, data_mask = datas[level_group]\n\n                if not self.fit:\n                    tmp = self.pca[level_group].transform(data)\n                    tmp = pd.DataFrame(tmp, columns=[f\"pca_{i}\" for i in range(pca_components)])\n\n                else:\n                    pca = PCA(n_components=pca_components, random_state=CFG.SEED)\n                    tmp = pca.fit_transform(data)\n                    tmp = pd.DataFrame(tmp, columns=[f\"pca_{i}\" for i in range(pca_components)])\n                    self.pca[level_group] = pca\n\n                data = data[top30_useful_lgb[level_group]]\n                data_mask = data_mask[top30_useful_lgb[level_group]]\n                for col in tmp.columns:\n                    data[col] = tmp[col].values\n                    data_mask[col] = 1\n\n                del tmp\n                gc.collect()\n\n                res[level_group] = (data, data_mask)\n\n            self.num_features = len(data.columns)\n        \n        return res\n        \n    def standardizing_and_masking(self, datas):\n        if self.level_group:\n            data_mask = datas.notnull().astype(int)\n            data = (datas - self.means[self.level_group]) / self.stds[self.level_group]\n            datas = (data.fillna(0), data_mask)\n            \n        else:\n            if self.fit:\n                self.means = {}\n                self.stds = {}\n\n            for k, data in datas.items():\n                if self.fit:\n                    self.means[k] = data.apply(lambda x: x.mean(axis=0))\n                    self.stds[k] = data.apply(lambda x: x.std(axis=0))\n\n                data_mask = data.notnull().astype(int)\n                #data_mask[\"CLS\"] = 1\n                #data_mask = data_mask[[\"CLS\"] + [col for col in data_mask.columns if col != \"CLS\"]]\n\n                data = (data - self.means[k]) / self.stds[k]\n                #data[\"CLS\"] = 0\n                #data = data[[\"CLS\"] + [col for col in data.columns if col != \"CLS\"]]\n\n                datas[k] = (data.fillna(0), data_mask)\n        return datas","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:44.969513Z","iopub.execute_input":"2023-05-21T22:21:44.970777Z","iopub.status.idle":"2023-05-21T22:21:45.027646Z","shell.execute_reply.started":"2023-05-21T22:21:44.970723Z","shell.execute_reply":"2023-05-21T22:21:45.026566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.2. Dataloader Creating Functions","metadata":{}},{"cell_type":"code","source":"def create_loader_train(X, X_mask, y, level_group=None, train_rate=.9, predict_all=False):\n    if level_group:\n        X = X.query(f'level_group==\"{level_group}\"')\n        X = X.reset_index().drop(\"level_group\", axis=1).set_index(\"session_id\")\n        X_mask = X_mask.query(f'level_group==\"{level_group}\"')\n        X_mask = X_mask.reset_index().drop(\"level_group\", axis=1).set_index(\"session_id\")\n    \n    if predict_all:\n        pass\n    else:\n        y = y[lq_dict[level_group]]\n        \n    msss = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=1-train_rate, random_state=CFG.SEED)\n    res = msss.split(X.values, y.values)\n    for train_index, val_index in res:\n        train_index = train_index.tolist()\n        val_index = val_index.tolist()\n\n    train_loader = DataLoader(\n        TensorDataset(\n            torch.Tensor(X.iloc[train_index, :].values),\n            torch.Tensor(X_mask.iloc[train_index, :].values).type(torch.int),\n            torch.Tensor(y.iloc[train_index, :].values),\n        ),\n        batch_size=CFG.BATCH_SIZE,\n        shuffle=True,\n        num_workers=4,\n    )\n\n    val_loader = DataLoader(\n        TensorDataset(\n            torch.Tensor(X.iloc[val_index, :].values),\n            torch.Tensor(X_mask.iloc[val_index, :].values).type(torch.int),\n            torch.Tensor(y.iloc[val_index, :].values),\n        ),\n        batch_size=CFG.BATCH_SIZE,\n        shuffle=True,\n        num_workers=4,\n    )\n\n    del train_index\n    gc.collect()\n\n    return train_loader, val_loader\n\ndef create_loader_test(X, X_mask, level_group=None):\n    if CFG.SUBMISSION_MODE:\n        pass\n    elif level_group:\n        X = X.query(f'level_group==\"{level_group}\"')\n        X = X.reset_index().drop(\"level_group\", axis=1).set_index(\"session_id\")\n        X_mask = X_mask.query(f'level_group==\"{level_group}\"')\n        X_mask = X_mask.reset_index().drop(\"level_group\", axis=1).set_index(\"session_id\")\n\n    test_loader = DataLoader(\n        TensorDataset(\n            torch.Tensor(X.values),\n            torch.Tensor(X_mask.values).type(torch.int),\n        ),\n        batch_size=1000, shuffle=False\n    )\n    \n    return test_loader","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.028728Z","iopub.execute_input":"2023-05-21T22:21:45.028980Z","iopub.status.idle":"2023-05-21T22:21:45.041201Z","shell.execute_reply.started":"2023-05-21T22:21:45.028953Z","shell.execute_reply":"2023-05-21T22:21:45.040108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.3. MultiOutputsModel","metadata":{}},{"cell_type":"code","source":"class MultiOutputsModel(nn.Module):\n    def __init__(self, num_features=10, num_outputs=10,\n                 units=[64, 128, 64, 64, 32],\n                 units_sub=[16, 32, 16, 8, 1],\n                 pretrained_model=None,\n                 dim=None\n                ):\n        super(MultiOutputsModel, self).__init__()\n        self.name = \"MultiOutputsModel\"\n        self.num_features = num_features\n        self.dim = dim\n        self.num_outputs = num_outputs\n        \n        if pretrained_model:\n            self.embedding = pretrained_model.embedding\n            self.saint = pretrained_model.saint\n            \n            self.each_dim_models =  nn.ModuleList([\n                EachSAINTDimModel(\n                    num_features=num_features,\n                    num_outputs=num_outputs,\n                    units=units,\n                    units_sub=units_sub,\n                    dim=dim\n                )\n                for _ in range(dim)\n            ])\n            \n            self.last_output_layers = nn.ModuleList([\n                SigmoidLayer(dim, 1) for _ in range(num_outputs)\n            ])\n        \n    def register_pretrained_model(self, pretrained_model):\n        for param in pretrained_model.parameters():\n            param.requires_grad = False\n        self.__init__(\n            num_features=pretrained_model.embedding.num_features,\n            dim=pretrained_model.dim,\n            num_outputs=self.num_outputs,\n            pretrained_model=pretrained_model,\n        )\n        \n    def forward(self, x, x_mask):\n        x = self.embedding(x.clone(), x_mask.clone())\n        x = self.saint(x)\n        #x = torch.mean(x, dim=1) #x[:,0,:]\n\n        dim_outputs = []\n        for i_dim, l in enumerate(self.each_dim_models):\n            dim_outputs.append(l(x[:,:,i_dim].clone()))\n        x = torch.stack(dim_outputs)\n        x = torch.transpose(x, 0, 1)\n        \n        last_outputs = []\n        for i_output, l in enumerate(self.last_output_layers):\n            last_outputs.append(l(x[:,:,i_output].clone()))\n            \n        x = torch.concat(last_outputs, dim=1)\n        return x\n\nclass EachSAINTDimModel(nn.Module):\n    def __init__(self, num_features=10, num_outputs=10,\n                 units=[128, 256, 128, 64, 32],\n                 units_sub=[64, 16, 32, 16, 1],\n                 dim=None\n                ):\n        super(EachSAINTDimModel, self).__init__()\n        self.num_features = num_features\n        self.dim = dim\n        self.units = [self.num_features] + units\n        self.units_sub = [self.units[-1]] + units_sub\n        \n        self.l1_bn = nn.BatchNorm1d(self.units[0])\n        self.l1 = nn.Linear(self.units[0], self.units[1])\n        nn.init.xavier_normal_(self.l1.weight)\n        \n        self.ls = nn.ModuleList([\n            LeakyReLULayer(self.units[i+1], self.units[i+2])\n            if i%2 == 1 else ResidualBlock(self.units[i+1], self.units[i+2])\n            for i in range(len(self.units)-2)\n        ])\n        self.num_outputs = num_outputs\n        \n        self.subs = nn.ModuleList([\n            MultiOutputsModelSub(self.units[-1])\n            for _ in range(num_outputs)\n        ])\n        \n    def forward(self, x):\n        x = F.leaky_relu(self.l1(self.l1_bn(x)))\n        x = F.dropout(x, .1, training=self.training)\n        for l in self.ls:\n            x = l(x)\n\n        res = []\n        for sub in self.subs:\n            res.append(sub(x.clone()))            \n        x = torch.concat(res, dim=1)\n        \n        return x\n    \nclass MultiOutputsModelSub(nn.Module):\n    def __init__(self, num_inputs, units_sub=[64, 16, 32, 16, 1]):\n        super(MultiOutputsModelSub, self).__init__()\n        self.geglu_layer = GeGLULayer(num_inputs, units_sub[0])\n        self.units_sub = units_sub\n        self.ls_sub =  nn.ModuleList([\n            LeakyReLULayer(self.units_sub[i], self.units_sub[i+1])\n            if i < len(self.units_sub)-1 else\n            SigmoidLayer(self.units_sub[i], self.units_sub[i+1])\n            for i in range(len(self.units_sub)-1)\n        ])\n                \n    def forward(self, x):\n        x = self.geglu_layer(x)\n        for i, l in enumerate(self.ls_sub):\n            if i+1 < len(self.units_sub)-1:\n                x = l(x)\n            else:\n                x = F.dropout(x, .2, training=self.training)\n                x = l(x)\n        return x\n\nclass LeakyReLULayer(nn.Module):\n    def __init__(self, num_inputs, num_outputs):\n        super(LeakyReLULayer, self).__init__()\n        self.bn = nn.BatchNorm1d(num_inputs)\n        self.linear = nn.Linear(num_inputs, num_outputs)\n                \n    def forward(self, x):\n        x = self.linear(self.bn(x))\n        x = F.leaky_relu(x)\n        return x\n\nclass SigmoidLayer(nn.Module):\n    def __init__(self, num_inputs, num_outputs):\n        super(SigmoidLayer, self).__init__()\n        self.bn = nn.BatchNorm1d(num_inputs)\n        self.linear = nn.Linear(num_inputs, num_outputs)\n                \n    def forward(self, x):\n        x = self.linear(self.bn(x))\n        x = F.sigmoid(x)\n        return x\n\nclass ResidualBlock(nn.Module):\n    def __init__(self, num_inputs, num_outputs):\n        super(ResidualBlock, self).__init__()\n        self.layer1 = LeakyReLULayer(num_inputs, 32)\n        self.layer2 = LeakyReLULayer(32, 16)\n        self.layer3 = LeakyReLULayer(16, num_inputs)\n        self.geglu = GeGLULayer(num_inputs, num_outputs)\n                \n    def forward(self, x):\n        x_mlp = self.layer1(x.clone())\n        x_mlp = self.layer2(x_mlp)\n        x += self.layer3(x_mlp)\n        x = self.geglu(x)\n        return x\n    \nclass GeGLULayer(nn.Module):\n    def __init__(self, num_inputs, num_outputs):\n        super(GeGLULayer, self).__init__()\n        self.bn = nn.BatchNorm1d(num_inputs)\n        self.linear = nn.Linear(num_inputs, num_outputs*2)\n        self.geglu = GEGLU()\n                \n    def forward(self, x):\n        x = self.linear(self.bn(x))\n        x = self.geglu(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.043296Z","iopub.execute_input":"2023-05-21T22:21:45.043787Z","iopub.status.idle":"2023-05-21T22:21:45.070534Z","shell.execute_reply.started":"2023-05-21T22:21:45.043757Z","shell.execute_reply":"2023-05-21T22:21:45.069440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.4. SAINT","metadata":{}},{"cell_type":"code","source":"class MLP(nn.Module):\n    def __init__(self, three_dim):\n        super(MLP, self).__init__()\n        self.layers = nn.Sequential(\n            nn.Linear(three_dim[0], three_dim[1]),\n            nn.ReLU(),\n            nn.Linear(three_dim[1], three_dim[2])\n        )\n        \n    def forward(self, x):\n        if len(x.shape)==1:\n            x = x.view(x.size(0), -1)\n        x = self.layers(x)\n        return x\n\nclass EmbeddingLayer(nn.Module):\n    def __init__(self, num_features, dim=32):\n        super(EmbeddingLayer, self).__init__()\n        self.dim = dim\n        self.num_features = num_features\n        self.MLPs = nn.ModuleList([MLP([1, 100, dim]) for _ in range(num_features)])\n        mask_offset = F.pad(\n            torch.Tensor(num_features).fill_(2).type(torch.int8),\n            (1,0),\n            value=0\n        ).cumsum(dim=-1)[:-1].to(CFG.DEVICE)\n        self.register_buffer('mask_offset', mask_offset)\n        self.embedding_mask = nn.Embedding(num_features*2, dim)\n        \n    def forward(self, x, x_mask):\n        x_enc = torch.empty(*x.shape, self.dim).to(CFG.DEVICE)\n        for i in range(self.num_features):\n            x_enc[:,i,:] = self.MLPs[i](x[:,i])\n        x_enc[x_mask==0] = self.embedding_mask(x_mask+self.mask_offset.type_as(x_mask))[x_mask==0]\n        return x_enc\n    \nclass SAINT(nn.Module):\n    def __init__(self, num_features, dim=32, heads=8, dim_head=16, attention_dropout=.1, ff_dropout=.1):\n        super(SAINT, self).__init__()\n        self.msa = Attention(dim, heads, dim_head, attention_dropout)\n        self.ff1 = FeedForward(dim, dropout=ff_dropout)\n        self.misa = Attention(dim*num_features, heads, 64, attention_dropout)\n        self.ff2 = FeedForward(dim*num_features, dropout=ff_dropout)\n        self.norm1 = nn.LayerNorm(dim)\n        self.norm2 = nn.LayerNorm(dim)\n        self.norm3 = nn.LayerNorm(dim*num_features)\n        self.norm4 = nn.LayerNorm(dim*num_features)\n        \n    def forward(self, x):\n        shape = x.shape\n        \n        x = self.norm1(x)\n        x += self.msa(x.clone())\n        \n        x = self.norm2(x)\n        x += self.ff1(x.clone())\n        x = x.view(1, shape[0], -1)\n        \n        x = self.norm3(x)\n        x += self.misa(x.clone())\n        \n        x = self.norm4(x)\n        x += self.ff2(x.clone())\n        x = x.view(*shape)\n        \n        return x\n\nclass Attention(nn.Module):\n    def __init__(self, dim=32, heads=8, dim_head=16, dropout=0.1):\n        super().__init__()\n        inner_dim = dim_head * heads\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n        self.reshaping = lambda x, shape: torch.permute(x, (0,2,1)).reshape(shape[0], heads, shape[1], -1)\n        self.to_qkv = nn.Linear(dim, inner_dim*3, bias=False)\n        self.to_out = nn.Linear(inner_dim, dim)\n\n    def forward(self, x):\n        q, k, v = self.to_qkv(x).chunk(3, dim =-1)\n        shape = q.shape\n        q, k, v = self.reshaping(q, shape), self.reshaping(k, shape), self.reshaping(v, shape)\n        sim = torch.einsum('b h i d, b h j d -> b h i j', (q,k)) * self.scale\n        attention = sim.softmax(dim=-1)\n        out = torch.einsum('b h i j, b h j d -> b h i d', attention, v)\n        out_shape = out.shape\n        out = out.view(out_shape[0], out_shape[2], -1)\n        out = self.to_out(out)\n        return out\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim=32, mult=4, dropout=.1):\n        super(FeedForward, self).__init__()\n        self.linear1 = nn.Linear(dim, dim*mult*2)\n        self.linear2 = nn.Linear(dim*mult, dim)\n        self.dropout = dropout\n        self.geglu = GEGLU()\n        \n    def forward(self, x):\n        x = self.linear1(x)\n        x = self.geglu(x)\n        x = F.dropout(x, self.dropout, training=self.training)\n        x = self.linear2(x)\n        return x\n    \nclass GEGLU(nn.Module):\n    def forward(self, x):\n        x, gates = x.chunk(2, dim=-1)\n        return x*F.gelu(gates)\n    \nclass Constrastive(nn.Module):\n    def __init__(self, num_features, dim=32, temperature=0.7):\n        super(Constrastive, self).__init__()\n        units = [\n            dim*num_features,\n            6*dim*num_features//5,\n            dim*num_features//2\n        ]\n        self.projection_head_true = MLP(units)\n        self.projection_head_false = MLP(units)\n        self.reshaping = lambda x: (x / x.norm(dim=-1, keepdim=True)).flatten(1,2)\n        self.temperature = temperature\n        \n    def forward(self, x_true, x_false):\n        x_true, x_false = self.reshaping(x_true), self.reshaping(x_false)\n        x_true = F.normalize(self.projection_head_true(x_true), dim=-1).flatten(1)\n        x_false = F.normalize(self.projection_head_false(x_false), dim=-1).flatten(1)  \n        logits = x_true @ x_false.t() / self.temperature\n        logits_ =  x_false @ x_true.t() / self.temperature\n        targets = torch.arange(logits.size(0)).to(CFG.DEVICE)\n        loss = F.cross_entropy(logits, targets, reduction=\"mean\")\n        loss_ = F.cross_entropy(logits_, targets, reduction=\"mean\")\n        loss = (loss + loss_) / 2\n        return loss\n    \nclass Denoising(nn.Module):\n    def __init__(self, num_features, dim=32):\n        super(Denoising, self).__init__()\n        self.num_features = num_features\n        self.MLPs = nn.ModuleList([\n            MLP([dim, dim*5, 1])\n            for i in range(num_features)\n        ])\n        \n    def forward(self, x, x_original):\n        x = [\n            self.MLPs[i](x[:,i,:])\n            for i in range(1, self.num_features) # except for CLS\n        ]\n        x = torch.cat(x,dim=1)\n        loss = F.mse_loss(x, x_original[:,1:], reduction=\"mean\")\n        return loss","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.071705Z","iopub.execute_input":"2023-05-21T22:21:45.072148Z","iopub.status.idle":"2023-05-21T22:21:45.101875Z","shell.execute_reply.started":"2023-05-21T22:21:45.072119Z","shell.execute_reply":"2023-05-21T22:21:45.100059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.5. Pretraining Class","metadata":{}},{"cell_type":"code","source":"class PretrainingSAINT(nn.Module):\n    def __init__(self, num_features, dim=32):\n        super(PretrainingSAINT, self).__init__()\n        self.dim = dim\n        self.embedding = EmbeddingLayer(num_features, dim)\n        self.saint = SAINT(num_features, dim)\n        self.contrastive = Constrastive(num_features, dim)\n        self.denoising = Denoising(num_features, dim)\n    \n    def cutmix(self, x, m=.1):\n        x = x.clone()\n        x_shuffle = x[torch.randperm(x.shape[0]),:]\n        random_choice = torch.from_numpy(np.random.choice(2,(x.shape),p=[m,1-m]))\n        x[random_choice==0] = x_shuffle[random_choice==0]\n        return x\n\n    def mixup(self, x_enc, alpha=.3):\n        index = torch.randperm(x_enc.shape[0])\n        x_enc = alpha*x_enc + (1-alpha)*x_enc[index, :]\n        return x_enc\n    \n    def forward(self, x, x_mask):\n        x_true = self.embedding(x.clone(), x_mask.clone())\n        x_true = self.saint(x_true)\n        \n        x_false = self.cutmix(x.clone())\n        x_false = self.embedding(x_false, x_mask.clone())\n        x_false = self.mixup(x_false)\n        x_false = self.saint(x_false)\n        \n        contrastive_loss = self.contrastive(x_true.clone(), x_false.clone())\n        denoising_loss = self.denoising(x_false.clone(), x.clone())\n        \n        return contrastive_loss, denoising_loss","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.103674Z","iopub.execute_input":"2023-05-21T22:21:45.104038Z","iopub.status.idle":"2023-05-21T22:21:45.120237Z","shell.execute_reply.started":"2023-05-21T22:21:45.104010Z","shell.execute_reply":"2023-05-21T22:21:45.118579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.6. Pretraining Function","metadata":{}},{"cell_type":"code","source":"def pretraining(train_loader, level_group, epochs=CFG.EPOCHS, lambda1=.5, lambda2=10, omit=CFG.PRETRAINING_OMIT):\n    num_features = train_loader.dataset.tensors[0].shape[1]\n    model = PretrainingSAINT(num_features).to(CFG.DEVICE)\n    checkpoint_path = f\"{CFG.CHECKPOINT_PATH}/pretrain_params_{level_group}.pth\"\n    \n    if omit:\n        model.load_state_dict(torch.load(checkpoint_path))\n    else:\n        optimizer = torch.optim.AdamW(model.parameters(),lr=0.0001)\n        losses = [10**10]\n        model.train()\n\n        print(\"Pretraining is starting...\") \n        for epoch in tqdm(range(epochs)):\n            loss_sum = 0.0\n            contrastive_loss_sum = 0.0\n            denoising_loss_sum = 0.0\n            for i, (X, X_mask, _) in enumerate(train_loader):\n                X, X_mask = X.to(CFG.DEVICE), X_mask.to(CFG.DEVICE)\n                contrastive_loss, denoising_loss = model(X, X_mask)\n                loss = lambda1 * contrastive_loss + lambda2 * denoising_loss\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                contrastive_loss_sum += (lambda1 * contrastive_loss).item()\n                denoising_loss_sum += (lambda2 * denoising_loss).item()\n                loss_sum += loss.item()\n\n            epoch_loss = loss_sum/i\n            if epoch_loss < min(losses):\n                best_epoch = epoch\n                torch.save(model.state_dict(), checkpoint_path)\n            losses.append(epoch_loss)\n\n            print(\n                f\"Epoch {epoch+1}/{epochs}: loss {epoch_loss: .4f} (contrastive_loss {contrastive_loss_sum/i: .4f}, denoising_loss {denoising_loss_sum/i: .4f})\"\n            )\n\n        model.load_state_dict(torch.load(checkpoint_path))\n        print(f\"Pretraining has finished.\\nBest Epoch is {best_epoch} with loss {min(losses)} !\")\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.122026Z","iopub.execute_input":"2023-05-21T22:21:45.122536Z","iopub.status.idle":"2023-05-21T22:21:45.137448Z","shell.execute_reply.started":"2023-05-21T22:21:45.122503Z","shell.execute_reply":"2023-05-21T22:21:45.136297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.7. Other Functions","metadata":{}},{"cell_type":"code","source":"def calc_weight(data_loader):\n    count1 = torch.sum(data_loader.dataset.tensors[2], dim=0)\n    count_all = data_loader.dataset.tensors[2].shape[0]\n    count0 = count_all - count1\n    weight = count0 / count1\n    \n    del count1, count_all, count0\n    gc.collect()\n    \n    return weight\n\ndef calc_matrix(x):\n    x = torch.where((x > CFG.THRESHOLD), 1, -1)\n    x_t = x.T\n    length = x_t.shape[0]\n    res = []\n\n    for i in range(length):\n        res.append(x *  x.T[i].unsqueeze(-1))\n    res = torch.concat(res, dim=1)\n    res = res.view(-1, length, length)\n    res = torch.where(res==-1, 0.0, 1.0)\n    res = torch.mean(res, axis=0).to(CFG.DEVICE)\n        \n    return res","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.138748Z","iopub.execute_input":"2023-05-21T22:21:45.139769Z","iopub.status.idle":"2023-05-21T22:21:45.155927Z","shell.execute_reply.started":"2023-05-21T22:21:45.139724Z","shell.execute_reply":"2023-05-21T22:21:45.154697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.8. Training Function","metadata":{}},{"cell_type":"code","source":"def training(train_loader, val_loader, model_class, pretrained_model, omit=CFG.TRAINING_OMIT):\n    num_features = train_loader.dataset.tensors[0].shape[1]\n    num_outputs = val_loader.dataset.tensors[2].shape[1]\n    level_group = '0-4' if num_outputs == 3 else '5-12' if num_outputs == 10 else '13-22' if num_outputs == 5 else None\n    model = model_class(num_features, num_outputs)\n    model.register_pretrained_model(pretrained_model)\n    model = model.to(CFG.DEVICE)\n    checkpoint_path = f\"{CFG.CHECKPOINT_PATH}/{model.name}_bestmodel{'_'+level_group if level_group else ''}{'_all' if CFG.PREDICT_ALL else ''}.pth\"\n    \n    if omit:\n        checkpoint_path = f\"{CFG.CHECKPOINT_PATH}/{model.name}_bestmodel{'_'+level_group if level_group else ''}{'_all' if CFG.PREDICT_ALL else ''}.pth\"\n        model.load_state_dict(torch.load(checkpoint_path))\n        with open(f\"{CFG.CHECKPOINT_PATH}/{model.name}_thresholds{'_all' if CFG.PREDICT_ALL else ''}.pickle\", \"rb\") as f:\n            best_thresholds = pickle.load(f)[level_group]\n    else:\n        print(\"Training is starting...\") \n        \n        weight_ratio = calc_weight(train_loader).to(CFG.DEVICE)\n        early_stopping_count = 0\n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG.LEARNING_RATE)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20, eta_min=.005)\n        loss_trains, loss_vals, f1_trains, f1_vals = [], [], [], []\n        loss_func = nn.BCELoss(reduction='none')\n\n        for epoch in range(CFG.EPOCHS):\n            #\"\"\"\n            #train\n            #\"\"\"\n            model.train()\n            loss_train = 0\n            preds, true_values = [], []\n            for i, (x, x_mask, t) in enumerate(train_loader):\n                x, x_mask, t = x.to(CFG.DEVICE), x_mask.to(CFG.DEVICE), t.to(CFG.DEVICE)\n                y = model(x, x_mask)\n                preds += y\n                true_values += t\n                \n                weight = (lambda w: torch.where(w==0, 1, w))(t*weight_ratio)\n                loss = torch.mean(loss_func(y, t)*weight) / num_outputs\n                loss_train += loss.item()\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n            scheduler.step()\n            loss_train /= i\n            loss_trains.append(loss_train)\n            preds = torch.stack(preds).view(-1)\n            true_values = torch.stack(true_values).view(-1).detach().cpu().numpy()\n            f1_train = f1_score(true_values, (preds > CFG.THRESHOLD).int().detach().cpu().numpy(), average=\"macro\")\n            f1_trains.append(f1_train)\n\n            #\"\"\"\n            #validation\n            #\"\"\"\n            model.eval()\n            loss_val = 0\n            preds, true_values = [], []\n            for i, (x, x_mask, t) in enumerate(val_loader):\n                x, x_mask, t = x.to(CFG.DEVICE), x_mask.to(CFG.DEVICE), t.to(CFG.DEVICE)\n                y = model(x, x_mask)\n                preds += y\n                true_values += t\n\n                weight = (lambda w: torch.where(w==0, 1, w))(t*weight_ratio)\n                loss = torch.mean(loss_func(y, t)*weight) / num_outputs\n                loss_val += loss.item()\n\n            loss_val /= i\n            loss_vals.append(loss_val)\n            preds = torch.stack(preds).view(-1)\n            true_values = torch.stack(true_values).view(-1).detach().cpu().numpy()\n            f1_val = f1_score(true_values, (preds > CFG.THRESHOLD).int().detach().cpu().numpy(), average=\"macro\")\n            f1_vals.append(f1_val)\n\n            print(f\"Epoch {epoch+1}/{CFG.EPOCHS}: loss_train {loss_train:.4f}, loss_val {loss_val:.4f}, f1_train {f1_train:.4f} f1_val - {f1_val:.4f}\")\n\n            #\"\"\"\n            #early stopping\n            #\"\"\"\n            if epoch+1 < CFG.WARM_UP: # warm up: do not stop early\n                best_score_before = None\n                continue\n            elif epoch+1 == CFG.WARM_UP:\n                best_score_before = min(loss_vals[CFG.WARM_UP-1:])\n            else:\n                best_score_before = min(loss_vals[CFG.WARM_UP-1:-1])\n\n            if (min(loss_vals[CFG.WARM_UP-1:]) == loss_val):# and (best_score_before != loss_val):\n                early_stopping_count = 0\n                torch.save(model.state_dict(), checkpoint_path)\n            else:\n                early_stopping_count += 1\n\n            if early_stopping_count == CFG.EARLY_STOPPING_ROUNDS:\n                best_epoch = loss_vals.index(min(loss_vals[CFG.WARM_UP-1:]))\n                early_stopping_message =\\\n                f\"Best Epoch: {best_epoch+1}\"\\\n                + f\", TrainLoss: {loss_trains[best_epoch]:.4f}\" + f\", ValLoss: {loss_vals[best_epoch]:.4f}\"\\\n                + f\", TrainF1: {f1_trains[best_epoch]:.4f}\" + f\", ValF1: {f1_vals[best_epoch]:.4f}\"\n                print(\"\\n!!!Early Stopping !!!\")\n                print(early_stopping_message)\n                try:\n                    model.load_state_dict(torch.load(checkpoint_path))\n                except Exception as e:\n                    print(e)\n                break\n            else:\n                best_epoch = epoch + 1\n\n        torch.save(model.state_dict(), checkpoint_path)\n\n        #\"\"\"\n        #checking train result\n        #\"\"\"\n        result = pd.DataFrame(\n            {\n                \"value\": loss_trains + loss_vals + f1_trains + f1_vals,\n                \"metric\": [\"loss\"] * (len(loss_trains)+len(loss_vals)) + [\"f1_score\"] * (len(loss_trains)+len(loss_vals)),\n                \"epoch\": ([i+1 for i in range(len(loss_trains))] + [i+1 for i in range(len(loss_vals))])*2,\n                \"train/val\": ([\"train\" for _ in range(len(loss_trains))] + [\"val\" for _ in range(len(loss_vals))])*2\n            }\n        )\n        px.line(result.query(\"metric=='loss'\"), x=\"epoch\", y=\"value\",\n                color=\"train/val\", height=250, title=\"loss\").show()\n        px.line(result.query(\"metric=='f1_score'\"), x=\"epoch\", y=\"value\",\n                color=\"train/val\", height=250, title=\"f1_score\").show()\n\n        #\"\"\"\n        #exploring best threshold\n        #\"\"\"\n        preds, true_values = [], []\n        for i, (x, x_mask, t) in enumerate(val_loader):\n            x, x_mask, t = x.to(CFG.DEVICE), x_mask.to(CFG.DEVICE), t.to(CFG.DEVICE)\n            y = model(x, x_mask).view(-1)\n            preds += y\n            true_values += t\n\n        preds = torch.stack(preds).view(-1).detach().cpu().numpy()\n        true_values = torch.stack(true_values).view(-1).detach().cpu().numpy()\n        px.box(x=true_values, y=preds, points=\"all\").show()\n\n        f1s = []\n        for th in range(0, 101, 1):\n            f1 = f1_score(true_values, (preds > (th/100)).astype(int), average=\"macro\")\n            f1s.append(f1)\n        f1s = pd.DataFrame({\"threshold\":[i/100 for i in range(0, 101, 1)], \"f1\":f1s}).set_index(\"threshold\")\n        best_threshold = f1s[f1s.f1.apply(lambda x: x//.01)==f1s.f1.apply(lambda x: x//.01).max()].index.max()\n        px.line(f1s, title=f\"Best Threshold is {best_threshold}.\").show()\n\n        #\"\"\"\n        #fine tuning with validation data\n        #\"\"\"\n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG.LEARNING_RATE*.1)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20*.5, eta_min=.005)\n\n        loss_trains, f1_trains = [], []\n        loss_func = nn.BCELoss(reduction='none')\n        model.train()\n\n        epochs = max(int(best_epoch*.1), 20)\n        for epoch in range(epochs):\n            loss_train = 0\n            preds, true_values = [], []\n            for i, (x, x_mask, t) in enumerate(val_loader):\n                x, x_mask, t = x.to(CFG.DEVICE), x_mask.to(CFG.DEVICE), t.to(CFG.DEVICE)\n                y = model(x, x_mask)\n                preds += y\n                true_values += t\n\n                weight = (lambda w: torch.where(w==0, 1, w))(t*weight_ratio)\n                loss = torch.mean(loss_func(y, t)*weight) / num_outputs\n                loss_train += loss.item()\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n            scheduler.step()\n            loss_train /= i\n            loss_trains.append(loss_train)\n            preds = torch.stack(preds).view(-1)\n            true_values = torch.stack(true_values).view(-1).detach().cpu().numpy()\n            f1_train = f1_score(true_values, (preds > best_threshold).int().detach().cpu().numpy(), average=\"macro\")\n            f1_trains.append(f1_train)\n\n            print(f\"Epoch {epoch+1}: loss_train {loss_train:.4f}, f1_train {f1_train:.4f}\")\n\n            if epoch == 0:\n                best_score_before = loss_train\n                if f1_train != 0:\n                    torch.save(model.state_dict(), checkpoint_path)                  \n            else:\n                best_score_before = min(loss_trains)\n\n            if (min(loss_trains) == loss_train) and (best_score_before != loss_train):\n                if f1_train != 0:\n                    torch.save(model.state_dict(), checkpoint_path)                  \n\n        model.load_state_dict(torch.load(checkpoint_path))\n\n        #\"\"\"\n        #checking train result\n        #\"\"\"\n        result = pd.DataFrame(\n            {\n                \"value\": loss_trains + f1_trains,\n                \"metric\": [\"loss\"] * len(loss_trains) + [\"f1_score\"] * len(loss_trains),\n                \"epoch\": [i+1 for i in range(len(loss_trains))]*2,\n            }\n        )\n        px.line(result.query(\"metric=='loss'\"), x=\"epoch\", y=\"value\",\n                height=250, title=\"FineTuning: loss\").show()\n        px.line(result.query(\"metric=='f1_score'\"), x=\"epoch\", y=\"value\",\n                height=250, title=\"FineTuning: f1_score\").show()\n\n    return model, best_threshold","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.157499Z","iopub.execute_input":"2023-05-21T22:21:45.158411Z","iopub.status.idle":"2023-05-21T22:21:45.194219Z","shell.execute_reply.started":"2023-05-21T22:21:45.158375Z","shell.execute_reply":"2023-05-21T22:21:45.193437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.9. Predicting Function","metadata":{}},{"cell_type":"code","source":"def predict(test, model, level_group=None, threshold=None, question=None):\n    \n    if CFG.SUBMISSION_MODE:\n        test_idx = test[0].index.unique()\n        if question:\n            cols = [question]\n        else:\n            if level_group and model.num_outputs != 18:\n                cols = lq_dict[level_group]\n            else:\n                cols = questions\n    elif level_group:\n        #test = test.query(f\"level_group=='{level_group}'\")\n        test_idx = [i[0] for i in test[0].index.unique()]\n        if threshold is None:\n            threshold = CFG.THRESHOLD\n        \n        if question:\n            cols = [question]\n        else:\n            if model.num_outputs == 18:\n                cols = questions\n            else:\n                cols = lq_dict[level_group]\n    else:\n        test_idx = test[0].index.unique()\n        cols = questions\n        if threshold is None:\n            threshold = CFG.THRESHOLD\n    \n    test_loader = create_loader_test(test[0], test[1], level_group)\n    preds = []\n    model.eval()\n\n    for x, x_mask in test_loader:\n        y = model(x.to(CFG.DEVICE), x_mask.to(CFG.DEVICE))\n        preds += y\n    preds = torch.stack(preds)\n    \n    submission = pd.DataFrame(\n        (preds > threshold).int().detach().cpu().numpy(),\n        columns = cols,\n    )\n    \n    submission = submission[cols]\n    submission[\"session_id\"] = test_idx\n    submission[\"session_id\"] = submission[\"session_id\"].astype(str)\n    submission = pd.melt(submission, id_vars=\"session_id\", var_name=\"question\", value_name=\"correct\")\n    submission[\"session_id\"] += \"_\" + submission.question\n    submission = submission[[\"session_id\", \"correct\"]]\n    \n    del preds, test_idx\n    gc.collect()\n    \n    return submission","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.196959Z","iopub.execute_input":"2023-05-21T22:21:45.197652Z","iopub.status.idle":"2023-05-21T22:21:45.213367Z","shell.execute_reply.started":"2023-05-21T22:21:45.197596Z","shell.execute_reply":"2023-05-21T22:21:45.212387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2.10. LightGBM for feature elimination","metadata":{}},{"cell_type":"code","source":"def lgb_training(X, y, params=None, train_rate=.9):\n    \n    lgb_models = {}\n    lgb_preds = {}\n    lgb_scores = {}\n    fi_df = []\n    \n    if params is None:\n        params = dict(\n            objective='binary',\n            metric='binary_logloss',\n            verbosity=1,\n            early_stopping_round=100,\n            random_state=CFG.SEED,\n            is_unbalance=True,\n            num_iterations=2000,\n            num_leaves=500,\n            lambda_l1=.2,\n            lambda_l2=.2,\n            bagging_freq=10,\n            bagging_seed=CFG.SEED,\n            force_col_wise=True\n        )\n    \n    for level_group in lq_dict.keys():\n        for q in lq_dict[level_group]:\n            X_train, X_val, y_train, y_val = tts(\n                X[level_group][0],\n                y[q],\n                train_size=train_rate,\n                random_state=CFG.SEED,\n                stratify=y[q]\n            )\n            lgb_train = lgb.Dataset(X_train, y_train)\n            lgb_val = lgb.Dataset(X_val, y_val)\n\n            del X_train, y_train\n            gc.collect()\n\n            lgb_model = lgb.train(\n                params=params,\n                train_set=lgb_train,\n                num_boost_round=1000,\n                valid_sets=(lgb_train, lgb_val),\n                callbacks=None,\n                verbose_eval=100\n            )\n\n            lgb_models[q] = lgb_model\n            lgb_preds[q] = lgb_model.predict(X_val)\n            lgb_scores[q] = f1_score(y_val,\n                                     [int(p > CFG.THRESHOLD) for p in lgb_preds[q]],\n                                     average=\"macro\")\n\n            fi_df.append(\n                pd.DataFrame(\n                    lgb_model.feature_importance(importance_type=\"gain\"),\n                    index=X[level_group][0].columns,\n                    columns=[q]\n                )\n            )\n\n    fi_df = pd.concat(fi_df, axis=1)\n    \n    feature_elimination_dict = {}\n    for level_group in lq_dict.keys():\n        fe_cols = fi_df[lq_dict[level_group]].mean(axis=1).sort_values(ascending=False)\n        fe_cols = fe_cols.iloc[:30].index.tolist()\n        feature_elimination_dict[level_group] = fe_cols\n    \n    return lgb_models, lgb_preds, lgb_scores, fi_df, feature_elimination_dict","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.214587Z","iopub.execute_input":"2023-05-21T22:21:45.214876Z","iopub.status.idle":"2023-05-21T22:21:45.230750Z","shell.execute_reply.started":"2023-05-21T22:21:45.214850Z","shell.execute_reply":"2023-05-21T22:21:45.229583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Execution","metadata":{}},{"cell_type":"markdown","source":"## 2.1. Preparing Objects","metadata":{}},{"cell_type":"code","source":"%%time\n\nif CFG.SUBMISSION_MODE:\n    with open(f\"{CFG.CHECKPOINT_PATH}/dp.pickle\", \"rb\") as f:\n        dp = pickle.load(f)\n\n    models = {}    \n    for level_group in lq_dict.keys():\n        num_features = dp.num_features\n        pretrained_model = PretrainingSAINT(num_features).to(CFG.DEVICE)\n        model = MultiOutputsModel(num_features, 18 if CFG.PREDICT_ALL else len(lq_dict[level_group]))\n        model.register_pretrained_model(pretrained_model)\n        checkpoint_path = f\"{CFG.CHECKPOINT_PATH}/{model.name}_bestmodel{'_'+level_group if level_group else ''}{'_all' if CFG.PREDICT_ALL else ''}.pth\"\n        model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu')))\n        models[level_group] = model\n        \n    with open(f\"{CFG.CHECKPOINT_PATH}/{model.name}_thresholds{'_all' if CFG.PREDICT_ALL else ''}.pickle\", \"rb\") as f:\n        thresholds = pickle.load(f)\n        \nelse:\n    if CFG.RESUME:\n        with open(f\"{CFG.CHECKPOINT_PATH}/dp.pickle\", \"rb\") as f:\n            dp = pickle.load(f)\n        \n        with open(f\"{CFG.CHECKPOINT_PATH}/train.pickle\", \"rb\") as f:\n            train = pickle.load(f)\n        \n        test = dp.transform(f\"{CFG.INPUT}/test.csv\")\n        \n        with open(f\"{CFG.CHECKPOINT_PATH}/cooccurence_rate.pickle\", \"rb\") as f:\n            cooccurence_rate = pickle.load(f)\n        \n        train_labels = pd.read_csv(f\"{CFG.INPUT}/train_labels.csv\", engine='python')\n        train_labels[[\"session_id\", \"question\"]] = train_labels.session_id.str.split(\"_\", expand=True)\n        train_labels = train_labels[[\"session_id\", \"question\", \"correct\"]]\n        train_labels[\"session_id\"] = train_labels[\"session_id\"].astype(int)\n        train_labels = train_labels.pivot(index=\"session_id\", columns=\"question\", values=\"correct\").sort_index()\n        train_labels = train_labels.reindex(columns=questions)\n    else:\n        dp = DataProcessing()\n\n        train = dp.fit_transform(f\"{CFG.INPUT}/train.csv\")\n        test = dp.transform(f\"{CFG.INPUT}/test.csv\")\n\n        if CFG.CHECKPOINT:\n            with open(f\"{CFG.CHECKPOINT_PATH}/dp.pickle\", \"wb\") as f:\n                pickle.dump(dp, f)\n            with open(f\"{CFG.CHECKPOINT_PATH}/train.pickle\", \"wb\") as f:\n                pickle.dump(train, f)\n\n        del dp\n        gc.collect()\n\n        train_labels = pd.read_csv(f\"{CFG.INPUT}/train_labels.csv\", engine='python')\n        train_labels[[\"session_id\", \"question\"]] = train_labels.session_id.str.split(\"_\", expand=True)\n        train_labels = train_labels[[\"session_id\", \"question\", \"correct\"]]\n        train_labels[\"session_id\"] = train_labels[\"session_id\"].astype(int)\n        train_labels = train_labels.pivot(index=\"session_id\", columns=\"question\", values=\"correct\").sort_index()\n        train_labels = train_labels.reindex(columns=questions)\n\n        check = train_labels.copy()\n        res = {}\n        for session_id, row in check[questions].iterrows():\n            tmp = []\n            for i1, q1 in enumerate(row):\n                for i2, q2 in enumerate(row):\n                    tmp.append((questions[i1], questions[i2], int(q1==q2)))\n            tmp = pd.DataFrame(tmp)\n            tmp = pd.pivot(data=tmp, index=0, columns=1, values=2)\n            res[session_id] = np.array(tmp.reindex(index=questions, columns=questions))\n\n        cooccurence_rate = np.mean(list(res.values()), axis=0)\n        cooccurence_rate = pd.DataFrame(data=cooccurence_rate, columns=questions)\n        cooccurence_rate.index = questions\n\n        if CFG.CHECKPOINT:\n            with open(f\"{CFG.CHECKPOINT_PATH}/cooccurence_rate.pickle\", \"wb\") as f:\n                pickle.dump(cooccurence_rate, f)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:45.232123Z","iopub.execute_input":"2023-05-21T22:21:45.232463Z","iopub.status.idle":"2023-05-21T22:21:52.171908Z","shell.execute_reply.started":"2023-05-21T22:21:45.232427Z","shell.execute_reply":"2023-05-21T22:21:52.170774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2. Deep Learning","metadata":{}},{"cell_type":"code","source":"%%time\n\nif CFG.SUBMISSION_MODE:    \n    env = jo_wilder_310.make_env()\n    iter_test = env.iter_test()\n\n    storage = []\n    for x, sample in iter_test:\n        level_group = x.level_group.values[0]\n        if level_group == \"0-4\":\n            storage.append(x)\n            x = dp.transform(data=x, decided_level_group=level_group)\n        elif level_group == \"5-12\":\n            storage.append(x)\n            x = pd.concat(storage, axis=0, ignore_index=True)\n            x.level_group = level_group\n            x = dp.transform(data=x, decided_level_group=level_group)\n        else: # \"13-22\"\n            x = pd.concat(storage + [x], axis=0, ignore_index=True)\n            x.level_group = level_group\n            x = dp.transform(data=x, decided_level_group=level_group)\n            storage = []\n        \n        submission = predict(x, models[level_group], level_group, thresholds[level_group])\n        submission = submission.set_index(\"session_id\").reindex(sample.session_id).reset_index()\n        env.predict(submission)                       \n\nelse:\n    thresholds = {}\n    for level_group in lq_dict.keys():\n        print(f\"\\nlevel_group {level_group} model...\")\n        \n        # create data loader\n        train_loader, val_loader = create_loader_train(\n                            train[level_group][0],\n                            train[level_group][1],\n                            train_labels,\n                            level_group,\n                            predict_all=CFG.PREDICT_ALL\n                        )\n        # pretraining\n        pretrained_model = pretraining(train_loader, level_group, epochs=50)\n            \n        # training\n        model, best_threshold = training(train_loader, val_loader, MultiOutputsModel, pretrained_model)\n            \n        # predicting\n        submission = predict(test[level_group], model, level_group, best_threshold)\n        display(submission)\n        submission.to_csv(\"submission.csv\", index=False)\n        \n        thresholds[level_group] = best_threshold\n        \n        torch.cuda.empty_cache()\n    \n    thresholds_path = f\"{CFG.CHECKPOINT_PATH}/{model.name}_thresholds{'_all' if CFG.PREDICT_ALL else ''}.pickle\"\n    with open(thresholds_path, \"wb\") as f:\n        pickle.dump(thresholds, f)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T22:21:52.172942Z","iopub.execute_input":"2023-05-21T22:21:52.173420Z","iopub.status.idle":"2023-05-21T22:22:24.486674Z","shell.execute_reply.started":"2023-05-21T22:21:52.173393Z","shell.execute_reply":"2023-05-21T22:22:24.484697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}