{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Neural Network Using Only Click Position","metadata":{}},{"cell_type":"markdown","source":"[Thanks to https://www.kaggle.com/code/chrisqiu/0-681-pytorch-using-only-1-column/notebook]\n\nWe can choose the column be 'room_coor_x' and 'room_coor_y'.\n\n- Here I used 1d-patch squence to transform the columns. And then use 'nn.embedding' to get the position. For example, you can divide a 9x9 image to 9 patches, each patches is 3x3, the patches are coded with number [1,2,3,4,5,6,7,8,9].\n- I designed the layer like the notebook mentioned before.\n- In games, click is an important actions. This feature is a little worser than 'elapsed time' with acc about 62-65 training with it alone, but it is obviously nonnegligible.","metadata":{}},{"cell_type":"markdown","source":"### Data Loading","metadata":{}},{"cell_type":"code","source":"data_dir=\"/home/yhchenmath/Dataset/Kaggle/predict-student-performance-from-game-play\"\n\nimport pandas as pd\nimport numpy as np\nimport os\nos.chdir(\"/import/home2/yhchenmath/Code/Kaggle/student_performance/\")\nimport torch\n\ndf = pd.read_csv(\n    os.path.join(data_dir, 'train.csv'),\n    usecols = ['session_id', 'level_group', 'room_coor_x', 'room_coor_y'],\n    low_memory = True\n)\n\n# seed everything\ntorch.manual_seed(101)\nnp.random.seed(101)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_height = df['room_coor_x'].max()\nmin_height = df['room_coor_x'].min()\nmax_width = df['room_coor_y'].max()\nmin_width = df['room_coor_y'].min()\n\nnum_px = 8\nnum_py = 6\n# Calculate patch index given a coordinate (x,y)\ndef get_patch_index(x, y, min_image_height, max_image_height, min_image_width, max_image_width, num_patches_height = 8, num_patches_width = 6):\n    if np.isnan(x):\n        return num_px * num_py + 1\n    patch_height = (max_image_height - min_image_height + 1) / num_patches_height\n    patch_width = (max_image_width - min_image_width + 1) / num_patches_width\n    patch_x = (x - min_image_height) // patch_height\n    patch_y = (y - min_image_width) // patch_width\n    patch_index = patch_x * num_patches_width + patch_y + 1\n    return patch_index\n\ngps = []\nfor _, session in df.groupby('session_id'):\n    for _, gp in session.groupby('level_group'):\n        gp['patch'] = gp.apply(lambda row : get_patch_index(row['room_coor_x'], row['room_coor_y'], min_height, max_height, min_width, max_width, num_px, num_py), axis = 1)\n        gps.append(gp)\n\nnum_patches = num_px * num_py + 2\ndf = pd.concat(gps)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = pd.read_csv(os.path.join(data_dir, 'train_labels.csv'))\n\nspilt =  targets.session_id.str.split(\"_\", expand = True)\ntargets['session_id'] = spilt[0].astype(np.int64)\ntargets['q'] = spilt[1].str.slice(1).astype(int)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pytorch requires the sequences inside a batch to have equal length, but different sessions have different lengths. That's why we have to trim the longer sequence and pad the shorter sequence.\n\nBut please notice that you don't have to do this. If you train a network with 1 batch_size, the conv1d can absolutely handle it. However, there are lots of drawbacks of using 1 sample per batch, such as long training time and bad gradient estimation.\n\nAnother thing to consider is how long per sequence. I choose 500 for simplicity. You could also try 95% quantile of every level_group.","metadata":{}},{"cell_type":"code","source":"def trim(X, trim_steps):\n    data_steps = X.shape[0]\n    if data_steps == trim_steps: return X\n\n    if data_steps < trim_steps:\n        shortage = trim_steps - data_steps\n        return np.pad(X, ((0,shortage),(0,0)),'constant')\n\n    start = int(np.random.random() * (data_steps - trim_steps))\n    return X[start:start+trim_steps]","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data into Level Group","metadata":{}},{"cell_type":"code","source":"GP1 = '0-4'\nGP2 = '5-12'\nGP3 = '13-22'\n\ndef group_session_id(df):\n    return np.stack([gp.correct.values for _,gp in df.groupby('session_id')])\n\nGROUP_DATA = {\n    GP1: {\n        'targets': group_session_id(targets[(targets.q >= 1) & (targets.q <= 3)]),\n        'pos_weight': 1/2\n    },\n    GP2: {\n        'targets': group_session_id(targets[(targets.q >= 4) & (targets.q <= 13)]),\n        'pos_weight': 1\n    },\n    GP3: {\n        'targets': group_session_id(targets[(targets.q >= 14) & (targets.q <= 18)]),\n        'pos_weight': 1/2\n    }\n}\n\nCLIK_POSITION_NUM = 500\n\nfor GP in [GP1, GP2, GP3]:\n    gdf = df[df.level_group == GP]\n\n    curr = np.zeros(shape = (gdf.session_id.nunique(), CLIK_POSITION_NUM, 1))\n\n    for i, (session_id, sdf) in enumerate(gdf.groupby('session_id')):\n        curr[i] = trim(sdf[[\"patch\"]].values, CLIK_POSITION_NUM)[None, ...]\n\n    GROUP_DATA[GP]['position'] = curr\n\nfor GP in GROUP_DATA.keys():\n    data, targets = GROUP_DATA[GP]['position'], GROUP_DATA[GP]['targets']\n    print(f'group {GP}')\n    print(f'Position shape: {data.shape}, target shape: {targets.shape}')\n    print()\n\nnum_patches = num_patches + 1 # For miss step : Coded as 0","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass PositionalEmb(nn.Module):\n    def __init__(self, num_patches, nembed, nfeatures):\n        super(PositionalEmb, self).__init__()\n        self.position_emb = nn.Embedding(num_patches, 2 * nembed)\n        # input: [Batch, CLIP_NUMS, 1]\n        # Out: [Batch, 1, 2 * nembed]\n        self.nfeatures = nfeatures\n        self.convs = nn.Sequential(\n            nn.Conv1d(2 * nembed, 64,  kernel_size = 5),\n            nn.BatchNorm1d(64),\n            nn.LeakyReLU(inplace = True),\n            nn.MaxPool1d(2),\n\n            nn.Conv1d(64, 128,  kernel_size = 5),\n            nn.BatchNorm1d(128),\n            nn.LeakyReLU(inplace = True),\n            nn.MaxPool1d(2),\n\n            nn.Conv1d(128, self.nfeatures,  kernel_size = 3),\n            nn.BatchNorm1d(self.nfeatures),\n            nn.LeakyReLU(inplace = True),\n        )\n\n        self.fcs = nn.Sequential(\n                # 2* because in the pooling layer we take mean AND the std.\n                nn.Linear(2*self.nfeatures, nembed),\n            )\n\n    def forward(self, positions):\n        positions = positions.squeeze(2)\n        e = self.position_emb(positions.long())\n        # [Batch, CLIP_NUMS, 2 * NEMBED]\n        e = torch.transpose(e, 1, 2)\n        # [Batch, 2 * NEMBED, CLIP_NUMS]\n        e = self.convs(e)\n        std = torch.std(e, dim = 2)\n        mean = torch.mean(e, dim = 2)\n        x = torch.cat([std, mean], dim = 1)\n        x = self.fcs(x)\n        return x\n\nclass PositionModel(nn.Module):\n    def __init__(self,nout, npatches, nembed, nfeatures):\n        super(PositionModel, self).__init__()\n        self.encoder = PositionalEmb(npatches, nembed, nfeatures)\n        self.clf = nn.Sequential(\n            nn.LeakyReLU(inplace = True),\n            nn.Dropout(0.2),\n\n            nn.Linear(nembed, nembed // 2),\n            nn.LeakyReLU(inplace = True),\n            nn.Dropout(0.2),\n\n            nn.Linear(nembed // 2, nout),\n        )\n        self.nout = nout\n        self.nembed = nembed\n        self.nfeatures = nfeatures\n\n    def forward(self, pos):\n        pos = self.encoder(pos)\n        l = self.clf(0 + pos)\n        return l","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model Config\nmodel_params = {}\nmodel_params[GP1] = dict(\n    nout = 3,\n    nembed = 64,\n    nfeatures = 64,\n)\nmodel_params[GP2] = dict(\n    nout = 10,\n    nembed = 64,\n    nfeatures = 64,\n)\nmodel_params[GP3] = dict(\n    nout = 5,\n    nembed = 64,\n    nfeatures = 64,\n)\n\n# test model\nfor GP in model_params:\n    model = PositionModel(**model_params[GP], npatches = num_patches)\n    print(f'group {GP}')\n    print(f'total params: {sum(p.numel() for p in model.parameters())}')\n    print(f'output shape: {model(torch.ones(32, 1000, 1).long()).shape}')\n    print()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Trainer","metadata":{}},{"cell_type":"code","source":"import torch.optim as optim\nfrom sklearn.metrics import f1_score\nfrom tqdm import tqdm\n\nos.environ['CUDA_VISIBLE_DEVICES'] = '8'\nDEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\nclass Trainer():\n    def __init__(self,\n                 model,\n                 save_path,\n                 pos_weight,\n                 num_labels,\n                 learning_rate\n                ):\n        self.model = model\n        self.optimizer = optim.Adam(self.model.parameters(), lr = learning_rate)\n        self.loss_fn = nn.BCEWithLogitsLoss(pos_weight = torch.ones(num_labels) * pos_weight).to(DEVICE)\n        self.save_path = save_path\n        self.num_labels = num_labels\n\n    def train_network(self, train_loader, test_loader):\n        EPOCHS = Config.EPOCHS\n        max_score =  0\n        losses = []\n        f1s = []\n\n        prev_score = 0\n        not_improve = 0\n\n        pbar = tqdm(range(1, EPOCHS+1))\n        for it in pbar:\n            lr = self.optimizer.param_groups[0]['lr']\n            loss = self.train_one_epoch(train_loader)\n\n            f1 = self.evaluate(test_loader)\n            f1_train = self.evaluate(train_loader)\n            if f1 > max_score:\n                max_score = f1\n                self.save_model(self.save_path)\n\n            if f1 > prev_score:\n                not_improve = 0\n            else:\n                not_improve += 1\n                if not_improve > Config.EARLY_STOP:\n                    pbar.set_description(f'epoch: {it}, early stop; max_f1: {max_score}')\n                    break\n\n            desc = f'epoch: {it}' + ' '\\\n                + f'lr: {lr:e}' + ' '\\\n                + f'avg_loss: {loss / len(train_loader) :e}'+' '\\\n                + f'avg_f1: {f1 :.4f}' + ' '\\\n                + f'max_f1: {max_score :.4f}' + ' '\\\n                + f'f1_train: {f1_train :.4f}'\n\n            pbar.set_description(desc)\n\n            losses.append(loss / len(train_loader))\n            f1s.append(f1)\n            prev_score = f1\n\n        return {'bce': losses, 'f1': f1s}\n\n    def train_one_epoch(self, loader):\n        self.model.train()\n        self.model.to(DEVICE)\n\n        loss = 0\n        for X,y in loader:\n            # X = X.to(DEVICE)\n            X = X.to(DEVICE)\n            y = y.to(DEVICE)\n\n            self.optimizer.zero_grad()\n            outp = self.model(X)\n            nloss = self.loss_fn(outp,y)\n            nloss.backward()\n            self.optimizer.step()\n\n            loss += nloss.detach().cpu().item()\n\n        return loss\n\n    def predict(self, loader):\n        self.model.eval()\n        self.model.to(DEVICE)\n\n        preds = []\n        labels = []\n        for X, y in loader:\n            X = X.to(DEVICE)\n            with torch.no_grad():\n                outp = torch.sigmoid(self.model(X))\n\n            preds.append(outp.detach().cpu().numpy())\n            labels.append(y.detach().cpu().numpy())\n\n        return np.concatenate(preds, axis = 0), np.concatenate(labels, axis = 0)\n\n    def evaluate(self, loader):\n        preds, labels = self.predict(loader)\n        return f1_score((preds.reshape(-1) > 0.5).astype(int), labels.reshape(-1), average = \"macro\")\n\n    def generate_prediction(self, loader):\n        self.model.load_state_dict(torch.load(self.save_path, map_location = DEVICE))\n        return self.predict(loader)[0]\n\n    def save_model(self, path):\n        torch.save(self.model.state_dict(), path)\n        torch.save(self.model.encoder.state_dict(), path + '-encoder')\n\n    def load_encoder(self, path):\n        self.model.encoder.load_state_dict(torch.load(path))\n\n    def load_model(self,path):\n        self.model.load_state_dict(torch.load(path))","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\n\nclass Ds(Dataset):\n    def __init__(self, data, targets):\n        self.data = data\n        self.targets = targets\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n\n        X = torch.tensor(self.data[idx], dtype = torch.float32)\n        y = torch.tensor(self.targets[idx], dtype = torch.float32)\n\n        return X,y","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Loop","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nfrom tqdm import tqdm\n\nclass Config:\n    BATCH_SIZE = 256\n    LEARNING_RATE = 1e-4\n    EPOCHS = 100\n    EARLY_STOP = 8\n\noofs = []\n\nfor level_group in [GP1,GP2,GP3]:\n    print(f'*** training group {level_group} ***')\n    data = GROUP_DATA[level_group]['position']\n    labels = GROUP_DATA[level_group]['targets']\n    pos_weight = GROUP_DATA[level_group]['pos_weight']\n\n    model_param = model_params[level_group]\n\n    oof = pd.DataFrame(data=np.zeros((targets.shape[0], model_param['nout'])))\n    gkf = GroupKFold(n_splits=10)\n\n    for nfold, (train_index, test_index) in enumerate(gkf.split(X = oof, groups = oof.index)):\n        print(f'--- fold {nfold+1} ---')\n\n        save_path = f'./ckpt/model-{level_group}-fold-{nfold}'\n\n        model = PositionModel(**model_param, npatches = num_patches)\n\n        trainer = Trainer(\n            model,\n            save_path = save_path,\n            num_labels = model.nout,\n            pos_weight = pos_weight,\n            learning_rate = Config.LEARNING_RATE\n        )\n\n        train_data, train_label = data[train_index], labels[train_index]\n        valid_data, valid_label = data[test_index], labels[test_index]\n\n        train_loader = DataLoader(\n            Ds(train_data, train_label), batch_size=Config.BATCH_SIZE, shuffle=True, num_workers = 2)\n        test_loader = DataLoader(\n            Ds(valid_data, valid_label), batch_size=Config.BATCH_SIZE, shuffle=False, num_workers = 2)\n\n        if level_group == GP2:\n            trainer.load_encoder(f'model-{GP1}-fold-{nfold}-encoder')\n        elif level_group == GP3:\n            trainer.load_encoder(f'model-{GP2}-fold-{nfold}-encoder')\n\n        scalars = trainer.train_network(train_loader, test_loader)\n\n        preds = trainer.generate_prediction(test_loader)\n        oof.iloc[test_index] = preds\n    oofs.append(oof)","metadata":{"collapsed":false,"pycharm":{"is_executing":true},"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]}]}