{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pytorch using only 1 column - submission","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-28T08:57:03.193294Z","iopub.execute_input":"2023-02-28T08:57:03.194073Z","iopub.status.idle":"2023-02-28T08:57:03.225552Z","shell.execute_reply.started":"2023-02-28T08:57:03.194019Z","shell.execute_reply":"2023-02-28T08:57:03.224570Z"}}},{"cell_type":"markdown","source":"This is the POC of [this notebook](https://www.kaggle.com/code/chrisqiu/0-681-pytorch-using-only-1-column).\n\n### update 2023-04-23 \nUpdate the submission logic because they have changed the time series api. See [this discussion](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/396202)","metadata":{}},{"cell_type":"code","source":"THRESHOLD = 0.50\nWPATH = '/kaggle/input/jowilder-1col-weights'\nFUSE = True","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:37:31.468694Z","iopub.execute_input":"2023-04-23T04:37:31.469086Z","iopub.status.idle":"2023-04-23T04:37:31.504729Z","shell.execute_reply.started":"2023-04-23T04:37:31.469052Z","shell.execute_reply":"2023-04-23T04:37:31.503369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch \n\ntorch.manual_seed(101)\nnp.random.seed(101)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:37:31.506833Z","iopub.execute_input":"2023-04-23T04:37:31.507191Z","iopub.status.idle":"2023-04-23T04:37:33.954072Z","shell.execute_reply.started":"2023-04-23T04:37:31.507157Z","shell.execute_reply":"2023-04-23T04:37:33.953102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv',\n                usecols = ['session_id', 'level_group', 'elapsed_time'],\n                low_memory = True)\nFEATURES = ['event_duration']","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:37:33.955696Z","iopub.execute_input":"2023-04-23T04:37:33.956508Z","iopub.status.idle":"2023-04-23T04:39:01.299168Z","shell.execute_reply.started":"2023-04-23T04:37:33.956471Z","shell.execute_reply":"2023-04-23T04:39:01.297831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport copy\n\n# trim input data\n\ndef trim(X, trim_steps):\n    data_steps = X.shape[0]\n    if data_steps == trim_steps: return X\n    \n    if data_steps < trim_steps:\n        shortage = trim_steps - data_steps\n        return np.pad(X, ((0,shortage),(0,0)),'constant') \n    \n    start = int(np.random.random() * (data_steps - trim_steps))    \n    return X[start:start+trim_steps]  ","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:39:01.301585Z","iopub.execute_input":"2023-04-23T04:39:01.302019Z","iopub.status.idle":"2023-04-23T04:39:01.309672Z","shell.execute_reply.started":"2023-04-23T04:39:01.301983Z","shell.execute_reply":"2023-04-23T04:39:01.308556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass Encoder(nn.Module):\n    def __init__(self, nembed, nfeatures = 128):\n        \n        super(Encoder, self).__init__()\n        \n        self.nfeatures = nfeatures\n        \n        self.convs = nn.Sequential(\n            nn.Conv1d(1, 64,  kernel_size = 5),\n            nn.BatchNorm1d(64),\n            nn.LeakyReLU(inplace = True),\n            nn.MaxPool1d(2),\n            \n            nn.Conv1d(64, 128,  kernel_size = 5),\n            nn.BatchNorm1d(128),\n            nn.LeakyReLU(inplace = True),\n            nn.MaxPool1d(2),\n            \n            nn.Conv1d(128, self.nfeatures,  kernel_size = 3),\n            nn.BatchNorm1d(self.nfeatures),\n            nn.LeakyReLU(inplace = True),\n        ) \n        \n\n        self.fcs = nn.Sequential(\n                # 2* because in the pooling layer we take mean AND the std.\n                nn.Linear(2*self.nfeatures, nembed),\n            )\n\n    \n    def forward(self,x):        \n        x = torch.transpose(x, 1,2)\n        x = self.convs(x)        \n        std = torch.std(x, dim = 2)\n        mean = torch.mean(x, dim = 2)\n        x = torch.cat([std, mean], dim = 1)\n        x = self.fcs(x)\n        return x\n    \nclass Model(nn.Module):\n    def __init__(self,nout, nembed, nfeatures):\n        super(Model, self).__init__()\n        self.encoder = Encoder(nembed, nfeatures)\n        self.clf = nn.Sequential(\n            \n            nn.LeakyReLU(inplace = True),\n            nn.Dropout(0.2),\n            \n            nn.Linear(nembed, nembed // 2),\n            nn.LeakyReLU(inplace = True),\n            nn.Dropout(0.2),\n            \n            nn.Linear(nembed // 2, nout),\n\n        ) \n        \n        self.nout = nout\n        self.nembed = nembed\n        self.nfeatures = nfeatures\n                \n    \n    def forward(self, x):\n        x = self.encoder(x)\n        x = self.clf(x)\n        return x\n    \n# assembler\nclass GroupModel(nn.Module):\n    def __init__(self, model, weights, fuse = False):\n        super(GroupModel, self).__init__()\n        \n        self.fuse = fuse\n        self.models = [copy.deepcopy(model) for _ in range(len(weights))]\n        self.weights = weights\n        \n        for i in range(len(self.weights)):\n            self.models[i].load_state_dict(torch.load(weights[i], map_location = 'cpu'))\n            self.models[i].eval()\n\n    def forward(self,x):\n        \n        if self.fuse:\n            with torch.no_grad():\n                preds = [model(x) for model in self.models]\n                \n            preds = torch.mean(torch.cat(preds, dim = 0), dim = 0, keepdims = True)\n        \n        else:\n            with torch.no_grad():\n                preds = self.models[0](x)\n                \n        return preds.sigmoid()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:39:01.311253Z","iopub.execute_input":"2023-04-23T04:39:01.311840Z","iopub.status.idle":"2023-04-23T04:39:01.458692Z","shell.execute_reply.started":"2023-04-23T04:39:01.311805Z","shell.execute_reply":"2023-04-23T04:39:01.457599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\nweights = [w for w in glob.glob(f'{WPATH}/model-*') if 'encoder' not in w]\n\nGP1 = '0-4'\nGP2 = '5-12'\nGP3 = '13-22'\n\nmodel_params = {}\nmodel_params[GP1] = dict(\n    nout = 3,\n    nembed = 128,\n    nfeatures = 128,\n)\nmodel_params[GP2] = dict(\n    nout = 10,\n    nembed = 128,\n    nfeatures = 128,\n)\nmodel_params[GP3] = dict(\n    nout = 5,\n    nembed = 128,\n    nfeatures = 128,\n)\n\n\nmodels = {\n    GP: GroupModel(\n        model = Model(**params),\n        weights = [w for w in weights if GP in w],\n        fuse = FUSE,\n    ) for GP,params in model_params.items()\n}\n\nfor GP in model_params:    \n    model = models[GP]\n    print(f'group {GP}')\n    print(f'output shape: {model(torch.ones(1, 1000, len(FEATURES))).shape}')\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:39:01.461423Z","iopub.execute_input":"2023-04-23T04:39:01.462169Z","iopub.status.idle":"2023-04-23T04:39:02.214867Z","shell.execute_reply.started":"2023-04-23T04:39:01.462125Z","shell.execute_reply":"2023-04-23T04:39:02.213836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:39:02.219314Z","iopub.execute_input":"2023-04-23T04:39:02.221781Z","iopub.status.idle":"2023-04-23T04:39:02.247856Z","shell.execute_reply.started":"2023-04-23T04:39:02.221734Z","shell.execute_reply":"2023-04-23T04:39:02.246742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"questions = {'0-4':(1,4), '5-12':(4,14), '13-22':(14,19)}\n\nfor (test, sample_submission) in iter_test:\n    \n    # test info\n    level_group = test.level_group.iloc[0]\n    session_id = test.session_id.iloc[0]\n    \n    # model\n    model = models[level_group]\n    \n    # data\n    event_duration = test.elapsed_time.fillna(-1).diff().fillna(0).clip(0, 3.6e6)\n    X = trim(event_duration.values[..., None], 1000)\n    X = torch.tensor(X, dtype = torch.float32)\n\n    # predict\n    preds = model(X.unsqueeze(0)).detach().numpy()\n    preds = (preds > THRESHOLD).astype(int)\n\n    # generate question label\n    mask = sample_submission.session_id.str.contains(f'{session_id}')\n    \n    try:\n        sample_submission.loc[mask, 'correct'] = preds.reshape(-1)\n    except:\n        sample_submission.loc[mask, 'correct'] = 0\n      \n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:39:02.252686Z","iopub.execute_input":"2023-04-23T04:39:02.254696Z","iopub.status.idle":"2023-04-23T04:39:02.539305Z","shell.execute_reply.started":"2023-04-23T04:39:02.254645Z","shell.execute_reply":"2023-04-23T04:39:02.538402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = pd.read_csv('/kaggle/working/submission.csv')\nres.correct.value_counts(dropna = False)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:39:02.540915Z","iopub.execute_input":"2023-04-23T04:39:02.541610Z","iopub.status.idle":"2023-04-23T04:39:02.554560Z","shell.execute_reply.started":"2023-04-23T04:39:02.541573Z","shell.execute_reply":"2023-04-23T04:39:02.553401Z"},"trusted":true},"execution_count":null,"outputs":[]}]}