{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\nimport gc\nfrom typing import List\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import make_grid\nfrom torch.utils.data import DataLoader, random_split\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import StepLR\n\nimport riiideducation\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\n\nuserActions = pd.read_pickle(\"../input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip\")\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\n\nprint(\"Train size:\", userActions.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# only care about the question interactions\nuserActions = userActions.loc[userActions['content_type_id'] == 0]\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# does no care about correct answer\nquestions = questions.drop(columns=['correct_answer'])\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess Question Tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert tags to array of numbers\nmaximumTag = -math.inf\nminimumTag = math.inf\ndef tagsToArray(x: str) -> List[int]:\n    if x is np.nan:\n        return []\n    res = [int(tag) for tag in x.split()]\n    global maximumTag\n    global minimumTag\n    maximumTag = max(maximumTag, *res)\n    minimumTag = min(minimumTag, *res)\n    return res\n    \n\nquestions.tags = questions.tags.apply(tagsToArray)\n\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('min tag: ' + str(minimumTag))\nprint('max tag: ' + str(maximumTag))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convertTagsToEncodedArray(x: List[int]) -> List[int]:\n    global maximumTag\n    encoded = np.zeros(maximumTag + 1)\n    for tag in x:\n        encoded[tag] = 1\n    return encoded\n\n\nquestions.tags = questions.tags.apply(convertTagsToEncodedArray)\n\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ninteractionFeatures = ['timestamp', 'content_type_id', 'task_container_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']\nquestionFeatures = ['bundle_id', 'part']\n\nnumberOfTrainingUsers = 1000\n\ndef toInt(x):\n    if pd.isna(x) or x is None or x == np.nan:\n        return -1\n    return int(x)\n\ndef cleanUpUserInteractions(userInteractions, garbageCollect=False, removeNoneQuestions=False):\n    # only look at interactions related to questions\n    if garbageCollect:\n        gc.collect()\n    interactions = userInteractions\n    \n    if garbageCollect:\n        gc.collect()\n    \n    if removeNoneQuestions:\n        interactions = userInteractions.loc[userInteractions['content_type_id'] == 0]\n    \n    if garbageCollect:\n        gc.collect()\n\n    \n    interactions['content_type_id'] = interactions['content_type_id'].apply(toInt)\n    \n    gc.collect()\n    \n    interactions['prior_question_elapsed_time'] = interactions['prior_question_elapsed_time'].apply(toInt)\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions['timestamp'] = interactions['timestamp'].apply(toInt)\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions['content_type_id'] = interactions['content_type_id'].apply(toInt)\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions['prior_question_had_explanation'] = interactions['prior_question_had_explanation'].apply(toInt)\n\n    if garbageCollect:\n        gc.collect()\n    \n    interactions[interactionFeatures] = interactions[interactionFeatures].astype('float').fillna(value = -1)\n    \n    if garbageCollect:\n        gc.collect()\n\n    \n    return interactions\n    \n\nclass UserDataset(Dataset):\n    \"\"\"Dataset class for column dataset.\n    Args:\n       cats (list of str): List of the name of columns contain\n                           categorical variables.\n       conts (list of str): List of the name of columns which \n                           contain continuous variables.\n       y (Tensor, optional): Target variables.\n       is_reg (bool): If the task is regression, set ``True``, \n                      otherwise (classification) ``False``.\n       is_multi (bool): If the task is multi-label classification, \n                        set ``True``.\n    \"\"\"\n    def __init__(self, userInteractions, questions):\n        gc.collect()\n        self.questions = {}\n        self.tags = {}\n        for index, row in questions.iterrows():\n            question_id = row.question_id\n            self.questions[question_id] = torch.from_numpy(row[questionFeatures].astype('float').values).float()\n            self.tags[question_id] = torch.from_numpy(row.tags.astype('float')).float()\n        gc.collect()\n        \n        userIds = userInteractions.user_id.unique()[:numberOfTrainingUsers]\n        dataWithUsers = cleanUpUserInteractions(userInteractions[userInteractions.answered_correctly != -1].loc[userInteractions.user_id.isin(userIds)], garbageCollect=True)\n        \n        self.blankTags = torch.from_numpy(convertTagsToEncodedArray([])).float()\n        self.blankQuestion = torch.from_numpy(np.ones(len(questionFeatures),)).float()\n        \n        gc.collect()\n        \n        self.length = len(userIds)\n        dataBank = []\n\n        for userId in userIds:\n            data = []\n            for index,row in dataWithUsers.loc[dataWithUsers.user_id == userId].iterrows():\n                data.append([*self.getData(row), torch.Tensor([row.answered_correctly]).long()])\n            dataBank.append(data)\n        \n        self.dataBank = dataBank\n        \n        gc.collect()\n        \n\n    \n    def getData(self, row):\n        questionId = int(row.content_id)\n        return torch.from_numpy(row[interactionFeatures].astype('float').values).float(), self.questions.get(questionId, self.blankQuestion), self.tags.get(questionId, self.blankTags)\n        \n    def __len__(self): \n        return self.length\n    \n    def __getitem__(self, idx):\n        return self.dataBank[idx]\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = UserDataset(userActions, questions)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 1\n\ngc.collect()\n\ntrain_dataset, val_dataset = random_split(data, [900, 100])\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size,\n                          shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, \n                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hiddenSateDimension = 10\n\nclass Predictor(nn.Module):\n  def __init__(self, tagCount: int, questionFeatureCount: int, interactionFeatureCount: int):\n    super(Predictor, self).__init__()\n    \n    reductionDimensions = 4\n    predictorDimensions = 16\n    \n\n    self.tagEncoder = nn.Sequential(\n      nn.Linear(tagCount, 64),\n      nn.ReLU(True),\n      nn.Linear(64, 32),\n      nn.ReLU(True), \n      nn.Linear(32, reductionDimensions)\n    )\n\n    self.tagDecoder = nn.Sequential(\n      nn.Linear(reductionDimensions, 32),\n      nn.ReLU(True),\n      nn.Linear(32, 64),\n      nn.ReLU(True),\n      nn.Linear(64, tagCount), \n#       nn.Tanh()\n    )\n\n    self.predictorEncoder = nn.Sequential(\n      nn.Linear(reductionDimensions + questionFeatureCount + interactionFeatureCount + hiddenSateDimension, 128),\n      nn.ReLU(True),\n      nn.Linear(128, 64),\n      nn.ReLU(True),\n      nn.Linear(64, 32),\n      nn.ReLU(True),\n      nn.Linear(32, predictorDimensions),\n    )\n    \n    self.hiddenStateCreator = nn.Sequential(\n      nn.Linear(predictorDimensions, 32),\n      nn.ReLU(True),\n      nn.Linear(32, 64),\n      nn.ReLU(True),\n      nn.Linear(64, 128),\n      nn.ReLU(True),\n      nn.Linear(128, hiddenSateDimension),  \n    )\n    \n    self.predictor = nn.Sequential(\n        nn.Linear(predictorDimensions, 32),\n        nn.ReLU(True),\n        nn.Linear(32, 16),\n        nn.ReLU(True),\n        nn.Linear(16, 2),\n        nn.Tanh()\n#         nn.Softmax()\n    )\n    \n  def getBlankHiddenState(self):\n    return Variable(torch.zeros(1, hiddenSateDimension))\n    \n\n\n  def forward(self, interaction, question, tags, hiddenState):\n    if hiddenState is None:\n        hiddenState = Variable(self.getBlankHiddenState())\n    encodedTag = self.tagEncoder(tags)\n    decodedTag = self.tagDecoder(encodedTag)\n    predictOn = torch.cat((encodedTag, question, interaction, hiddenState), -1)\n    predictEncoded = self.predictorEncoder(predictOn)\n    prediction = self.predictor(predictEncoded)\n    newHiddenState = self.hiddenStateCreator(predictEncoded)\n    return prediction, decodedTag, newHiddenState\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lossBase = nn.MSELoss()\nclassificationLoss = nn.CrossEntropyLoss()\n\ndef loss_criterion(tags, decodedTags, tagRegularizationTerm, answeredCorrectly, prediction):\n  return lossBase(tags, decodedTags) * tagRegularizationTerm + classificationLoss(prediction, answeredCorrectly.squeeze(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tagRegularizationTerm = 0.001\npredictionRegularizationTerm = 0.001\nepochs = 2\n\nsae = Predictor(maximumTag + 1, len(questionFeatures), len(interactionFeatures))\nsae_optimizer = torch.optim.Adam(\n    sae.parameters(), lr=0.001, weight_decay=1e-5)\n\n\ntrain_loss  = []\ntrain_acc = []\nval_loss = []\nval_acc = []\n\ngc.collect()\n\nfor epoch in range(epochs):\n    sae.train()\n    running_acc = 0.0\n    batch_loss = []\n    for index, samples in enumerate(train_loader):\n        if index % 100 == 0:\n            gc.collect()\n        hiddenState = None\n        loss = None\n        for interaction, question, tags, y in samples:\n            interaction = Variable(interaction)\n            question = Variable(question)\n            tags = Variable(tags)\n            y = Variable(y)\n            # ===================forward=====================\n            (prediction, decodedTag, newHiddenState) = sae(interaction, question, tags, hiddenState)\n            currentLoss = loss_criterion(tags, decodedTag, tagRegularizationTerm, y, prediction)\n            if loss is None:\n                loss = currentLoss\n            else:\n                loss += currentLoss\n            hiddenState = newHiddenState\n        # ===================backward====================\n        sae_optimizer.zero_grad()\n        if loss is not None:\n            loss.backward()\n        sae_optimizer.step()\n\n        # print statistics\n        batch_loss.append(loss.item())\n        \n        out = torch.argmax(prediction.detach(),dim=1).unsqueeze(1)\n        assert out.shape==y.shape\n        running_acc += (out==y).sum().item()\n    train_loss.append(np.mean(batch_loss))\n    train_acc.append(running_acc*100/len(train_dataset))\n    print(f\"Train loss {epoch+1}: {train_loss[-1]},Train Acc:{running_acc*100/len(train_dataset)}%\")\n\n\n\n    sae.eval()\n    batch_loss  = []\n    correct = 0.0\n    sampleCount = 0\n    with torch.no_grad():\n        totalLoss = 0\n        for index, samples in enumerate(val_loader):\n            hiddenState = None\n            sampleCount += len(samples)\n            for interaction, question, tags, y in samples:\n                interaction = Variable(interaction)\n                question = Variable(question)\n                tags = Variable(tags)\n                y = Variable(y)\n\n                (prediction, decodedTag, newHiddenState) = sae(interaction, question, tags, hiddenState)\n                loss = loss_criterion(tags, decodedTag, tagRegularizationTerm, y, prediction).item()\n                \n                hiddenState = newHiddenState\n\n                batch_loss.append(loss)\n\n                out = torch.argmax(prediction,dim=1).unsqueeze(1)\n                acc = (y==out).sum().item()\n                correct += acc\n    val_loss.append(np.mean(batch_loss))\n    val_acc.append(correct*100/sampleCount)\n    print(f\"Val accuracy:{correct*100/sampleCount}% Val loss:{np.mean(batch_loss)}\")\n    \n    gc.collect()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_loss)\nprint(train_acc)\nprint(val_loss)\nprint(val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hiddenStates = {}\nsae.eval()\n\nwith torch.no_grad():\n    for (test_df, sample_prediction_df) in env.iter_test():\n        test_df = cleanUpUserInteractions(test_df, removeNoneQuestions=True, garbageCollect=True)\n        n = len(test_df)\n        answeredCorrectly = [0] * n\n        index = 0\n        for _, row in test_df.iterrows():\n            hiddenState = hiddenStates.get(row.user_id, None)\n            interaction, question, tags = data.getData(row)\n            interaction = Variable(interaction.unsqueeze(0))\n            question = Variable(question.unsqueeze(0))\n            tags = Variable(tags.unsqueeze(0))\n            prediction, decodedTag, newHiddenState = sae(interaction, question, tags, hiddenState)\n            hiddenStates[row.user_id] = newHiddenState\n            prediction = torch.abs(prediction.squeeze(0))\n            prediction = float(prediction[1].item() / prediction.sum().item())\n            answeredCorrectly[index] = prediction\n            index += 1\n        test_df['answered_correctly'] = answeredCorrectly\n        env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}