{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\nimport gc\nfrom typing import List\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torchvision.utils import make_grid\nfrom torch.utils.data import DataLoader, random_split\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import StepLR\n\nimport riiideducation\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\n\nuserActions = pd.read_pickle(\"../input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip\")\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\n\nprint(\"Train size:\", userActions.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# only look at those actions related to questions\nnumberOfActions = 1000000\nuserActions = userActions.loc[userActions['content_type_id'] == 0].head(numberOfActions)\nuserActions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# does no care about correct answer\nquestions = questions.drop(columns=['correct_answer'])\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess Question Tags"},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert tags to array of numbers\nmaximumTag = -math.inf\nminimumTag = math.inf\ndef tagsToArray(x: str) -> List[int]:\n    if x is np.nan:\n        return []\n    res = [int(tag) for tag in x.split()]\n    global maximumTag\n    global minimumTag\n    maximumTag = max(maximumTag, *res)\n    minimumTag = min(minimumTag, *res)\n    return res\n    \n\nquestions.tags = questions.tags.apply(tagsToArray)\n\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('min tag: ' + str(minimumTag))\nprint('max tag: ' + str(maximumTag))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convertTagsToEncodedArray(x: List[int]) -> List[int]:\n    global maximumTag\n    encoded = np.zeros(maximumTag + 1)\n    for tag in x:\n        encoded[tag] = 1\n    return encoded\n\n\nquestions.tags = questions.tags.apply(convertTagsToEncodedArray)\n\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create DataSet"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ninteractionFeatures = ['timestamp', 'content_type_id', 'task_container_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']\nquestionFeatures = ['bundle_id', 'part']\n\ndef toInt(x):\n    if pd.isna(x) or x is None or x == np.nan:\n        return -1\n    return int(x)\n\ndef cleanUpUserInteractions(userInteractions, garbageCollect=False):\n    interactions = userInteractions.loc[userInteractions['content_type_id'] == 0]\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions['content_type_id'] = interactions['content_type_id'].apply(toInt)\n    interactions['prior_question_elapsed_time'] = interactions['prior_question_elapsed_time'].apply(toInt)\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions['timestamp'] = interactions['timestamp'].apply(toInt)\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions['content_type_id'] = interactions['content_type_id'].apply(toInt)\n    interactions['prior_question_had_explanation'] = interactions['prior_question_had_explanation'].apply(toInt)\n    \n    if garbageCollect:\n        gc.collect()\n    \n    interactions = interactions.replace([np.inf, -np.inf], np.nan)\n    interactions = interactions.fillna(-1)\n\n    if garbageCollect:\n        gc.collect()\n    \n    interactions[interactionFeatures] = interactions[interactionFeatures].astype('float').fillna(value = -1)\n    \n    if garbageCollect:\n        gc.collect()\n\n    \n    return interactions\n    \n\nclass UserDataset(Dataset):\n    \"\"\"Dataset class for column dataset.\n    Args:\n       cats (list of str): List of the name of columns contain\n                           categorical variables.\n       conts (list of str): List of the name of columns which \n                           contain continuous variables.\n       y (Tensor, optional): Target variables.\n       is_reg (bool): If the task is regression, set ``True``, \n                      otherwise (classification) ``False``.\n       is_multi (bool): If the task is multi-label classification, \n                        set ``True``.\n    \"\"\"\n    def __init__(self, userInteractions, questions):\n        self.isTest = False\n        self.length = len(userInteractions)\n        self.userInteractions = cleanUpUserInteractions(userInteractions[userInteractions.answered_correctly != -1], garbageCollect=True).iloc\n        gc.collect()\n        self.questions = {}\n        self.tags = {}\n        for index, row in questions.iterrows():\n            question_id = int(row.question_id)\n            self.questions[question_id] = torch.from_numpy(row[questionFeatures].astype('float').values).float()\n            self.tags[question_id] = torch.from_numpy(row.tags.astype('float')).float()\n        gc.collect()\n        \n        self.blankTags = torch.from_numpy(convertTagsToEncodedArray([])).float()\n        self.blankQuestion = torch.from_numpy(np.ones(len(questionFeatures),)).float()\n        \n    \n    def getData(self, row):\n        questionId = int(row.content_id)\n        return torch.from_numpy(row[interactionFeatures].astype('float').values).float(), self.questions.get(questionId, self.blankQuestion), self.tags.get(questionId, self.blankTags)\n        \n    def __len__(self): \n        return self.length\n    \n    def switchToTest(self, df):\n        self.isTest = True\n        self.length = len(df)\n        self.userInteractions = df.iloc\n        \n    \n    def __getitem__(self, idx):\n        row = self.userInteractions[idx]\n        if not self.isTest:\n            return [torch.from_numpy(row[interactionFeatures].values).float(), self.questions[row.content_id], self.tags[row.content_id], torch.Tensor([row.answered_correctly]).long()]\n        return [torch.from_numpy(row[interactionFeatures].values.astype('float')).float(), self.questions[row.content_id], self.tags[row.content_id]]\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainData = UserDataset(userActions, questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\n\ngc.collect()\n\nvalidationCount = int(numberOfActions * 0.1)\n\ntrain_dataset, val_dataset = random_split(trainData, [numberOfActions - validationCount, validationCount])\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size,\n                          shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, \n                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Predictor(nn.Module):\n  def __init__(self, tagCount: int, questionFeatureCount: int, interactionFeatureCount: int):\n    super(Predictor, self).__init__()\n    \n    reductionDimensions = 4\n    predictorDimensions = 16\n    \n\n    self.tagEncoder = nn.Sequential(\n      nn.Linear(tagCount, 64),\n      nn.ReLU(True),\n      nn.Linear(64, 128),\n      nn.Dropout(),\n      nn.ReLU(True),\n      nn.Linear(128, 32),\n      nn.ReLU(True), \n      nn.Linear(32, reductionDimensions)\n    )\n\n    self.tagDecoder = nn.Sequential(\n      nn.Linear(reductionDimensions, 32),\n      nn.ReLU(True),\n      nn.Linear(32, 128),\n      nn.Dropout(),\n      nn.ReLU(True),\n      nn.Linear(128, 64),\n      nn.ReLU(True),\n      nn.Linear(64, tagCount), \n    )\n\n    self.predictorEncoder = nn.Sequential(\n      nn.Linear(reductionDimensions + questionFeatureCount + interactionFeatureCount, 64),\n      nn.ReLU(True),\n      nn.Linear(64, 64),\n      nn.Dropout(),\n      nn.ReLU(True),\n      nn.Linear(64, 32),\n      nn.ReLU(True),\n      nn.Linear(32, predictorDimensions),\n    )\n    \n    self.predictorDecoder = nn.Sequential(\n      nn.Linear(predictorDimensions, 16),\n      nn.ReLU(True),\n      nn.Linear(16, 32),\n      nn.ReLU(True),\n      nn.Linear(32, 64),\n      nn.ReLU(True),\n      nn.Linear(64, reductionDimensions + questionFeatureCount + interactionFeatureCount),  \n    )\n    \n    self.predictor = nn.Sequential(\n        nn.Linear(predictorDimensions, 32),\n        nn.ReLU(True),\n        nn.Linear(32, 16),\n        nn.ReLU(True),\n        nn.Linear(16, 2)\n    )\n\n  def forward(self, interaction, question, tags):\n    encodedTag = self.tagEncoder(tags)\n    decodedTag = self.tagDecoder(encodedTag)\n    predictOn = torch.cat((encodedTag, question, interaction), -1)\n    predictEncoded = self.predictorEncoder(predictOn)\n    predictDecoded = self.predictorDecoder(predictEncoded)\n    prediction = self.predictor(predictEncoded)\n    return prediction, decodedTag, predictOn, predictDecoded\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lossBase = nn.MSELoss()\nclassificationLoss = nn.CrossEntropyLoss()\n\ndef loss_criterion(tags, decodedTags, tagRegularizationTerm, predictionOriginal, predictionDecoded, predictionRegularizationTerm, answeredCorrectly, prediction):\n    return lossBase(tags, decodedTags) * tagRegularizationTerm + lossBase(predictionOriginal, predictionDecoded) * predictionRegularizationTerm + classificationLoss(prediction, answeredCorrectly.squeeze(1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"tagRegularizationTerm = 0.001\npredictionRegularizationTerm = 0.001\nepochs = 3\n\nmodel = Predictor(maximumTag + 1, len(questionFeatures), len(interactionFeatures)).cuda()\nmodel_optimizer = torch.optim.Adam(\n    model.parameters(), lr=0.0001, weight_decay=1e-5)\n\n\ntrain_loss  = []\ntrain_acc = []\nval_loss = []\nval_acc = []\n\ngc.collect()\n\nfor epoch in range(epochs):\n    model.train()\n    running_acc = 0.0\n    batch_loss = []\n    for index, (interaction, question, tags, y) in enumerate(train_loader):\n        interaction = Variable(interaction).cuda()\n        question = Variable(question).cuda()\n        tags = Variable(tags).cuda()\n        y = Variable(y).cuda()\n        # ===================forward=====================\n        prediction, decodedTag, predictOn, predictDecoded = model(interaction, question, tags)\n        loss = loss_criterion(tags, decodedTag, tagRegularizationTerm, predictOn, predictDecoded, predictionRegularizationTerm, y, prediction)\n        # ===================backward====================\n        model_optimizer.zero_grad()\n        loss.backward()\n        model_optimizer.step()\n\n        # print statistics\n        batch_loss.append(loss.item())\n\n        out = torch.argmax(prediction.detach(),dim=1).unsqueeze(1)\n        assert out.shape==y.shape\n        running_acc += (out==y).sum().item()\n        \n        if index % 50000 == 0:\n            gc.collect()\n    train_loss.append(np.mean(batch_loss))\n    train_acc.append(running_acc*100/len(train_dataset))\n    print(f\"Train loss {epoch+1}: {train_loss[-1]},Train Acc:{running_acc*100/len(train_dataset)}%\")\n\n\n\n    model.eval()\n    batch_loss  = []\n    correct = 0.0\n    with torch.no_grad():\n        totalLoss = 0\n        for interaction, question, tags, y in val_loader:\n            interaction = Variable(interaction).cuda()\n            question = Variable(question).cuda()\n            tags = Variable(tags).cuda()\n            y = Variable(y).cuda()\n            \n            prediction, decodedTag, predictOn, predictDecoded = model(interaction, question, tags)\n            loss = loss_criterion(tags, decodedTag, tagRegularizationTerm, predictOn, predictDecoded, predictionRegularizationTerm, y, prediction)\n            \n            batch_loss.append(loss.item())\n\n            out = torch.argmax(prediction,dim=1).unsqueeze(1)\n            acc = (y==out).sum().item()\n            correct += acc\n    val_loss.append(np.mean(batch_loss))\n    val_acc.append(correct*100/len(val_dataset))\n    print(f\"Val accuracy:{correct*100/len(val_dataset)}% Val loss:{np.mean(batch_loss)}\")\n    \n    if correct/len(val_dataset) > 0.65:\n        break\n    \n    gc.collect()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_loss)\nprint(train_acc)\nprint(val_loss)\nprint(val_acc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    for (test_df, sample_prediction_df) in env.iter_test():\n        test_df = cleanUpUserInteractions(test_df, garbageCollect=True)\n        trainData.switchToTest(test_df)\n        n = len(test_df)\n        answeredCorrectly = np.zeros((n,))\n        testdata = test_df.iloc\n        for index, (interactions, question, tags) in enumerate(DataLoader(trainData, batch_size=batch_size, shuffle=False)):\n            interactions = Variable(interactions).cuda()\n            question = Variable(question).cuda()\n            tags = Variable(tags).cuda()\n            prediction, *rest = model(interactions, question, tags)\n            prediction = torch.argmax(prediction,dim=1).cpu().detach().numpy()\n            answeredCorrectly[(index * batch_size):(index * batch_size + len(prediction))] = prediction\n        test_df['answered_correctly'] = answeredCorrectly\n        env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}