{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nimport pandas as pd\n\n# You can only call make_env() once, so don't lose it!\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"max_num = 1000000\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=True,nrows = max_num,\n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n                             'prior_question_had_explanation': 'boolean',\n                             }\n                      )\n# we destimate 'lectures' information due we don't have access in the real scenario and will be noisy input.\n# additional useful information\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv', low_memory=True)\n# we must point into \"content_type_id == 0\" thus those are the real scenario cases\n# 0 means the event was a question being posed to the user\ntrain_df = train_df.loc[train_df['content_type_id'] == 0]\n# we target a nominal value [1,0]\nprint(train_df['answered_correctly'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# additional information for real training\nquestions_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# examples for real training\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the whole dataset joining \"questions\" \ntrain_df = train_df.merge(questions_df, left_on='content_id', right_on='question_id', how= 'left', copy = False)\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pune subsets into target columns + additional question columns\ntrain_df = train_df[['timestamp','prior_question_elapsed_time','prior_question_had_explanation', 'correct_answer','part' ,'tags', 'answered_correctly']]\n# remove noisy inputs (NaN, Null .......)\ntrain_df = train_df.dropna()\n# convert 'prior_question_had_explanation' to numerical.\ntrain_df[\"prior_question_had_explanation\"] = train_df[\"prior_question_had_explanation\"].astype(int)\n# Those constitude the enginered features to learn from\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nfrom transformers import DistilBertForSequenceClassification, DistilBertConfig, set_seed, Trainer, TrainingArguments\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score\n\n\nfrom tokenizers import Tokenizer\nfrom tokenizers.models import BPE\nfrom tokenizers.pre_tokenizers import Whitespace\nfrom tokenizers.trainers import BpeTrainer\n\ntokenizer = Tokenizer(BPE())\ntokenizer.pre_tokenizer = Whitespace()\ntrainer = BpeTrainer(special_tokens=[\"[UNK]\", \"[CLS]\", \"[SEP]\", \"[PAD]\", \"[MASK]\"])\n\nimport torch\nimport numpy as np\nimport logging\nfrom transformers.trainer_utils import is_main_process\n\ntraining_args = TrainingArguments(\n    output_dir='/kaggle/working/results',          # output directory\n    num_train_epochs=3,              # total number of training epochs\n    per_device_train_batch_size=128,  # batch size per device during training\n    per_device_eval_batch_size=128,   # batch size for evaluation\n    logging_dir='/kaggle/working/logs',            # directory for storing logs\n    load_best_model_at_end= True,\n    metric_for_best_model= \"f1\",\n    evaluation_strategy=\"steps\",\n    eval_steps= 2000,\n)\n\nlogger = logging.getLogger(__name__)\n\n# Setup logging\nlogging.basicConfig(\n    format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n    datefmt=\"%m/%d/%Y %H:%M:%S\",\n    level=logging.INFO if is_main_process(training_args.local_rank) else logging.WARN,\n)\n\n# Log on each process the small summary:\nlogger.warning(\n    f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}\"\n    + f\"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}\"\n)\n# Set the verbosity to info of the Transformers logger (on main process only):\nif is_main_process(training_args.local_rank):\n    transformers.utils.logging.set_verbosity_info()\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\nlogger.info(\"Training/evaluation parameters %s\", training_args)\n# replicability status\nset_seed(0)\n\n# we abstract from numbers to strings (we think that this new encoding brings further precision)\ntrain_texts = []\ntrain_labels = []\n# Iterate over each row to generate the dataset\n\nfor index, rows in train_df.iterrows(): \n    # Create text for current numeric values\n    tmp_txt = str(rows.timestamp)+\" \"+str(rows.prior_question_elapsed_time)+\" \"+str(rows.prior_question_had_explanation)+\" \"+str(rows.correct_answer)+\" \"+str(rows.part) + \" \"+str(rows.tags)\n    tmp_label = rows.answered_correctly\n    train_labels.append(int(tmp_label))  \n    train_texts.append(tmp_txt) \n    \n# prepare custom tokenizer training\nvocab_file =\"/kaggle/working/riid.train.raw\"\nwith open(vocab_file, 'w') as file:\n    for item in train_texts:\n        file.write(\"%s\\n\" % item)\n        \n# split data (5 % dev and 95 % train)\ntrain_texts, val_texts, train_labels, val_labels = train_test_split(train_texts, train_labels, test_size=.05)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_texts[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# change to a non-internet version\ntokenizer.train(trainer, [vocab_file])\ntrain_encodings = tokenizer.encode_batch(train_texts)\nval_encodings = tokenizer.encode_batch(val_texts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for elem in train_encodings[:10]: \n    print(elem.ids)\n    print(elem.attention_mask)\n    print(\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, encodings, labels, max_len):\n        self.encodings = encodings\n        self.labels = labels\n        self.max_len = max_len\n\n    def __getitem__(self, idx):\n        item = {}\n        item['input_ids'] = torch.tensor(self.encodings[idx].ids + [0]*(self.max_len - len(self.encodings[idx].ids))) \n        item['attention_mask'] = torch.tensor(self.encodings[idx].attention_mask + [0]*(self.max_len - len(self.encodings[idx].attention_mask))) \n        item['labels'] = torch.tensor(self.labels[idx])\n        return item\n\n    def __len__(self):\n        return len(self.labels)\n\nmax_len = max([len(lst.ids) for lst in (train_encodings + val_encodings)])\ntrain_dataset = Dataset(train_encodings, train_labels, max_len)\nval_dataset = Dataset(val_encodings, val_labels, max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(2):\n    print(train_dataset.__getitem__(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(2):\n    print(val_dataset.__getitem__(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# change to a non-internet version\nconfiguration = DistilBertConfig(vocab_size=tokenizer.get_vocab_size())\nprint(configuration)\nmodel = DistilBertForSequenceClassification(configuration)\n\ndef compute_metrics(p):\n    predictions, labels = p\n    predictions = np.argmax(predictions, axis=1)\n    try:    \n        return {\n            \"roc_auc_score\": roc_auc_score(labels, predictions),\n            \"accuracy_score\": accuracy_score(labels, predictions),\n            \"precision\": precision_score(labels, predictions),\n            \"recall\": recall_score(labels, predictions),\n            \"f1\": f1_score(labels, predictions),\n        }\n    except:\n        return {\n            \"roc_auc_score\": 0,\n            \"accuracy_score\": 0,\n            \"precision\": 0,\n            \"recall\": 0,\n            \"f1\": 0,\n        }\n\ntrainer = Trainer(\n    model=model,                         # the instantiated 🤗 Transformers model to be trained\n    args=training_args,                  # training arguments, defined above\n    train_dataset=train_dataset,         # training dataset\n    eval_dataset=val_dataset,             # evaluation dataset\n    compute_metrics=compute_metrics\n)\n\n\ntrainer.train()\ntrainer.save_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can only iterate through a result from `env.iter_test()` once\n# so be careful not to lose it once you start iterating.\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    # we loop for every prediction batch adding the \"answered_correctly\" column with predicted info.\n    # first we avoid the lectures\n    test_df = test_df.loc[test_df['content_type_id'] == 0]\n    # we abstract from numbers to strings (we think that this new encoding brings further precision)\n    tmp_df = test_df.merge(questions_df, left_on='content_id', right_on='question_id', how= 'left')\n    \n    test_texts = []\n    test_labels = []\n    # Iterate over each row to generate the dataset\n    for index, rows in tmp_df.iterrows(): \n        # Create text for current numeric values\n        tmp_txt = str(rows.timestamp)+\" \"+str(rows.prior_question_elapsed_time)+\" \"+str(rows.prior_question_had_explanation)+\" \"+str(rows.correct_answer)+\" \"+str(rows.part) + \" \"+str(rows.tags)\n        test_labels.append(0)  # unknown\n        test_texts.append(tmp_txt) \n        \n    test_encodings = tokenizer.encode_batch(train_texts)\n    test_dataset = Dataset(test_encodings, test_labels, max_len)\n    predictions, labels, metrics = trainer.predict(test_dataset)\n    predictions = np.argmax(predictions, axis=1)\n    test_df['answered_correctly'] = predictions\n    print(test_df[['row_id', 'answered_correctly']])\n    env.predict(test_df[['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}