{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Time-series API (iter_test) Emulator \n\nThis script emulate all the features that have been officially announced about Time-series API:\n<pre>\nThe hidden test set contains new users but not new questions.\nThe test data follows chronologically after the train data. The test iterations give interactions of users chronologically.\nEach group will contain interactions from many different users, but no more than one task_container_id of questions from any single user. \nEach group has between 1 and 1000 users.\nExpect to see roughly 2.5 million questions in the hidden test set.\nThe API will also consume roughly 15 minutes of runtime for loading and serving the data.\nThe API loads the data using the types specified in Data Description page.\n</pre>\n\nI hope this helps to validation, especially for reducing \"Submission Scoring Error\".\n\nThis emulator may help to check following which can't check with official visible test:\n* Memory usage\n* Disk size consumed\n* The time it took to inference\n* Handling of New Users\n* Handling of not only questions but also lectures.\n* etc.\n\nTo deal with \"Submission Scoring Error\", you'd better to refer to [this discussion](https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/192124).\n\nAnd, of course, this will help you to check your validation score.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport time\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn import preprocessing\nimport os\nimport gc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## loading validation file.\nThis file is made by following notebook:\nhttps://www.kaggle.com/its7171/cv-strategy"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_df = pd.read_pickle('../input/riiid-cross-validation-files/cv2_valid.pickle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## iter_test emulator\nThis class emulate iter_test()"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n        pre_content_type_id = -1\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n                # known user(not prev user or (differnt task container and both question))\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            pre_content_type_id = crr_content_type_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## emulator setting"},{"metadata":{"trusted":true},"cell_type":"code","source":"validaten_flg = True\nif validaten_flg:\n    iter_test = Iter_Valid(target_df,max_user=1000)\n    predicted = []\n    def set_predict(df):\n        predicted.append(df)\nelse:\n    import riiideducation\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Feature and Exctracting"},{"metadata":{"trusted":true},"cell_type":"code","source":"def Feature_and_extracting():\n    dtype = {\n        'timestamp':'int64',\n        'content_type_id':'bool',\n        'content_id':'int16',\n        'answered_correctly':'int8',\n        'prior_question_elapsed_time':'float32',\n        'prior_question_had_explanation':'int8'\n    }\n    cols = [\n        'timestamp',\n        'content_type_id',\n        'content_id',\n        'answered_correctly',\n        'prior_question_elapsed_time',\n        'prior_question_had_explanation'\n    ]\n    path=\"/kaggle/input/riiid-test-answer-prediction/\"\n    if(os.path.exists('train.pkl')):\n        df_train = pd.read_pickle('train.pkl')\n    else:\n        df_train = pd.read_csv(path+'train.csv',sep=',',usecols=cols,dtype=dtype)\n        df_train.to_pickle('train.pkl')\n    df_q = pd.read_csv(path+'questions.csv', sep=',')\n    print(1)\n    df_train['prior_question_had_explanation'] = df_train['prior_question_had_explanation'].astype('int8')\n    df_train.info() \n    df_train = df_train[df_train.answered_correctly != -1]\n    correct = df_train.groupby([\"content_id\", 'answered_correctly'], as_index=False).size()\n    correct.info()\n    gc.collect()\n    correct = correct.pivot(index= \"content_id\", columns='answered_correctly', values='size')\n    \n    print(12)\n    correct.columns = ['Wrong', 'Right']\n    correct = correct.fillna(0)\n    print(2)\n    correct[['Wrong', 'Right']] = correct[['Wrong', 'Right']].astype(int)\n    df_q = df_q.merge(correct, left_on = \"question_id\", right_on = \"content_id\", how = \"left\")\n    print(3)\n    del df_train['content_type_id']\n    del df_q['bundle_id']\n    del df_q['part']\n    df_q['dif'] = df_q['Wrong']/(df_q['Right']+df_q['Wrong'])\n    del df_q['correct_answer']\n    del df_q['tags']\n    del df_q['Wrong']\n    del df_q['Right']\n    print(5)\n    df_train.info()\n    df_train = df_train.merge(df_q,how = 'inner',left_on='content_id',right_on='question_id')\n    del df_train['content_id']\n    answers = df_train['answered_correctly']\n    del df_train['answered_correctly']\n    print(4)\n    \n    del df_train['question_id']\n    df_train = df_train.fillna(0)\n    \n    \n    answers = answers.to_numpy(dtype='bool')  \n    answers = np.array([answers, ~answers],dtype='int8').transpose()\n    \n    \n    from sklearn import preprocessing\n    df_train = df_train.values\n    min_max_scaler = preprocessing.MinMaxScaler()\n    df_train = min_max_scaler.fit_transform(df_train)\n    \n    \n    \n    import tensorflow as tf\n    from tensorflow import keras\n    model = keras.Sequential()\n    act = 'relu'\n    model.add(keras.layers.Dense(350, input_dim=4, activation=act))\n\n    model.add(keras.layers.Dense(100, activation=act))\n\n    model.add(keras.layers.Dense(2, activation='softmax'))\n\n    model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['AUC'])\n    model.fit(df_train, answers, epochs=5, batch_size=10000,verbose=1,validation_split=0.3)\n\n    model.save('model')\n    return model\ndef Predict(df_test, df_q, model):\n    df_test = df_test.merge(df_q,how = 'inner',left_on='content_id',right_on='bundle_id')\n    row_id = df_test.row_id\n    df_test = df_test[['timestamp','prior_question_elapsed_time','prior_question_had_explanation','dif']]\n    df_test['timestamp'] = df_test['timestamp'].astype('float32')\n    df_test['prior_question_had_explanation'] = df_test['prior_question_had_explanation'].astype('float32')\n    df_test = df_test.values\n    min_max_scaler = preprocessing.MinMaxScaler()\n    df_test = min_max_scaler.fit_transform(df_test)\n    pred = model.predict(df_test)\n    return pred.reshape((2,len(pred)))[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## iterator\nNow we can use iter_test(wrapper for env.iter_test) and set_predict(wrapper for env.predict) as usual."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype = {\n    'timestamp':'int64',\n    'content_type_id':'bool',\n    'content_id':'int16',\n    'answered_correctly':'int8',\n    'prior_question_elapsed_time':'float32',\n    'prior_question_had_explanation':'int8'\n}\ncols = [\n    'timestamp',\n    'content_type_id',\n    'content_id',\n    'answered_correctly',\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation'\n]\npath=\"/kaggle/input/riiid-test-answer-prediction/\"\nif(os.path.exists('train.pkl')):\n    df_train = pd.read_pickle('train.pkl')\nelse:\n    df_train = pd.read_csv(path+'train.csv',sep=',',usecols=cols,dtype=dtype)\n    df_train.to_pickle('train.pkl')\ndf_q = pd.read_csv(path+'questions.csv', sep=',')\n\ncorrect = df_train[df_train.answered_correctly != -1].groupby([\"content_id\", 'answered_correctly'], as_index=False).size()\ncorrect = correct.pivot(index= \"content_id\", columns='answered_correctly', values='size')\ncorrect.columns = ['Wrong', 'Right']\ncorrect = correct.fillna(0)\ncorrect[['Wrong', 'Right']] = correct[['Wrong', 'Right']].astype(int)\ndf_q = df_q.merge(correct, left_on = \"question_id\", right_on = \"content_id\", how = \"left\")\n\ndel correct\ndf_q['dif'] = df_q['Wrong']/(df_q['Right']+df_q['Wrong'])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_q = df_q[['question_id','dif']]\ndf_q.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Feature_and_extracting()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pbar = tqdm(total=2500000)\nprevious_test_df = None\ncounter = 0\nfor (current_test, current_prediction_df) in iter_test:          \n    if previous_test_df is not None:\n        answers = eval(current_test[\"prior_group_answers_correct\"].iloc[0])\n        responses = eval(current_test[\"prior_group_responses\"].iloc[0])\n        previous_test_df['answered_correctly'] = answers\n        previous_test_df['user_answer'] = responses\n    previous_test_df = current_test.copy()\n    current_test = current_test[current_test.content_type_id == 0]\n    # your prediction code here\n    current_test['answered_correctly'] = Predict(current_test,df_q,model)\n    set_predict(current_test.loc[:,['row_id', 'answered_correctly']])\n    pbar.update(len(current_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if validaten_flg:\n#     #validation score\n#     y_true = target_df[target_df.content_type_id == 0].answered_correctly\n#     y_pred = pd.concat(predicted).answered_correctly\n#     print('validation auc:',roc_auc_score(y_true, y_pred))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}