{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom pathlib import Path\nimport gc\nimport random\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nfrom pathlib import Path\nimport datatable as dt\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input')\nassert path.exists()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 构造特征\nimport pandas as pd\n\ndef add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum,\n                  timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum,\n                  elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update=True):\n    \n     # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n    # -----------------------------------------------------------------------\n    for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):\n        \n#         print(row[0])\n#         print(answered_correctly_u_count)\n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n        print(timestamp_u[row[0]])\n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n             \n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})\n    \n    \n    # 防止index 错序造成的concate错误\n    # 合并user_df 与 train_df\n    # 由于train_df 做过一次挑选,因此需要reset index\n    df.reset_index(drop=True, inplace=True)\n    \n    user_df.reset_index(drop=True, inplace=True)\n\n    df = pd.concat([df, user_df], axis=1)\n\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n            \n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# question\nimport pandas as pd\n\nquestions_df = pd.read_csv(path/'riiid-test-answer-prediction/questions.csv')\nquestions_df['part'] = questions_df['part'].astype(np.int32)\nquestions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\ngbm = lgb.Booster(model_file='../input/lightgbmmodel/model.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\n\n\njson_file = \"../input/lightgbm-features-dicts/lightgbm_features_dicts.json\"\n\nf = open(json_file)\nfeatures_dicts = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del f","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_u_count_ = features_dicts['answered_correctly_u_count']\nanswered_correctly_u_sum_ = features_dicts['answered_correctly_u_sum']\nelapsed_time_u_sum_ = features_dicts['elapsed_time_u_sum']\nexplanation_u_sum_ = features_dicts['explanation_u_sum']\nanswered_correctly_q_count_ = features_dicts['answered_correctly_q_count']\nanswered_correctly_q_sum_ = features_dicts['answered_correctly_q_sum']\nelapsed_time_q_sum_ = features_dicts['elapsed_time_q_sum']\nexplanation_q_sum_ = features_dicts['explanation_q_sum']\nanswered_correctly_uq_ = features_dicts['answered_correctly_uq']\ntimestamp_u_ = features_dicts['timestamp_u']\ntimestamp_u_incorrect_ = features_dicts['timestamp_u_incorrect']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\ndef change_type(temp):\n    result = defaultdict(int)\n    for k,v in temp.items():\n        key = float(k)\n        value = int(float(v))\n        result[key] =value\n    return result\n\n\nanswered_correctly_u_count = change_type(answered_correctly_u_count_)\nanswered_correctly_u_sum = change_type(answered_correctly_u_sum_)\nelapsed_time_u_sum = change_type(elapsed_time_u_sum_)\nexplanation_u_sum = change_type(explanation_u_sum_)\n\nanswered_correctly_q_count = change_type(answered_correctly_q_count_)\nanswered_correctly_q_sum = change_type(answered_correctly_q_sum_)\nelapsed_time_q_sum = change_type(elapsed_time_q_sum_)\nexplanation_q_sum = change_type(explanation_q_sum_)\n\n\ndel answered_correctly_u_count_\ndel answered_correctly_u_sum_\ndel elapsed_time_u_sum_\ndel explanation_u_sum_\ndel answered_correctly_q_count_\ndel answered_correctly_q_sum_\ndel elapsed_time_q_sum_\ndel explanation_q_sum_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def change_answered_correctly_uq_type(t):\n    result = defaultdict(lambda: defaultdict(int))\n    for k,v in t.items():\n        key = float(k)\n        temp = defaultdict(int)\n        for k1,v1 in v.items():\n            k1 = float(k1)\n            temp[k1] = v1\n        result[key] = temp\n    return result\n\nanswered_correctly_uq = change_answered_correctly_uq_type(answered_correctly_uq_)\ndel answered_correctly_uq_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def changetolist(t):\n    result = defaultdict(list)\n    for k,v in t.items():\n        key = float(k)\n        value = list(v)\n        result[key] = value\n    return result\n\ntimestamp_u = changetolist(timestamp_u_)\ntimestamp_u_incorrect = changetolist(timestamp_u_incorrect_)\n\ndel timestamp_u_\ndel timestamp_u_incorrect_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# 测试\n# Get feature dict\nfrom collections import defaultdict\n\nTARGET = 'answered_correctly'\nFEATURES = ['answered_correctly_u_avg', 'explanation_u_avg', 'elapsed_time_u_avg',\n            'answered_correctly_q_avg', 'explanation_q_avg', 'elapsed_time_q_avg',\n            'answered_correctly_uq_count', 'timestamp_u_recency_1',\n            'timestamp_u_recency_2', 'timestamp_u_recency_3',\n            'timestamp_u_incorrect_recency']\n# answered_correctly_u_count = features_dicts['answered_correctly_u_count']\n# answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\n# elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\n# explanation_u_sum = features_dicts['explanation_u_sum']\n# answered_correctly_q_count = features_dicts['answered_correctly_q_count']\n# answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\n# elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\n# explanation_q_sum = features_dicts['explanation_q_sum']\n# answered_correctly_uq = features_dicts['answered_correctly_uq']\n# timestamp_u = features_dicts['timestamp_u']\n# timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\nprior_question_elapsed_time_mean = 25423.810042960366\n# Get api iterator and predictor\nimport riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict\n\nprevious_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)\n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    test_df[TARGET] = 0\n    test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)\n    test_df[TARGET] =  gbm.predict(test_df[FEATURES],num_iteration=gbm.best_iteration)\n    set_predict(test_df[['row_id', TARGET]])\n\nprint('Job Done')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}