{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":1823617,"sourceType":"datasetVersion","datasetId":945277},{"sourceId":10797436,"sourceType":"datasetVersion","datasetId":6701332}],"dockerImageVersionId":30886,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install trueskill\nimport dill\nimport pickle\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\nimport gc\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport time\nimport pickle\nimport seaborn as sns\nimport dill\nimport re\nimport sqlite3\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom trueskill import Rating, quality_1vs1, rate_1vs1\nimport trueskill\nimport math\nfrom sklearn.preprocessing import LabelEncoder\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:41.78487Z","iopub.execute_input":"2025-02-19T16:12:41.785168Z","iopub.status.idle":"2025-02-19T16:12:53.027908Z","shell.execute_reply.started":"2025-02-19T16:12:41.785144Z","shell.execute_reply":"2025-02-19T16:12:53.026868Z"}},"outputs":[{"name":"stdout","text":"Collecting trueskill\n  Downloading trueskill-0.4.5.tar.gz (21 kB)\n  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\nRequirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from trueskill) (1.17.0)\nBuilding wheels for collected packages: trueskill\n  Building wheel for trueskill (setup.py) ... \u001b[?25l\u001b[?25hdone\n  Created wheel for trueskill: filename=trueskill-0.4.5-py3-none-any.whl size=18049 sha256=5b663693fda668c7a6eaef74f4de68ac0db590f8f92a5377c0cf0de6c129747a\n  Stored in directory: /root/.cache/pip/wheels/b9/4f/29/c79f0a2956775524c7a23638ac2b6fbb516c680f8e5eed9b53\nSuccessfully built trueskill\nInstalling collected packages: trueskill\nSuccessfully installed trueskill-0.4.5\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"def pickle_dump_dill(obj, path):\n    with open(path, mode='wb') as f:\n        dill.dump(obj,f)\n\ndef pickle_load_dill(path):\n    with open(path, mode='rb') as f:\n        data = dill.load(f)\n        return data  \n\ndef pickle_dump(obj, path):\n    with open(path, mode='wb') as f:\n        pickle.dump(obj,f)\n\ndef pickle_load(path):\n    with open(path, mode='rb') as f:\n        data = pickle.load(f)\n        return data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:53.029319Z","iopub.execute_input":"2025-02-19T16:12:53.030008Z","iopub.status.idle":"2025-02-19T16:12:53.036025Z","shell.execute_reply.started":"2025-02-19T16:12:53.029972Z","shell.execute_reply":"2025-02-19T16:12:53.034781Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"class Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n        pre_content_type_id = -1\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n                # known user(not prev user or (differnt task container and both question))\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            pre_content_type_id = crr_content_type_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:53.039048Z","iopub.execute_input":"2025-02-19T16:12:53.039659Z","iopub.status.idle":"2025-02-19T16:12:53.080865Z","shell.execute_reply.started":"2025-02-19T16:12:53.039603Z","shell.execute_reply":"2025-02-19T16:12:53.07965Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:26:46.589458Z","iopub.execute_input":"2025-02-19T16:26:46.589806Z","iopub.status.idle":"2025-02-19T16:26:46.605239Z","shell.execute_reply.started":"2025-02-19T16:26:46.589777Z","shell.execute_reply":"2025-02-19T16:26:46.604026Z"}},"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)","\u001b[0;32m<ipython-input-7-140a36a9155e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mriiideducation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mriiideducation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_env\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0miter_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mset_predict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/kaggle/input/riiid-test-answer-prediction/riiideducation/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mcompetition\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmake_env\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0m__all__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'make_env'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'riiideducation.competition'"],"ename":"ModuleNotFoundError","evalue":"No module named 'riiideducation.competition'","output_type":"error"}],"execution_count":7},{"cell_type":"code","source":"prior_b_dict = pickle_load_dill('../input/riid-preprocessed/prior_b_dict.pickle')\ndiagnostic_u_dict = pickle_load_dill('../input/riid-preprocessed/diagnostic_u_dict.pickle')    \nquestion_trueskill_dict = pickle_load_dill('../input/riid-preprocessed/question_trueskill_dict.pickle')\nuser_trueskill_dict = pickle_load_dill('../input/riid-preprocessed/user_trueskill_dict.pickle')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:53.269445Z","iopub.status.idle":"2025-02-19T16:12:53.269818Z","shell.execute_reply":"2025-02-19T16:12:53.269656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_model = pickle.load(open('/kaggle/input/riid-preprocessed/lgb_model.pkl', 'rb'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:53.270423Z","iopub.status.idle":"2025-02-19T16:12:53.270751Z","shell.execute_reply":"2025-02-19T16:12:53.270614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cp -r '../input/riid-preprocessed/feat_new.db' ./ \ndbname = './feat_new.db'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:53.271884Z","iopub.status.idle":"2025-02-19T16:12:53.272386Z","shell.execute_reply":"2025-02-19T16:12:53.272188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pbar = tqdm(total=2500000)\nprevious_test_df = None\ncounter = 0\nconn = sqlite3.connect(dbname) \nfor (test_df, sample_prediction_df) in iter_test:\n    counter += 1\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_feats_sqlite(previous_test_df, conn)\n        update_trueskill(previous_test_df, user_trueskill_dict, question_trueskill_dict, verbose=False)\n        if not counter % 100:\n            conn.commit()\n    test_df.timestamp=(test_df.timestamp/1000).astype('int32')\n    test_df = add_prior_b(test_df, questions_cb, questions_bpt, prior_b_dict,verbose=False)\n    test_df['col_for_merge']=(test_df['content_id'].astype('int32')*10+test_df['content_type_id']).astype('int32')\n    test_df = test_df.merge(content_df, how = 'left', on = 'col_for_merge')\n    test_df.content_id=test_df.content_id.astype('int32')\n    test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'].fillna(25439.41)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False).astype('int8')\n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_feats_without_update_sqlite(test_df, conn)\n    test_df = add_is_diagnostic(test_df, diagnostic_u_dict,verbose=False)\n    test_df = add_trueskill_without_update(test_df, user_trueskill_dict, question_trueskill_dict, verbose=False)\n    test_df = add_new_columns(test_df)\n    test_df = fillna_df(test_df)\n    lgb_result =  lgb_model.predict(test_df[FEATS])\n    test_df[TARGET] = lgb_result\n    set_predict(test_df[['row_id', TARGET]])\n    pbar.update(len(test_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T16:12:53.273141Z","iopub.status.idle":"2025-02-19T16:12:53.273485Z","shell.execute_reply":"2025-02-19T16:12:53.273328Z"}},"outputs":[],"execution_count":null}]}