{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n!cp ../input/rapids/rapids.0.15.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path \n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cudf  # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cupy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(props):\n    start_mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in props.columns:\n        if props[col].dtype != object:  # Exclude strings\n            \n          \n            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = props[col].max()\n            mn = props[col].min()\n            \n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(props[col]).all(): \n                NAlist.append(col)\n                props[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = props[col].fillna(0).astype(np.int64)\n            result = (props[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True\n\n            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        props[col] = props[col].astype(np.uint8)\n                    elif mx < 65535:\n                        props[col] = props[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        props[col] = props[col].astype(np.uint32)\n                    else:\n                        props[col] = props[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        props[col] = props[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        props[col] = props[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        props[col] = props[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        props[col] = props[col].astype(np.int64)    \n            \n            # Make float datatypes 32 bit\n            else:\n                props[col] = props[col].astype(np.float32)\n            \n           \n    \n    \n    \n   \n    \n    return props","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train =pd.read_pickle('../input/riid-part-3/whole.pkl')\n\n\n\n\n\nprint(train.columns)\n\n\ntrain = cudf.from_pandas(train)\n\nattempt = train[[\"attempt_no\",'user_id','content_id']].drop_duplicates(subset=['content_id','user_id'], keep='last').set_index(['user_id','content_id'],drop=False)\n\n#lb_make.fit(train['prior_question_had_explanation'])\n\ncorrect_user = train[['answered_correctly_user','user_id']].set_index('user_id',drop=False)\n\ncorrect_container = train[['task_container_id','answered_correctly_container']].set_index('task_container_id',drop=False)\n\ncorrect_question =train[['content_id','answered_correctly_question']].set_index('content_id',drop=False)\n\nmean_explanations = train[['mean_explanations','user_id']].set_index('user_id',drop=False)\n\nuser_questions = train[['User_questions','user_id']].set_index('user_id',drop=False)\n\n\nuser_acc = correct_user.copy()\nuser_acc['user_acc'] = correct_user['answered_correctly_user'] - correct_user['answered_correctly_user'].shift(1)\nuser_acc.drop('answered_correctly_user',axis=1,inplace=True)\nuser_acc = user_acc.drop_duplicates(subset='user_id', keep='last')\n\ndel train\n\ncorrect_user = correct_user.drop_duplicates(subset=['user_id'], keep='last')\ncorrect_container = correct_container.drop_duplicates(subset=['task_container_id'], keep='last')\ncorrect_question = correct_question.drop_duplicates(subset=['content_id'], keep='last')\nuser_questions = user_questions.drop_duplicates(subset=['user_id'] , keep='last' )\nmean_explanations = mean_explanations.drop_duplicates(subset=['user_id'], keep='last')\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer_wrong_question  = correct_question.copy()\nanswer_wrong_question['answer_wrong_question'] = [c for c in (1 - correct_question['answered_correctly_question'].to_pandas())]\nanswer_wrong_question.drop('answered_correctly_question',axis=1,inplace=True)\n\n\n\nanswer_wrong_user = correct_user.copy()\nanswer_wrong_user['answer_wrong_user'] = [c for c in (1 - correct_user['answered_correctly_user'].to_pandas())]\nanswer_wrong_user.drop('answered_correctly_user',axis=1,inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type_of_sq =  cudf.from_pandas(pd.read_pickle('../input/sq-lectures-user2/type_of_sq.pkl')[['type_of_sq','user_id']].drop_duplicates(subset=['user_id'] , keep='last' ))\ntype_of_con = cudf.from_pandas(pd.read_pickle('../input/conc-lectures-user/con_lec_user.pkl')[['type_of_con','user_id']].drop_duplicates(subset=['user_id'] , keep='last' ))\ntype_of_start =  cudf.from_pandas(pd.read_pickle('../input/starter-lectures-user/starter_lec_user.pkl')[['type_of_start','user_id']].drop_duplicates(subset=['user_id'] , keep='last' ))\ntype_of_int = cudf.from_pandas(pd.read_pickle('../input/intent-lectures-user/int_lec_user.pkl')[['type_of_int','user_id']].drop_duplicates(subset=['user_id'] , keep='last' ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = cudf.read_csv('../input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags = cudf.from_pandas(pd.read_pickle('../input/tagss/tags.pkl'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\n\n\n\n\nquestions_ = [c for c in correct_question.to_pandas()['content_id'].unique()]\n\ncorrect_question = correct_question[['content_id','answered_correctly_question']].to_pandas().set_index('content_id').T.to_dict('list')\npart = questions[['question_id','part']].set_index('question_id').to_pandas().T.to_dict('list')\ntags1 = tags[['question_id','tags1']].set_index('question_id').to_pandas().T.to_dict('list')\ntags2 = tags[['question_id','tags2']].set_index('question_id').to_pandas().T.to_dict('list')\n\nfor q in questions_:\n    correct_question[q] = correct_question[q][0]\n    part[q] = part[q][0]\n    tags1[q] = tags1[q][0]\n    tags2[q] = tags2[q][0]\n\nusers = [c for c in correct_user.to_pandas()['user_id'].unique()]\ntype_of_sq = type_of_sq[['user_id','type_of_sq']].set_index('user_id').to_pandas().T.to_dict('list')\ntype_of_int = type_of_int[['user_id','type_of_int']].set_index('user_id').to_pandas().T.to_dict('list')\ntype_of_con = type_of_con[['user_id','type_of_con']].set_index('user_id').to_pandas().T.to_dict('list')\ntype_of_start = type_of_start[['user_id','type_of_start']].set_index('user_id').to_pandas().T.to_dict('list')\nuser_questions = user_questions.to_pandas().set_index('user_id').T.to_dict('list')\ncorrect_user = correct_user.to_pandas().T.to_dict('list')\nanswer_wrong_user = answer_wrong_user.to_pandas().T.to_dict('list')\nmean_explanations = mean_explanations.to_pandas().T.to_dict('list')\nuser_acc = user_acc[['user_id','user_acc']].to_pandas().set_index('user_id').T.to_dict('list')\n\n\nfor u in users:\n    type_of_sq[u] = type_of_sq[u][0]\n    type_of_int[u] = type_of_int[u][0]\n    type_of_con[u] = type_of_con[u][0]\n    type_of_start[u] = type_of_start[u][0]\n    user_questions[u] = user_questions[u][0]\n    correct_user[u] = correct_user[u][0]\n    answer_wrong_user[u] = answer_wrong_user[u][0]\n    mean_explanations[u] = mean_explanations[u][0]\n    user_acc[u] = user_acc[u][0]\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"from tqdm import tqdm\nattempts= defaultdict(int)\nfor cnt,row in enumerate(tqdm(cupy.array(attempt[['user_id','content_id','attempt_no']].values))):\n    attempts[int(row[0]),int(row[1])] = int(row[2])\n    "},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"def treat_users():"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nclf = lgb.Booster(model_file='../input/final-model-5d523e/lgb_classifier.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\n\n\ncorrect_user\n\nuser_acc\n\nattempt\n\nanswer_wrong_user\n\ncorrrect_user_sum\n\nwrong_user_sum\n\nuser_questions\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_data(df):\n    uq = np.zeros(len(df))\n    cu = np.zeros(len(df))\n    wu = np.zeros(len(df))\n    me = np.zeros(len(df))\n    \n    tg1 = np.zeros(len(df))\n    tg2 = np.zeros(len(df))\n    pt = np.zeros(len(df))\n    corQ = np.zeros(len(df))\n    wronQ = np.zeros(len(df))\n    userac = np.zeros(len(df))\n    \n    sq= np.zeros(len(df))\n    init= np.zeros(len(df))\n    start= np.zeros(len(df))\n    con= np.zeros(len(df))\n    \n    for cnt,row in enumerate(cupy.array(df[['user_id','content_id']].values)):\n        if user_questions.get(int(row[0])) != None:\n            uq[cnt] = user_questions[int(row[0])]\n            cu[cnt] = correct_user[int(row[0])]\n            wu[cnt] = answer_wrong_user[int(row[0])]\n            me[cnt] = mean_explanations[int(row[0])]\n            userac[cnt] = user_acc[int(row[0])]\n                        \n            \n        else:\n            uq[cnt] = 0\n            cu[cnt] =0.643215\n            wu[cnt] = 1-0.643215\n            me[cnt] = 0\n            userac[cnt] = 0\n            \n            \n            \n            \n            \n        if type_of_sq.get(int(row[0])) != None:\n            sq[cnt] = type_of_sq[int(row[0])]\n        else: sq[cnt] = 0\n        \n        if type_of_int.get(int(row[0])) != None:\n            init[cnt] = type_of_int[int(row[0])]\n        else: init[cnt] = 0\n         \n        if type_of_start.get(int(row[0])) != None:\n            start[cnt] = type_of_start[int(row[0])]\n        else: start[cnt] = 0\n           \n        \n        if type_of_con.get(int(row[0])) != None:\n             con[cnt] = type_of_con[int(row[0])]\n        else:con[cnt] = 0\n        \n        if tags1.get(int(row[1])) != None:\n            tg1[cnt] = tags1[int(row[1])]\n        else: \n            tg1[cnt]=0\n        if tags2.get(int(row[1])) != None:\n            tg2[cnt] = tags2[int(row[1])]\n        else: \n            tg2[cnt]=0\n        if part.get(int(row[1]))!=None:\n            pt[cnt] = part[int(row[1])]\n        else: \n            pt[cnt] = -1\n        \n        if correct_question.get(int(row[1]))!=None:\n            corQ[cnt] = correct_question[int(row[1])]\n            wronQ[cnt] = 1 - correct_question[int(row[1])]\n        else:\n            corQ[cnt] = 0.5\n            wronQ[cnt] = 0.5\n            \n        \n       \n\n    df['User_questions'] = uq\n    df['answer_wrong_user'] = wu\n    df['answered_correctly_user'] = cu\n    df['mean_explanations'] = me\n    df['user_acc'] = userac\n    \n    df['part'] = pt\n    df['tags1'] = tg1\n    df['tags2'] = tg2\n    df['answered_correctly_question'] = corQ\n    df['answer_wrong_question'] = wronQ\n    \n    df['type_of_start'] = start\n    df['type_of_con'] = con\n    df['type_of_sq'] = sq\n    df['type_of_int'] = init\n    \n    \n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef registrar(reg):\n    \n    \n        \n   \n    for cnt,row in enumerate(cupy.array(reg[['user_id','answered_correctly','prior_question_had_explanation','attempt_no']].values)):\n        if user_questions.get(int(row[0])) != None:\n            user_questions[int(row[0])] = user_questions[int(row[0])]+1\n            prev = correct_user[int(row[0])]\n            correct_user[int(row[0])] = ((correct_user[int(row[0])]*user_questions[int(row[0])])+int(row[1]))/(user_questions[int(row[0])]+1)\n            user_acc[int(row[0])] = correct_user[int(row[0])] - prev\n                                                 \n            answer_wrong_user[int(row[0])] = 1-correct_user[int(row[0])]\n            mean_explanations[int(row[0])] = (mean_explanations[int(row[0])]*user_questions[int(row[0])]+int(row[2]))/(user_questions[int(row[0])]+1)\n        else:\n            user_questions[int(row[0])] = 0\n            correct_user[int(row[0])] =0.643215\n            answer_wrong_user[int(row[0])] = 1-0.643215\n            mean_explanations[int(row[0])] = 0\n            user_acc[int(row[0])] = 0                                   \n        \n        \n\n        \n    \n\n    reg['attempt_no'] = reg['attempt_no']+1\n    \n    \n    \n    \n    \n    \n    attempt.append(reg[[\"attempt_no\",'user_id','content_id']].to_pandas()).drop_duplicates(subset=['content_id','user_id'], keep='last')\n   \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i=False\nfor (test_df, sample_prediction_df) in iter_test:\n    start = time.time()\n    \n    test_df = cudf.DataFrame(test_df)\n    \n    test_df['saw_lecture'] = [c for c in (test_df.to_pandas().groupby('user_id')['content_type_id'].shift(1).fillna(0))]\n    test_df['saw_lecture'] = test_df['saw_lecture']==1\n    \n    \n    \n    test_df = test_df.loc[test_df['content_type_id'] == 0]\n    \n    if i ==True :\n        \n      \n        previous['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        registrar(previous)\n        \n    \n    test_df['prior_question_had_explanation'].fillna(False,inplace=True)\n    del test_df['prior_question_elapsed_time']\n    \n    \n    \n    \n    test_df = cudf.merge(test_df,cudf.DataFrame(correct_container[['task_container_id','answered_correctly_container']]),on='task_container_id', how='left')\n  \n \n    \n    \n    \n    \n    test_df = add_data(test_df)\n    \n   \n    test_df = cudf.merge(test_df, cudf.DataFrame(attempt[[\"attempt_no\",'user_id','content_id']]),on=['user_id','content_id'],how='left')\n    \n    \n    \n    \n    test_df[\"attempt_no\"].fillna(1,inplace=True)\n   \n    \n    test_df = test_df.sort_values('row_id')\n    previous = test_df.copy()\n    test_df = test_df.to_pandas()\n    \n   \n    \n    \n    test_df['answered_correctly'] = clf.predict(test_df[['part', 'content_id', 'task_container_id','answered_correctly_container','answer_wrong_user','type_of_con','type_of_sq'\n       ,'answer_wrong_question','user_acc','User_questions','mean_explanations','tags1','tags2','attempt_no','type_of_int','type_of_start',\n                        'answered_correctly_question','answered_correctly_user','saw_lecture']])\n    \n    \n    \n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    i=True\n    stop = time.time()\n    duration = stop-start\n    print(duration)\n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}