{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   usecols=[2, 3,   7, 8, 9],\n                   dtype={\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          \n                          \n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n                            usecols=[0,1, 3,4],\n                            dtype={'question_id': 'int16',\n                              'part': 'int8','bundle_id': 'int8','tags': 'str'}\n                          )\ntag = questions_df[\"tags\"].str.split(\" \", n = 10, expand = True) \ntag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']\ntag['tagsum'] = 6 - tag.isnull().sum(axis=1)\ntag['content_id'] = questions_df['question_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user = pd.DataFrame(train,columns = ['user_id','content_id','answered_correctly','prior_question_elapsed_time'])\n#位移一列位置\nuser.prior_question_elapsed_time = user.prior_question_elapsed_time.shift(-1)\nuser.prior_question_elapsed_time = user.prior_question_elapsed_time.fillna(0)\nuser_lectures = user[user.answered_correctly == -1]\nuser_question = user[user.answered_correctly != -1]\n#增加空间\n\ndel user","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#bundle_id是指每道题的编号，有些大问题包含多个子问题，每个子问题编号未question_id,大题目编号是bundle_id\n\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nquestions = questions.rename(columns = {'question_id':'content_id'})\nlectures = lectures.rename(columns = {'lecture_id':'content_id'})\nquestions = pd.DataFrame(questions,columns = ['content_id','part'])\nuser_lectures = pd.merge(user_lectures,lectures,on ='content_id',how = 'left')\nuser_question = pd.merge(user_question,questions,on ='content_id',how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question_part = pd.DataFrame(user_question,columns = ['user_id','part','answered_correctly'])\nuser_question_time = pd.DataFrame(user_question,columns = ['user_id','part','prior_question_elapsed_time'])\ndel user_question","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question_part_1 = user_question_part.groupby(['user_id','part'],as_index = False).sum()\nuser_question_part_2 = user_question_part.groupby(['user_id','part'],as_index = False).count()\ndel user_question_part","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question_part  = pd.merge(user_question_part_1,user_question_part_2,on =['user_id','part'],how = 'left')\nuser_question_part['correctly'] = user_question_part.answered_correctly_x/user_question_part.answered_correctly_y\nuser_question_part = user_question_part.rename(columns = {'answered_correctly_x':'answered_correctly','answered_correctly_y':'answered_count'})\nuser_question_part = user_question_part.drop(columns=['answered_correctly'])\nuser_question_time = user_question_time.groupby(['user_id','part'],as_index = False).sum()\nuser_question = pd.merge(user_question_part,user_question_time,on =['user_id','part'],how = 'left')\ndel user_question_part_1,user_question_part_2,user_question_part,user_question_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function\n\nimport math\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_question['answered_count'].mean()\nstd = user_question['answered_count'].std()\nerror = user_question[np.abs(user_question['answered_count'] - u) > 3*std]\nuser_question.loc[user_question.answered_count >= error.answered_count.min(),'answered_count'] = error.answered_count.min()\naverage = float(sum(user_question.answered_count))/len(user_question.answered_count)\ntotal = 0\n\nfor value in user_question.answered_count:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_question.answered_count))\nuser_question.answered_count = [(x-average)/stddev for x in user_question.answered_count]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_question['prior_question_elapsed_time'].mean()\nstd = user_question['prior_question_elapsed_time'].std()\nerror = user_question[np.abs(user_question['prior_question_elapsed_time'] - u) > 3*std]\nuser_question.loc[user_question.prior_question_elapsed_time >= error.prior_question_elapsed_time.min(),'prior_question_elapsed_time'] = error.prior_question_elapsed_time.min()\naverage = float(sum(user_question.prior_question_elapsed_time))/len(user_question.prior_question_elapsed_time)\ntotal = 0\n\nfor value in user_question.prior_question_elapsed_time:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_question.prior_question_elapsed_time))\nuser_question.prior_question_elapsed_time = [(x-average)/stddev for x in user_question.prior_question_elapsed_time]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_c = pd.DataFrame(train,columns = ['user_id','content_id','answered_correctly','prior_question_elapsed_time'])\nquestion_c.prior_question_elapsed_time = question_c.prior_question_elapsed_time.shift(-1)\nquestion_c.prior_question_elapsed_time = question_c.prior_question_elapsed_time.fillna(0)\nquestion_c = question_c[question_c.answered_correctly != -1]\nquestion_c = pd.merge(question_c,questions,on ='content_id',how = 'left')\nquestion_c_part = pd.DataFrame(question_c,columns = ['content_id','part','answered_correctly'])\nquestion_c_part_1 = question_c_part.groupby(['content_id','part'],as_index = False).sum()\nquestion_c_part_2 = question_c_part.groupby(['content_id','part'],as_index = False).count()\ndel question_c_part","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_c_part  = pd.merge(question_c_part_1,question_c_part_2,on =['content_id','part'],how = 'left')\nquestion_c_part['correctly'] = question_c_part.answered_correctly_x/question_c_part.answered_correctly_y\nquestion_c_part = question_c_part.rename(columns = {'answered_correctly_x':'answered_correctly','answered_correctly_y':'answered_count'})\nquestion_c_part = question_c_part.drop(columns=['answered_correctly'])\nquestion_c_part = pd.merge(question_c_part,tag,on = 'content_id',how = 'left')\nquestion_c_part = question_c_part.drop(columns=['tags1','tags2','tags3','tags4','tags5','tags6'])\ndel question_c_part_1,question_c_part_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = question_c_part['answered_count'].mean()\nstd = question_c_part['answered_count'].std()\nerror = question_c_part[np.abs(question_c_part['answered_count'] - u) > 3*std]\nquestion_c_part.loc[question_c_part.answered_count >= error.answered_count.min(),'answered_count'] = error.answered_count.min()\naverage = float(sum(question_c_part.answered_count))/len(question_c_part.answered_count)\ntotal = 0\n\nfor value in question_c_part.answered_count:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(question_c_part.answered_count))\nquestion_c_part.answered_count = [(x-average)/stddev for x in question_c_part.answered_count]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = question_c_part['tagsum'].mean()\nstd = question_c_part['tagsum'].std()\nerror = question_c_part[np.abs(question_c_part['tagsum'] - u) > 3*std]\nquestion_c_part.loc[question_c_part.tagsum >= error.tagsum.min(),'tagsum'] = error.tagsum.min()\naverage = float(sum(question_c_part.tagsum))/len(question_c_part.tagsum)\ntotal = 0\n\nfor value in question_c_part.tagsum:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(question_c_part.tagsum))\nquestion_c_part.tagsum = [(x-average)/stddev for x in question_c_part.tagsum]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lectures_tag = pd.DataFrame(user_lectures,columns = ['user_id','part','tag'])\nuser_lectures_tag['temp'] = 1\nuser_lectures_tag = user_lectures_tag.groupby(['user_id','part','tag'],as_index = False).count()\nuser_lectures_tag = pd.DataFrame(user_lectures_tag,columns = ['user_id','part','tag'])\nuser_lectures_tag = user_lectures_tag.groupby(['user_id','part'],as_index = False).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lectures_temp= pd.DataFrame(user_lectures,columns = ['user_id','part','type_of'])\nuser_lectures_intention = user_lectures_temp[user_lectures_temp.type_of == 'intention']\nuser_lectures_concept = user_lectures_temp[user_lectures_temp.type_of == 'concept']\nuser_lectures_solving = user_lectures_temp[user_lectures_temp.type_of == 'solving question']\nuser_lectures_intention = user_lectures_intention.groupby(['user_id','part'],as_index = False).count()\nuser_lectures_concept = user_lectures_concept.groupby(['user_id','part'],as_index = False).count()\nuser_lectures_solving = user_lectures_solving.groupby(['user_id','part'],as_index = False).count()\nuser_lectures_intention = user_lectures_intention.rename(columns = {'type_of':'intention'})\nuser_lectures_concept = user_lectures_concept.rename(columns = {'type_of':'concept'})\nuser_lectures_solving = user_lectures_solving.rename(columns = {'type_of':'solving'})\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lectures_lt = pd.DataFrame(user_lectures,columns = ['user_id','part','prior_question_elapsed_time'])\nuser_lectures_lt = user_lectures_lt.groupby(['user_id','part'],as_index = False).sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_lectures_intention['intention'].mean()\nstd = user_lectures_intention['intention'].std()\nerror = user_lectures_intention[np.abs(user_lectures_intention['intention'] - u) > 3*std]\nuser_lectures_intention.loc[user_lectures_intention.intention >= error.intention.min(),'intention'] = error.intention.min()\naverage = float(sum(user_lectures_intention.intention))/len(user_lectures_intention.intention)\ntotal = 0\n\nfor value in user_lectures_intention.intention:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_lectures_intention.intention))\nuser_lectures_intention.intention = [(x-average)/stddev for x in user_lectures_intention.intention]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_lectures_concept['concept'].mean()\nstd = user_lectures_concept['concept'].std()\nerror = user_lectures_concept[np.abs(user_lectures_concept['concept'] - u) > 3*std]\nuser_lectures_concept.loc[user_lectures_concept.concept >= error.concept.min(),'concept'] = error.concept.min()\naverage = float(sum(user_lectures_concept.concept))/len(user_lectures_concept.concept)\ntotal = 0\n\nfor value in user_lectures_concept.concept:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_lectures_concept.concept))\nuser_lectures_concept.concept = [(x-average)/stddev for x in user_lectures_concept.concept]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_lectures_solving['solving'].mean()\nstd = user_lectures_solving['solving'].std()\nerror = user_lectures_solving[np.abs(user_lectures_solving['solving'] - u) > 3*std]\nuser_lectures_solving.loc[user_lectures_solving.solving >= error.solving.min(),'solving'] = error.solving.min()\naverage = float(sum(user_lectures_solving.solving))/len(user_lectures_solving.solving)\ntotal = 0\n\nfor value in user_lectures_solving.solving:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_lectures_solving.solving))\nuser_lectures_solving.solving = [(x-average)/stddev for x in user_lectures_solving.solving]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_lectures_tag['tag'].mean()\nstd = user_lectures_tag['tag'].std()\nerror = user_lectures_tag[np.abs(user_lectures_tag['tag'] - u) > 3*std]\nuser_lectures_tag.loc[user_lectures_tag.tag >= error.tag.min(),'tag'] = error.tag.min()\naverage = float(sum(user_lectures_tag.tag))/len(user_lectures_tag.tag)\ntotal = 0\n\nfor value in user_lectures_tag.tag:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_lectures_tag.tag))\nuser_lectures_tag.tag = [(x-average)/stddev for x in user_lectures_tag.tag]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u = user_lectures_lt['prior_question_elapsed_time'].mean()\nstd = user_lectures_lt['prior_question_elapsed_time'].std()\nerror = user_lectures_lt[np.abs(user_lectures_lt['prior_question_elapsed_time'] - u) > 3*std]\nuser_lectures_lt.loc[user_lectures_lt.prior_question_elapsed_time >= error.prior_question_elapsed_time.min(),'prior_question_elapsed_time'] = error.prior_question_elapsed_time.min()\naverage = float(sum(user_lectures_lt.prior_question_elapsed_time))/len(user_lectures_lt.prior_question_elapsed_time)\ntotal = 0\n\nfor value in user_lectures_lt.prior_question_elapsed_time:\n\n    total += (value - average) ** 2\n    \nstddev = math.sqrt(total/len(user_lectures_lt.prior_question_elapsed_time))\nuser_lectures_lt.prior_question_elapsed_time = [(x-average)/stddev for x in user_lectures_lt.prior_question_elapsed_time]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question = pd.merge(user_question,user_lectures_intention,on =['user_id','part'],how = 'left')\nuser_question = pd.merge(user_question,user_lectures_concept,on =['user_id','part'],how = 'left')\nuser_question = pd.merge(user_question,user_lectures_solving,on =['user_id','part'],how = 'left')\nuser_question = pd.merge(user_question,user_lectures_tag,on =['user_id','part'],how = 'left')\nuser_question = pd.merge(user_question,user_lectures_lt,on =['user_id','part'],how = 'left')\nuser_question = user_question.rename(columns = {'prior_question_elapsed_time_x':'question_time','prior_question_elapsed_time_y':'lectrue_time'})\n\nuser_question = user_question.fillna(0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt #Data Visualization \nimport seaborn as sns  #Python library for Vidualization\nfrom sklearn.cluster import KMeans\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question_part1 = user_question[user_question.part == 1]\nuser_question_part2 = user_question[user_question.part == 2]\nuser_question_part3 = user_question[user_question.part == 3]\nuser_question_part4 = user_question[user_question.part == 4]\nuser_question_part5 = user_question[user_question.part == 5]\nuser_question_part6 = user_question[user_question.part == 6]\nuser_question_part7 = user_question[user_question.part == 7]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question_part1_m = pd.DataFrame(user_question_part1,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])\nuser_question_part2_m = pd.DataFrame(user_question_part2,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])\nuser_question_part3_m = pd.DataFrame(user_question_part3,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])\nuser_question_part4_m = pd.DataFrame(user_question_part4,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])\nuser_question_part5_m = pd.DataFrame(user_question_part5,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])\nuser_question_part6_m = pd.DataFrame(user_question_part6,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])\nuser_question_part7_m = pd.DataFrame(user_question_part7,columns = ['answered_count','correctly','question_time','intention','concept','solving','tag','lectrue_time'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kmeansmodel = KMeans(n_clusters= 5,max_iter = 100000,random_state = 0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_kmeans_part1= kmeansmodel.fit_predict(user_question_part1_m)\ny_kmeans_part2= kmeansmodel.fit_predict(user_question_part2_m)\ny_kmeans_part3= kmeansmodel.fit_predict(user_question_part3_m)\ny_kmeans_part4= kmeansmodel.fit_predict(user_question_part4_m)\ny_kmeans_part5= kmeansmodel.fit_predict(user_question_part5_m)\ny_kmeans_part6= kmeansmodel.fit_predict(user_question_part6_m)\ny_kmeans_part7= kmeansmodel.fit_predict(user_question_part7_m)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_question_part1['type'] = y_kmeans_part1\nuser_question_part2['type'] = y_kmeans_part2\nuser_question_part3['type'] = y_kmeans_part3\nuser_question_part4['type'] = y_kmeans_part4\nuser_question_part5['type'] = y_kmeans_part5\nuser_question_part6['type'] = y_kmeans_part6\nuser_question_part7['type'] = y_kmeans_part7\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_tz = pd.concat([user_question_part1,user_question_part2,user_question_part3,user_question_part4,user_question_part5,user_question_part6,user_question_part7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del user_question,user_question_part1_m,user_question_part2_m,user_question_part3_m,user_question_part4_m,user_question_part5_m\ndel user_question_part6_m,user_question_part7_m,user_question_part1,user_question_part2,user_question_part3,user_question_part4\ndel user_question_part5,user_question_part6,user_question_part7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_tz =pd.DataFrame(user_tz,columns = ['user_id','part','type'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_c_part_part1 = question_c_part[question_c_part.part == 1]\nquestion_c_part_part2 = question_c_part[question_c_part.part == 2]\nquestion_c_part_part3 = question_c_part[question_c_part.part == 3]\nquestion_c_part_part4 = question_c_part[question_c_part.part == 4]\nquestion_c_part_part5 = question_c_part[question_c_part.part == 5]\nquestion_c_part_part6 = question_c_part[question_c_part.part == 6]\nquestion_c_part_part7 = question_c_part[question_c_part.part == 7]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_c_part_part1 = pd.DataFrame(question_c_part_part1,columns = ['answered_count','correctly','tagsum'])\nquestion_c_part_part2 = pd.DataFrame(question_c_part_part2,columns = ['answered_count','correctly','tagsum'])\nquestion_c_part_part3 = pd.DataFrame(question_c_part_part3,columns = ['answered_count','correctly','tagsum'])\nquestion_c_part_part4 = pd.DataFrame(question_c_part_part4,columns = ['answered_count','correctly','tagsum'])\nquestion_c_part_part5 = pd.DataFrame(question_c_part_part5,columns = ['answered_count','correctly','tagsum'])\nquestion_c_part_part6 = pd.DataFrame(question_c_part_part6,columns = ['answered_count','correctly','tagsum'])\nquestion_c_part_part7 = pd.DataFrame(question_c_part_part7,columns = ['answered_count','correctly','tagsum'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_kmeans_part1= kmeansmodel.fit_predict(question_c_part_part1)\ny_kmeans_part2= kmeansmodel.fit_predict(question_c_part_part2)\ny_kmeans_part3= kmeansmodel.fit_predict(question_c_part_part3)\ny_kmeans_part4= kmeansmodel.fit_predict(question_c_part_part4)\ny_kmeans_part5= kmeansmodel.fit_predict(question_c_part_part5)\ny_kmeans_part6= kmeansmodel.fit_predict(question_c_part_part6)\ny_kmeans_part7= kmeansmodel.fit_predict(question_c_part_part7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_c_part_part1['type'] = y_kmeans_part1\nquestion_c_part_part2['type'] = y_kmeans_part2\nquestion_c_part_part3['type'] = y_kmeans_part3\nquestion_c_part_part4['type'] = y_kmeans_part4\nquestion_c_part_part5['type'] = y_kmeans_part5\nquestion_c_part_part6['type'] = y_kmeans_part6\nquestion_c_part_part7['type'] = y_kmeans_part7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_tz = pd.concat([question_c_part_part1,question_c_part_part2,question_c_part_part3,question_c_part_part4,question_c_part_part5,question_c_part_part6,question_c_part_part7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_tz['content_id'] = question_c_part.content_id\nquestion_tz['part'] = question_c_part.part\nquestion_tz =pd.DataFrame(question_tz,columns = ['content_id','part','type'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del question_c_part,question_c_part_part1,question_c_part_part2 ,question_c_part_part3,question_c_part_part4\ndel question_c_part_part5,question_c_part_part6,question_c_part_part7\ndel y_kmeans_part1,y_kmeans_part2,y_kmeans_part3,y_kmeans_part4,y_kmeans_part5,y_kmeans_part6,y_kmeans_part7\ndel questions_df,tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.prior_question_elapsed_time = train.prior_question_elapsed_time.shift(-1)\ntrain.prior_question_elapsed_time = train.prior_question_elapsed_time.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain = pd.merge(train,question_tz,on = ['content_id'],how = 'left')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train,user_tz,on = ['user_id','part'],how = 'left')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['answered_correctly']\ntrain = train.drop(['answered_correctly'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn import datasets\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train,y,test_size = 0.4,random_state = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train,y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlb_make = LabelEncoder()\n\nX_train.prior_question_had_explanation.fillna(False, inplace = True)\nX_test.prior_question_had_explanation.fillna(False, inplace = True)\nX_train[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(X_train[\"prior_question_had_explanation\"])\nX_test[\"prior_question_had_explanation_enc\"] = lb_make.fit_transform(X_test[\"prior_question_had_explanation\"])\n\nX_train = X_train.drop(['prior_question_had_explanation'], axis=1)\nX_test = X_test.drop(['prior_question_had_explanation'], axis=1)\n\n                       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.to_csv(\"X_train.csv\", index=0)\nX_test.to_csv(\"X_test.csv\", index=0)\ny_train.to_csv(\"y_train.csv\", index=0)\ny_test.to_csv(\"y_test.csv\", index=0)\nquestion_tz.to_csv(\"question_tz.csv\", index=0)\nuser_tz.to_csv(\"user_tz.csv\", index=0)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}