{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Let's start by cleaning the data as in part1 but using a pipe for cleaner code"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#loading data\n\nimport numpy as np \nimport pandas as pd \nimport riiideducation \nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nimport warnings \nwarnings.filterwarnings('ignore')\n\nfor dirname, _, filenames in os.walk('/kaggle/input/riiid-test-answer-prediction'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=3*(10**6), \n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n                             'prior_question_had_explanation': 'boolean',\n                             }\n                      )\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nquestion.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lecture = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\nlecture.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seperate_interactions(train_df):\n    questions_interactions = train_df.merge(question, left_on = 'content_id', right_on = 'question_id', how = 'left')\n    questions_interactions = questions_interactions[questions_interactions.content_type_id == 0]\n    questions_interactions.rename(columns = {'part': 'test_part'}, inplace = True)\n\n    lectures_interactions = train_df.merge(lecture, left_on = 'content_id', right_on = 'lecture_id', how = 'left') \n    lectures_interactions.rename(columns = {'part': 'category'}, inplace = True)\n    lectures_interactions = lectures_interactions[lectures_interactions.content_type_id == 1]\n    return questions_interactions,lectures_interactions\ndef fill_question_nulls(questions_interactions,elapse_val=-1,expl_val=-1):\n    '''Fixes the nulls in prior_question_elapsed_time and prior_question_had_explanation'''\n    indeces = questions_interactions[questions_interactions.prior_question_had_explanation.isnull()].index\n    values = {'prior_question_elapsed_time': elapse_val, 'prior_question_had_explanation': False}\n    questions_interactions.fillna(value=values,inplace=True)\n    questions_interactions.prior_question_had_explanation = questions_interactions.prior_question_had_explanation.astype('int8')\n    questions_interactions.loc[indeces,'prior_question_had_explanation'] = expl_val\n    return questions_interactions\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(props):\n    start_mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in props.columns:\n        if props[col].dtype != object:  # Exclude strings\n            # make variables for Int, max and min\n            IsInt = False\n            mx = props[col].max()\n            mn = props[col].min()\n            \n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(props[col]).all(): \n                NAlist.append(col)\n                props[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = props[col].fillna(0).astype(np.int64)\n            result = (props[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True\n\n            \n\n            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        props[col] = props[col].astype(np.uint8)\n                    elif mx < 65535:\n                        props[col] = props[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        props[col] = props[col].astype(np.uint32)\n                    else:\n                        props[col] = props[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        props[col] = props[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        props[col] = props[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        props[col] = props[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        props[col] = props[col].astype(np.int64)    \n            \n            # Make float datatypes 32 bit\n            else:\n                props[col] = props[col].astype(np.float32)\n            \n            # Print new column type\n           # print(\"dtype after: \",props[col].dtype)\n           # print(\"******************************\")\n    \n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return props","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions,lectures_interactions = seperate_interactions(train_df)\nquestions_interactions_cleaned = (questions_interactions.pipe(fill_question_nulls).pipe(reduce_mem_usage))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions_cleaned.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_sense_columns_lecture = ['content_type_id','user_answer','answered_correctly','prior_question_elapsed_time','prior_question_had_explanation']\nnon_sense_columns_question =['content_id','content_type_id']\nquestions_interactions_cleaned.drop(columns=non_sense_columns_question,inplace=True)\nlectures_interactions.drop(columns=non_sense_columns_lecture,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now let's try to answer some questions"},{"metadata":{},"cell_type":"markdown","source":"# Are students getting better after they spend much time on the app ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"# let's first get the maximum time a student spent on the app\nprint('The maximum time is {} hour'.format(questions_interactions_cleaned.timestamp.max()/(1000*60*60*24)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's see the distribution of the overall time spent by the students in hours"},{"metadata":{"trusted":true},"cell_type":"code","source":"x=questions_interactions_cleaned.groupby('user_id').timestamp.max()/(1000*60*60*24)\nplt.hist(x=x,bins=100);\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> We can see that a lot of users don't continue using the app \n\n> Let's see how are they doing compared with those who use the app for longer times"},{"metadata":{},"cell_type":"markdown","source":"## Let's see the number of persons who use the app for less than half month"},{"metadata":{"trusted":true},"cell_type":"code","source":"students_quit_early = x[x<24*15]\ndf_early_quit = questions_interactions_cleaned[questions_interactions_cleaned.user_id.isin(students_quit_early.index)]\ndf_early_quit.groupby('user_id').answered_correctly.mean().hist(grid=False);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The users mean performance is slightly right skewed and has mean at about 0.6"},{"metadata":{},"cell_type":"markdown","source":"## Now let's check other students performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_cont = questions_interactions_cleaned[~questions_interactions_cleaned.user_id.isin(students_quit_early.index)]\ndf_cont.groupby('user_id').answered_correctly.mean().hist(bins=19,grid=False);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"100*df_cont.shape[0]/questions_interactions_cleaned.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x[x>24*15].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> There are just 511 student who continued for more than one month with just 18% percent of the total records.\n\n> Their distribution is more right skewed which suggests that they have better grades"},{"metadata":{},"cell_type":"markdown","source":"Let's see the relation between time spent and score for these students"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_new = df_cont.copy()\ndf_new['timestamp'] = round(df_new['timestamp']/(1000*60*60*24))\ndf_new = df_new.groupby('user_id').agg({'answered_correctly':'mean','timestamp':'std'})\nsns.regplot(x='timestamp',y='answered_correctly',data=df_new,fit_reg=True);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"  > The standard deviation values start from 50 to 400 and looks to have negative relationship with the performance with high outliers."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_new = df_early_quit.copy()\ndf_new['timestamp'] = round(df_new['timestamp']/(1000*60*60*24))\ndf_new = df_new.groupby('user_id').agg({'answered_correctly':'mean','timestamp':'std'})\nsns.regplot(x='timestamp',y='answered_correctly',data=df_new,fit_reg=True);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The relation looks to be positive with a lot of students with low time standard deviation and maximum standard deviation of 175."},{"metadata":{},"cell_type":"markdown","source":"## Is there a relation between the part (relevant section of the TOEIC test) being answered and the performance ? "},{"metadata":{"trusted":true},"cell_type":"code","source":"color = sns.color_palette()[0]\nsns.countplot(data=questions_interactions_cleaned,x='test_part',color=color);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most of the questions lies on the 5th part followed by the 2nd part."},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions_cleaned.groupby('test_part').answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The first 5 parts look to a decreasing score pattern."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_early_quit.groupby('test_part').answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_cont.groupby('test_part').answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The position of these parts is  independent on the student category."},{"metadata":{},"cell_type":"markdown","source":"# Let's check the performance against the task_container_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"containers_perf = questions_interactions_cleaned.groupby('task_container_id').answered_correctly.mean()\ncontainers_perf.hist(grid=False,bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":">A lot of the task containers have only one record that they have just a 0 or 1 mean."},{"metadata":{"trusted":true},"cell_type":"code","source":"containers_perf_filtered = containers_perf[containers_perf>0][containers_perf<1]\ncontainers_perf_filtered.hist(grid=False,bins=200);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"containers_perf_filtered.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"containers_perf_filtered.value_counts().nlargest(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The mean of the filtered is high at about 0.66\n\n> It can be observed we have peaks at 0.667. 0.5,0.75,0.33\n\n> The peak of 0.667 can appear when a container has only 3 questions, one of them answered wrongly to have a 2/3 score.\n"},{"metadata":{},"cell_type":"markdown","source":"# Are students performing well when they see the lecture related to the question before answering ?\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lectures_info = lectures_interactions.groupby(['user_id','category']).timestamp.min().reset_index()\nuser_lectures_info['user_cat'] = user_lectures_info['user_id'].astype(str)+'_'+user_lectures_info['category'].astype('int32').astype(str)\nuser_lectures_info = user_lectures_info[['user_cat','timestamp']]\nuser_lectures_info.rename(columns = {'timestamp': 'min_time'}, inplace = True)\nuser_lectures_info.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions_cleaned_copy = questions_interactions_cleaned.copy()\nquestions_interactions_cleaned_copy['user_cat'] = questions_interactions_cleaned_copy['user_id'].astype(str)+'_'+questions_interactions_cleaned_copy['test_part'].astype(str)\nquestions_interactions_cleaned_copy = questions_interactions_cleaned_copy.merge(user_lectures_info, on = 'user_cat', how = 'left') \nquestions_interactions_cleaned_copy.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions_cleaned_copy.fillna(questions_interactions_cleaned_copy.timestamp.max()+1,inplace=True)\nquestions_interactions_cleaned_copy['is_lec_watched'] = questions_interactions_cleaned_copy['timestamp'] > questions_interactions_cleaned_copy['min_time']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(questions_interactions_cleaned_copy.is_lec_watched);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions_cleaned_copy.groupby('is_lec_watched').answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### How much the early quitting students watch the lectures before answering compared with those who didn't quit early ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"early_lec_watch = questions_interactions_cleaned_copy[questions_interactions_cleaned_copy.user_id.isin(students_quit_early.index)]\nsns.countplot(early_lec_watch.is_lec_watched);\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_lec_watch = questions_interactions_cleaned_copy[~questions_interactions_cleaned_copy.user_id.isin(students_quit_early.index)]\n\nsns.countplot(cont_lec_watch.is_lec_watched);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The early quitting students watched lectures less"},{"metadata":{"trusted":true},"cell_type":"code","source":"early_lec_watch.groupby('is_lec_watched').answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_lec_watch.groupby('is_lec_watched').answered_correctly.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Watching the lecture before answering looks to improve the early quitting students performance and doesn't affect the others much."},{"metadata":{},"cell_type":"markdown","source":"# Let's see how the type of the lecture affects the previous results"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_lectures_info = lectures_interactions.groupby(['user_id','category','type_of']).timestamp.min().reset_index()\nuser_lectures_info['user_cat'] = user_lectures_info['user_id'].astype(str)+'_'+user_lectures_info['category'].astype('int32').astype(str)\nuser_lectures_info = user_lectures_info[['user_cat','type_of','timestamp']]\nuser_lectures_info.rename(columns = {'timestamp': 'min_time'}, inplace = True)\nuser_lectures_info.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_interactions_cleaned_copy = questions_interactions_cleaned.copy()\nquestions_interactions_cleaned_copy['user_cat'] = questions_interactions_cleaned_copy['user_id'].astype(str)+'_'+questions_interactions_cleaned_copy['test_part'].astype(str)\nquestions_interactions_cleaned_copy = questions_interactions_cleaned_copy.merge(user_lectures_info, on = 'user_cat', how = 'left') \nvalues = {'min_time':questions_interactions_cleaned_copy.timestamp.max()+1,'type_of':'na'} \nquestions_interactions_cleaned_copy.fillna(value=values,inplace=True)\nquestions_interactions_cleaned_copy['is_lec_watched'] = questions_interactions_cleaned_copy['timestamp'] > questions_interactions_cleaned_copy['min_time']\nquestions_interactions_cleaned_copy.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for val in questions_interactions_cleaned_copy.type_of.unique():\n    df = questions_interactions_cleaned_copy[questions_interactions_cleaned_copy.type_of==val]\n    y=  df.groupby('is_lec_watched').answered_correctly.mean()\n    x = y.index\n    sns.barplot(x=x,y=y);\n    plt.title(val)\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" The type_of doesn't look to have an effect on the correct answers"},{"metadata":{},"cell_type":"markdown","source":"## Let's now check the base student level against his overall performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"records_per_student = questions_interactions_cleaned.groupby('user_id').row_id.count()\nrecords_per_student.hist(bins=100,grid=False);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"records_per_student_count = records_per_student.value_counts()\nrecords_per_student_count[records_per_student_count<10 ].shape[0]/records_per_student_count.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> 88% of the students has less than 10 records !!\n\n> We may focus on looking at different features independent from students."},{"metadata":{},"cell_type":"markdown","source":"## Can we get the questions easiness ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"100*questions_interactions_cleaned_copy.question_id.nunique()/questions_interactions_cleaned_copy.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"quest_perf = questions_interactions_cleaned_copy.groupby('question_id').answered_correctly.mean()\nquest_perf.hist(grid=False,bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Many questions has only one answered right record\n\n> The mean is 0.8 when we filter out the always right questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"filtered_quest_per = quest_perf[quest_perf<1]\nfiltered_quest_per.hist(grid=False,bins=20);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filtered_questions = filtered_quest_per.index\nfiltered_df = questions_interactions_cleaned_copy[questions_interactions_cleaned_copy.question_id.isin(filtered_questions)]\n100*filtered_df.shape[0]/questions_interactions_cleaned_copy.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Finally let's check the tags relation with the performance."},{"metadata":{"trusted":true},"cell_type":"code","source":"df = questions_interactions_cleaned_copy.copy()\ndf = df.assign(tags2=df['tags'].str.split(' ')).explode('tags2')\ndf['tags2'] = df['tags2'].astype('int32') \ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_perf = df.groupby('tags2').answered_correctly.mean()\ntags_perf.hist(grid=False,bins=20);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> some tags look to have high scores at 0.7 and less tags has higher scores\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = questions_interactions_cleaned_copy.copy()\ndf = df.assign(tags2=df['tags'].str.split(' '))\ndf['num_tags'] = df['tags2'].str.len()\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(df.groupby('num_tags').answered_correctly.mean().index,df.groupby('num_tags').answered_correctly.mean(),color=color);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The relation between the number of the tags and the performance is positive"},{"metadata":{},"cell_type":"raw","source":"sns.countplot(df['num_tags'],color=color)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* We can see that the statistics across the data are different depending on the student quit time.\n* The standard deviation of time for each student is related to his performance\n* The features that addresses the question difficulity (eg. task,container_id,tag) are more related to the performance than the features related to the students.\n* The most related feature to the performance is the number of tags per question."},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"# Refrences\n\n1- memory used reduction: https://www.kaggle.com/cdeotte/dae-book3c from the cool grand master: Chris Deotte\n\n2- https://stackoverflow.com/questions/12680754/split-explode-pandas-dataframe-string-entry-to-separate-rows\n\n"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}