{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dataset Prepartion is from the Kernel: https://www.kaggle.com/rohanrao/tutorial-on-reading-large-datasets"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_pickle('../input/riiid-trainpkl/riiid_train.pkl.gzip')\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number Of Users\", len(train.user_id.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.content_type_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax=sns.countplot(train.content_type_id)\nax.set(xlabel='Question/Lecture Content', ylabel='Count')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Number of Questions posed releatively high compared to the lectures.\n\nTotal Posed Question - 99271300\n\nTotal Watched lectures - 1959032"},{"metadata":{},"cell_type":"markdown","source":"# Question is Answered Correctly or not"},{"metadata":{},"cell_type":"markdown","source":"This is the Target we want to find the prediction.\n\nanswered_correctly(0) --> wrong Answer\n\nanswered_correctly(1) --> Right Answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train.answered_correctly)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Percent Of Questions Answered Correctly:', \n      100*(len(train[train.answered_correctly==1])/len(train[train.answered_correctly>=0]))\n     )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Arount 65.7% of the Questions are Answered Correctly in the dataset."},{"metadata":{},"cell_type":"markdown","source":"# user_answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.user_answer.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train.user_answer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.pie(\n    list(train.user_answer.value_counts()),\n    labels=list(train.user_answer.value_counts().index),\n    shadow=True,\n    autopct='%1.1f%%'\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"100*train[train.user_answer>=0].user_answer.value_counts()/len(train[train.user_answer>=0])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Multi choice questions\n\n* Percentage of the Answer choices the User hits for option:2 is reletively less compared to the other options"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# TimeStamp"},{"metadata":{"trusted":true},"cell_type":"code","source":"milliseconds_to_days=1000*60*60*24","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(max(train.timestamp/milliseconds_to_days))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Max Number Of days user is active after first interaction : 1011 ~2.8 Years"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n(train[['timestamp']]/milliseconds_to_days).hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" The above Histogram represents the Number Of Interactions as the days preceed.\n* \n* We can observe a sharp fall in the histogram starting around ~80 days\n* Looks like only a the number of interactions got reduced"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.user_id.unique()[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train[train.user_id == 115].timestamp/milliseconds_to_days)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number Of Samples For user:115', train[train.user_id == 115].shape)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looks the user interaction of 2 days is stored.\n\nlets look at other user."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train[train.user_id == 124].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train[train.user_id == 124].timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train[train.user_id == 8623].shape)\nplt.plot(train[train.user_id == 8623].timestamp/milliseconds_to_days)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train[train.user_id == 81429].shape)\nplt.plot(train[train.user_id == 81429].timestamp/milliseconds_to_days)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Number Of Records for the User: 8623 --> 112\n\nDays Gap Between first and last visit: 10 days\n\nWe can see some users are more consistent in days than the other."},{"metadata":{},"cell_type":"markdown","source":"1. timestamp indicates if the user is new/old.\n2. User Interacting frequently/not\n3. Paired with the Questions answered; Interacting single day for a long time or not etc."},{"metadata":{},"cell_type":"markdown","source":"# lets Check the content_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('content_type_id')[['content_id']].nunique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"questions - 13523\nlectures - 415"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"1. Any Questions/Lectures Re-attempted?\n2. Top 20 questions answered correct/wrong\n3. Is there a tough question that many users atteps answered wrong.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncontent_id_df=train[train.content_type_id==0].groupby('content_id')[['user_id', 'answered_correctly']].agg({'user_id': 'nunique', 'answered_correctly': 'sum'}).reset_index()\ncontent_id_df.rename(columns={'user_id': 'total_answered'}, inplace=True)\ncontent_id_df['percent_answered_correctly']=100*content_id_df['answered_correctly'].div(content_id_df['total_answered'])\ncontent_id_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_id_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_id_df.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(content_id_df.corr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.scatterplot(data=content_id_df,\n                x='total_answered',\n                y='answered_correctly',\n                hue=[0]*len(content_id_df),\n                palette='CMRmap_r'\n               )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.scatterplot(data=content_id_df,\n                x='total_answered',\n                y='percent_answered_correctly',\n                hue=[0]*len(content_id_df),\n                palette='CMRmap_r'\n               )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_id_df[content_id_df.total_answered>=75000].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_id_df[content_id_df.total_answered<=10].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_id_df[content_id_df.total_answered<=100].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(content_id_df[(content_id_df.total_answered<=50000) \n                          ].total_answered,\n             bins=200,\n             kde=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Some of the questions are answered more than one time by the user --> with percent_answered_correctly>100\n\n2. There is a high amount of variance for the question answered less.\n\n3. 44 Questions answered by more than 75000 users.\n\n4. Observed that the variance reduction in the graph as move from 0-->50000 on y-axis"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train.content_type_id==0].groupby(['user_id', ''])[['content_id']].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.pairplot(train[['']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}