Import the required librairies
"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import seaborn as sns \nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df):\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n \n for col in df.columns:\n col_type = df[col].dtype\n \n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n else:\n df[col] = df[col].astype('category')\n\n end_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n \n return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" We will imput missing values for the variables prior_question_elapsed_time and prior_question_had_explanation."},{"metadata":{"trusted":true},"cell_type":"code","source":"#We know for users first question bundle, prior_question_elapsed_time should be null and prior_question_had_explanation should be False.\ntrain['prior_question_elapsed_time'].fillna(0,inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_had_explanation'].fillna(False,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=train.sort_values(by=[\"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" More than 98 % of the events in our training datas are question events. Feature engineering
"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nb_previous_lectures(i):\n idd=train.loc[i,'user_id']\n indices=[j for j in train.index if j<=i]\n ext=train.loc[indices]\n dd=ext[(ext['user_id']==idd) & (ext['content_type_id']==1)]\n nb=dd['content_type_id'].sum()\n return nb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" Datas Explorations
"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of students on this dataset.\n\nusers=len(train['user_id'].unique())\nprint('There is data of {} students in the training set '.format(users))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# percentil of the question and lecture events in the training dataset.\nevents={0:\"question\",1:\"lecture\"}\nlabels=list(events.values())\nval=[train['content_type_id'].value_counts().loc[0],train['content_type_id'].value_counts().loc[1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=px.pie(names=labels,values=val,title=\"events percentile\")\nfig.update_layout(title={'x':0.475,'y':0.9,'xanchor':'center','yanchor':'top'})\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"