{"cells": [{"cell_type": "markdown", "metadata": {}, "source": ["# **Students performance EDA** \n", "*Javier Hern\u00e1ndez - Guatemala*\n", "\n", "<h2><center> <img src=\"https://i.vimeocdn.com/filter/overlay?src0=https%3A%2F%2Fi.vimeocdn.com%2Fvideo%2F731763771-bde18cc6d5a825b00541f8c62166e5c4a34c1c97d7efcd994ce1c96a3cc22d69-d_1280x720&src1=https%3A%2F%2Ff.vimeocdn.com%2Fimages_v6%2Fshare%2Fplay_icon_overlay.png\" alt=\"EducationGame\" width=\"40%\" height=\"40%\"></center></h2>"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## Table of Contents\n", "\n", "* [1. Train EDA](#train-eda)\n", "   * [1.1 Missing Values](#missing-values)\n", "   * [1.2 Unique Values](#unique-values)\n", "   * [1.3 Event Names](#event-names)\n", "   * [1.4 Level Groups](#level-groups)\n", "   * [1.5 Levels](#levels)\n", "   "]}, {"cell_type": "code", "execution_count": 22, "metadata": {"_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}, "outputs": [], "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load\n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt #we use this to visualize the dataset\n", "import seaborn as sns #we use this to make countplots\n", "import warnings\n", "\n", "# Input data files are available in the read-only \"../input/\" directory\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n", "\n", "import os\n", "for dirname, _, filenames in os.walk('/kaggle/input'):\n", "    for filename in filenames:\n", "        print(os.path.join(dirname, filename))\n", "\n", "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n", "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n", "\n", "# settings\n", "sns.set(style=\"whitegrid\", color_codes=True)\n", "warnings.filterwarnings(\"ignore\")"]}, {"cell_type": "code", "execution_count": 23, "metadata": {"tags": []}, "outputs": [], "source": ["files_status = True\n", "\n", "dtypes={'session_id':'int', \n", "'elapsed_time':np.int32,\n", "    'event_name':'category',\n", "    'name':'category',\n", "    'level':np.int32,\n", "    'page':'category',\n", "    'room_coor_x':np.float32,\n", "    'room_coor_y':np.float32,\n", "    'screen_coor_x':np.float32,\n", "    'screen_coor_y':np.float32,\n", "    'hover_duration':np.float32,\n", "     'text':'category',\n", "     'fqid':'category',\n", "     'room_fqid':'category',\n", "     'text_fqid':'category',\n", "     'fullscreen':'category',\n", "     'hq':'category',\n", "     'music':'category',\n", "     'level_group':'category'}\n", "\n", "try:    \n", "    train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', dtype=dtypes)\n", "    train_labels = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\n", "    test = pd.read_csv(\"'/kaggle/input/predict-student-performance-from-game-play/test.csv'\")\n", "except OSError as e:\n", "    print(\"files not found in Kaggle environment: \",e.errno)\n", "    files_status = False"]}, {"cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": ["#If environment is not kaggle notebook, read them from a folder\n", "\n", "try:\n", "    if files_status == False:\n", "        train = pd.read_csv(\"../predict-student-performance-from-game-play/train.csv\",dtype=dtypes)\n", "        train_labels = pd.read_csv('../predict-student-performance-from-game-play/train_labels.csv')\n", "        test = pd.read_csv(\"../predict-student-performance-from-game-play/test.csv\")\n", "except OSError as e:\n", "    print(\"Files not found\")"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## 1. Train EDA <a class=\"anchor\" id=\"train-eda\"></a>\n"]}, {"cell_type": "code", "execution_count": 25, "metadata": {"tags": []}, "outputs": [], "source": ["train.head(5)"]}, {"cell_type": "code", "execution_count": 5, "metadata": {"tags": []}, "outputs": [], "source": ["train.info(show_counts = True)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### 1.1 Missing Values <a class=\"anchor\" id=\"missing-values\"></a>"]}, {"cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": ["# Count NaN values in multiple columns of DataFrame\n", "nan_count = (train.isna().sum() / len(train)) \n", "nan_count = nan_count.round(2)\n", "print(\"Quantity of null values\")\n", "print(\"\")\n", "nan = nan_count.map('{:.2%}'.format)\n", "print(nan)\n"]}, {"cell_type": "code", "execution_count": 7, "metadata": {"tags": []}, "outputs": [], "source": ["gcolor = \"#3d9979\""]}, {"cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": ["fig = plt.figure(figsize =(16, 8))\n", "plt.xlabel('Variable')\n", "plt.ylabel('% of missing values')\n", "plt.title('Share of missing values',fontsize= 20)\n", "plt.grid(True)\n", "\n", "g = sns.barplot(x=nan_count.index, y=nan_count,facecolor = gcolor)\n", "g.xaxis.set_tick_params(rotation=35)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### 1.2 Unique Values <a class=\"anchor\" id=\"unique-values\"></a>"]}, {"cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": ["# counting unique values of Session_ID\n", "n = len(pd.unique(train['session_id']))\n", "print(\"Unique Session IDs: \",n)"]}, {"cell_type": "code", "execution_count": 10, "metadata": {"tags": []}, "outputs": [], "source": ["max_elapsed_time = train[\"elapsed_time\"].max()\n", "print(\"Max elapsed time: \",max_elapsed_time)"]}, {"cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": ["#Now we will try a Box & Wiskers plot\n", "fig = plt.figure(figsize =(16, 8))\n", "ax = fig.add_subplot()\n", "plt.title(\"Box & whisker Elapsed time\",fontsize=20)\n", "green_diamond = dict(markerfacecolor='g', marker='D')\n", "green_whisker = dict(facecolor=gcolor, color=\"green\")\n", "\n", "\n", "plt.boxplot(train['elapsed_time'],flierprops=green_diamond,notch=True,patch_artist=True,boxprops=green_whisker)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### 1.3 Distinct Event Names <a class=\"anchor\" id=\"event-names\"></a>"]}, {"cell_type": "code", "execution_count": 12, "metadata": {"tags": []}, "outputs": [], "source": ["event_name_unique = train[\"event_name\"].unique()\n", "#print(event_name_unique)"]}, {"cell_type": "code", "execution_count": 13, "metadata": {"tags": []}, "outputs": [], "source": ["countplt, ax = plt.subplots(figsize = (16,8))\n", "ax =sns.countplot(x = 'event_name', data=train,facecolor=gcolor,\n", "                  order = train['event_name'].value_counts().index)\n", "ax.xaxis.set_tick_params(rotation=35)\n", "plt.title(\"Count Event Names\",fontsize=20)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### 1.4 Distinct Level Groups  <a class=\"anchor\" id=\"level-groups\"></a>"]}, {"cell_type": "code", "execution_count": 14, "metadata": {"tags": []}, "outputs": [], "source": ["level_group_unique = train[\"level_group\"].unique()\n", "#print(level_group_unique)"]}, {"cell_type": "code", "execution_count": 15, "metadata": {"tags": []}, "outputs": [], "source": ["countplt, ax = plt.subplots(figsize = (16,8))\n", "ax =sns.countplot(x = 'level_group', data=train,facecolor=gcolor,orient= 'h',order=train[\"level_group\"].unique())\n", "plt.title(\"Count Level Group\",fontsize=20)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### 1.5 Distinct Levels  <a class=\"anchor\" id=\"levels\"></a>"]}, {"cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": ["level_unique = train[\"level\"].unique()\n", "#print(level_unique)"]}, {"cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": ["countplt, ax = plt.subplots(figsize = (16,8))\n", "ax =sns.countplot(x = 'level', data=train,facecolor=gcolor)\n", "plt.title(\"Count Levels\",fontsize=20)"]}, {"cell_type": "code", "execution_count": 18, "metadata": {"tags": []}, "outputs": [], "source": ["\n", "train[[\"elapsed_time\",\"level\",\"room_coor_x\",\"room_coor_y\",\"screen_coor_x\"\n", "       ,\"screen_coor_y\",\"hover_duration\"]].hist(bins=60, figsize=(16,12),color=gcolor);\n", "plt.show()"]}, {"cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": ["def corrMat(df,id=False):\n", "    ''' Function to plot correlation of features '''\n", "    \n", "    corr_mat = df.corr().round(2)\n", "    f, ax = plt.subplots(figsize=(16,8))\n", "    mask = np.zeros_like(corr_mat,dtype=np.bool)\n", "    mask[np.triu_indices_from(mask)] = True\n", "    sns.heatmap(corr_mat,mask=mask,vmin=-1,vmax=1,center=0, \n", "                cmap='Greens',square=False,lw=2,annot=True,cbar=False);\n", "    plt.show() "]}, {"cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": ["corrMat(train) # plot masked numpy correlation matrix"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## 2. Train labels EDA"]}, {"cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": ["train_labels.head(5)"]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": []}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8"}}, "nbformat": 4, "nbformat_minor": 4}