{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Have A Look\n* [TOEIC](https://www.iibc-global.org/english/toeic/test/lr.html) is a very popular English language test in Japan.\n* The Reading and Listening exams are divided into 7 parts.\n* The structure of the TOEIC test can be seen [here](https://www.iibc-global.org/english/toeic/test/lr/about/format.html).\n* [Riiid](https://sunryse.co/app/startups/r_6WjOnZ6mV7JPbpY) is the first Korean startup to offer a TOEIC learning app that uses AI to optimize learning for each individual.\n* [riiid](https://www.riiid.co/en/product)link shows [TOEIC learning app・santatoeic](https://santatoeic.jp/intro)\n* The TOEIC learning app, [santatoeic](https://santatoeic.jp/intro), allows you to try tests and lectures with free registration."},{"metadata":{},"cell_type":"markdown","source":"# Installation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Regular Libraries\nimport riiideducation\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.express as px\nfrom matplotlib.ticker import FuncFormatter\nimport os\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nfrom scipy.stats import pearsonr\nimport tqdm\nimport copy\nimport re\n\n# You can only call make_env() once, so don't lose it!\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Color Palette\ncustom_colors = ['#7400ff', '#a788e4', '#d216d2', '#ffb500', '#36c9dd']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rapids Imports\nimport cupy # CuPy is an open-source array library accelerated with NVIDIA CUDA.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Checking data statistics of training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=10**5, \n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n                             'prior_question_had_explanation': 'boolean',\n                             }\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup the paths to multiple data format\nTRAIN_FEATHER_PATH = '../input/riiid-train-data-multiple-formats/riiid_train.feather'\nTRAIN_H5_PATH = '../input/riiid-train-data-multiple-formats/riiid_train.h5'\nTRAIN_JAY_PATH = '../input/riiid-train-data-multiple-formats/riiid_train.jay'\nTRAIN_PARQUET_PATH = '../input/riiid-train-data-multiple-formats/riiid_train.parquet'\nTRAIN_PKL_PATH = '../input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#used for changing color of text in print statement\nfrom colorama import Fore, Back, Style\ny_ = Fore.YELLOW\nr_ = Fore.RED\ng_ = Fore.GREEN\nb_ = Fore.BLUE\nm_ = Fore.MAGENTA\nsr_ = Style.RESET_ALL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display some of the training data\ntrain_df.head(5).style.applymap(lambda x: 'background-color:lightsteelblue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's come up with a unique number.\ntrain_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the data type\ntrain_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Monitor memory usage\ntrain_df.memory_usage(deep=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Type conversion for efficient use of memory\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = reduce_mem_usage(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check for missing values in the training data\ntrain_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are missing values in two columns."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display of training data\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Concerning 'answered_correctly', Let's leave out the line including -1 because [Data Description](https://www.kaggle.com/c/riiid-test-answer-prediction/data) shows that ’Read -1 as null, for lectures’"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display Summary Statistics\ntrain_df.describe().style.applymap(lambda x: 'background-color:lightgreen')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display information by user ID \nprint(pd.pivot_table(train_df, index='user_id', values=['timestamp', 'prior_question_elapsed_time', 'answered_correctly'], aggfunc='mean'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display information by user ID\ntrain_df_pivot = pd.pivot_table(train_df, index='user_id', columns='answered_correctly')\ntrain_df_pivot.head(5).style.applymap(lambda x: 'background-color:lightgreen')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read -1 as null, for lectures.\n(train_df['answered_correctly']==-1).mean()\n# We should exclude information about lectures.\ntrain_df_questions = train_df[train_df['answered_correctly']!=-1]\ntrain_df_questions['answered_correctly'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Display the average percentage of correct answers per user\ntrain_df_questions.groupby('user_id')['answered_correctly'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display the unique number of elements in a specific column\ntrain_df['user_answer'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* The questions are multiple choice, and the answers are expected to take a value between 0 and 3.\n* According to [the data description](https://www.kaggle.com/c/riiid-test-answer-prediction/data), -1 is for lectures and is excluded"},{"metadata":{},"cell_type":"markdown","source":"# Checking data statistics of test and sample data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can only iterate through a result from `env.iter_test()` once\n# so be careful not to lose it once you start iterating.\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's get the data for the first test batch and check it out."},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_prediction_df = pd.read_csv('../input/riiid-test-answer-prediction/example_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_prediction_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that we'll get an error if we try to continue on to the next batch without making our predictions for the current batch."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number of rows in traing set: ', train_df.shape[0])\nprint('Number of columns in traing set: ', train_df.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next(iter_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env.predict(sample_prediction_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df['answered_correctly'] = 0.5\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.groupby('user_id').agg(['min', 'max', 'mean']).head(5).style.applymap(lambda x: 'background-color:lightgreen')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Checking data statistics of question's data"},{"metadata":{},"cell_type":"markdown","source":"* [questions.csv](https://www.kaggle.com/c/riiid-test-answer-prediction/data) is that Metadata for the questions posed to users."},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Confirmation of the format of question's data\nprint('Number of rows in question data set: ', questions.shape[0])\nprint('Number of columns in question data set: ', questions.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display some of the question's data\nquestions.head(5).style.applymap(lambda x: 'background-color:lightsteelblue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's come up with a unique number.\nquestions.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check statistics in the question's data\nquestions.head(5).style.applymap(lambda x: 'background-color:lightgreen')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check for missing values in the question's data\nquestions.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There is missing values in one column."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display of question's data\nquestions.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.groupby('question_id').agg(['min', 'max', 'mean']).head(5).style.applymap(lambda x: 'background-color:lightgreen')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let's take a look at the data for bundle_id=7796\n#I think we can assume that major problem=bundle_id, minor problem=question_id.\n#We can see that all the parts match and the tags are almost identical.\nquestions[questions[\"bundle_id\"] == 7796]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correct = train_df[train_df.answered_correctly != -1].answered_correctly.value_counts(ascending=True)\n\nfig = plt.figure(figsize=(12,4))\ncorrect.plot.barh()\nfor i, v in zip(correct.index, correct.values):\n    plt.text(v, i, '{:,}'.format(v), color='white', fontweight='bold', fontsize=14, ha='right', va='center')\nplt.title(\"Questions answered correctly\")\nplt.xticks(rotation=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Checking data statistics of lecture's data"},{"metadata":{},"cell_type":"markdown","source":"* [lectures.csv](https://www.kaggle.com/c/riiid-test-answer-prediction/data) is that Metadata for the lectures watched by users as they progress in their education."},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the data type\nlectures.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's come up with a unique number.\nlectures.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Confirmation of the format of lecture's data\nprint('Number of rows in lecture data set: ', lectures.shape[0])\nprint('Number of columns in lecture data set: ', lectures.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check statistics in the lecture's data\nlectures.head(5).style.applymap(lambda x: 'background-color:lightgreen')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check for missing values in the lecture's data\nlectures.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are no columns with missing values."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display of the lecture's data\nlectures.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures[\"type_of\"].drop_duplicates()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"WIDTH = 800","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cids = train_df.content_id.value_counts()[:30]\n\nfig = plt.figure(figsize=(12,6))\nax = cids.plot.bar()\nplt.title(\"Thirty most used content id's\")\nplt.xticks(rotation=90)\nax.get_yaxis().set_major_formatter(FuncFormatter(lambda x, p: format(int(x), ','))) #add thousands separator\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nds = train_df['content_type_id'].value_counts().reset_index()\n\nds.columns = [\n    'content_type_id', \n    'percent'\n]\n\nds['percent'] /= len(train_df)\n\nfig = px.pie(\n    ds, \n    names='content_type_id', \n    values='percent', \n    title='Lecures & questions', \n    width=WIDTH,\n    height=500 \n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = train_df['user_id'].value_counts().reset_index()\nds.columns = ['user_id', 'count']\nds = ds.sort_values('user_id')\n\nfig = px.line(\n    ds, \n    x='user_id', \n    y='count', \n    title='User action distribution', \n    height=600, \n    width=800\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the unique number \nn = train_df['prior_question_elapsed_time'].nunique()\nprint(n)\n# import math module\nimport math\n# import data visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n# First, I'll use Sturgess's formula to find the appropriate number of classes in the histogram \nk = 1 + math.log2(n)\n# Display a histogram of the ElapsedTime of the training data\nsns.distplot(train_df['prior_question_elapsed_time'], kde=True, rug=False, bins=int(k)) \n# Graph Title\nplt.title('ElapsedTime')\n# Show Histogram\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = train_df['prior_question_elapsed_time'].value_counts().reset_index()\nds.columns = ['prior_question_elapsed_time', 'mean']\nds = ds.sort_values('prior_question_elapsed_time')\n\nfig = px.line(\n    ds, \n    x='prior_question_elapsed_time', \n    y='mean', \n    title='Distribution of Prior_question_elapsed_time', \n    height=600, \n    width=900\n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the unique number \nn = train_df['timestamp'].nunique()\n# First, I'll use Sturgess's formula to find the appropriate number of classes in the histogram \nk = 1 + math.log2(n)\n# Graph Title\nplt.title('Timestamp')\n# Show Histogram\ntrain_df['timestamp'].hist(bins=int(k))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Timestamp is the time in milliseconds between this user interaction and the first event completion from that user."},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y=\"part\", data=questions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* The structure of the TOEIC test can be seen [here](https://www.iibc-global.org/english/toeic/test/lr/about/format.html).\n* There are a lot of Part 5's.Part 5 is a grammar and grammar problem."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display the distribution of each part of the question\nsns.countplot(y=\"part\", hue=None, data=questions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* The structure of the TOEIC test can be seen [here](https://www.iibc-global.org/english/toeic/test/lr/about/format.html).\n* There are a lot of Part 5's.Part 5 is a grammar and grammar problem."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display the distribution of lecture types.\nsns.countplot(x=\"part\", hue=\"type_of\",data=lectures)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y=\"user_answer\", hue=None, data=train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* The questions are multiple choice, and the answers are expected to take a value between 0 and 3.\n* According to [the data description](https://www.kaggle.com/c/riiid-test-answer-prediction/data), -1 is for lectures and is excluded"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Distribution of correct answers by userID\ngrouped_by_user_df = train_df_questions.groupby('user_id')\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count'] })\n\nuser_answers_df[('answered_correctly','mean')].hist(bins = int(k))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* We'd like to compare the groups with the highest percentage of correct answers and the groups with the lowest percentage of correct answers about prior_question_elapsed_time."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Draw a pie chart about gender.\nplt.pie(lectures[\"type_of\"].value_counts(),labels=[\"concept\",\"solving question\",\"intention\",\"starter\"],autopct=\"%.1f%%\")\nplt.title(\"Type of Lectures\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# coding: utf-8\nfrom tqdm import tqdm\nimport time\n\n# Set the total value \nbar = tqdm(total = 1000)\n# Add description\nbar.set_description('Progress rate')\nfor i in range(100):\n    # Set the progress\n    bar.update(25)\n    time.sleep(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Acknowledgements\n* [Competition API Detailed Introduction](https://www.kaggle.com/sohier/competition-api-detailed-introduction)\n* [Riiid: Comprehensive EDA + Baseline](https://www.kaggle.com/erikbruin/riiid-comprehensive-eda-baseline)\n* [Riiid! Answer Correctness Prediction EDA. Modeling](https://www.kaggle.com/isaienkov/riiid-answer-correctness-prediction-eda-modeling)\n* [Riiid - EDA&Baseline](https://www.kaggle.com/yutohisamatsu/riiid-eda-baseline)\n* [Answer Correctness - Modeling with RapidsAI](https://www.kaggle.com/andradaolteanu/answer-correctness-modeling-with-rapidsai)\n* [Riiid: EDA of full datase](https://www.kaggle.com/erikbruin/riiid-eda-of-full-dataset#Loading-the-data)\n* [日本語EDA for biginner](https://www.kaggle.com/chumajin/eda-for-biginner)\n* [【日本語】[Japanese] Riiid コンペに取り組む前の準備・随時更新](https://www.kaggle.com/takiyu/japanese-riiid)\n* [Answer Correctness - RAPIDS, XGB, LGBM](https://www.kaggle.com/andradaolteanu/answer-correctness-rapids-xgb-lgbm)\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}