{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Riiid! Answer Correctness Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport warnings\nwarnings.simplefilter('ignore')\n\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom lightgbm import LGBMClassifier\n\nimport pandas as pd\nfrom pandas.plotting import scatter_matrix\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.simplefilter('ignore')\nfrom sklearn.metrics import roc_auc_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, learning_curve\nfrom sklearn.utils import shuffle\n\nimport riiideducation\n\n%matplotlib inline\n# for heatmap and other plots\ncolorMap1 = sns.color_palette(\"RdBu_r\")\n# for countplot and others plots\ncolorMap2 = 'Blues_r'\n\nWIDTH=800","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"types = {\n        'row_id': 'int64', \n        'timestamp': 'int64', \n        'user_id': 'int32', \n        'content_id': 'int16', \n        'content_type_id': 'int8',\n        'task_container_id': 'int16', \n        'user_answer': 'int8', \n        'answered_correctly': 'int8', \n        'prior_question_elapsed_time': 'float32', \n        'prior_question_had_explanation': 'boolean'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/train.csv', \n    low_memory=False, \n    nrows=10**7, \n    dtype=types\n)\nquestions=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\ntest=pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#                  DATA EXPLORATION & EDA"},{"metadata":{},"cell_type":"markdown","source":"**TRAIN**"},{"metadata":{},"cell_type":"markdown","source":"**row_id**: (int64) ID code for the row.\n\n**timestamp**: (int64) the time in milliseconds between this user interaction and the first event completion from that user.\n\n**user_id**: (int32) ID code for the user.\n\n**content_id**: (int16) ID code for the user interaction\n\n**content_type_id**: (int8) 0 if the event was a question being posed to the user, 1 if the event was the user watching a lecture.\n\n**task_container_id**: (int16) Id code for the batch of questions or lectures. For example, a user might see three questions in a row before seeing the explanations for any of them. Those three would all share a task_container_id.\n\n**user_answer**: (int8) the user's answer to the question, if any. Read -1 as null, for lectures.\n\n**answered_correctly**: (int8) if the user responded correctly. Read -1 as null, for lectures.\n\n**prior_question_elapsed_time**: (float32) The average time in milliseconds it took a user to answer each question in the previous question bundle, ignoring any lectures\nin between. Is null for a user's first question bundle or lecture. Note that the time is the average time a user took to solve each question in the previous bundle.\n\n**prior_question_had_explanation**: (bool) Whether or not the user saw an explanation and the correct response(s) after answering the previous question bundle, ignoring any lectures in between. The value is shared across a single question bundle, and is null for a user's first question bundle or lecture. Typically the first several questions a user sees were part of an onboarding diagnostic test where they did not get any feedback."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Train shape: {train.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**QUESTIONS.CSV**"},{"metadata":{},"cell_type":"markdown","source":"\nquestion_id: foreign key for the train/test content_id column, when the content type is question (0).\n\nbundle_id: code for which questions are served together.\n\ncorrect_answer: the answer to the question. Can be compared with the train user_answer column to check if the user was right.\n\npart: top level category code for the question.\n\ntags: one or more detailed tag codes for the question. The meaning of the tags will not be provided, but these codes are sufficient for clustering the questions together."},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions.describe().style.background_gradient(cmap='Blues')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(questions.isnull().sum() / len(questions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\n\n\nquestions['tag'] = questions['tags'].str.split(' ')\nquestions = questions.explode('tag')\nquestions = pd.merge(\n    questions, \n    questions.groupby('question_id')['tag'].count().reset_index(), \n    on='question_id'\n    )\n\nquestions = questions.drop(['tag_x'], axis=1)\n\nquestions.columns = [\n    'question_id', \n    'bundle_id', \n    'correct_answer', \n    'part', \n    'tags', \n    'tags_number'\n]\nquestions = questions.drop_duplicates()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = questions['correct_answer'].value_counts().reset_index()\n\nds.columns = [\n    'correct_answer', \n    'number_of_answers'\n]\n\nds['correct_answer'] = ds['correct_answer'].astype(str) + '-'\nds = ds.sort_values(['number_of_answers'])\n\nfig = px.bar(\n    ds, \n    x='number_of_answers', \n    y='correct_answer', \n    orientation='h', \n    title='Number of correct answers per group', \n    width=WIDTH,\n    height=300\n)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = questions['part'].value_counts().reset_index()\n\nds.columns = [\n    'part', \n    'count'\n]\n\nds['part'] = ds['part'].astype(str) + '-'\nds = ds.sort_values(['count'])\n\nfig = px.bar(\n    ds, \n    x='count', \n    y='part', \n    orientation='h', \n    title='Parts distribution',\n    width=WIDTH,\n    height=400\n)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = questions['tags_number'].value_counts().reset_index()\n\nds.columns = [\n    'tags_number', \n    'count'\n]\n\nds['tags_number'] = ds['tags_number'].astype(str) + '-'\nds = ds.sort_values(['tags_number'])\n\nfig = px.bar(\n    ds, \n    x='count', \n    y='tags_number', \n    orientation='h', \n    title='Number tags distribution', \n    width=WIDTH,\n    height=400 \n    )\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ncheck = questions['tags'].str.split(' ').explode('tags').reset_index()\ncheck = check['tags'].value_counts().reset_index()\n\ncheck.columns = [\n    'tag', \n    'count'\n]\n\ncheck['tag'] = check['tag'].astype(str) + '-'\ncheck = check.sort_values(['count']).tail(40)\n\nfig = px.bar(\n    check, \n    x='count', \n    y='tag', \n    orientation='h', \n     title='Top 40 most useful tags', \n    width=WIDTH,\n    height=900 \n)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nLECTURES.CSV"},{"metadata":{},"cell_type":"markdown","source":"Metadata for the lectures watched by users as they progress in their education.\n\n**lecture_id**: foreign key for the train/test content_id column, when the content type is lecture (1).\n\n**part**: top level category code for the lecture.\n\n**tag**: one tag codes for the lecture. The meaning of the tags will not be provided, but these codes are sufficient for clustering the lectures together.\n\n**type_of**: brief description of the core purpose of the lecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Part of missing values for every column')\nprint(lectures.isnull().sum() / len(lectures))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures['type_of'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Test.csv**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FEATURE ENGINEERING"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_df = train.iloc[:int(9/10 * len(train))]\ntrain = train.iloc[int(9/10 * len(train)):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_questions_only_df = features_df[features_df['answered_correctly']!=-1]\ngrouped_by_user_df = train_questions_only_df.groupby('user_id')\nfeatures_full_df = features_df\n\nuser_answers_df = grouped_by_user_df.agg(\n    {\n        'answered_correctly': [\n            'mean', \n            'count', \n            'sum',\n            'std', \n            'median', \n            'skew']}\n).copy()\n\nuser_answers_df.columns = [\n    'mean_user_accuracy',\n    'questions_answered',\n    'user_correct_answers',\n    'std_user_accuracy', \n    'median_user_accuracy', \n    'skew_user_accuracy'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = train_questions_only_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg(\n    {\n        'answered_correctly': [\n            'mean', \n            'count', \n            'sum',\n            'std', \n            'median', \n            'skew'\n        ]\n    }\n).copy()\n\ncontent_answers_df.columns = [\n    'mean_accuracy', \n    'question_asked', \n    'content_positive_answers',\n    'std_accuracy', \n    'median_accuracy', \n    'skew_accuracy'\n]\ncontent_answers_df['content_positive_answers'] = content_answers_df['content_positive_answers'].astype(int)\n\ncontent_answers_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_container_df = features_full_df.groupby('task_container_id')\ncontainer_df = grouped_by_container_df.agg(\n    {'answered_correctly': [\n            'mean', \n            'count'],\n    'content_type_id':['sum']\n    }).copy()\n#lectures = train_full_df.groupby('task_container_id')[\"content_type_id\"].sum()\ncontainer_df.columns = [\n    'mean_container', \n    'questions_container',\n    'lecs_in_container']\ncontainer_df[\"lecs_in_container\"] = container_df[\"lecs_in_container\"].astype(int)\ncontainer_df[\"lecs_to_qs\"] = container_df[\"lecs_in_container\"]/container_df[\"questions_container\"]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del features_df\ndel grouped_by_user_df\ndel grouped_by_content_df\ndel grouped_by_container_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'mean_user_accuracy', \n    'questions_answered',\n    'user_correct_answers',\n    'std_user_accuracy', \n    'median_user_accuracy',\n    'skew_user_accuracy',\n    'mean_accuracy', \n    'question_asked',\n    'std_accuracy', \n    'content_positive_answers',\n    'median_accuracy',\n    'prior_question_elapsed_time', \n    'prior_question_had_explanation',\n    'skew_accuracy' ,  \n    'mean_container', \n    'questions_container',\n    'lecs_in_container',\n    'lecs_to_qs']\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[train[target] != -1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(user_answers_df, how='left', on='user_id')\ntrain = train.merge(content_answers_df, how='left', on='content_id')\ntrain = train.merge(container_df, how='left', on='task_container_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(value=False).astype(bool)\ndf = train.fillna(train.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col_to_drop = set(train.columns.values.tolist()).difference(features + [target])\nfor col in col_to_drop:\n    del df[col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.replace([np.inf, -np.inf], np.nan)\ndf = df.fillna(df.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df, y_train, y_test = train_test_split(df[features], df[target], random_state=3, test_size=0.25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Random forest classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nmodel = RandomForestClassifier(n_estimators = 100, max_depth=5, criterion = \"entropy\")\nmodel.fit(train_df, y_train)\nprint('Forest ROC-AUC score: ', roc_auc_score(y_test.values, model.predict_proba(test_df)[:, 1]))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submission preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    # merge\n    test_df = test_df.merge(user_answers_df, on = \"user_id\", how = \"left\")\n    test_df = test_df.merge(content_answers_df, on = \"content_id\", how = \"left\")\n    test_df = test_df.merge(container_df, how='left', on='task_container_id')\n    \n    # type transformation\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value=False).astype(bool)\n    test_df.fillna(df.mean(), inplace = True)\n    \n    # preds\n    test_df['answered_correctly'] = model.predict_proba(test_df[features])[:, 1]\n    cols_to_submission = ['row_id', 'answered_correctly', 'group_num']\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}