{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h1><center>Riiid! Answer Correctness Prediction. Keras NN Model.</center></h1>\n\n<center><img src=\"https://www.riiid.co/assets/opengraph.png\"></center>"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import roc_auc_score\n        \nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Input, Dense\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\n    '/kaggle/input/riiid-test-answer-prediction/train.csv',\n    usecols=[\n        'timestamp', \n        'user_id', \n        'content_id', \n        'user_answer', \n        'answered_correctly', \n        'prior_question_elapsed_time',\n        'prior_question_had_explanation'\n    ],\n       dtype={\n           'timestamp': 'int64',\n           'user_id': 'int32',\n           'content_id': 'int16',\n           'user_answer': 'int8',\n           'answered_correctly': 'int8',\n           'prior_question_elapsed_time': 'float32', \n           'prior_question_had_explanation': 'boolean'\n       }\n)\ntrain = train.sort_values(['timestamp'], ascending=True)\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.loc[train['answered_correctly'] != -1].reset_index(drop=True)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_had_explanation'] = train['prior_question_had_explanation'].fillna(value = False).astype(bool)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train['timestamp']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_df = train.iloc[:int(9 /10 * len(train))]\ntrain_df = train.iloc[int(9 /10 * len(train)):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_user_df = features_df.groupby('user_id')\nuser_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'std', 'median', 'skew', 'var']}).copy()\nuser_answers_df.columns = ['mean_user_accuracy', 'questions_answered', 'std_user_accuracy', 'median_user_accuracy', 'skew_user_accuracy', 'var_user_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_by_content_df = features_df.groupby('content_id')\ncontent_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count', 'std', 'median', 'skew', 'var']}).copy()\ncontent_answers_df.columns = ['mean_accuracy', 'question_asked', 'std_accuracy', 'median_accuracy', 'skew_accuracy', 'var_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel features_df\ndel grouped_by_user_df\ndel grouped_by_content_df\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(user_answers_df, how='left', on='user_id')\ntrain_df = train_df.merge(content_answers_df, how='left', on='content_id')\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'mean_user_accuracy', \n    'questions_answered',\n    'std_user_accuracy', \n    'median_user_accuracy',\n    'skew_user_accuracy',\n    'var_user_accuracy',\n    'mean_accuracy', \n    'question_asked',\n    'std_accuracy', \n    'median_accuracy',\n    'prior_question_elapsed_time', \n    'prior_question_had_explanation',\n    'skew_accuracy',\n    'var_accuracy'\n]\ntarget = 'answered_correctly'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[features + [target]]\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.replace([np.inf, -np.inf], np.nan)\ntrain_df = train_df.fillna(0)\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype(np.int8)\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    model = tf.keras.Sequential([\n        tf.keras.layers.Input(14),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dense(100, activation=\"relu\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(20, activation=\"relu\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(1, activation=\"sigmoid\")\n    ])\n    model.compile(optimizer='adam', loss=\"binary_crossentropy\", metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = pd.DataFrame()\nres['row_id'] = [i for i in range(9927130)]\nres.loc[:, ['answered_correctly']] = 0\nmodels = []\n\nfor n, (tr, te) in enumerate(KFold(n_splits=5, random_state=666, shuffle=True).split(train_df[target])):\n    print(f'Fold {n}')\n    \n    model = create_model()\n    \n    model.fit(\n        train_df[features].values[tr],\n        train_df[target].values[tr],\n        validation_split=0.2,\n        epochs=50, \n        batch_size=2048\n    )\n\n    res.loc[te, ['answered_correctly']] = model.predict(train_df[features].values[te])\n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('NN score: ', roc_auc_score(train_df[target].values, res[target].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    y_preds = []\n    test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')\n    test_df = test_df.merge(content_answers_df, how = 'left', on = 'content_id')\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False).astype(bool)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(np.int8)\n    test_df = test_df.replace([np.inf, -np.inf], np.nan)\n    test_df.fillna(value=0, inplace = True)\n\n    for model in models:\n        y_pred = model.predict(test_df[features].values)\n        y_preds.append(y_pred)\n\n    y_preds = sum(y_preds) / len(y_preds)\n    test_df['answered_correctly'] = y_preds\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}