{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# datatable installation\n!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl\n\nimport riiideducation\nimport gc\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nimport datatable as dt\nimport lightgbm as lgb\nfrom matplotlib import pyplot as plt\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load data using datatable"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_data_types = {\n    'row_id': 'int32',\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'answered_correctly': 'int8', \n    'prior_question_had_explanation': 'bool',\n}\ntrain_df = dt.fread(\"../input/riiid-test-answer-prediction/train.csv\", columns=set(train_data_types.keys())).to_pandas()\nfor column, dtype in train_data_types.items():\n    train_df[column] = train_df[column].astype(dtype) \ntrain_df.memory_usage(deep=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def correct(field):\n    correct = train_df[train_df.answered_correctly != -1].groupby([field, 'answered_correctly'], as_index=False).size()\n    correct = correct.pivot(index= field, columns='answered_correctly', values='size')\n    correct['Percent_correct'] = round(correct.iloc[:,1]/(correct.iloc[:,0] + correct.iloc[:,1]),2)\n    correct = correct.sort_values(by = \"Percent_correct\", ascending = False)\n    correct = correct.iloc[:,2]\n    return correct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup_labels_6 = ['Group_1', 'Group_2', 'Group_3', 'Group_4', 'Group_5', 'Group_6']\ntrain_df['timestamp_group'] = pd.qcut(train_df['timestamp'], q=6, labels=group_labels_6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_user_cut_point = 177725917.833","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df['new_users'] = np.where(train_df['timestamp_group'] == 'Group_1', True, False)\ndel train_df['timestamp_group']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\nquestions = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"str_tag = questions.tags\ntags_list = [x.split() for x in str_tag.astype(str)]\nquestions['tags'] = tags_list\ncorrect = train_df[train_df.answered_correctly != -1].groupby([\"content_id\", 'answered_correctly'], as_index=False).size()\ncorrect = correct.pivot(index= \"content_id\", columns='answered_correctly', values='size')\ncorrect.columns = ['Wrong', 'Right']\ncorrect = correct.fillna(0)\ncorrect[['Wrong', 'Right']] = correct[['Wrong', 'Right']].astype(int)\nquestions = questions.merge(correct, left_on = \"question_id\", right_on = \"content_id\", how = \"left\")\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags = [\" \".join(x).split() for x in questions[questions.tags != \"nan\"].tags.values]\ntags = [item for elem in tags for item in elem]\ntags = set(tags)\ntags = list(tags)\ntags_df = pd.DataFrame()\nfor x in range(len(tags)):\n    df = questions[questions.tags.apply(lambda l: tags[x] in l)]\n    df1 = df.agg({'Wrong': ['sum'], 'Right': ['sum']})\n    df1['Total_questions'] = df1.Wrong + df1.Right\n    df1['Question_ids_with_tag'] = len(df)\n    df1['tag'] = tags[x]\n    df1 = df1.set_index('tag')\n    tags_df = tags_df.append(df1)\n\ntags_df[['Wrong', 'Right', 'Total_questions']] = tags_df[['Wrong', 'Right', 'Total_questions']].astype(int)\ntags_df['Percent_correct'] = tags_df.Right/tags_df.Total_questions\ntags_df = tags_df.sort_values(by = \"Percent_correct\")\n\ntags_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"select_rows = list(range(0,10)) + list(range(178, len(tags_df)))\ntags_select = tags_df.iloc[select_rows,4]\n\nfig = plt.figure(figsize=(12,6))\nx = tags_select.index\ny = tags_select.values\nclrs = ['red' if y < 0.6 else 'green' for y in tags_select.values]\ntags_select.plot.bar(x, y, color=clrs)\nplt.title(\"Ten hardest and ten easiest tags\")\nplt.xlabel(\"Tag\")\nplt.ylabel(\"Percent answers correct of questions with the tag\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#adding user features\nuser_df = train_df[train_df.answered_correctly != -1].groupby('user_id').agg({'answered_correctly': ['count', 'mean']}).reset_index()\nuser_df.columns = ['user_id', 'user_questions', 'user_mean']\n\n\nuser_lect = train_df.groupby([\"user_id\", \"answered_correctly\"]).size().unstack()\n# Changed [-1, 0, 1] to ['Lecture', 'Wrong', 'Right']\nuser_lect.columns = ['Lecture', 'Wrong', 'Right']\nuser_lect['Lecture'] = user_lect['Lecture'].fillna(0)\n\n# Add another column to indicate whether the user watch lectures or not\nuser_lect = user_lect.astype('Int64')\nuser_lect['Watched_lecture'] = np.where(user_lect.Lecture > 0, True, False)\nuser_lect = user_lect.reset_index()\nuser_lect = user_lect[['user_id', 'Watched_lecture']]\nuser_df = user_df.merge(user_lect, on = \"user_id\", how = \"left\")\ndel user_lect\nuser_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#adding content features\ncontent_df = train_df[train_df.answered_correctly != -1].groupby('content_id').agg({'answered_correctly': ['count', 'mean']}).reset_index()\ncontent_df.columns = ['content_id', 'content_questions', 'content_mean']\ncontent_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncv2_train = pickle.load(open(\"../input/cv-index-for-riiid/train_index.pkl\", 'rb'))\ncv2_valid = pickle.load(open(\"../input/cv-index-for-riiid/valid_index.pkl\", 'rb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Split the train set as train and validation set.\nvalidation_df = train_df[train_df.row_id.isin(cv2_valid)]\ntrain_df = train_df[train_df.row_id.isin(cv2_train)]\n\nvalidation_df = validation_df.drop(columns = \"row_id\")\ntrain_df = train_df.drop(columns = \"row_id\")\n\ndel cv2_train, cv2_valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#train_df = pd.merge(train_df, user_lect[['user_id', 'Watched_lecture']], how='left', on=['user_id', 'user_id'])\ntrain_df = train_df.merge(user_df, on = \"user_id\", how = \"left\")\ntrain_df = train_df.merge(content_df, on = \"content_id\", how = \"left\")\ntrain_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nvalidation_df = validation_df.merge(user_df, on = \"user_id\", how = \"left\")\nvalidation_df = validation_df.merge(content_df, on = \"content_id\", how = \"left\")\nvalidation_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_fill_na(df):\n    df['content_questions'].fillna(0, inplace = True)\n    df['content_mean'].fillna(0.5, inplace = True)\n    df['user_questions'].fillna(0, inplace = True)\n    df['user_mean'].fillna(0.5, inplace = True)\n    df[['content_questions', 'user_questions']] = df[['content_questions', 'user_questions']].astype(int)\n    df['new_users'].fillna(True, inplace = True)\n    df['Watched_lecture'].fillna(False, inplace = True)\n    df['prior_question_had_explanation'].fillna(True, inplace = True)\n    return(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\nexample_sample_submission = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df = merge_fill_na(train_df)\nvalidation_df = merge_fill_na(validation_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#build final train/validation set\nfeatures = ['content_id', 'prior_question_had_explanation', 'Watched_lecture', 'new_users', \n            'user_questions', 'user_mean', 'content_questions', 'content_mean']\n\n\ntrain_df = train_df.sample(n=10000000, random_state = 1)\ny_train = train_df['answered_correctly']\ntrain = train_df[features]\n\ny_val = validation_df['answered_correctly']\nvalidation = validation_df[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nparams = {'objective': 'binary',\n          'metric': 'auc',\n          'seed': 2020,\n          'learning_rate': 0.1, #default\n          \"boosting_type\": \"gbdt\" #default\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_train = lgb.Dataset(train, y_train, categorical_feature = ['prior_question_had_explanation'])\nlgb_eval = lgb.Dataset(validation, y_val, categorical_feature = ['prior_question_had_explanation'])\ndel train, y_train, validation, y_val\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#train\nmodel = lgb.train(\n    params, lgb_train,\n    valid_sets=[lgb_train, lgb_eval],\n    verbose_eval=50,\n    num_boost_round=10000,\n    early_stopping_rounds=8\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df['new_users'] = np.where(test_df['timestamp'] <= new_user_cut_point, True, False)\n    test_df = test_df.merge(user_df, on = \"user_id\", how = \"left\")\n    test_df = test_df.merge(content_df, on = \"content_id\", how = \"left\")\n    test_df = merge_fill_na(test_df)\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype('bool')\n    test_df['answered_correctly'] =  model.predict(test_df[features])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}