{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Example submission and test\nsample_prediction_df_dtype = {'row_id': 'int64', 'answered_correctly': 'float16','group_num': 'int64'}\ntest_df_dtype = {'row_id': 'int64','group_num': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n            'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n            'prior_question_had_explanation': 'boolean'\n                     }\ntrain_dtype= {'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n            'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n            'prior_question_had_explanation': 'boolean'\n            }\nquestions_dtype = {'question_id': 'int16','bundle_id': 'int16','correct_answer': 'int8','part': 'int8','tags':str}\nlectures_dtype = {'lecture_id': 'int16','tags':str,'part': 'int8','type_of':str}\n\nsample_prediction_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_sample_submission.csv',dtype=sample_prediction_df_dtype, low_memory=False)\ntest_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv',dtype=test_df_dtype, low_memory=False)\ntrain = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',dtype=train_dtype, low_memory=False, nrows=10**5,skiprows=range(1, 0))\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',dtype=questions_dtype, low_memory=False)\nlectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv',dtype=lectures_dtype, low_memory=False)\n\nprint(\"DONE LOADING DATA.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom statsmodels.stats.proportion import proportion_confint\n\ndef transform_pqhe(df):\n    df['prior_question_had_explanation'].fillna(True,inplace=True)\n    df['prior_question_had_explanation']= df['prior_question_had_explanation'].apply(lambda x: int(x))\n    return df\n\ntrain = transform_pqhe(train)\ntrain = train[train['content_type_id'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, val_df = train_test_split(train, test_size = 0.2, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nfeatures = ['prior_question_had_explanation'\n           ,'answered_correctly_perc_question','answered_correctly_perc_question_lower_bound_95%','answered_correctly_perc_question_upper_bound_95%','upper_bound_diff_question','lower_bound_diff_question'\n           ,'answered_correctly_perc_user', 'answered_correctly_perc_user_lower_bound_95%','answered_correctly_perc_user_upper_bound_95%','upper_bound_diff_user','lower_bound_diff_user']\nfeatures = ['prior_question_had_explanation']\ntarget = ['answered_correctly']\n\n\nlgb_train = lgb.Dataset(train_df[features], train_df[target])\nlgb_eval = lgb.Dataset(val_df[features], val_df[target])\n\nparam = {'num_leaves':5,\n         'max_depth':5,\n         'num_leaves':5,\n         'n_estimators ':5,\n         'max_bin':5,\n         'num_leaves':5,\n         'reg_lambda':1,\n         'metric': 'auc',\n         'objective':'binary'}\n\n# Train the model\nclf = lgb.train(param,\n                lgb_train,\n                valid_sets=[lgb_train, lgb_eval],\n                verbose_eval=5000,\n                early_stopping_rounds=100)\npredictions = clf.predict(val_df[features])\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\n\niter_test = env.iter_test()\nfor (test_df, sample_prediction_df) in iter_test:\n    #preprocessing 'pipeline'\n    test_df = transform_pqhe(test_df)\n    #add column with prediction\n    test_df['answered_correctly'] = clf.predict(test_df[features])\n#    test_df['answered_correctly'].fillna(0.5)\n    #add prediction to environment\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}