{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport category_encoders as ce\nimport gc\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import preprocessing, metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%\n# reading the dataset from raw csv file\n\n!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl\nimport datatable as dt\ntrain = dt.fread(\"../input/riiid-test-answer-prediction/train.csv\" ,max_nrows=20555).to_pandas() #, max_nrows=2000555","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = pd.read_csv('../input/riiid-test-answer-prediction/train.csv',nrows=10**6)#, nrows=10**","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndata_types_dict = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'boolean'\n}\ntrain = train.astype(data_types_dict)\ntrain['prior_question_had_explanation'].fillna(False, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train= train.drop(columns=['row_id','task_container_id','user_answer'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain = train[train.content_type_id == 0]\ntrain['attempt'] = train.assign(dif = ((train.content_id.diff() != 0) | (train.user_id.diff() != 0))).groupby([\"user_id\",\"content_id\"]).dif.cumsum()\ntrain['attempt']=train['attempt'] - 1\ntrain['attempt'] = train['attempt'].apply(lambda x: 4 if x >= 4 else x)\n#train = train[(train['content_id']==405) & (train['user_id']==1108148)]\ntrain= train.drop(columns=['content_type_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#1- UUUUUUUUUUUUUUUUUUUUUUUUUUUUU\ngrouped_by_df = train[['user_id','answered_correctly']].groupby('user_id')\ntrain_user = grouped_by_df.agg({'answered_correctly': ['mean', 'sum','count']}).copy()\ntrain_user.columns=['user_mean_accuracy','user_count_correct','user_count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#QQQQQQQQQQQQQQQQ\ngrouped_by_df = train[['content_id','answered_correctly']].groupby('content_id')\ntrain_Q = grouped_by_df.agg({'answered_correctly': ['mean']}).copy()\ntrain_Q.columns=['Q_mean_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Questions\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag = questions[\"tags\"].str.split(\" \", expand = True)\ntag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']\n\nquestions =  pd.concat([questions,tag],axis=1)\nquestions['tags1'] = pd.to_numeric(questions['tags1'], errors='coerce')\nquestions['tags2'] = pd.to_numeric(questions['tags2'], errors='coerce')\nquestions['tags3'] = pd.to_numeric(questions['tags3'], errors='coerce')\nquestions['tags4'] = pd.to_numeric(questions['tags4'], errors='coerce')\nquestions['tags5'] = pd.to_numeric(questions['tags5'], errors='coerce')\nquestions['tags6'] = pd.to_numeric(questions['tags6'], errors='coerce')\n\n\n\nquestions['tags'] = questions['tags'].astype(str)\ncat_features = ['tags']\nencoder = LabelEncoder()\nlabel_encoder = preprocessing.LabelEncoder()\nfor feature in cat_features:\n    encoded = label_encoder.fit_transform(questions[cat_features])\n    questions[feature + '_labels'] = encoded\n    \n\nquestions= questions.drop(columns=['tags1','tags2','tags3','tags4','tags5','tags6','bundle_id','correct_answer'])\nquestions.fillna(0, inplace=True)\nquestions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_Q=train_Q.merge(\n    questions,\n    how='left',\n    left_on='content_id',\n    right_on='question_id')\ntrain_Q.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = None\ndel(questions)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_Q","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convertBoolean(x):\n    if str(x) == \"False\":\n        return 0\n    elif str(x) == \"True\":\n        return 1\n    else:\n        return 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def handling(_dt_train, _train_user, _train_Q):#\n    _dt_train['prior_question_had_explanation'].fillna(False, inplace=True)\n    _dt_train[\"prior_question_had_explanation_enc\"] = _dt_train['prior_question_had_explanation'].apply(convertBoolean)\n    \n    \n    _dt_train = _dt_train.merge(_train_user, how = 'left', on = 'user_id')\n    del _train_user\n    gc.collect()\n    \n    _dt_train = _dt_train.merge(_train_Q, how = 'left', left_on = 'content_id',right_on = 'question_id')\n    del _train_Q\n    gc.collect()\n    \n    _dt_train['mean_user_content_accuracy'] =2 * (_dt_train['user_mean_accuracy'] *  _dt_train['Q_mean_accuracy']) / (_dt_train['user_mean_accuracy'] + _dt_train['Q_mean_accuracy'])\n  \n    _dt_train = _dt_train.drop(columns=['user_id', 'content_id','prior_question_had_explanation','question_id'])\n       \n    _dt_train.fillna(0.5, inplace=True)\n    \n    return _dt_train\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain = train.sort_values(['timestamp'], ascending=True)\ntrain_part_len=int(len(train)*0.66)\ntrain = train.iloc[train_part_len:,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dt_train=handling(train, train_user, train_Q)#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain = None\ndel(train)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dt_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\nfeatures = [\n    # user features\n    'user_mean_accuracy',\n    'user_count_correct',\n    'user_count',\n    # content features\n    'Q_mean_accuracy',\n    # part features\n    'part',\n    # other features\n    'Q_mean_accuracy',\n    'attempt',\n    'tags_label',\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation'\n]\n\n\n\ntarget = 'answered_correctly'\n    \ndt_y = dt_train[target]\ndt_x = dt_train[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dt_x.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dt_train = None\ndel(dt_train)\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_fraction = 0.05\nvalid_size = int(len(dt_x) * valid_fraction)\n\ntrain_x = dt_x[:-1 * valid_size]\nvalid_x = dt_x[-valid_size:]\n\n\ntrain_y = dt_y[:-1 * valid_size]\nvalid_y = dt_y[-valid_size:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"dt_x shape\" + str(dt_x.shape))\nprint(\"train_x shape\" + str(train_x.shape))\nprint(\"valid_x shape\" + str(valid_x.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features=train_x.columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n\ndtrain = lgb.Dataset(train_x, label=train_y)\ndvalid = lgb.Dataset(valid_x, label=valid_y)\n\nparams= {\n    'objective': 'binary',\n    'seed': 42,\n    'metric': 'auc',\n    'learning_rate': 0.05,\n    'max_bin': 1500,\n    'num_leaves': 70 \n    }\n    \nmodel = lgb.train(\n        params, \n        dtrain, \n        num_boost_round=2500, \n        valid_sets=[dtrain,dvalid], \n        early_stopping_rounds=20, \n        verbose_eval=50,\n        feature_name = features,\n        )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#displaying the most important features\nlgb.plot_importance(model)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = test_df['prior_question_had_explanation'].apply(convertBoolean)\n\n    \n    test_df = test_df[test_df.content_type_id == 0]\n    test_df['attempt'] = test_df.assign(dif = ((test_df.content_id.diff() != 0) | (test_df.user_id.diff() != 0))).groupby([\"user_id\",\"content_id\"]).dif.cumsum()\n    test_df['attempt']=test_df['attempt'] - 1\n    test_df['attempt'] = test_df['attempt'].apply(lambda x: 4 if x >= 4 else x)\n    \n    \n    test_df = test_df.merge(train_user, how = 'left', on = 'user_id')\n    test_df = test_df.merge(train_Q, how = 'left', left_on = 'content_id',right_on = 'question_id')\n    \n    \n    test_df['mean_user_content_accuracy'] =2 * (test_df['user_mean_accuracy'] *  test_df['Q_mean_accuracy']) / (test_df['user_mean_accuracy'] + test_df['Q_mean_accuracy'])\n\n\n    test_df.fillna(0.5, inplace=True)\n   \n\n    test_df['answered_correctly'] =  model.predict(test_df[features])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}