{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Read Data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import logging\n\nRUN_ON_KAGGLE = True\nRANDOM_SEED = 3\n\nimport gc\nimport os\nimport pickle\nimport random\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nrandom.seed(RANDOM_SEED)\nnp.random.seed(seed=RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if RUN_ON_KAGGLE:\n    data_folder = \"/kaggle/input/riiid-test-answer-prediction\"\n    \n    formatter = logging.Formatter('\\x1b[1m\\x1b[33m[%(levelname)s %(asctime)s.%(msecs)03d %(name)s]\\x1b[0m: %(message)s', '%Y-%m-%d %H:%M:%S')\n    console = logging.StreamHandler()\n    console.setLevel(logging.INFO)\n    console.setFormatter(formatter)\n    logger = logging.getLogger('I')\n    logger.addHandler(console)\nelse:\n    data_folder = \"../data\"\n    \n    logging.basicConfig(level=logging.INFO,\n                    format='\\x1b[1m\\x1b[33m[%(levelname)s %(asctime)s.%(msecs)03d %(name)s]\\x1b[0m: %(message)s',\n                    datefmt='%Y-%m-%d %H:%M:%S')\n    logger = logging.getLogger('I')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"processed_data_dir = \"/kaggle/input/riid-acp-data-joined-user-feat\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"reading training data...\")\ntrain = pd.read_pickle(\n    os.path.join(processed_data_dir, \"train-with-user-fea.pkl\")\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"reading validation data...\")\nvalid = pd.read_pickle(\n    os.path.join(processed_data_dir, \"eval-with-user-fea.pkl\")\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop(\"row_id\", inplace=True, axis=1)\nvalid.drop(\"row_id\", inplace=True, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pickle_load(file_path):\n    with open(file_path, \"rb\") as f:\n        return pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer_cnt = defaultdict(int)\nanswer_cnt.update(\n    pickle_load(os.path.join(processed_data_dir, \"answer_cnt.pkl\"))\n)\ncorrect_sum = defaultdict(int)\ncorrect_sum.update(\n    pickle_load(os.path.join(processed_data_dir, \"correct_sum.pkl\"))\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"reading questions...\")\nquestions = pd.read_csv(\n    os.path.join(data_folder, \"questions.csv\"),\n    dtype={\n        \"question_id\": \"int16\",\n        \"bundle_id\": \"int32\",\n        \"correct_answer\": \"int8\",\n        \"part\": \"int8\"\n    }\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"reading lectures...\")\nlectures = pd.read_csv(\n    os.path.join(data_folder, \"lectures.csv\"),\n    dtype={\n        \"lecture_id\": \"int16\",\n        \"tag\": \"int16\",\n        \"part\": \"int8\"\n    }\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_user_feats(\n    df,\n    user_correct_sum: dict,\n    user_answer_cnt: dict,\n    update=True\n):\n    correct_sum = np.zeros(len(df), dtype=np.int32)\n    answer_cnt = np.zeros(len(df), dtype=np.int32)\n    cols = [\"user_id\", \"answered_correctly\"] if update else [\"user_id\"]\n    for i, row in df[cols].iterrows():\n        user_id = row[\"user_id\"]\n        correct_sum[i] = user_correct_sum[user_id]\n        answer_cnt[i] = user_answer_cnt[user_id]\n        if update:\n            answer_res = row[\"answered_correctly\"]\n            user_correct_sum[user_id] += answer_res\n            user_answer_cnt[user_id] += 1\n    user_feats = pd.DataFrame({\n        \"correct_sum\": correct_sum,\n        \"answer_cnt\": answer_cnt,\n        \"acc\": (correct_sum / answer_cnt).astype(np.float32)\n    })\n    return pd.concat([df, user_feats], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_user_feats(df, user_correct_sum: dict, user_answer_cnt: dict):\n    df = df[df[\"content_type_id\"] == 0]\n    for _, row in df[[\"user_id\", \"answered_correctly\"]].iterrows():\n        user_id = row[\"user_id\"]\n        answer_res = row[\"answered_correctly\"]\n        user_correct_sum[user_id] += answer_res\n        user_answer_cnt[user_id] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_left_join(left, right, on=None, left_on=None, right_on=None):\n    left_on = left_on or on\n    right_on = right_on or on\n    assert left_on in left.columns, f\"{on} not found in left DF\"\n    assert right_on == right.index.name, f\"{right_on} not equals to right's index_name\"\n    return pd.concat([\n        left.reset_index(drop=True),\n        right.reindex(left[left_on].values).reset_index(drop=True)\n    ], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"generating content features...\")\ncontent_df = train[\n    [\"content_id\", \"answered_correctly\"]\n].groupby(\"content_id\").agg(\n    content_acc=(\"answered_correctly\", \"mean\")\n).astype({\"content_acc\": \"float32\"})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"joining content features to train...\")\ntrain = fast_left_join(\n    train,\n    content_df,\n    on=\"content_id\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"join content features to validation...\")\nvalid = fast_left_join(\n    valid,\n    content_df,\n    on=\"content_id\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\nvalid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"generating questions features...\")\nquestion_df = questions[[\n    \"question_id\", \"part\"\n]].set_index(\"question_id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"joining questions features to validation...\")\nvalid = fast_left_join(\n    valid,\n    question_df,\n    left_on = \"content_id\",\n    right_on = \"question_id\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"joining questions features to train...\")\ntrain = fast_left_join(\n    train,\n    question_df,\n    left_on = \"content_id\",\n    right_on = \"question_id\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logger.info(\"filling missing values...\")\ntrain['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\nvalid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = \"answered_correctly\"\nFEATS = [\n    \"correct_sum\",\n    \"answer_cnt\",\n    \"acc\",\n    \"part\",\n    \"content_acc\",\n    \"prior_question_elapsed_time\",\n    \"prior_question_had_explanation\"\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dro_cols = list(set(train.columns) - set(FEATS))\ny_tr = train[TARGET]\ny_va = valid[TARGET]\ntrain.drop(dro_cols, axis=1, inplace=True)\nvalid.drop(dro_cols, axis=1, inplace=True)\n_ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_train = lgb.Dataset(train[FEATS], y_tr, categorical_feature=[\"part\"])\nlgb_valid = lgb.Dataset(valid[FEATS], y_va, reference=lgb_train, categorical_feature=[\"part\"])\ndel train, y_tr\n_=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"USE_OFFLINE_MODEL = True\nmodel_path = os.path.join(processed_data_dir, \"model_7fea.pkl\")\n\nif USE_OFFLINE_MODEL:\n    logger.info(\"loading offline trained model...\")\n    model = pickle_load(model_path)\nelse:\n    logger.info(\"start training model...\")\n    model = lgb.train(\n        {\n            \"objective\": \"binary\",\n            \"metric\": \"auc\",\n            \"boosting_type\": \"gbdt\",\n            \"learning_rate\": 0.3,\n            \"seed\": 3\n        }, \n        lgb_train,\n        valid_sets=[lgb_train, lgb_valid],\n        verbose_eval=20,\n        num_boost_round=1000,\n        early_stopping_rounds=10\n    )\nauc = roc_auc_score(y_va, model.predict(valid[FEATS]))\nlogger.info(f\"auc on validation: {auc}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nlgb.plot_importance(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del lgb_train, lgb_valid\n_ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre_test_df = None\ninfer_start, iter_cnt = time.time(), 0\nactuals = []\npreds = []\n\nfor (test_df, sample_pred_df) in iter_test:\n    if pre_test_df is not None:\n        targets = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        actuals.extend(targets)\n        pre_test_df[TARGET] = targets\n        update_user_feats(pre_test_df, correct_sum, answer_cnt)\n    pre_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    test_df = add_user_feats(\n        test_df, correct_sum, answer_cnt, update=False\n    )\n    test_df = fast_left_join(\n        test_df, content_df, on=\"content_id\"\n    )\n    test_df = fast_left_join(\n        test_df, question_df, left_on=\"content_id\", right_on=\"question_id\"\n    )\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n    preds_ =  model.predict(test_df[FEATS])\n    test_df[TARGET] = preds_\n    preds.extend(preds_.tolist())\n    set_predict(test_df[['row_id', TARGET]])\n    iter_cnt += 1\n\ninfer_duration = time.time() - infer_start\nlogger.info(f\"total iterations: {iter_cnt}, cost {infer_duration:.4f} secs, {infer_duration/iter_cnt:.4f} secs/iteration\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(actuals, preds[:len(actuals)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}