{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport gc\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\n\nimport riiideducation","metadata":{"execution":{"iopub.status.busy":"2021-06-23T00:01:00.62201Z","iopub.execute_input":"2021-06-23T00:01:00.622489Z","iopub.status.idle":"2021-06-23T00:01:01.750283Z","shell.execute_reply.started":"2021-06-23T00:01:00.622389Z","shell.execute_reply":"2021-06-23T00:01:01.749223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)\n\n\n# Funcion for user stats with loops\ndef add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = True):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n    # -----------------------------------------------------------------------\n    \n    for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):\n        \n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n            \n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n             \n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})\n    \n    df = pd.concat([df, user_df], axis = 1)\n    return df\n        \ndef update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n            \n    return\n\ndef read_and_preprocess(feature_engineering = False):\n    \n    train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n    valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n    question_file = '../input/riiid-test-answer-prediction/questions.csv'\n    \n    # Read data\n    feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-40000000:]\n    \n    # Filter by content_type_id to discard lectures\n    train = train.loc[train.content_type_id == False].reset_index(drop = True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_file)\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n    \n    train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    \n    # Client dictionaries\n    answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    \n    # Question dictionaries\n    answered_correctly_q_count = defaultdict(int)\n    answered_correctly_q_sum = defaultdict(int)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    \n    # Client Question dictionary\n    answered_correctly_uq = defaultdict(lambda: defaultdict(int))\n    \n    print('User feature calculation started...')\n    print('\\n')\n    train = add_features(train, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)\n    valid = add_features(valid, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)\n    gc.collect()\n    print('User feature calculation completed...')\n    print('\\n')\n    \n    features_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect\n    }\n    \n    return train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts\n\n# Function for training and evaluation\ndef train_and_evaluate(train, valid, feature_engineering = False):\n    \n    TARGET = 'answered_correctly'\n    # Features to train and predict\n    FEATURES = ['prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',\n                'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n                'timestamp_u_incorrect_recency']\n    \n    # Delete some training data to experiment faster\n    if feature_engineering:\n        train = train.sample(15000000, random_state = SEED)\n    gc.collect()\n    print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n    drop_cols = list(set(train.columns) - set(FEATURES))\n    y_train = train[TARGET]\n    y_val = valid[TARGET]\n    # Drop unnecessary columns\n    train.drop(drop_cols, axis = 1, inplace = True)\n    valid.drop(drop_cols, axis = 1, inplace = True)\n    gc.collect()\n    \n    lgb_train = lgb.Dataset(train[FEATURES], y_train)\n    lgb_valid = lgb.Dataset(valid[FEATURES], y_val)\n    del train, y_train\n    gc.collect()\n    \n    params = {'objective': 'binary', \n              'seed': SEED,\n              'metric': 'auc',\n              'num_leaves': 200,\n              'feature_fraction': 0.75,\n              'bagging_freq': 10,\n              'bagging_fraction': 0.80\n             }\n    \n    model = lgb.train(\n        params = params,\n        train_set = lgb_train,\n        num_boost_round = 10000,\n        valid_sets = [lgb_train, lgb_valid],\n        early_stopping_rounds = 10,\n        verbose_eval = 50\n    )\n    \n    print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n    \n    feature_importance = model.feature_importance()\n    feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n    \n    fig = plt.figure(figsize = (10, 10))\n    fig.suptitle('Feature Importance', fontsize = 20)\n    plt.tick_params(axis = 'x', labelsize = 12)\n    plt.tick_params(axis = 'y', labelsize = 12)\n    plt.xlabel('Importance', fontsize = 15)\n    plt.ylabel('Features', fontsize = 15)\n    sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')\n    plt.show()\n    \n    return TARGET, FEATURES, model\n\n# Using time series api that simulates production predictions\ndef inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):\n    \n    # Get feature dict\n    answered_correctly_u_count = features_dicts['answered_correctly_u_count']\n    answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\n    elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\n    explanation_u_sum = features_dicts['explanation_u_sum']\n    answered_correctly_q_count = features_dicts['answered_correctly_q_count']\n    answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\n    elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\n    explanation_q_sum = features_dicts['explanation_q_sum']\n    answered_correctly_uq = features_dicts['answered_correctly_uq']\n    timestamp_u = features_dicts['timestamp_u']\n    timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\n    \n    # Get api iterator and predictor\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict\n    \n    previous_test_df = None\n    for (test_df, sample_prediction_df) in iter_test:\n        if previous_test_df is not None:\n            previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n            update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)\n        previous_test_df = test_df.copy()\n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n        test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n        test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n        test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n        test_df[TARGET] = 0\n        test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)\n        test_df[TARGET] =  model.predict(test_df[FEATURES])\n        set_predict(test_df[['row_id', TARGET]])\n        \n    print('Job Done')\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-23T00:01:01.752204Z","iopub.execute_input":"2021-06-23T00:01:01.752557Z","iopub.status.idle":"2021-06-23T00:01:01.818044Z","shell.execute_reply.started":"2021-06-23T00:01:01.752522Z","shell.execute_reply":"2021-06-23T00:01:01.817152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain, valid, questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T00:01:01.819686Z","iopub.execute_input":"2021-06-23T00:01:01.820213Z","iopub.status.idle":"2021-06-23T00:27:24.382083Z","shell.execute_reply.started":"2021-06-23T00:01:01.820178Z","shell.execute_reply":"2021-06-23T00:27:24.381003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nTARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = True)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T00:27:24.383909Z","iopub.execute_input":"2021-06-23T00:27:24.384364Z","iopub.status.idle":"2021-06-23T01:03:22.357284Z","shell.execute_reply.started":"2021-06-23T00:27:24.384318Z","shell.execute_reply":"2021-06-23T01:03:22.356003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T01:03:22.359289Z","iopub.execute_input":"2021-06-23T01:03:22.359597Z","iopub.status.idle":"2021-06-23T01:03:22.506751Z","shell.execute_reply.started":"2021-06-23T01:03:22.359568Z","shell.execute_reply":"2021-06-23T01:03:22.505893Z"},"trusted":true},"execution_count":null,"outputs":[]}]}