{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\nThanks to tito and ragnar for this kernels https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\nhttps://www.kaggle.com/ragnar123/riiid-model-lgbm\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\n#import riiideducation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\n\n# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)\n\nqs = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nqs['tag'] = qs['tags'].apply(lambda ts: [int(x) for x in str(ts).split() if x != 'nan'])\nqs_tags = qs[['question_id', 'tag']].set_index('question_id').to_dict()['tag']\nqs_parts = qs[['question_id', 'part']].set_index('question_id').to_dict()['part']\ndel qs\ngc.collect()\n\n\n# Funcion for user stats with loops\ndef add_features(df, len_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, last_wrong_container_u, last_container_u, previous_qu, previous_qa, answered_correctly_pu_count, answered_correctly_pu_sum, last20_answers_u, last20_uq_diff, container_uq_diff, diff_u_sum, diff_u_cnt, diff_pu_sum, diff_pu_cnt, row_from_lecture, update = True):\n    # -----------------------------------------------------------------------\n    # Client features\n    #len_df = len(df)\n    print(len_df)\n    answered_correctly_u_avg = np.zeros(len_df, dtype = np.float32)\n    answered_correctly_pu_avg = np.zeros(len_df, dtype = np.float32)\n    answered_correctly_u_avg20 = np.zeros(len_df, dtype = np.float32)\n    difficulty_u_avg20 = np.zeros(len_df, dtype = np.float32)\n    difficulty_u_avg = np.zeros(len_df, dtype = np.float32)\n    difficulty_pu_avg = np.zeros(len_df, dtype = np.float32)\n    row_from_lecture_u = np.zeros(len_df, dtype = np.int32)\n    answered_u_cnt = np.zeros(len_df, dtype = np.float32)\n    answered_pu_cnt = np.zeros(len_df, dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len_df, dtype = np.float32)\n    explanation_u_avg = np.zeros(len_df, dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len_df, dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len_df, dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len_df, dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len_df, dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len_df, dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len_df, dtype = np.float32)\n    explanation_q_avg = np.zeros(len_df, dtype = np.float32)\n    #------------------------------------------------------------------\n    # Container features\n    wrong_container_u = np.zeros(len_df, dtype = np.int8)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len_df, dtype = np.int32)\n    # -----------------------------------------------------------------------\n    num = -1\n    \n    for row in df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp', 'task_container_id', 'tag', 'part']].values:\n        \n        #New question or lecture\n        if row[1] != -1:\n            num += 1\n            row_from_lecture[row[0]] += 1\n        else:\n            row_from_lecture[row[0]] = 1\n            continue\n        \n        # Previous container update if new container\n        if last_container_u[row[0]] != row[6]:\n            # Client features updates\n            answered_correctly_u_count[row[0]] += len(previous_qu[row[0]])\n            explanation_u_sum[row[0]] += int(row[4])*len(previous_qu[row[0]])\n            elapsed_time_u_sum[row[0]] += row[3]*len(previous_qu[row[0]])\n            if len(timestamp_u[row[0]]) == 4:\n                timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n            # ------------------------------------------------------------------\n            #Client question features updates\n            for (qid, q_diff) in zip(previous_qu[row[0]], container_uq_diff[row[0]]):\n                if not np.isnan(q_diff):\n                    diff_pu_sum[str(row[0]) + '_' + str(qs_parts[qid])] += q_diff\n                    diff_pu_cnt[str(row[0]) + '_' + str(qs_parts[qid])] += 1\n            container_uq_diff[row[0]] = [x for x in container_uq_diff[row[0]]\n                                        if not np.isnan(x)]\n            if len(container_uq_diff[row[0]]) > 0:\n                last20_uq_diff[row[0]] += container_uq_diff[row[0]]\n                diff_u_sum[row[0]] += sum(container_uq_diff[row[0]])\n                diff_u_cnt[row[0]] += len(container_uq_diff[row[0]])\n            if len(last20_uq_diff[row[0]]) > 20:\n                for k in range(len(last20_uq_diff[row[0]]) - 20):\n                    last20_uq_diff[row[0]].pop(0)\n            # Question features updates\n            for qid in previous_qu[row[0]]:\n                answered_correctly_q_count[qid] += 1\n                explanation_q_sum[qid] += int(row[4])\n                elapsed_time_q_sum[qid] += row[3]\n                # ------------------------------------------------------------------\n                # Client Question updates\n                answered_correctly_uq[str(row[0]) + '_' + str(qid)] += 1\n                # ------------------------------------------------------------------\n            if update:\n                # ------------------------------------------------------------------\n                # Client features updates\n                answered_correctly_u_sum[row[0]] += sum(previous_qa[row[0]])\n                if len(previous_qa[row[0]]) != 0:\n                    last20_answers_u[row[0]] += previous_qa[row[0]]\n                if len(last20_answers_u[row[0]]) > 20:\n                    for k in range(len(last20_answers_u[row[0]]) - 20):\n                        last20_answers_u[row[0]].pop(0)\n\n                # ------------------------------------------------------------------\n                # Question features updates\n                for (qid, qa) in zip(previous_qu[row[0]], previous_qa[row[0]]):\n                    answered_correctly_q_sum[qid] += qa\n                    answered_correctly_pu_sum[str(row[0]) + '_' + str(qs_parts[qid])] += qa \n                    answered_correctly_pu_count[str(row[0]) + '_' + str(qs_parts[qid])] += 1\n                # ------------------------------------------------------------------\n                     \n            \n            \n        # Client features assignation\n        # ------------------------------------------------------------------\n        if row_from_lecture[row[0]] != 0:\n            row_from_lecture_u[num] = row_from_lecture[row[0]]\n        else:\n            row_from_lecture_u[num] = np.nan\n            \n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            answered_u_cnt[num] = answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            answered_u_cnt[num] = np.nan\n            explanation_u_avg[num] = np.nan\n            \n        if answered_correctly_pu_count[str(row[0]) + '_' + str(row[8])] != 0:\n            answered_correctly_pu_avg[num] = answered_correctly_pu_sum[str(row[0]) + '_' + str(row[8])] / answered_correctly_pu_count[str(row[0]) + '_' + str(row[8])]\n        else:\n            answered_correctly_pu_avg[num] = np.nan\n            \n        if len(last20_answers_u[row[0]]) != 0:\n            answered_correctly_u_avg20[num] = np.mean(last20_answers_u[row[0]])\n        else:\n            answered_correctly_u_avg20[num] = np.nan\n            \n        if len(last20_uq_diff[row[0]]) > 0:\n            difficulty_u_avg20[num] = np.mean(last20_uq_diff[row[0]])\n        else:\n            difficulty_u_avg20[num] = np.nan\n            \n        if diff_u_cnt[row[0]] != 0:\n            difficulty_u_avg[num] = diff_u_sum[row[0]]/diff_u_cnt[row[0]]\n        else:\n            difficulty_u_avg[num] = np.nan\n            \n        if diff_pu_cnt[str(row[0]) + '_' + str(qs_parts[row[2]])] != 0:\n            difficulty_pu_avg[num] = diff_pu_sum[str(row[0]) + '_' + str(qs_parts[row[2]])]/diff_pu_cnt[str(row[0]) + '_' + str(qs_parts[row[2]])]\n        else:\n            difficulty_pu_avg[num] = np.nan\n            \n        answered_pu_cnt[num] = answered_correctly_pu_count[str(row[0]) + '_' + str(row[8])]\n                \n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif timestamp_u[row[0]][-1] == row[5]:\n            if len(timestamp_u[row[0]]) == 1:\n                timestamp_u_recency_1[num] = np.nan\n                timestamp_u_recency_2[num] = np.nan\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 2:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n                timestamp_u_recency_2[num] = np.nan\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 3:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n                timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 4:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n                timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n                timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        else:\n            if len(timestamp_u[row[0]]) >= 3:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][-1]\n                timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][-2]\n                timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][-3]\n            elif len(timestamp_u[row[0]]) == 2:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][-1]\n                timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][-2]\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 1:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][-1]\n                timestamp_u_recency_2[num] = np.nan\n                timestamp_u_recency_3[num] = np.nan\n            \n\n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        elif timestamp_u_incorrect[row[0]][-1] == row[5]:\n            if len(timestamp_u_incorrect[row[0]]) == 1:\n                timestamp_u_incorrect_recency[num] = np.nan\n            else:\n                timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][-1]\n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]       \n        else:\n            answered_correctly_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Container features assignation\n        if row[6] < last_container_u[row[0]] or \\\n        row[6] == last_wrong_container_u[row[0]]:\n            last_wrong_container_u[row[0]] = row[6]\n            wrong_container_u[num] = 1\n        else:\n            wrong_container_u[num] = 0\n                \n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[str(row[0]) + '_' + str(row[2])]\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        \n        # Current row updates\n        if last_container_u[row[0]] != row[6]:\n            previous_qu[row[0]] = [row[2]]\n            if answered_correctly_q_count[row[2]] != 0:\n                container_uq_diff[row[0]] = [answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]]\n            else:\n                container_uq_diff[row[0]]  = [np.nan]\n            # Flag for training and inference\n            if update:\n                previous_qa[row[0]] = [row[1]]\n                \n        else:\n            previous_qu[row[0]].append(row[2])\n            if answered_correctly_q_count[row[2]] != 0:\n                container_uq_diff[row[0]].append(\n                    answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]])\n            else:\n                container_uq_diff[row[0]].append(np.nan)\n            if update:\n                previous_qa[row[0]].append(row[1])\n        if update:\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 0 \\\n                or row[5] != timestamp_u_incorrect[row[0]][-1]:\n                    if len(timestamp_u_incorrect[row[0]]) == 2:\n                        timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n\n\n        last_container_u[row[0]] = row[6]\n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency,\n                            'wrong_container_u': wrong_container_u,\n                            'answered_u_cnt': answered_u_cnt,\n                            'answered_correctly_pu_avg': answered_correctly_pu_avg,\n                            'answered_pu_cnt': answered_pu_cnt,\n                            'answered_correctly_u_avg20': answered_correctly_u_avg20,\n                            'difficulty_u_avg20': difficulty_u_avg20,\n                            'difficulty_u_avg': difficulty_u_avg,\n                            'difficulty_pu_avg': difficulty_pu_avg,\n                            'row_from_lecture_u': row_from_lecture_u})\n    \n    df = pd.concat([df.loc[df.answered_correctly != -1].reset_index(drop=True),\n                    user_df], axis = 1)\n    return df\n        \ndef update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect, last_container_u, previous_qa, answered_correctly_pu_count, answered_correctly_pu_sum):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp', 'task_container_id']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 2:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            answered_correctly_pu_sum[str(row[0]) + '_' + str(qs_parts[row[2]])] += row[1] \n            answered_correctly_pu_count[str(row[0]) + '_' + str(qs_parts[row[2]])] += 1\n            # ------------------------------------------------------------------\n            \n    return\n\ndef read_and_preprocess(feature_engineering = False):\n    \n    train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n    valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n    question_file = '../input/riiid-test-answer-prediction/questions.csv'\n    lecture_file = '../input/riiid-test-answer-prediction/lectures.csv'\n    \n    # Read data\n    feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'task_container_id']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-40000000:]\n    \n    # Filter by content_type_id to discard lectures\n    train['row_n'] = 1\n    train['row_n'] = train['row_n'].cumsum()\n    valid['row_n'] = 1\n    valid['row_n'] = valid['row_n'].cumsum()\n    train_lectures = train.loc[train.content_type_id == True]\n    train = train.loc[train.content_type_id == False]#.reset_index(drop = True)\n    valid_lectures = valid.loc[valid.content_type_id == True]\n    valid = valid.loc[valid.content_type_id == False]#.reset_index(drop = True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_file)\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n    questions_df['tag'] = questions_df['tags'].apply(lambda ts: [int(x) for x in str(ts).split() if x != 'nan'])\n    lectures_df = pd.read_csv(lecture_file)\n    lectures_df['part'] = lectures_df['part'].astype(np.int32)\n    #\n    \n    train = pd.merge(train, questions_df[['question_id', 'part', 'tag']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    len_train = len(train)\n    train_lectures = pd.merge(train_lectures, lectures_df[['lecture_id', 'part', 'tag']], left_on = 'content_id', right_on = 'lecture_id', how = 'left')\n    train = pd.concat([train, train_lectures]).sort_values(['row_n'])\n    #train = train.sort_values(['row_n'])\n    train.drop(['question_id', 'lecture_id', 'row_n'], axis=1, inplace=True)\n    valid = pd.merge(valid, questions_df[['question_id', 'part', 'tag']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    len_valid = len(valid)\n    valid_lectures = pd.merge(valid_lectures, lectures_df[['lecture_id', 'part', 'tag']], left_on = 'content_id', right_on = 'lecture_id', how = 'left')\n    valid = pd.concat([valid, valid_lectures]).sort_values(['row_n'])\n    valid.drop(['question_id', 'lecture_id', 'row_n'], axis=1, inplace=True)\n    #valid = valid.sort_values(['row_n'])\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    \n    \n    \n    # Client dictionaries\n    answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    answered_correctly_pu_count = defaultdict(int)\n    answered_correctly_pu_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    last20_answers_u = defaultdict(list)\n    diff_u_sum = defaultdict(float) \n    diff_u_cnt = defaultdict(int)\n    diff_pu_sum = defaultdict(float)\n    diff_pu_cnt = defaultdict(int)\n    row_from_lecture = defaultdict(int)\n                                                        \n    \n    # Question dictionaries\n    answered_correctly_q_count = defaultdict(int)\n    answered_correctly_q_sum = defaultdict(int)\n    previous_qu = defaultdict(list)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    previous_qa = defaultdict(list)\n    \n    #Container dicts\n    last_container_u = defaultdict(lambda: -1)\n    last_wrong_container_u = defaultdict(lambda: -1)\n    \n    # Client Question dictionary\n    answered_correctly_uq = defaultdict(int)\n    last20_uq_diff = defaultdict(list)\n    container_uq_diff = defaultdict(list)\n    \n    print('User feature calculation started...')\n    print('\\n')\n    train = add_features(train, len_train, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, last_wrong_container_u, last_container_u, previous_qu, previous_qa, answered_correctly_pu_count, answered_correctly_pu_sum, last20_answers_u, last20_uq_diff, container_uq_diff, diff_u_sum, diff_u_cnt, diff_pu_sum, diff_pu_cnt, row_from_lecture)\n    valid = add_features(valid, len_valid, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, last_wrong_container_u, last_container_u, previous_qu, previous_qa, answered_correctly_pu_count, answered_correctly_pu_sum, last20_answers_u, last20_uq_diff, container_uq_diff, diff_u_sum, diff_u_cnt, diff_pu_sum, diff_pu_cnt, row_from_lecture)\n    gc.collect()\n    print('User feature calculation completed...')\n    print('\\n')\n    \n    features_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect,\n        'last_wrong_container_u': last_wrong_container_u,\n        'last_container_u': last_container_u,\n        'previous_qu': previous_qu,\n        'previous_qa': previous_qa,\n        'answered_correctly_pu_count': answered_correctly_pu_count,\n        'answered_correctly_pu_sum': answered_correctly_pu_sum,\n        'last20_answers_u': last20_answers_u,\n        'last20_uq_diff': last20_uq_diff,\n        'container_uq_diff': container_uq_diff,\n        'diff_u_sum': diff_u_sum,\n        'diff_u_cnt': diff_u_cnt,\n        'diff_pu_sum': diff_pu_sum,\n        'diff_pu_cnt': diff_pu_cnt,\n        'row_from_lecture': row_from_lecture\n    }\n    \n    return train, valid, questions_df, lectures_df, prior_question_elapsed_time_mean, features_dicts\n\n# Function for training and evaluation\ndef train_and_evaluate(train, valid, feature_engineering = False):\n    \n    TARGET = 'answered_correctly'\n    # Features to train and predict\n    FEATURES = ['prior_question_elapsed_time', 'prior_question_had_explanation',\n                'part', 'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',\n                'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n                'timestamp_u_incorrect_recency',\n               'wrong_container_u', 'answered_u_cnt',\n               'answered_correctly_pu_avg',\n               'answered_pu_cnt', 'answered_correctly_u_avg20',\n                'difficulty_u_avg20', 'difficulty_u_avg',\n                'difficulty_pu_avg', 'row_from_lecture_u'\n               ]\n    \n    # Delete some training data to experiment faster\n    if feature_engineering:\n        train = train.sample(15000000, random_state = SEED)\n    gc.collect()\n    print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n    drop_cols = list(set(train.columns) - set(FEATURES))\n    y_train = train[TARGET]\n    y_val = valid[TARGET]\n    # Drop unnecessary columns\n    train.drop(drop_cols, axis = 1, inplace = True)\n    valid.drop(drop_cols, axis = 1, inplace = True)\n    gc.collect()\n    \n    lgb_train = lgb.Dataset(train[FEATURES], y_train)\n    lgb_valid = lgb.Dataset(valid[FEATURES], y_val)\n    del train, y_train\n    gc.collect()\n    \n    params = {'objective': 'binary', \n              'seed': SEED,\n              'metric': 'auc',\n              'num_leaves': 500,\n              'feature_fraction': 0.8,\n              'bagging_freq': 10,\n              'bagging_fraction': 0.80\n             }\n    \n    model = lgb.train(\n        params = params,\n        train_set = lgb_train,\n        num_boost_round = 10000,\n        valid_sets = [lgb_train, lgb_valid],\n        early_stopping_rounds = 10,\n        verbose_eval = 50\n    )\n    \n    print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n    \n    feature_importance = model.feature_importance()\n    feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n    \n    fig = plt.figure(figsize = (10, 10))\n    fig.suptitle('Feature Importance', fontsize = 20)\n    plt.tick_params(axis = 'x', labelsize = 12)\n    plt.tick_params(axis = 'y', labelsize = 12)\n    plt.xlabel('Importance', fontsize = 15)\n    plt.ylabel('Features', fontsize = 15)\n    sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')\n    plt.show()\n    \n    return TARGET, FEATURES, model\n\n    \ntrain, valid, questions_df, lectures_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = True)\n#TARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = True)\n#inference(TARGET, FEATURES, model, questions_df, lectures_df, prior_question_elapsed_time_mean, features_dicts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"prior_question_elapsed_time_mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import gc\ntrain.to_pickle('train3.pkl')\nvalid.to_pickle('valid3.pkl')\n#del train\n#del valid\n#train = pd.read_pickle('train3.pkl')\n#valid = pd.read_pickle('valid3.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"TARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = True)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#defaultdict to dict, drop nested dicts\ndef default_to_dict(features_dict):\n    res = {}\n    for k1 in features_dict.keys():\n        #res[k1] = {}\n        if k1 == 'answered_correctly_uq':\n            continue\n        for k2 in features_dict[k1].keys():\n            if type(features_dict[k1][k2]) == type(defaultdict()):\n                break#res[k1][k2] = dict(features_dict[k1][k2])\n            else:\n                res[k1] = dict(features_dict[k1])\n                break\n            \n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#save dict to file\nimport pickle\nres_dict = default_to_dict(features_dicts)\nwith open('features_f2l.pickle', 'wb') as handle:\n    pickle.dump(res_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import joblib\n# save model\njoblib.dump(model, 'lgbf2.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#save nested dict to file\nres = pd.DataFrame({'uc_id': sorted(list(features_dicts['answered_correctly_uq'].keys())),\n                    'count': [features_dicts['answered_correctly_uq'][k]\n                        for k in sorted(\n                            list(features_dicts['answered_correctly_uq'].keys()))]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"res.to_csv('uq.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}