{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\nThanks to tito for this great script\n\nhttps://www.kaggle.com/ragnar123/riiid-model-lgbm\n\nhttps://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pickle\nimport pandas as pd\nimport random\nimport riiideducation\nimport seaborn as sns\n\nfrom collections import defaultdict\nfrom sklearn.metrics import roc_auc_score\nfrom tqdm import tqdm\n\n\ntrain_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\nvalid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\nquestion_file= '../input/riiid-test-answer-prediction/questions.csv'\ncontents_feat_file= '../input/riiid-offline-features/content_feats.pkl'\nparts_feat_file   = '../input/riiid-offline-features/part_feats.pkl'\nquestion_tags_file='../input/riiid-offline-features/question_tags_feat.csv'\n\n# \nisDebug = False\nfeature_engineering_rows = 4000000 if isDebug else 40000000 \ntraining_rows = 150000 if isDebug else 10000000 \n\nprint(f'Number of rows for feature engineering: {str(feature_engineering_rows)}')\nprint(f'Number of rows for training: {str(training_rows)}')\n\nquestion_tags_df_dtypes = {\n    'question_id': np.int64,\n    'tags_lsi': np.int8,\n    'tag_acc_max': np.float16,\n    'tag_count': np.int8,\n    'tag_acc_min': np.float16\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)\n\n\n# Funcion for user stats with loops\ndef add_features(\n        df, answered_correctly_u_count, answered_correctly_u_sum,\n        elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n        answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum,\n        explanation_q_sum, answered_correctly_uq, part_user_count, part_user_sum, \n        answered_correctly_ut, answered_correctly_ut_sum,\n        update=True\n):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float16)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float16)\n    explanation_u_avg  = np.zeros(len(df), dtype = np.float16)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float16)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float16)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float16)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float16)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float16)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float16)\n    explanation_q_avg  = np.zeros(len(df), dtype = np.float16)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int16)\n    # -----------------------------------------------------------------------\n    # Part-User \n    part_u_count= np.zeros(len(df), dtype = np.int32)\n    part_u_mean = np.zeros(len(df), dtype = np.float16)\n    # -----------------------------------------------------------------------\n    # User-Tags \n    answered_correctly_ut_count   = np.zeros(len(df), dtype = np.int16)\n    answered_correctly_ut_avg_max = np.zeros(len(df), dtype = np.float16)\n    answered_correctly_ut_avg_min = np.zeros(len(df), dtype = np.float16)\n    # -----------------------------------------------------------------------\n    \n    # Loop for feature excpet part and tags related features\n    for num, row in enumerate(zip(*df[[\n        'user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time',\n        'prior_question_had_explanation', 'timestamp', 'part', 'tags'\n    ]].to_dict(\"list\").values())):\n        \n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            # answered_correctly_u_avg[num] = np.nan\n            answered_correctly_u_avg[num] = .637\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n        \n        # Timestampe features assignation\n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]    \n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n            \n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        \n        # ------------------------------------------------------------------\n        # Part-User features assignation\n        if part_user_count[row[0]][row[6]]==0:\n            part_u_count[num] = 0\n            part_u_mean[num]  = .637\n        else:\n            part_u_count[num] = part_user_count[row[0]][row[6]]\n            part_u_mean[num]  = part_user_sum[row[0]][row[6]]/part_user_count[row[0]][row[6]]\n            \n        # ------------------------------------------------------------------\n        # Tag-User features assignation\n        tags = row[7].split()\n        \n        tag_user_mean  = []\n        for tag in tags:\n            tag_user_count = answered_correctly_ut[row[0]][tag]\n            tag_user_sum   = answered_correctly_ut_sum[row[0]][tag]\n            \n            if tag_user_count == 0:\n                tag_user_mean.append(.637)\n            else:\n                tag_user_mean.append(tag_user_sum / tag_user_count)\n            \n            answered_correctly_ut_count[num] += tag_user_count\n            \n        answered_correctly_ut_avg_max[num] = max(tag_user_mean)\n        answered_correctly_ut_avg_min[num] = min(tag_user_mean)\n\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Client Part updates\n        part_user_count[row[0]][row[6]] += 1\n        # ------------------------------------------------------------------\n        # Client Tags updates\n        for tag in tags:\n            answered_correctly_ut[row[0]][tag] += 1\n        \n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            \n            # ------------------------------------------------------------------\n            # Part features updates\n            part_user_sum[row[0]][row[6]] += row[1]\n            \n            # ------------------------------------------------------------------\n            # Part features updates\n            for tag in tags:\n                answered_correctly_ut_sum[row[0]][tag] += row[1]\n            \n                \n    user_dict = {\n        'answered_correctly_u_avg': answered_correctly_u_avg,\n        'elapsed_time_u_avg': elapsed_time_u_avg,\n        'explanation_u_avg': explanation_u_avg,\n        'answered_correctly_q_avg': answered_correctly_q_avg,\n        'elapsed_time_q_avg': elapsed_time_q_avg,\n        'explanation_q_avg': explanation_q_avg,\n        'answered_correctly_uq_count': answered_correctly_uq_count,\n        'timestamp_u_recency_1': timestamp_u_recency_1,\n        'timestamp_u_recency_2': timestamp_u_recency_2,\n        'timestamp_u_recency_3': timestamp_u_recency_3,\n        'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency,\n        'part_u_count': part_u_count,\n        'part_u_mean': part_u_mean,\n        'answered_correctly_ut_count': answered_correctly_ut_count,\n        'answered_correctly_ut_avg_max': answered_correctly_ut_avg_max,\n        'answered_correctly_ut_avg_min': answered_correctly_ut_avg_min,\n    }\n    \n    for k, v in user_dict.items():\n        df[k] = v\n        \n    return df\n \n        \ndef update_features(\n        df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect, \n        part_user_sum, answered_correctly_ut_sum\n):\n    for row in df[[\n        'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp', 'part', 'tags'\n    ]].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            \n            # ------------------------------------------------------------------\n            # Part features updates\n            part_user_sum[row[0]][row[5]] += row[1]\n            \n            # ------------------------------------------------------------------\n            # Part features updates\n            tags = row[6].split()\n            for tag in tags:\n                answered_correctly_ut_sum[row[0]][tag] += row[1]\n            \n    return\n\n\ndef defaultdictInt():\n    return defaultdict(int)\n\n\ndef read_and_preprocess(feature_engineering=False):    \n    # Read data\n    feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-feature_engineering_rows:]\n    \n    # Filter by content_type_id to discard lectures\n    train = train.loc[train.content_type_id == False].reset_index(drop=True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)\n    \n    # REduce scale to avoid numerical overflow\n    train['prior_question_elapsed_time'] = train['prior_question_elapsed_time'] / 10000.\n    valid['prior_question_elapsed_time'] = valid['prior_question_elapsed_time'] / 10000.\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_file)\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n    \n    # questions_df['tags'].fillna('-1', inplace=True)\n\n    # Merge questions.csv\n    train = pd.merge(train, questions_df[['question_id', 'part', 'tags']], left_on='content_id', right_on='question_id', how='left')\n    valid = pd.merge(valid, questions_df[['question_id', 'part', 'tags']], left_on='content_id', right_on='question_id', how='left')\n\n    # Client dictionaries\n    answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    \n    # Question dictionaries\n    answered_correctly_q_count = defaultdict(int) \n    answered_correctly_q_sum = defaultdict(int)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    \n    # Client Question dictionary\n    answered_correctly_uq = defaultdict(defaultdictInt)\n    \n    # User-Part dictionaries\n    part_user_count = defaultdict(defaultdictInt)\n    part_user_sum   = defaultdict(defaultdictInt)\n    \n    # User-tags dictionaries\n    answered_correctly_ut = defaultdict(defaultdictInt)\n    answered_correctly_ut_sum = defaultdict(defaultdictInt)\n    \n    print('User feature calculation started...')\n    print('\\n')\n    train = add_features(\n        train, answered_correctly_u_count, answered_correctly_u_sum,\n        elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n        answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum,\n        explanation_q_sum, answered_correctly_uq, part_user_count, part_user_sum, \n        answered_correctly_ut, answered_correctly_ut_sum\n    )\n    valid = add_features(\n        valid, answered_correctly_u_count, answered_correctly_u_sum,\n        elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n        answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum,\n        explanation_q_sum, answered_correctly_uq, part_user_count, part_user_sum, \n        answered_correctly_ut, answered_correctly_ut_sum\n    )\n    gc.collect()\n    print('User feature calculation completed...')\n    print('\\n')\n\n    # Merge offline features\n    train.set_index('content_id', inplace=True), valid.set_index('content_id', inplace=True)\n    \n    # Content features\n    content_feat = pickle.load(open(contents_feat_file, 'rb'))\n    train = train.join(content_feat)\n    valid = valid.join(content_feat)\n    \n    # Question tag features\n    question_tags_df = pd.read_csv(question_tags_file, dtype=question_tags_df_dtypes)\n    question_tags_df.set_index('question_id', inplace=True)\n    train = train.join(question_tags_df)\n    valid = valid.join(question_tags_df)\n    \n    train.reset_index(inplace=True), valid.reset_index(inplace=True)\n    \n    # Part features\n    # part_feat = pickle.load(open(parts_feat_file, 'rb'))\n    # train.set_index('part', inplace=True), valid.set_index('part', inplace=True)\n    # train = train.join(part_feat)\n    # valid = valid.join(part_feat)\n    # train.reset_index(inplace=True), valid.reset_index(inplace=True)\n    \n    features_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect,\n        'part_user_count': part_user_count,\n        'part_user_sum': part_user_sum,\n        'answered_correctly_ut': answered_correctly_ut,\n        'answered_correctly_ut_sum': answered_correctly_ut_sum\n    }\n    \n    return train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain, valid, questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# # if os.path.exists(r'./tmp'):\n# #     os.mkdir(r'./tmp')\n# for col, feat in features_dicts.items():\n#     pickle.dump(feat, open(f'./{col}.pkl', 'wb'))\n    \n#     features_dicts[col] = None\n    \n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create features of features\ndef features4features(df):\n    df['correctness_u_q_avg_diff']  = df['answered_correctly_u_avg'] - df['answered_correctly_q_avg']\n    df['correctness_u_pu_avg_diff'] = df['answered_correctly_u_avg'] - df['part_u_mean']\n    df['correctness_u_uv_avg_max_diff'] = df['answered_correctly_u_avg'] - df['answered_correctly_ut_avg_max']\n    df['correctness_u_uv_avg_min_diff'] = df['answered_correctly_u_avg'] - df['answered_correctly_ut_avg_min']\n    df['correctness_uv_avg_max_min_diff'] = df['answered_correctly_ut_avg_max'] - df['answered_correctly_ut_avg_min']\n    df['correctness_u_t_avg_max_diff'] = df['answered_correctly_u_avg'] - df['tag_acc_max']\n    df['correctness_u_t_avg_min_diff'] = df['answered_correctly_u_avg'] - df['tag_acc_min']\n    \n    df['elapsed_time_u_q_diff'] = df['elapsed_time_u_avg'] - df['elapsed_time_q_avg']\n    df['explanation_u_q_diff']  = df['explanation_u_avg']  - df['explanation_q_avg']\n    \n    return df\n\n\n# Function for training and evaluation\ndef train_and_evaluate(train, valid, feature_engineering=False):\n    \n    TARGET = 'answered_correctly'\n    # Features to train and predict\n    FEATURES = [\n        'prior_question_elapsed_time', 'answered_correctly_u_avg', 'part', \n        # 'prior_question_had_explanation', \n        'elapsed_time_u_avg', 'explanation_u_avg', 'answered_correctly_q_avg', \n        'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', \n        'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n        'timestamp_u_incorrect_recency', 'part_u_count', 'part_u_mean',\n        'answered_correctly_ut_count', 'answered_correctly_ut_avg_max',\n        'answered_correctly_ut_avg_min',\n        # offlines:\n        'question_elapsed_time_mean', 'question_had_explanation_mean',\n        'question_correctly_q_count', 'question_correctly_q_mean',\n        # 'part_elapsed_time_mean','part_had_explanation_mean','part_correctly_q_mean',\n        'tags_lsi', 'tag_acc_max', 'tag_acc_min', \n        # 'tag_count'\n    ]\n    \n    CATEGORICAL_FEATURES = [\n        'tags_lsi', 'part'\n    ]\n    \n    for col in CATEGORICAL_FEATURES:\n        train[col] = train[col].astype('category')\n        valid[col] = valid[col].astype('category')\n    \n    # Delete some training data to experiment faster\n    if feature_engineering:\n        train = train.sample(training_rows, random_state=SEED)\n        \n    gc.collect()\n    \n    print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n    \n    drop_cols = list(set(train.columns) - set(FEATURES))\n    y_train = train[TARGET]\n    y_val = valid[TARGET]\n    \n    # Drop unnecessary columns\n    for col in drop_cols:\n        del train[col], valid[col]\n    gc.collect()\n    \n    # Create features of features\n    train = features4features(train)\n    valid = features4features(valid)\n    \n    FEATURES = train.columns\n    \n    # To lgb dataset format\n    lgb_train = lgb.Dataset(train[FEATURES], y_train, categorical_feature=CATEGORICAL_FEATURES)\n    lgb_valid = lgb.Dataset(valid[FEATURES], y_val, categorical_feature=CATEGORICAL_FEATURES)\n    \n    del y_train, train\n    gc.collect()\n    \n    params = {\n        'objective': 'binary', \n        'seed': SEED,\n        'learning_rate': .1,\n        'max_depth': 15,\n        'metric': 'auc',\n        'num_leaves': 300,\n        'feature_fraction': 0.75,\n        'bagging_freq': 10,\n        # 'bagging_fraction': 0.80,\n        'subsample': 0.80,\n    }\n\n    \n    model = lgb.train(\n        params=params,\n        train_set=lgb_train,\n        num_boost_round=10000,\n        valid_sets=[lgb_train, lgb_valid],\n        early_stopping_rounds=10,\n        verbose_eval=50,\n    )\n    \n    print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n\n    feature_importance = model.feature_importance()\n    feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n    \n    fig = plt.figure(figsize=(10, 10))\n    fig.suptitle('Feature Importance', fontsize=20)\n    plt.tick_params(axis='x', labelsize=12)\n    plt.tick_params(axis='y', labelsize=12)\n    plt.xlabel('Importance', fontsize=15)\n    plt.ylabel('Features', fontsize=15)\n    sns.barplot(x=feature_importance['Importance'], y=feature_importance['Features'], orient='h')\n    plt.show()\n    \n    return TARGET, FEATURES, CATEGORICAL_FEATURES, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# TARGET, FEATURES, CATEGORICAL_FEATURES, model = train_and_evaluate(train, valid, feature_engineering=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\n# Features to train and predict\nFEATURES = [\n    'prior_question_elapsed_time', 'answered_correctly_u_avg', 'part', \n    # 'prior_question_had_explanation', \n    'elapsed_time_u_avg', 'explanation_u_avg', 'answered_correctly_q_avg', \n    'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', \n    'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n    'timestamp_u_incorrect_recency', 'part_u_count', 'part_u_mean',\n    'answered_correctly_ut_count', 'answered_correctly_ut_avg_max',\n    'answered_correctly_ut_avg_min',\n    # offlines:\n    'question_elapsed_time_mean', 'question_had_explanation_mean',\n    'question_correctly_q_count', 'question_correctly_q_mean',\n    # 'part_elapsed_time_mean','part_had_explanation_mean','part_correctly_q_mean',\n    'tags_lsi', 'tag_acc_max', 'tag_acc_min', \n    # 'tag_count'\n]\n\nCATEGORICAL_FEATURES = [\n    'tags_lsi', 'part'\n]\n\nfor col in CATEGORICAL_FEATURES:\n    train[col] = train[col].astype('category')\n    valid[col] = valid[col].astype('category')\n\n# Delete some training data to experiment faster\n# if feature_engineering:\ntrain = train.sample(training_rows, random_state=SEED)\n\ngc.collect()\n\nprint(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n\ndrop_cols = list(set(train.columns) - set(FEATURES))\ny_train = train[TARGET]\ny_val = valid[TARGET]\n\n# Drop unnecessary columns\nfor col in drop_cols:\n    del train[col], valid[col]\ngc.collect()\n\n# Create features of features\ntrain = features4features(train)\nvalid = features4features(valid)\n\nFEATURES = train.columns\n\n# To lgb dataset format\nlgb_train = lgb.Dataset(train[FEATURES], y_train, categorical_feature=CATEGORICAL_FEATURES)\nlgb_valid = lgb.Dataset(valid[FEATURES], y_val, categorical_feature=CATEGORICAL_FEATURES)\n\ndel y_train, train\ngc.collect()\n\nparams = {\n    'objective': 'binary', \n    'seed': SEED,\n    'learning_rate': .1,\n    'max_depth': 15,\n    'metric': 'auc',\n    'num_leaves': 300,\n    'feature_fraction': 0.75,\n    'bagging_freq': 10,\n    # 'bagging_fraction': 0.80,\n    'subsample': 0.80,\n}\n\n\nmodel = lgb.train(\n    params=params,\n    train_set=lgb_train,\n    num_boost_round=10000,\n    valid_sets=[lgb_train, lgb_valid],\n    early_stopping_rounds=10,\n    verbose_eval=50,\n)\n\nprint('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n\nfeature_importance = model.feature_importance()\nfeature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n\nfig = plt.figure(figsize=(10, 10))\nfig.suptitle('Feature Importance', fontsize=20)\nplt.tick_params(axis='x', labelsize=12)\nplt.tick_params(axis='y', labelsize=12)\nplt.xlabel('Importance', fontsize=15)\nplt.ylabel('Features', fontsize=15)\nsns.barplot(x=feature_importance['Importance'], y=feature_importance['Features'], orient='h')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using time series api that simulates production predictions\ndef inference(TARGET, FEATURES, CATEGORICAL_FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):\n    \n    # Get feature dict\n    answered_correctly_u_count = features_dicts['answered_correctly_u_count']\n    answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\n    elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\n    explanation_u_sum = features_dicts['explanation_u_sum']\n    answered_correctly_q_count = features_dicts['answered_correctly_q_count']\n    answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\n    elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\n    explanation_q_sum = features_dicts['explanation_q_sum']\n    answered_correctly_uq = features_dicts['answered_correctly_uq']\n    timestamp_u = features_dicts['timestamp_u']\n    timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\n    part_user_count = features_dicts['part_user_count']\n    part_user_sum = features_dicts['part_user_sum']\n    answered_correctly_ut = features_dicts['answered_correctly_ut']\n    answered_correctly_ut_sum = features_dicts['answered_correctly_ut_sum']\n    \n    # Get api iterator and predictor\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict\n\n    # Get offline features and merge\n    content_feat = pickle.load(open(contents_feat_file, 'rb'))\n    # part_feat = pickle.load(open(parts_feat_file, 'rb'))\n    question_tags_df = pd.read_csv(question_tags_file, dtype=question_tags_df_dtypes)\n    question_tags_df.set_index('question_id', inplace=True)\n    \n    previous_test_df = None\n    for (test_df, sample_prediction_df) in iter_test:\n        if previous_test_df is not None:\n            previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n            update_features(\n                previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect,\n                part_user_sum, answered_correctly_ut_sum\n            )\n        \n        test_df = pd.merge(test_df, questions_df[['question_id', 'part', 'tags']], left_on='content_id', right_on='question_id', how='left')\n        \n        previous_test_df = test_df.copy()\n        \n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n        \n        test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'] / 10000.\n        test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n        test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n\n        # test_df.set_index('content_id', inplace=True)\n        # test_df = test_df.join(content_feat)\n        # test_df.reset_index(inplace=True)\n        # \n        # test_df.set_index('part', inplace=True)\n        # test_df = test_df.join(part_feat)\n        # test_df.reset_index(inplace=True)\n        # \n        # test_df = pd.merge(test_df, question_tags_df, left_on='content_id', right_on='question_id', how='left')\n        \n        # Merge offline features\n        test_df.set_index('content_id', inplace=True)\n        \n        # Content features\n        test_df = test_df.join(content_feat)\n        \n        # Question tag features\n        test_df = test_df.join(question_tags_df)\n        \n        test_df.reset_index(inplace=True)\n        \n        test_df[TARGET] = 0\n        test_df = add_features(\n            test_df, answered_correctly_u_count, answered_correctly_u_sum,\n            elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n            answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum,\n            explanation_q_sum, answered_correctly_uq, part_user_count, part_user_sum, \n            answered_correctly_ut, answered_correctly_ut_sum,\n            update=False\n        )\n        for col in CATEGORICAL_FEATURES:\n            test_df[col] = test_df[col].astype('category')\n            \n        test_df = features4features(test_df)\n        test_df[TARGET] =  model.predict(test_df[FEATURES])\n        set_predict(test_df[['row_id', TARGET]])\n        \n    print('Job Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# for col, _ in features_dicts.items():\n#     features_dicts[col] = pickle.load(open(f'./{col}.pkl', 'rb'))\n    \n#     os.remove(f'./{col}.pkl')\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ninference(TARGET, FEATURES, CATEGORICAL_FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}