{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\nThanks to tito for this great script https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n\n* Creating predictive feature is very important, here I just used 14 features and 15M data points to train the model.\n* The dataset is big to preprocess using python with a for loop, their are other tools and frameworks like (SQL, Spark, Apache Beam, Dask) where you could make feature engineering much faster but if we are smart and make predictive feature it's ok to just use for loops.\n* Foward feature engineering seems a good technique to try in this problem (create 1 new feature that you think it could be predective based on the problem, run the pipeline and check if val score increase, if it increase that feature is predictive and you should add it. Care when you just get some minor improvement, sometime is better to discard that feature because your experimentation process is going to get slower)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport riiideducation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)\n\n\n# Funcion for user stats with loops\ndef add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = True):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n    # -----------------------------------------------------------------------\n    \n    for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):\n        \n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n            \n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n             \n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})\n    \n    df = pd.concat([df, user_df], axis = 1)\n    return df\n        \ndef update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n            \n    return\n\ndef read_and_preprocess(feature_engineering = False):\n    \n    train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n    valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n    question_file = '../input/riiid-test-answer-prediction/questions.csv'\n    \n    # Read data\n    feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-40000000:]\n    \n    # Filter by content_type_id to discard lectures\n    train = train.loc[train.content_type_id == False].reset_index(drop = True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_file)\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n    \n    train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    \n    # Client dictionaries\n    answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    \n    # Question dictionaries\n    answered_correctly_q_count = defaultdict(int)\n    answered_correctly_q_sum = defaultdict(int)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    \n    # Client Question dictionary\n    answered_correctly_uq = defaultdict(lambda: defaultdict(int))\n    \n    print('User feature calculation started...')\n    print('\\n')\n    train = add_features(train, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)\n    valid = add_features(valid, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)\n    gc.collect()\n    print('User feature calculation completed...')\n    print('\\n')\n    \n    features_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect\n    }\n    \n    return train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts\n\n# Function for training and evaluation\ndef train_and_evaluate(train, valid, feature_engineering = False):\n    \n    TARGET = 'answered_correctly'\n    # Features to train and predict\n    FEATURES = ['prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',\n                'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n                'timestamp_u_incorrect_recency']\n    \n    # Delete some training data to experiment faster\n    if feature_engineering:\n        train = train.sample(15000000, random_state = SEED)\n    gc.collect()\n    print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n    drop_cols = list(set(train.columns) - set(FEATURES))\n    y_train = train[TARGET]\n    y_val = valid[TARGET]\n    # Drop unnecessary columns\n    train.drop(drop_cols, axis = 1, inplace = True)\n    valid.drop(drop_cols, axis = 1, inplace = True)\n    gc.collect()\n    \n    lgb_train = lgb.Dataset(train[FEATURES], y_train)\n    lgb_valid = lgb.Dataset(valid[FEATURES], y_val)\n    del train, y_train\n    gc.collect()\n    \n    params = {'objective': 'binary', \n              'seed': SEED,\n              'metric': 'auc',\n              'num_leaves': 200,\n              'feature_fraction': 0.75,\n              'bagging_freq': 10,\n              'bagging_fraction': 0.80\n             }\n    \n    model = lgb.train(\n        params = params,\n        train_set = lgb_train,\n        num_boost_round = 10000,\n        valid_sets = [lgb_train, lgb_valid],\n        early_stopping_rounds = 10,\n        verbose_eval = 50\n    )\n    \n    print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n    \n    feature_importance = model.feature_importance()\n    feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n    \n    fig = plt.figure(figsize = (10, 10))\n    fig.suptitle('Feature Importance', fontsize = 20)\n    plt.tick_params(axis = 'x', labelsize = 12)\n    plt.tick_params(axis = 'y', labelsize = 12)\n    plt.xlabel('Importance', fontsize = 15)\n    plt.ylabel('Features', fontsize = 15)\n    sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')\n    plt.show()\n    \n    return TARGET, FEATURES, model\n\n# Using time series api that simulates production predictions\ndef inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):\n    \n    # Get feature dict\n    answered_correctly_u_count = features_dicts['answered_correctly_u_count']\n    answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\n    elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\n    explanation_u_sum = features_dicts['explanation_u_sum']\n    answered_correctly_q_count = features_dicts['answered_correctly_q_count']\n    answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\n    elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\n    explanation_q_sum = features_dicts['explanation_q_sum']\n    answered_correctly_uq = features_dicts['answered_correctly_uq']\n    timestamp_u = features_dicts['timestamp_u']\n    timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\n    \n    # Get api iterator and predictor\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict\n    \n    previous_test_df = None\n    for (test_df, sample_prediction_df) in iter_test:\n        if previous_test_df is not None:\n            previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n            update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)\n        previous_test_df = test_df.copy()\n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n        test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n        test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n        test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n        test_df[TARGET] = 0\n        test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)\n        test_df[TARGET] =  model.predict(test_df[FEATURES])\n        set_predict(test_df[['row_id', TARGET]])\n        \n    print('Job Done')\n    \ntrain, valid, questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = True)\nTARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = True)\ninference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}