{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\nThanks to tito for this great script https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n\n* Creating predictive feature is very important, here I just used 14 features and 15M data points to train the model.\n* The dataset is big to preprocess using python with a for loop, their are other tools and frameworks like (SQL, Spark, Apache Beam, Dask) where you could make feature engineering much faster but if we are smart and make predictive feature it's ok to just use for loops.\n* Foward feature engineering seems a good technique to try in this problem (create 1 new feature that you think it could be predective based on the problem, run the pipeline and check if val score increase, if it increase that feature is predictive and you should add it. Care when you just get some minor improvement, sometime is better to discard that feature because your experimentation process is going to get slower)."},{"metadata":{"trusted":true},"cell_type":"code","source":"datasize =    30000000\nstatesize =   20000000 # train[:-statesize] for store state\nsamplesize =   8000000 # need lower than datasize - statesize","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm import tqdm\nimport lightgbm as lgb\nimport riiideducation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Random seed\nSEED = 456\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)\n\n\n# Funcion for user stats with loops\ndef add_features(df, #answered_correctly_u_count, \n                 answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n                 #answered_correctly_q_count, \n                 answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq,\n                 uccm_sum, cucm_sum, user_part_sum, user_part_count, bundle_sum, bundle_count, ubcm_sum, timestamp_u_correct, #bucm_sum, \n                 cucm0_sum, cucm0_count, cucm1_sum, cucm1_count, \n                 uccm0_sum, uccm0_count, uccm1_sum, uccm1_count,\n                 assign, update = True):\n    \n    if assign == True:\n        # -----------------------------------------------------------------------\n        # Client features\n        answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n        elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n        explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n        timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n        timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n        timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n        timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n        \n        tucr = np.zeros(len(df), dtype = np.float32)\n        \n        \n        # -----------------------------------------------------------------------\n        # Question features\n        answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n        elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n        explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n        # -----------------------------------------------------------------------\n        # User Question\n        answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n        # -----------------------------------------------------------------------\n\n        ### +++++\n        uccm_mean = np.zeros(len(df), dtype = np.float32)\n        ### +++++\n        cucm_mean = np.zeros(len(df), dtype = np.float32)\n        ### +++++\n        user_part_mean = np.zeros(len(df), dtype = np.float32)\n\n        ### +++++\n        ubcm_mean = np.zeros(len(df), dtype = np.float32)\n        bundle_mean = np.zeros(len(df), dtype = np.float16)\n        #bucm_mean = np.zeros(len(df), dtype = np.float16)\n        ### \n        uco = np.zeros(len(df), dtype = np.int16)\n        ### \n        upco = np.zeros(len(df), dtype = np.int16)\n        \n        ### \n        uccm0_mean = np.zeros(len(df), dtype = np.float16)\n        uccm1_mean = np.zeros(len(df), dtype = np.float16)\n        cucm0_mean = np.zeros(len(df), dtype = np.float16)\n        cucm1_mean = np.zeros(len(df), dtype = np.float16)\n\n    \n    \n    for num, row in enumerate(tqdm(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation',\n                                  'timestamp', 'part', 'bundle_id', 'task_container_id']].values)):\n        \n        if assign == True:\n            \n            # Client features assignation\n            # ------------------------------------------------------------------\n            if (uccm0_count[row[0]] + uccm1_count[row[0]]) != 0:\n                answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n                elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n                explanation_u_avg[num] = explanation_u_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n                ### +++++\n                uccm_mean[num] = uccm_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n                ### ubcm\n                ubcm_mean[num] = ubcm_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n                #### user count\n                uco[num] = (uccm0_count[row[0]] + uccm1_count[row[0]])\n            else:\n                answered_correctly_u_avg[num] = np.nan\n                elapsed_time_u_avg[num] = np.nan\n                explanation_u_avg[num] = np.nan\n                ### +++++\n                uccm_mean[num] = np.nan\n                ### ubcm\n                ubcm_mean[num] = np.nan\n                ### user count\n                uco[num] = 0\n            ### +++++ user part correctness\n            if user_part_count[(row[0], row[6])] != 0:\n                user_part_mean[num] = user_part_sum[(row[0], row[6])] / user_part_count[(row[0], row[6])]\n                upco[num] = user_part_count[(row[0], row[6])]\n            else:\n                user_part_mean[num] = np.nan\n                upco[num] = 0\n\n\n            if len(timestamp_u[row[0]]) == 0:\n                timestamp_u_recency_1[num] = np.nan\n                timestamp_u_recency_2[num] = np.nan\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 1:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n                timestamp_u_recency_2[num] = np.nan\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 2:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n                timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n                timestamp_u_recency_3[num] = np.nan\n            elif len(timestamp_u[row[0]]) == 3:\n                timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n                timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n                timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n\n            if len(timestamp_u_incorrect[row[0]]) == 0:\n                timestamp_u_incorrect_recency[num] = np.nan\n            else:\n                timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n                \n                \n            if len(timestamp_u_correct[row[0]]) == 0:\n                tucr[num] = np.nan\n            else:\n                tucr[num] = row[5] - timestamp_u_correct[row[0]][0]\n\n            # ------------------------------------------------------------------\n            # Question features assignation\n            if (cucm0_count[row[2]] + cucm1_count[row[2]]) != 0:\n                answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n                elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n                explanation_q_avg[num] = explanation_q_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n\n                ### +++++\n                cucm_mean[num] = cucm_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n\n            else:\n                answered_correctly_q_avg[num] = np.nan\n                elapsed_time_q_avg[num] = np.nan\n                explanation_q_avg[num] = np.nan\n\n                ### +++++\n                cucm_mean[num] = np.nan\n            \n            if bundle_count[row[7]] == 0:\n                bundle_mean[num] = np.nan\n                \n                #bucm_mean[num] = np.nan\n                \n            else:\n                bundle_mean[num] = bundle_sum[row[7]] / bundle_count[row[7]]\n                    \n                #bucm_mean[num] = bucm_sum[row[7]] / bundle_count[row[7]]\n\n            ### cucm0 cucm1\n            if cucm0_count[row[2]] != 0:\n                cucm0_mean[num] = cucm0_sum[row[2]] / cucm0_count[row[2]]\n            else:\n                cucm0_mean[num] = np.nan\n            \n            if cucm1_count[row[2]] != 0:\n                cucm1_mean[num] = cucm1_sum[row[2]] / cucm1_count[row[2]]\n            else: \n                cucm1_mean[num] = np.nan\n                \n\n            ### uccm0 uccm1 \n            if uccm0_count[row[0]] != 0:\n                uccm0_mean[num] = uccm0_sum[row[0]] / uccm0_count[row[0]]\n            else:\n                uccm0_mean[num] = np.nan\n            \n            if uccm1_count[row[0]] != 0:\n                uccm1_mean[num] = uccm1_sum[row[0]] / uccm1_count[row[0]]\n            else: \n                uccm1_mean[num] = np.nan\n                \n            # ------------------------------------------------------------------\n            # Client Question assignation\n            answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n            # ------------------------------------------------------------------ \n\n        ### +++++ uccm cucm update +++++\n        if (cucm0_count[row[2]] + cucm1_count[row[2]]) == 0:\n            uccm_sum[row[0]] += 0.5\n        else:\n            uccm_sum[row[0]] += answered_correctly_q_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n\n        if (uccm0_count[row[0]] + uccm1_count[row[0]]) == 0:\n            cucm_sum[row[2]] += 0.5\n            #bucm_sum[row[7]] += 0.5\n        else:\n            cucm_sum[row[2]] += answered_correctly_u_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n            #bucm_sum[row[7]] += answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n        ### +++++ ubcm update\n        if bundle_count[row[7]] == 0:\n            ubcm_sum[row[0]] += 0.5\n        else:\n            ubcm_sum[row[0]] += (bundle_sum[row[7]] / bundle_count[row[7]])\n\n        \n        \n        # ------------------------------------------------------------------\n        # Client features updates\n        #answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # user part\n        user_part_count[(row[0], row[6])] += 1\n        \n        \n        # ------------------------------------------------------------------\n        # Question features updates\n        #answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        \n        ### bundle count\n        bundle_count[row[7]] += 1\n        \n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            \n            \n            if row[1] == 1:\n                if len(timestamp_u_correct[row[0]]) == 1:\n                    timestamp_u_correct[row[0]].pop(0)\n                    timestamp_u_correct[row[0]].append(row[5])\n                else:\n                    timestamp_u_correct[row[0]].append(row[5])\n            \n            \n            \n            \n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            \n            ## bundle sum\n            bundle_sum[row[7]] += row[1]\n            \n            \n            # ------------------------------------------------------------------\n            # user part sum, count\n            user_part_sum[(row[0], row[6])] += row[1]\n            \n            # cucm0, cucm1\n            if row[1] == 1:\n                cucm1_count[row[2]] += 1\n                if (uccm0_count[row[0]] + uccm1_count[row[0]]) == 0:\n                    cucm1_sum[row[2]] += 0.5\n                else:\n                    cucm1_sum[row[2]] += answered_correctly_u_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n                    \n                    \n                    \n            if row[1] == 0:\n                cucm0_count[row[2]] += 1\n                if (uccm0_count[row[0]] + uccm1_count[row[0]]) == 0:\n                    cucm0_sum[row[2]] += 0.5\n                else:\n                    cucm0_sum[row[2]] += answered_correctly_u_sum[row[0]] / (uccm0_count[row[0]] + uccm1_count[row[0]])\n            \n            # uccm0, uccm1\n            if row[1] == 1:\n                uccm1_count[row[0]] += 1\n                \n                if (cucm0_count[row[2]] + cucm1_count[row[2]]) == 0:\n                    uccm1_sum[row[0]] += 0.5\n                else:\n                    uccm1_sum[row[0]] += answered_correctly_q_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n             \n            if row[1] == 0:\n                uccm0_count[row[0]] += 1\n                if (cucm0_count[row[2]] + cucm1_count[row[2]]) == 0:\n                    uccm0_sum[row[0]] += 0.5\n                else:\n                    uccm0_sum[row[0]] += answered_correctly_q_sum[row[2]] / (cucm0_count[row[2]] + cucm1_count[row[2]])\n              \n    \n    if assign == True:\n        \n        user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                                'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                                'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                                'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency,\n                                'uccm' : uccm_mean,\n                                'cucm' : cucm_mean,\n                                'upm' : user_part_mean, \n                                'ubcm' : ubcm_mean,\n                                'user_count' : uco,\n                                'user_part_count' : upco,\n                                'bm' : bundle_mean, \n                                'tucr': tucr,\n                                #'bucm': bucm_mean,\n                                'cucm0' : cucm0_mean,\n                                'cucm1' : cucm1_mean,\n                                'uccm0' : uccm0_mean,\n                                'uccm1' : uccm1_mean,\n                               })\n        df = df.reset_index(drop=True)\n        user_df = user_df.reset_index(drop=True)\n        df = pd.concat([df, user_df], axis = 1)\n        return df\n    else:\n        return 0 \n        \ndef update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect, user_part_sum, bundle_sum,\n                    cucm0_sum, cucm0_count, cucm1_sum, cucm1_count,\n                    uccm0_sum, uccm0_count, uccm1_sum, uccm1_count):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp', 'part', 'bundle_id' , \n                   'answered_correctly_u_avg', 'answered_correctly_q_avg']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            bundle_sum[row[6]] += row[1]\n            # ------------------------------------------------------------------\n            user_part_sum[(row[0], row[5])] += row[1]\n        \n        # cucm 0 1\n        if row[1] == 1:\n            cucm1_count[row[2]] += 1\n            if np.isnan(row[7]):\n                cucm1_sum[row[2]] += 0.5\n            else:\n                cucm1_sum[row[2]] += row[7]\n\n            uccm1_count[row[0]] += 1\n            if np.isnan(row[8]):\n                uccm1_sum[row[0]] += 0.5\n            else:\n                uccm1_sum[row[0]] += row[8]\n\n        if row[1] == 0:\n            cucm0_count[row[2]] += 1\n            if np.isnan(row[7]):\n                cucm0_sum[row[2]] += 0.5\n            else:\n                cucm0_sum[row[2]] += row[7]\n            \n            uccm0_count[row[0]] += 1\n            if np.isnan(row[8]):\n                uccm0_sum[row[0]] += 0.5\n            else:\n                uccm0_sum[row[0]] += row[8]\n\n\n\n            \n            \n            \n            \n    return\n\ndef read_and_preprocess(feature_engineering = False):\n    \n    train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n    valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n    question_file = '../input/riiid-test-answer-prediction/questions.csv'\n    \n    # Read data\n    feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'task_container_id']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    print('train shape', train.shape)\n    print('valid shape', valid.shape)\n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-datasize:]\n    \n    # Filter by content_type_id to discard lectures\n    train = train.loc[train.content_type_id == False].reset_index(drop = True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_file)\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n    \n    train = pd.merge(train, questions_df[['question_id', 'part', 'bundle_id']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    valid = pd.merge(valid, questions_df[['question_id', 'part', 'bundle_id']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    \n    # Client dictionaries\n    #answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    timestamp_u_correct = defaultdict(list)\n    \n    \n    \n    \n    # Question dictionaries\n    #answered_correctly_q_count = defaultdict(int)\n    answered_correctly_q_sum = defaultdict(int)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    \n    # Client Question dictionary\n    answered_correctly_uq = defaultdict(lambda: defaultdict(int))\n    \n    \n    ### +++++ uccm \n    uccm_sum = defaultdict(int)\n    \n    uccm0_sum = defaultdict(int)\n    uccm0_count = defaultdict(int)\n    uccm1_sum = defaultdict(int)\n    uccm1_count = defaultdict(int)   \n    \n    \n    \n    \n    \n    ### +++++ cucm\n    cucm_sum = defaultdict(int)\n    \n    cucm0_sum = defaultdict(int)\n    cucm0_count = defaultdict(int)\n    cucm1_sum = defaultdict(int)\n    cucm1_count = defaultdict(int)\n    \n    \n    \n    \n    ### +++++ user part sum, count\n    user_part_sum = defaultdict(int)\n    user_part_count = defaultdict(int)\n    ### +++++ bundle sum count, ubcm\n    bundle_sum = defaultdict(int)\n    bundle_count = defaultdict(int)\n    ubcm_sum = defaultdict(int)\n    #bucm_sum = defaultdict(int)\n    \n    print('User feature calculation started...')\n    print('\\n')\n    \n    temp = train[:-statesize].copy()\n    train = train[-statesize:]\n    \n    gc.collect()\n    temp = add_features(temp, #answered_correctly_u_count, \n                        answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n                        #answered_correctly_q_count, \n                        answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq,\n                        uccm_sum, cucm_sum, user_part_sum, user_part_count, bundle_sum, bundle_count, ubcm_sum, timestamp_u_correct, #bucm_sum, \n                        cucm0_sum, cucm0_count, cucm1_sum, cucm1_count, \n                        uccm0_sum, uccm0_count, uccm1_sum, uccm1_count,\n                        assign = False)\n    \n    gc.collect()\n    train = add_features(train, #answered_correctly_u_count, \n                         answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n                         #answered_correctly_q_count, \n                         answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq,\n                         uccm_sum, cucm_sum, user_part_sum, user_part_count, bundle_sum, bundle_count, ubcm_sum, timestamp_u_correct, #bucm_sum, \n                         cucm0_sum, cucm0_count, cucm1_sum, cucm1_count, \n                         uccm0_sum, uccm0_count, uccm1_sum, uccm1_count,\n                         assign = True)\n    gc.collect()\n    \n    \n    \n    \n    \n    \n    valid = add_features(valid, #answered_correctly_u_count, \n                         answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n                         #answered_correctly_q_count, \n                         answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq,\n                         uccm_sum, cucm_sum, user_part_sum, user_part_count, bundle_sum, bundle_count, ubcm_sum, timestamp_u_correct, #bucm_sum,\n                         cucm0_sum, cucm0_count, cucm1_sum, cucm1_count, \n                         uccm0_sum, uccm0_count, uccm1_sum, uccm1_count,\n                         assign = True)\n    \n    \n    \n    \n    \n    \n    \n    \n    gc.collect()\n    print('User feature calculation completed...')\n    print('\\n')\n    \n    features_dicts = {\n        #'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        #'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect,\n        ### +++++\n        'uccm_sum' : uccm_sum,\n        'cucm_sum' : cucm_sum,\n        'ups' : user_part_sum,\n        'upc' : user_part_count,\n        'bundle_sum' : bundle_sum,\n        'bundle_count' : bundle_count,\n        'ubcm_sum' : ubcm_sum,\n        'timestamp_u_correct': timestamp_u_correct,\n        #'bucm_sum' : bucm_sum,\n        'cucm0_sum': cucm0_sum,\n        'cucm0_count' : cucm0_count,\n        'cucm1_sum' : cucm1_sum,\n        'cucm1_count': cucm1_count,\n        'uccm0_sum': uccm0_sum,\n        'uccm0_count' : uccm0_count,\n        'uccm1_sum' : uccm1_sum,\n        'uccm1_count': uccm1_count,\n        \n    }\n    \n    return train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts\n\n# Function for training and evaluation\ndef train_and_evaluate(train, valid, feature_engineering = False):\n    \n    TARGET = 'answered_correctly'\n    # Features to train and predict\n    FEATURES = ['prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',\n                'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n                'timestamp_u_incorrect_recency',\n               ### +++++\n               'uccm', 'cucm', 'upm', 'ubcm', 'user_count', 'user_part_count', 'bm', 'tucr', #'bucm'\n              # 'task_container_id',\n                'cucm0', 'cucm1', 'uccm0', 'uccm1', 'bundle_id'\n               ]\n    \n    # Delete some training data to experiment faster\n    if feature_engineering:\n        train = train.sample(samplesize, random_state = SEED)\n        #train = train[-12000000:]\n    gc.collect()\n    print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n    drop_cols = list(set(train.columns) - set(FEATURES))\n    y_train = train[TARGET]\n    y_val = valid[TARGET]\n    # Drop unnecessary columns\n    train.drop(drop_cols, axis = 1, inplace = True)\n    valid.drop(drop_cols, axis = 1, inplace = True)\n    gc.collect()\n    \n    \n    train['bundle_id'] = train['bundle_id'].astype('category')\n    valid['bundle_id'] = valid['bundle_id'].astype('category')\n    \n    lgb_train = lgb.Dataset(train[FEATURES], y_train)\n    lgb_valid = lgb.Dataset(valid[FEATURES], y_val)\n    del train, y_train\n    gc.collect()\n    \n    params = {'objective': 'binary', \n              'seed': SEED,\n              'metric': 'auc',\n            #  'learning_rate': 0.03,\n              'num_leaves': 256,\n              'feature_fraction': 0.75,\n              'bagging_freq': 10,\n              'bagging_fraction': 0.80\n             }\n    \n    model = lgb.train(\n        params = params,\n        train_set = lgb_train,\n        num_boost_round = 1000,\n        valid_sets = [lgb_train, lgb_valid],\n        early_stopping_rounds = 10,\n        verbose_eval = 50\n    )\n    \n    print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n    \n    feature_importance = model.feature_importance()\n    feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n    \n    fig = plt.figure(figsize = (10, 10))\n    fig.suptitle('Feature Importance', fontsize = 20)\n    plt.tick_params(axis = 'x', labelsize = 12)\n    plt.tick_params(axis = 'y', labelsize = 12)\n    plt.xlabel('Importance', fontsize = 15)\n    plt.ylabel('Features', fontsize = 15)\n    sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')\n    \n    \n    lgb.plot_importance(model, importance_type = 'gain', figsize = (14, 10))\n    lgb.plot_importance(model, importance_type = 'split', figsize = (14, 10))\n \n    plt.show()\n    \n    \n    return TARGET, FEATURES, model\n\n# Using time series api that simulates production predictions\ndef inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):\n    \n    # Get feature dict\n    #answered_correctly_u_count = features_dicts['answered_correctly_u_count']\n    answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\n    elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\n    explanation_u_sum = features_dicts['explanation_u_sum']\n    #answered_correctly_q_count = features_dicts['answered_correctly_q_count']\n    answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\n    elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\n    explanation_q_sum = features_dicts['explanation_q_sum']\n    answered_correctly_uq = features_dicts['answered_correctly_uq']\n    timestamp_u = features_dicts['timestamp_u']\n    timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\n    ### ++++++++++\n    uccm_sum = features_dicts['uccm_sum']\n    ### ++++++++++\n    cucm_sum = features_dicts['cucm_sum']\n    ### ++++++++++\n    user_part_sum = features_dicts['ups']\n    user_part_count = features_dicts['upc']\n    ### +++++++++\n    bundle_sum = features_dicts['bundle_sum']\n    bundle_count = features_dicts['bundle_count']\n    ubcm_sum = features_dicts['ubcm_sum']\n    #bucm_sum = features_dicts['bucm_sum']\n    \n    timestamp_u_correct = features_dicts['timestamp_u_correct']\n    \n    ##\n    uccm0_sum = features_dicts['uccm0_sum']\n    uccm0_count = features_dicts['uccm0_count']\n    uccm1_sum = features_dicts['uccm1_sum']\n    uccm1_count = features_dicts['uccm1_count']\n    \n    cucm0_sum = features_dicts['cucm0_sum']\n    cucm0_count = features_dicts['cucm0_count']\n    cucm1_sum = features_dicts['cucm1_sum']\n    cucm1_count = features_dicts['cucm1_count']\n    \n    # Get api iterator and predictor\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict\n    \n    previous_test_df = None\n    for (test_df, sample_prediction_df) in iter_test:\n        if previous_test_df is not None:\n            previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n            update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect, user_part_sum, bundle_sum,\n                            cucm0_sum, cucm0_count, cucm1_sum, cucm1_count,\n                            uccm0_sum, uccm0_count, uccm1_sum, uccm1_count)\n        \n        ## \n        previous_test_df = pd.merge(previous_test_df, questions_df[['question_id', 'part', 'bundle_id']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n        \n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n        test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n        test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n        test_df = pd.merge(test_df, questions_df[['question_id', 'part', 'bundle_id']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n        test_df[TARGET] = 0\n        test_df = add_features(test_df, #answered_correctly_u_count, \n                               answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, \n                               #answered_correctly_q_count, \n                               answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq,\n                               uccm_sum, cucm_sum, user_part_sum, user_part_count, bundle_sum, bundle_count, ubcm_sum, timestamp_u_correct, #bucm_sum, \n                               cucm0_sum, cucm0_count, cucm1_sum, cucm1_count, \n                               uccm0_sum, uccm0_count, uccm1_sum, uccm1_count,\n                               assign = True, update = False)\n        test_df[TARGET] =  model.predict(test_df[FEATURES])\n        \n        previous_test_df = test_df.copy()\n        set_predict(test_df[['row_id', TARGET]])\n        \n    print('Job Done')\n    \ntrain, valid, questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = True)\n\n\ndel features_dicts\ngc.collect()\n\nTARGET, FEATURES, model = train_and_evaluate(train, valid, feature_engineering = True)\n\nmodel.save_model('lgb_model.txt', num_iteration=model.best_iteration)\n# model = lgb.Booster(model_file='lgb_model.txt')\n\n\n#inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}