{"cells":[{"metadata":{"_uuid":"e02bd0c7-5f0c-4090-9e5e-56b0c5832ac4","_cell_guid":"f63d7197-dc39-4bd6-9483-049e67e3cf12","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nimport gc\nimport riiideducation\nenv = riiideducation.make_env()\n\ndir_path = '/kaggle/input/riiid-test-answer-prediction/'\nfile_train = 'train.csv'\nfile_questions = 'questions.csv'\n\nnrows =  100 * 10000\n# nrows = None\n\ntrain = pd.read_csv(\n                    dir_path + file_train, \n                    nrows=nrows, \n                    usecols=['row_id', 'timestamp', 'user_id', 'content_id', \n                             'content_type_id', 'task_container_id', 'answered_correctly',\n                            'prior_question_elapsed_time','prior_question_had_explanation'],\n                    dtype={\n                            'row_id': 'int64',\n                            'timestamp': 'int64',\n                            'user_id': 'int32',\n                            'content_id': 'int16',\n                            'content_type_id': 'int8',\n                            'task_container_id': 'int8',\n                            'answered_correctly': 'int8',\n                            'prior_question_elapsed_time': 'float32',\n                            'prior_question_had_explanation': 'str'\n                        }\n                   )\n\nquestions = pd.read_csv(\n                        dir_path + file_questions, \n                        nrows=nrows,\n                        usecols=['question_id','bundle_id','part'], \n                        dtype={\n                           'question_id': 'int16',\n                           'bundle_id': 'int16',\n                           'part': 'int8',\n                       }\n                    )\n\n\ntrain['prior_question_had_explanation'] = train['prior_question_had_explanation'].map({'True':1,'False':0}).fillna(-1).astype(np.int8)\n\ntrain = train[train['content_type_id']==0]\n\ngc.collect()\n\n# 压缩内存\nmax_num = 100\ntrain = train.groupby(['user_id']).tail(max_num)\n\ntrain = pd.merge(\n        left=train,\n        right=questions,\n        how='left',\n        left_on='content_id',\n        right_on='question_id'\n        )\n\ntrain = train.fillna(0)\n\nclass cat_deal:\n    def __init__(self):\n        self.max_len = 0\n        self.dict_map = {}\n    \n    def fit(self, cat_list):\n        index = 1 \n        for cat_i in cat_list:\n            if cat_i not in self.dict_map:\n                self.dict_map[cat_i] = index\n                index += 1\n        self.max_len = index + 1\n        \n    def transform(self, cat_list):\n        cat_transform_list = []\n        for cat_i in cat_list:\n            if cat_i in self.dict_map:\n                cat_transform_list.append(self.dict_map[cat_i])\n            else:\n                cat_transform_list.append(0)\n        return cat_transform_list\n\nclass float_deal:\n    def __init__(self):\n        self.max = 0\n        self.min = 0\n        self.max_min = 0 \n        \n    def fit(self, float_list):\n        for float_i in float_list:\n            if float_i < self.min:\n                self.min = float_i\n            if float_i > self.max:\n                self.max = float_i\n        self.max_min = self.max - self.min\n        \n    def transform(self, float_list):\n        float_transform_list = []\n        for float_i in float_list:\n            if float_i < self.min:\n                float_transform_list.append(0)\n            elif float_i > self.max:\n                float_transform_list.append(1)\n            else:\n                float_transform_list.append(float_i/self.max_min)\n        return float_transform_list\n\ndict_cat_class = {}\nfor columns in ['user_id','content_id',\\\n                'task_container_id','prior_question_had_explanation',\\\n                'bundle_id','part']:\n    dict_cat_class[columns] = cat_deal()\n    dict_cat_class[columns].fit(train[columns])\n\n    train[columns] = dict_cat_class[columns].transform(train[columns])\n    print(columns)\n\n\ndict_float_class = {}\nfor columns in ['timestamp','prior_question_elapsed_time']:\n    dict_float_class[columns] = float_deal()\n    dict_float_class[columns].fit(train[columns])\n    \n    train[columns] = dict_float_class[columns].transform(train[columns])\n    print(columns)\n\ndef squeeze(embedding):\n    embedding = tf.squeeze(embedding,axis=1)\n    return embedding\ndef concat(embedding_list):\n    embedding = tf.concat(embedding_list, axis=1)\n    return embedding\ndef multiply(multi_x_y):\n    multi_x = multi_x_y[0]\n    multi_y = multi_x_y[1]\n    multi_x_y = tf.multiply(multi_x, multi_y)\n    return multi_x_y\n\n# 模型\ninput_timestamp = tf.keras.Input(shape=(1,))\ninput_prior_question_elapsed_time = tf.keras.Input(shape=(1,))\n\n# input int\ninput_user = tf.keras.Input(shape=(1,))\ninput_content = tf.keras.Input(shape=(1,))\ninput_task_container = tf.keras.Input(shape=(1,))\ninput_prior_question_had_explanation = tf.keras.Input(shape=(1,))\ninput_bundle = tf.keras.Input(shape=(1,))\ninput_part = tf.keras.Input(shape=(1,))\n\ninputs = [input_timestamp,input_prior_question_elapsed_time,\\\n         input_user,input_content,\\\n         input_task_container,input_prior_question_had_explanation,\\\n         input_bundle,input_part]\n# inputs = tf.keras.layers.Lambda(concat)(inputs)\n\n# input session\n# input_tags = Input(shape=(1))\n\n# embedding float\nembedding_timestamp = tf.keras.layers.Dense(64, activation=tf.nn.sigmoid)(input_timestamp)\nembedding_prior_question_elapsed_time = tf.keras.layers.Dense(64, activation=tf.nn.sigmoid)(input_prior_question_elapsed_time)\n\n# embedding int \nembedding_user = tf.keras.layers.Embedding(dict_cat_class['user_id'].max_len,\n                                           64, input_length=1)(input_user)\nembedding_user = tf.keras.layers.Lambda(squeeze)(embedding_user)\n\nembedding_content = tf.keras.layers.Embedding(dict_cat_class['content_id'].max_len,\n                                              64, input_length=1)(input_content)\nembedding_content = tf.keras.layers.Lambda(squeeze)(embedding_content)\n\nembedding_task_container = tf.keras.layers.Embedding(dict_cat_class['task_container_id'].max_len,\n                                                     64, input_length=1)(input_task_container)\nembedding_task_container = tf.keras.layers.Lambda(squeeze)(embedding_task_container)\n\nembedding_prior_question_had_explanation = tf.keras.layers.Embedding(dict_cat_class['prior_question_had_explanation'].max_len, \n                                                                     64, input_length=1)(input_prior_question_had_explanation)\nembedding_prior_question_had_explanation = tf.keras.layers.Lambda(squeeze)(embedding_prior_question_had_explanation)\n\nembedding_bundle = tf.keras.layers.Embedding(dict_cat_class['bundle_id'].max_len,\n                                             64, input_length=1)(input_bundle)\nembedding_bundle = tf.keras.layers.Lambda(squeeze)(embedding_bundle)\n\nembedding_part = tf.keras.layers.Embedding(dict_cat_class['part'].max_len,\n                                           64, input_length=1)(input_part)\nembedding_part = tf.keras.layers.Lambda(squeeze)(embedding_part)\n\nembedding_all = [embedding_timestamp,embedding_prior_question_elapsed_time,\\\n                embedding_user, embedding_content, embedding_task_container,\\\n                embedding_prior_question_had_explanation, embedding_bundle, embedding_part]\n\n\nnffm1, nffm2 = [], []\nfor i, embedding_i in enumerate(embedding_all):\n    for j, embedding_j in enumerate(embedding_all):\n        if i > j:\n            nffm1.append(embedding_i), nffm2.append(embedding_j)\nnffm1_layer = tf.keras.layers.Lambda(concat)(nffm1)\nnffm2_layer = tf.keras.layers.Lambda(concat)(nffm2)     \n\nnffm_all = tf.keras.layers.Lambda(multiply)([nffm1_layer,nffm2_layer])\n    \nlogit = tf.keras.layers.Dense(1, activation=tf.nn.sigmoid)(nffm_all)\n\n\nmodel = tf.keras.models.Model(inputs=inputs, outputs=logit)\n\n# 编译模型 设置参数\nmodel.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['binary_crossentropy'])\n\nplateau = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',\n                            verbose=0,\n                            mode='min',\n                            factor=0.1,\n                            patience=6)\n\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n                               verbose=0,\n                               mode='min',\n                               patience=10)\n\n# 保存\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(f'fold.h5',\n                             monitor='val_loss',\n                             verbose=0,\n                             mode='min',\n                             save_best_only=True)\n\n\n# 训练 验证\nvalid = pd.DataFrame()\nfor i in range(6):\n    \n    # 获取训练标签数据\n    last_records = train.drop_duplicates('user_id', keep='last')\n    \n    # 获取训练标签以前的数据\n    map__last_records__user_row = dict(zip(last_records['user_id'],last_records['row_id']))\n    train['filter_row'] = train['user_id'].map(map__last_records__user_row)\n    train = train[train['row_id']<train['filter_row']]\n\n    # 特征加入训练集\n    valid = valid.append(last_records)\n    print(len(valid))\n\n\nfeatures_columns = ['timestamp','prior_question_elapsed_time',\\\n                    'user_id','content_id',\\\n                    'task_container_id','prior_question_had_explanation',\\\n                    'bundle_id','part']\n\nX_valid, y_valid = [valid[columns].values for columns in features_columns], valid['answered_correctly'].values\ndel valid\n\nX_train, y_train = [train[columns].values for columns in features_columns], train['answered_correctly'].values\ndel train\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\nmodel.fit(X_train, y_train,\n          epochs=1,\n          batch_size=512 * 500 * 2,\n          verbose=1,\n          shuffle=True,\n          validation_data=(X_valid, y_valid),\n          callbacks=[plateau, early_stopping, checkpoint])\n\ny_valid_proba = model.predict(X_valid, verbose=0, batch_size=512)\nauc = roc_auc_score(y_valid, y_valid_proba)\nprint(auc)\n\n\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].map({'True':1,'False':0}).fillna(-1).astype(np.int8)\n    test_df = pd.merge(\n        left=test_df,\n        right=questions,\n        how='left',\n        left_on='content_id',\n        right_on='question_id'\n        )\n    test_df = test_df.fillna(0)\n    \n    for columns in ['user_id','content_id','task_container_id','prior_question_had_explanation','bundle_id','part']:\n        test_df[columns] = dict_cat_class[columns].transform(test_df[columns])\n        print(columns)\n    \n    for columns in ['timestamp','prior_question_elapsed_time']:\n        test_df[columns] = dict_float_class[columns].transform(test_df[columns])\n        print(columns)\n        \n    X_test = [test_df[columns].values for columns in features_columns]\n    \n    test_df['answered_correctly'] =  model.predict(X_test, verbose=0, batch_size=512)\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}