{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is heavily based on\n\n- https://www.kaggle.com/ulrich07/riiid-keras-starter\n\nPlease upvote this notebook too.\n\nHere I added logistic regression to get better insight into features and diversity to the ensemble."},{"metadata":{},"cell_type":"markdown","source":"# Libaries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# useful\nimport os\nimport random as rn\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\nfrom typing import List, NoReturn, Union, Tuple, Optional, Text, Generic, Callable, Dict\nfrom logging import getLogger, Formatter, StreamHandler, FileHandler, INFO\n\n# neural nets\nfrom sklearn import linear_model\nfrom tensorflow.keras.layers import (Dropout, BatchNormalization, Flatten, Convolution1D, Activation, Input, Dense, GaussianNoise, Lambda, Bidirectional,\n                                     Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,\n                                     RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)\nfrom tensorflow.keras.layers import Reshape, Concatenate, Layer\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, Callback, ReduceLROnPlateau, LearningRateScheduler\nfrom tensorflow.keras.losses import binary_crossentropy, categorical_crossentropy, mean_squared_error\nfrom tensorflow.keras.optimizers import Adam, RMSprop, SGD\nfrom tensorflow.keras.utils import Sequence, to_categorical\nfrom tensorflow.keras import losses, models, optimizers\nfrom tensorflow.keras import backend as K\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\n# visualize\nimport matplotlib.pyplot as plt\nimport matplotlib.style as style\nimport seaborn as sns\nfrom matplotlib import pyplot\nfrom matplotlib.ticker import ScalarFormatter\nsns.set_context(\"talk\")\nstyle.use('fivethirtyeight')\n\n# custom\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nLR = 0.008\nBATCH_SIZE = 24000\nWEIGHTS = [0.9, 0.1]\nEPOCHS = 8\nVAL_SIZE = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed : int) -> NoReturn :\n    \n    rn.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data organization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# PIVOT DATAFRAMES\npiv1 = pd.read_csv(\"../input/riiid-fixed-infos/content.csv\")\npiv2 = pd.read_csv(\"../input/riiid-fixed-infos/task.csv\")\npiv3 = pd.read_csv(\"../input/riiid-fixed-infos/user.csv\")\n\nfor col, df in zip([\"content_sum\", \"task_container_sum\", \"user_sum\"], [piv1, piv2, piv3]):\n    df[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min())\n#\nm1 = piv1[\"content_sum\"].median()\nm2 = piv2[\"task_container_sum\"].median()\nm3 = piv3[\"user_sum\"].median()\n\n\n# OTHER CONSTABTS\nTARGET = \"answered_correctly\"\nTIME_MEAN = 21000.0\nTIME_MIN = 0.0\nTIME_MAX = 300000.0\nmap_prior = {True:1, False:0}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df):\n    df = df.merge(piv1, how=\"left\", on=\"content_id\")\n    df[\"content_emb\"] = df[\"content_emb\"].fillna(0.5)\n    df[\"content_sum\"] = df[\"content_sum\"].fillna(m1)\n    df = df.merge(piv2, how=\"left\", on=\"task_container_id\")\n    df[\"task_container_emb\"] = df[\"task_container_emb\"].fillna(0.5)\n    df[\"task_container_sum\"] = df[\"task_container_sum\"].fillna(m2)\n    df = df.merge(piv3, how=\"left\", on=\"user_id\")\n    df[\"user_emb\"] = df[\"user_emb\"].fillna(0.5)\n    df[\"user_sum\"] = df[\"user_sum\"].fillna(m3)\n    df[\"prior_question_elapsed_time\"] = df[\"prior_question_elapsed_time\"].fillna(TIME_MEAN)\n    df[\"duration\"] = (df[\"prior_question_elapsed_time\"] - TIME_MIN) / (TIME_MAX - TIME_MIN)\n    df[\"prior_answer\"] = df[\"prior_question_had_explanation\"].map(map_prior)\n    df[\"prior_answer\"] = df[\"prior_answer\"].fillna(0.5)\n    #df = df.fillna(-1)\n    epsilon = 1e-6\n    df[\"score\"] = 2*df[\"content_emb\"]*df[\"user_emb\"] / (df[\"content_emb\"]+ df[\"user_emb\"] + epsilon)\n    return df\n#=========","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntr = pd.read_csv(\"../input/riiid-test-answer-prediction/train.csv\", \n                 low_memory=False, nrows=10**7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntr = preprocess(tr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FE = [\"content_emb\",\"content_sum\" ,\"task_container_emb\", \"task_container_sum\",\n      \"user_emb\", \"user_sum\",\"duration\", \"prior_answer\",\"score\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = tr.loc[tr.answered_correctly!=-1, FE].values\ny = tr.loc[tr.answered_correctly!=-1, TARGET].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fit Neural Net"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_ann(n_in):\n    inp = Input(shape=(n_in,), name=\"inp\")\n    x = Dense(512, activation='relu')(inp)\n    x = Dropout(0.08)(x)\n    x = GaussianNoise(0.01)(x)\n    x = BatchNormalization()(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(0.08)(x)\n    x = GaussianNoise(0.01)(x)\n    x = BatchNormalization()(x)\n    x = Dense(128, activation='relu')(x)\n    x = Dropout(0.08)(x)\n    x = GaussianNoise(0.01)(x)\n    x = BatchNormalization()(x)\n    preds = Dense(1, activation=\"sigmoid\", name = \"out\")(x)\n    model = models.Model(inp, preds)\n    \n    opt = tfa.optimizers.RectifiedAdam(lr=LR)\n    opt = tfa.optimizers.SWA(opt)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    model.compile(loss=loss, optimizer=opt, metrics=[\"accuracy\"])\n    return model\n#===================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = make_ann(x.shape[1])\nprint(net.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(SEED)\nK.clear_session()\nhistory = net.fit(x, y, validation_split=VAL_SIZE, batch_size=BATCH_SIZE, epochs=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot training & validation loss values\ndef plot_history(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('Model loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Val'], loc='upper right', frameon=False)\n    plt.show()\n    \nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Logistic regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\n        \"C\": 8.0, \n        \"solver\": \"lbfgs\", \n        \"warm_start\": False,\n        \"max_iter\": 8000,\n        \"fit_intercept\": True,\n        \"random_state\": SEED,\n        \"tol\": 1e-04,\n        \"n_jobs\": -1, \n        \"verbose\": 1, \n}\nlin_model = linear_model.LogisticRegression(**params)\nlin_model.fit(x, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fi = pd.DataFrame()\nfi['features'] = FE\nfi['linear_weights'] = lin_model.coef_.ravel()\n\nsns.barplot(x='linear_weights', y='features', data=fi.sort_values(by='linear_weights', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#it = 0\nfor test_df, sample_prediction_df in iter_test:\n    #it += 1\n    #if it % 100 == 0:\n    #    print(it)\n    test_df = preprocess(test_df)\n    x_te = test_df[FE].values\n    nn_pred = net.predict(x_te, batch_size=BATCH_SIZE, verbose=0)[:, 0]\n    lin_pred = lin_model.predict(x_te)\n    test_df['answered_correctly'] = WEIGHTS[0] * nn_pred + WEIGHTS[1] * lin_pred\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n#=================================================\n#print(it)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}