{"cells":[{"metadata":{},"cell_type":"markdown","source":"* The idea is to train the network on the full dataset by progressively keeping the best performing weights as we read the consecutive chunks. At the end we bag with all the weights in the inference part.\n\n* The inference notebook is [here](https://www.kaggle.com/ulrich07/riiid-keras-nnet-on-full-dataset-inference)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# useful\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport gc\n\n# neural nets\nimport tensorflow as tf\nimport tensorflow.keras.models as M\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n\n# custom\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CONSTANTS"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# PIVOT DATAFRAMES\npiv1 = pd.read_csv(\"../input/riiid-fixed-infos/content.csv\")\npiv2 = pd.read_csv(\"../input/riiid-fixed-infos/task.csv\")\npiv3 = pd.read_csv(\"../input/riiid-fixed-infos/user.csv\")\n\nfor col, df in zip([\"content_sum\", \"task_container_sum\", \"user_sum\"], [piv1, piv2, piv3]):\n    df[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min())\n#\nm1 = piv1[\"content_sum\"].median()\nm2 = piv2[\"task_container_sum\"].median()\nm3 = piv3[\"user_sum\"].median()\n\n\n# OTHER CONSTABTS\nTARGET = \"answered_correctly\"\nTIME_MEAN = 21000.0\nTIME_MIN = 0.0\nTIME_MAX = 300000.0\nmap_prior = {True:1, False:0}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df):\n    df = df.merge(piv1, how=\"left\", on=\"content_id\")\n    df[\"content_emb\"] = df[\"content_emb\"].fillna(0.5)\n    df[\"content_sum\"] = df[\"content_sum\"].fillna(m1)\n    \n    df = df.merge(piv2, how=\"left\", on=\"task_container_id\")\n    df[\"task_container_emb\"] = df[\"task_container_emb\"].fillna(0.5)\n    df[\"task_container_sum\"] = df[\"task_container_sum\"].fillna(m2)\n    \n    df = df.merge(piv3, how=\"left\", on=\"user_id\")\n    df[\"user_emb\"] = df[\"user_emb\"].fillna(0.5)\n    df[\"user_sum\"] = df[\"user_sum\"].fillna(m3)\n    \n    df[\"prior_question_elapsed_time\"] = df[\"prior_question_elapsed_time\"].fillna(TIME_MEAN)\n    df[\"duration\"] = (df[\"prior_question_elapsed_time\"] - TIME_MIN) / (TIME_MAX - TIME_MIN)\n    df[\"prior_answer\"] = df[\"prior_question_had_explanation\"].map(map_prior)\n    df[\"prior_answer\"] = df[\"prior_answer\"].fillna(0.5)\n    #df = df.fillna(-1)\n    epsilon = 1e-6\n    df[\"score\"] = 2*df[\"content_emb\"]*df[\"user_emb\"] / (df[\"content_emb\"]+ df[\"user_emb\"] + epsilon)\n    return df\n#=========","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_ann(n_in):\n    inp = L.Input(shape=(n_in,), name=\"inp\")\n    d1 = L.Dense(100, activation=\"relu\", name=\"d1\")(inp)\n    d2 = L.Dense(100, activation=\"relu\", name=\"d2\")(d1)\n    preds = L.Dense(1, activation=\"sigmoid\", name=\"preds\")(d2)\n    \n    model = M.Model(inp, preds, name=\"ANN\")\n    model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n    return model\n#===================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FE = [\"content_emb\",\"content_sum\" ,\"task_container_emb\", \"task_container_sum\",\n      \"user_emb\", \"user_sum\",\"duration\", \"prior_answer\",\"score\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TRAINING"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = make_ann(len(FE))\nEPOCHS = 20\nSCORES = []\nfor it in range(6):\n    print(f\"============FOLD {it}========\")\n    print(\"Loading dataset\")\n    df = pd.read_csv(f\"../input/riiiid-folds-data/FOLD{it}.csv\")\n    print(\"loaded\")\n    print(\"Sorting...\")\n    df = df[df.content_type_id==False].sort_values('timestamp', ascending=True).reset_index(drop = True)\n    print(\"Processing...\")\n    df = preprocess(df)\n    print(\"Processed\")\n    \n    ckpt = ModelCheckpoint(f\"w{it}.h5\", monitor='val_loss', verbose=1, save_best_only=True,mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_lr=0.001)\n    es = EarlyStopping(monitor='val_loss', patience=10)\n    \n    val = df.groupby(\"user_id\").tail(5)\n    df = df[~df.index.isin(val.index)]\n    \n    tr = df.groupby(\"user_id\").tail(18)\n    df = df[~df.index.isin(tr.index)]\n    \n    x_tr = tr[ FE].values\n    y_tr = tr[TARGET].values\n\n    x_val = val[ FE].values\n    y_val = val[TARGET].values\n    \n    net.fit(x_tr, y_tr, validation_data=(x_val, y_val), batch_size=30_000, epochs=EPOCHS, \n            callbacks=[ckpt, es, reduce_lr])\n    print(\"Evaluating...\")\n    p_val = net.predict(x_val, batch_size=30_000, verbose=1)[:, 0]\n    score = roc_auc_score(y_val, p_val)\n    print(f\"Val Score: {score}\")\n    SCORES.append(score)\n    del x_tr, y_tr, x_val, y_val, p_val\n    gc.collect()\n#=========","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for it, score in enumerate(SCORES):\n    print(f\"CHUNK {it+1}: {np.round(score, 4)}\")\n#========#","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## PREDICT"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nenv = riiideducation.make_env()\niter_test = env.iter_test()\n\nfor test_df, sample_prediction_df in iter_test:\n    test_df = preprocess(test_df)\n    x_te = test_df[FE].values\n    test_df['answered_correctly'] = net.predict(x_te, batch_size=50_000, verbose=0)[:, 0]\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n\"\"\"\n#=================================================\nprint(\"TRAINING NOTEBOOK\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}