{"cells":[{"metadata":{"id":"zWFPwwQiJHkK","executionInfo":{"status":"ok","timestamp":1609834438893,"user_tz":-480,"elapsed":3557,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"import psutil\nimport joblib\nimport random\nfrom tqdm import tqdm\n\nimport numpy as np\nimport gc\nimport pandas as pd\nimport time\n\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"id":"XCN8rj_IXkk5","executionInfo":{"status":"ok","timestamp":1609834443410,"user_tz":-480,"elapsed":756,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"def feature_time_lag(df, time_dict):\n\n    tt = np.zeros(len(df), dtype=np.int64)\n\n    for ind, row in enumerate(df[['user_id','timestamp','task_container_id']].values):\n\n        if row[0] in time_dict.keys():\n            if row[2]-time_dict[row[0]][1] == 0:\n\n                tt[ind] = time_dict[row[0]][2]\n\n            else:\n                t_last = time_dict[row[0]][0]\n                task_ind_last = time_dict[row[0]][1]\n                tt[ind] = row[1]-t_last\n                time_dict[row[0]] = (row[1], row[2], tt[ind])\n        else:\n            # time_dict : timestamp, task_container_id, lag_time\n            time_dict[row[0]] = (row[1], row[2], -1)\n            tt[ind] =  0\n\n    df[\"time_lag\"] = tt\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"id":"VAPKPK6VdjpJ","executionInfo":{"status":"ok","timestamp":1609834443698,"user_tz":-480,"elapsed":727,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"2bd66398-ba10-4c52-b797-a0d8f7d677b3","trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"id":"2pX27onNJijU","executionInfo":{"status":"ok","timestamp":1609834448569,"user_tz":-480,"elapsed":946,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"MAX_SEQ = 100\nD_MODEL = 256\nN_LAYER = 2\nBATCH_SIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"id":"fGs5lsr5JlKp","executionInfo":{"status":"ok","timestamp":1609834498260,"user_tz":-480,"elapsed":48300,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"6a4bed8d-048f-47c1-93e0-251217d35006","trusted":true},"cell_type":"code","source":"%%time\nimport pickle\nwith open(\"../input/saint-plus-data-new/train_df.pkl\",\"rb\") as f:\n    train_df = pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{"id":"UPByaJr85LHD","executionInfo":{"status":"ok","timestamp":1609834501015,"user_tz":-480,"elapsed":50270,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"question = pd.read_csv(\"../input/riiid-test-answer-prediction/questions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"id":"ts3Ocm18lD53","executionInfo":{"status":"ok","timestamp":1609834501016,"user_tz":-480,"elapsed":47281,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"85f16d70-11de-48eb-f2a4-c250c7005da0","trusted":true},"cell_type":"code","source":"question.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"P9ZWJoatpK5U","executionInfo":{"status":"ok","timestamp":1609834501017,"user_tz":-480,"elapsed":46863,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"9f628e7b-ef82-432d-feb9-5f60ad616f6a","trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"qXx52EDcXu0j","executionInfo":{"status":"ok","timestamp":1609834501018,"user_tz":-480,"elapsed":44566,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"%%time\ntime_dict = dict()\ntrain_df = feature_time_lag(train_df, time_dict)\n# del time_dict","execution_count":null,"outputs":[]},{"metadata":{"id":"jt4EKKwOZXPc","executionInfo":{"status":"ok","timestamp":1609834501019,"user_tz":-480,"elapsed":43933,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"45ae6824-4814-42eb-e90c-bfbbfd91aa6e","trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"O6_5CnyCK25o","executionInfo":{"status":"ok","timestamp":1609834505446,"user_tz":-480,"elapsed":39173,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"train_df = train_df[[\"timestamp\",\"user_id\",\"content_id\",\"content_type_id\",\"answered_correctly\",\"prior_question_elapsed_time\",\"prior_question_had_explanation\",\"time_lag\"]]","execution_count":null,"outputs":[]},{"metadata":{"id":"-p0yLPjjeuW3","executionInfo":{"status":"ok","timestamp":1609834511243,"user_tz":-480,"elapsed":39511,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df.content_type_id == 0]\n# train_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"BzsoCI73pnVx","executionInfo":{"status":"ok","timestamp":1609834512074,"user_tz":-480,"elapsed":35971,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"train_df.prior_question_elapsed_time = train_df.prior_question_elapsed_time.fillna(0)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].fillna(value = False).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"id":"AcLu2a135PNu","executionInfo":{"status":"ok","timestamp":1609834545419,"user_tz":-480,"elapsed":33334,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(question[[\"question_id\",\"part\", \"bundle_id\"]], how = \"left\", left_on = 'content_id', right_on = 'question_id')","execution_count":null,"outputs":[]},{"metadata":{"id":"M1oKKF6r_cPi","executionInfo":{"status":"ok","timestamp":1609834546566,"user_tz":-480,"elapsed":34476,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"b7a131af-0b0d-4c99-9234-25f8efefe403","trusted":true},"cell_type":"code","source":"len(train_df.content_id.unique())","execution_count":null,"outputs":[]},{"metadata":{"id":"h45BXfZ2QH_B","executionInfo":{"status":"ok","timestamp":1609834546999,"user_tz":-480,"elapsed":34902,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"54f6c987-8969-4fea-f5e8-7dd3edbba471","trusted":true},"cell_type":"code","source":"train_df.prior_question_had_explanation.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"id":"SdLO_x64YkP0","trusted":true},"cell_type":"code","source":"## drop outlier\n# temp_df = train_df.groupby(\"user_id\").count().reset_index()\n# outlier = temp_df[(temp_df.timestamp>9000)].user_id.tolist()\n# train_df = train_df[~train_df.user_id.isin(outlier)]","execution_count":null,"outputs":[]},{"metadata":{"id":"SCPAPhZyagsW","executionInfo":{"status":"ok","timestamp":1609834563768,"user_tz":-480,"elapsed":1884,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"0bf26fb0-2bb0-4163-a2a9-b32d347ac32d","trusted":true},"cell_type":"code","source":"skills = train_df[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))","execution_count":null,"outputs":[]},{"metadata":{"id":"Hk1E3toaOdz_","executionInfo":{"status":"ok","timestamp":1609834564213,"user_tz":-480,"elapsed":874,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"n_part = len(train_df[\"part\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"id":"51hlsDF8OoMP","executionInfo":{"status":"ok","timestamp":1609834564477,"user_tz":-480,"elapsed":595,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"3381b18e-791f-4c94-f124-d282cb252427","trusted":true},"cell_type":"code","source":"n_part","execution_count":null,"outputs":[]},{"metadata":{"id":"nPywXjBQ6GG7","executionInfo":{"status":"ok","timestamp":1609834569084,"user_tz":-480,"elapsed":4641,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"133f7390-d39b-49a6-c1b0-faf9a23d90a5","trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"id":"2uVUt61bKosK","executionInfo":{"status":"ok","timestamp":1609834798796,"user_tz":-480,"elapsed":111995,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"trusted":true},"cell_type":"code","source":"train_group = train_df[['user_id', 'content_id', 'answered_correctly', 'part', 'prior_question_elapsed_time', 'time_lag', 'prior_question_had_explanation', 'bundle_id']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values[-MAX_SEQ:],\n            r['answered_correctly'].values[-MAX_SEQ:],\n            r['part'].values[-MAX_SEQ:],\n            r['prior_question_elapsed_time'].values[-MAX_SEQ:],\n            r['time_lag'].values[-MAX_SEQ:],\n            r['prior_question_had_explanation'].values[-MAX_SEQ:],\n            r['bundle_id'].values[-MAX_SEQ:]))\n","execution_count":null,"outputs":[]},{"metadata":{"id":"5gjQa8jlADTT","trusted":true},"cell_type":"code","source":"# del train_df\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"lHK0KxoBDpWL","executionInfo":{"status":"ok","timestamp":1609834922743,"user_tz":-480,"elapsed":60060,"user":{"displayName":"林得恩","photoUrl":"","userId":"01685505290946491737"}},"outputId":"40735707-ded0-41fb-ad57-e62558be2b5d","trusted":true},"cell_type":"code","source":"joblib.dump(train_group,\"./group.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"id":"ylYcYMJOk7DU","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}