{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport time\nimport pickle\nimport gc\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nprint(tf.__version__)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_pickle(\"../input/riiid-cross-validation-files/cv1_train.pickle\")\nval = pd.read_pickle(\"../input/riiid-cross-validation-files/cv1_valid.pickle\")\n\ntrain.drop([\"max_time_stamp\", \"rand_time_stamp\", \"row_id\"], axis=1, inplace=True)\nval.drop([\"max_time_stamp\", \"rand_time_stamp\", \"row_id\"], axis=1, inplace=True)\ntrain_df = pd.concat([train, val], ignore_index=True)\ntrain_df = train_df.sort_values(by=[\"viretual_time_stamp\"])\ntrain_df.drop(\"viretual_time_stamp\", axis=1, inplace=True)\ndel train, val\ngc.collect()\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_time_lag(df):\n    \"\"\"\n    Compute time_lag feature, same task_container_id shared same timestamp for each user\n    \"\"\"\n    time_dict = {}\n    time_lag = np.zeros(len(df), dtype=np.float32)\n    for idx, row in enumerate(df[[\"user_id\", \"timestamp\", \"task_container_id\"]].values):\n        if row[0] not in time_dict:\n            time_lag[idx] = 0\n            time_dict[row[0]] = [row[1], row[2], 0] # last_timestamp, last_task_container_id, last_lagtime\n        else:\n            if row[2] == time_dict[row[0]][1]:\n                time_lag[idx] = time_dict[row[0]][2]\n            else:\n                time_lag[idx] = row[1] - time_dict[row[0]][0]\n                time_dict[row[0]][0] = row[1]\n                time_dict[row[0]][1] = row[2]\n                time_dict[row[0]][2] = time_lag[idx]\n\n    df[\"time_lag\"] = time_lag/1000/60 # convert to miniute\n    df[\"time_lag\"] = df[\"time_lag\"].clip(0, 1440) # clip to 1440 miniute which is one day\n    return time_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pre_process(train_df, ques_path, row_start=30e6, num_rows=50e6, split_ratio=0.9, seq_len=100):\n    print(\"Start pre-process\")\n    t_s = time.time()\n\n    Features = [\"timestamp\", \"user_id\", \"content_id\", \"content_type_id\", \"task_container_id\", \"user_answer\", \n                \"answered_correctly\", \"prior_question_elapsed_time\", \"prior_question_had_explanation\"]\n    train_df.index = train_df.index.astype('uint32')\n\n    # shift prior elapsed_time and had_explanation to make current elapsed_time and had_explanation\n    train_df = train_df[train_df.content_type_id == 0].reset_index()\n    train_df[\"prior_question_elapsed_time\"].fillna(0, inplace=True)\n    train_df[\"prior_question_elapsed_time\"] /= 1000 # convert to sec\n    train_df[\"prior_question_elapsed_time\"] = train_df[\"prior_question_elapsed_time\"].clip(0, 300)\n    train_df[\"prior_question_had_explanation\"].fillna(False, inplace=True)\n    train_df[\"prior_question_had_explanation\"] = train_df[\"prior_question_had_explanation\"].astype('int8')\n\n    print(\"Start merge dataframe\")\n    # merge with question dataframe to get part feature\n    ques_df = pd.read_csv(ques_path)[[\"question_id\", \"part\"]]\n    train_df = train_df.merge(ques_df, how='left', left_on='content_id', right_on='question_id')\n    train_df.drop([\"question_id\"], axis=1, inplace=True)\n    train_df[\"part\"] = train_df[\"part\"].astype('uint8')\n    print(train_df.head(10))\n    print(\"Complete merge dataframe\")\n    print(\"====================\")\n    \n    # get time_lag feature\n    print(\"Start compute time_lag\")\n    time_dict = get_time_lag(train_df)\n    with open(\"time_dict.pkl.zip\", 'wb') as pick:\n        pickle.dump(time_dict, pick)\n    print(\"Complete compute time_lag\")\n    print(\"====================\")\n    train_df.drop(\"timestamp\", axis=1, inplace=True)\n    \n    # plus 1 for cat feature which starts from 0\n    train_df[\"content_id\"] += 1\n    train_df[\"task_container_id\"] += 1\n    train_df[\"answered_correctly\"] += 1\n    train_df[\"prior_question_had_explanation\"] += 1\n    train_df[\"user_answer\"] += 1\n\n    Train_features = [\"user_id\", \"content_id\", \"part\", \"task_container_id\", \"time_lag\", \"prior_question_elapsed_time\",\n                      \"answered_correctly\", \"prior_question_had_explanation\", \"user_answer\"]\n\n    if num_rows == -1:\n        num_rows = train_df.shape[0]\n    \n    print(\"Start Inference group\")\n    infer_groups = train_df[Train_features].groupby(\"user_id\").apply(lambda df: (\n        df[\"content_id\"].values[-seq_len:],\n        df[\"part\"].values[-seq_len:],\n        df[\"task_container_id\"].values[-seq_len:],\n        df[\"time_lag\"].values[-seq_len:],\n        df[\"prior_question_elapsed_time\"].values[-seq_len:],\n        df[\"answered_correctly\"].values[-seq_len:],\n        df[\"prior_question_had_explanation\"].values[-seq_len:],\n        df[\"user_answer\"].values[-seq_len:]\n    ))\n    with open(\"infer_groups.pkl.zip\", 'wb') as pick:\n        pickle.dump(infer_groups, pick)\n    del infer_groups\n    \n    train_df = train_df.iloc[int(row_start):int(row_start+num_rows)]\n    val_df = train_df[int(num_rows*split_ratio):]\n    train_df = train_df[:int(num_rows*split_ratio)]\n\n    print(\"Train dataframe shape after process ({}, {})/ Val dataframe shape after process({}, {})\".format(train_df.shape[0], train_df.shape[1], val_df.shape[0], val_df.shape[1]))\n    print(\"====================\")\n\n    # Check data balance\n    num_new_user = val_df[~val_df[\"user_id\"].isin(train_df[\"user_id\"])][\"user_id\"].nunique()\n    num_new_content = val_df[~val_df[\"content_id\"].isin(train_df[\"content_id\"])][\"content_id\"].nunique()\n    train_content_id = train_df[\"content_id\"].nunique()\n    train_part = train_df[\"part\"].nunique()\n    train_correct = train_df[\"answered_correctly\"].mean()-1\n    val_correct = val_df[\"answered_correctly\"].mean()-1\n    print(\"Number of new users {}/ Number of new contents {}\".format(num_new_user, num_new_content))\n    print(\"Number of content_id {}/ Number of part {}\".format(train_content_id, train_part))\n    print(\"train correctness {:.3f}/val correctness {:.3f}\".format(train_correct, val_correct))\n    print(\"====================\")\n\n    print(\"Start train and Val grouping\")\n    train_group = train_df[Train_features].groupby(\"user_id\").apply(lambda df: (\n        df[\"content_id\"].values,\n        df[\"part\"].values,\n        df[\"task_container_id\"].values,\n        df[\"time_lag\"].values,\n        df[\"prior_question_elapsed_time\"].values,\n        df[\"answered_correctly\"].values,\n        df[\"prior_question_had_explanation\"].values,\n        df[\"user_answer\"].values,\n    ))\n    with open(\"train_group.pkl.zip\", 'wb') as pick:\n        pickle.dump(train_group, pick)\n    del train_group, train_df\n\n    val_group = val_df[Train_features].groupby(\"user_id\").apply(lambda df: (\n        df[\"content_id\"].values,\n        df[\"part\"].values,\n        df[\"task_container_id\"].values,\n        df[\"time_lag\"].values,\n        df[\"prior_question_elapsed_time\"].values,\n        df[\"answered_correctly\"].values,\n        df[\"prior_question_had_explanation\"].values,\n        df[\"user_answer\"].values,\n    ))\n    with open(\"val_group.pkl.zip\", 'wb') as pick:\n        pickle.dump(val_group, pick)\n    print(\"Complete pre-process, execution time {:.2f} s\".format(time.time()-t_s))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"pre_process(train_df, \"../input/riiid-test-answer-prediction/questions.csv\", 0, -1, 0.95)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}