{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Riiid TFRecords\nCreating TFRecord files for Riiid."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import roc_auc_score\n\nimport time\nimport random","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load and prepare data"},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype = {\n    'answered_correctly': 'int8',\n    # 'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    #'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    # 'user_answer': 'int8',\n    'prior_question_elapsed_time': 'float32'\n    #'prior_question_had_explanation': 'boolean'\n    }\n\ndtype_questions = {\n    'question_id': 'int32',\n    # 'bundle_id': 'int32',\n    # 'correct_answer': 'int8',\n    'part': 'int8'\n    #'tags': 'object'\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                    usecols=dtype.keys(),\n                    dtype=dtype)\n\n#only keep final 1000 interactions of each user\ntrain = train.groupby('user_id').tail(500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',\n                        usecols = dtype_questions.keys(),\n                        dtype = dtype_questions)\n\n#encode tags\n#tag_map = {tag : i + 1 for i, tag in enumerate(questions.tags.unique())}\n#questions['tags'] = questions['tags'].map(tag_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# pre processing\n#remove lectures\ntrain = train.loc[train.answered_correctly != -1, :]\n\n#part\ntrain = train.join(questions, on = 'content_id')\n\n#prior q\n#0 padding, 1 start, 2...n bins\ntrain.prior_question_elapsed_time = train.prior_question_elapsed_time.fillna(0)\ntrain.prior_question_elapsed_time, bins = pd.qcut(train.prior_question_elapsed_time, 150,\n                                           labels = False, retbins = True, duplicates = 'drop')\ntrain.prior_question_elapsed_time = train.prior_question_elapsed_time + 1\nbins[-1] = float('inf')\n\n#lagtime\n#0 padding, 1 start, 2...n bins\n#train['lagtime'] = train['timestamp'] - train.groupby('user_id')['timestamp'].shift(1, fill_value = 0.0)\n#train.lagtime, bins_lt = pd.qcut(train.lagtime, 250, labels = False, retbins = True, duplicates = 'drop')\n#train.lagtime = train.lagtime + 1\n#bins_lt[-1] = float('inf')\n\n#timestamp\n#0 padding, 1 start, 2...n bins\ntrain.timestamp, bins_ts = pd.qcut(train.timestamp, 250, labels = False, retbins = True, duplicates = 'drop')\ntrain.timestamp = train.timestamp + 1\nbins_ts[-1] = float('inf')\n\n#content_id\ntrain['content_id'] += 1\n\n#container_id\ntrain['task_container_id'] += 1\n\n#answered correctly: 0 start, 1 incorrect, 2 correct\ntrain['answered_correctly'] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"window_size = 100\nPADDING_TOKEN = 0\n\n#features\nx_cols = ['content_id', 'part', 'task_container_id']\ny_cols = ['answered_correctly', 'prior_question_elapsed_time', 'timestamp']\n\ncols = x_cols + y_cols\n\n#vocab sizes\ncontent_id_size = questions.question_id.max() + 2\nprior_q_size = train.prior_question_elapsed_time.max() + 1\n#tags_size = train.tags.max() + 1\ntimestamp_size = train.timestamp.max() + 1\npart_size = train.part.max() + 1\ncontainer_size = 10001\n\ninput_vocab_sizes = [content_id_size, part_size, container_size]\ninput_vocab_sizes = [int(x) for x in input_vocab_sizes]\n\ntarget_vocab_sizes = [3, prior_q_size, timestamp_size]\ntarget_vocab_sizes = [int(x) for x in target_vocab_sizes]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(input_vocab_sizes)\nprint(target_vocab_sizes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def strided_window(a, w, s, ret_weights = False):\n    '''\n    Applies a rolling window to the array which moves with a given stride length.\n    a : 2d np.array\n    w : window length int\n    s : stride length int\n    ret_weights : wether to return the weights for each row\n    \n    Returns: array (n_windows, w, a.shape[1]), weights (n_windows, w)\n    \n    s0 is the number of bytes moved to advance a row in a. s1 is the analog for columns\n    m = number of rows in a\n    n = number of columns (features) in a\n    \n    We are moving with stride length s so the number of windows is np.ceil((m-w+1)/s)\n    \n    Sample weights:\n    Weight of each row is 1/n where n is the number of times it appears across all windows.\n    \n    \n    '''\n    s0, s1 = a.strides\n    m, n = a.shape\n    \n    windowed_arr = np.lib.stride_tricks.as_strided(\n        a,\n        shape = (int(np.ceil((m-w+1)/s)), w, n),\n        strides = (s * s0, s0, s1)\n    )\n        \n    if ret_weights:\n        m_lower = s * ((m - w) // s)\n        rows = np.arange(m_lower)\n        weights = 1/np.minimum(np.minimum(1 + rows//s, w // s), 1 + (m_lower - rows)//s)\n        all_weights = np.zeros((m,1), np.float32)\n        all_weights[:m_lower,0] = weights\n                        \n        s0, s1 = all_weights.strides\n        \n        windowed_weights = np.lib.stride_tricks.as_strided(\n            all_weights,\n            shape = (int(np.ceil((m-w+1)/s)), w, 1),\n            strides = (s * s0, s0, s1)\n        )\n        \n        return windowed_arr, windowed_weights\n    else:\n        return windowed_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n    '''\n    Data generator for processing user time series and returning batches of\n    user histories and the correctness of their answer to the next question.\n    Takes a full dataset of user interactions and creates a training set from\n    a subset of the users (of given size) of time series and targets.\n    '''\n    \n    def __init__(self, data, x_cols, y_cols, window_size, subset_size = 0.2,\n                 stride = 10, batch_size = 32, pad_token = 0):\n        '''\n        data : full dataset\n        x_cols : cols to use for input\n        y_cols : col for target\n        window_size : length of history to use\n        stride : stride length for windowing\n        pad_token : value to pad with\n        subset_size : proportion of users to use for training data\n        batch_size : batch size for training\n        '''\n        \n        all_users = data.user_id.unique()\n        random.shuffle(all_users)\n        \n        self.data = data\n        self.all_users = all_users\n        self.n_users = data.user_id.nunique()\n        self.x_cols = x_cols\n        self.y_cols = y_cols\n        self.window_size = window_size\n        self.stride = stride\n        self.pad_token = pad_token\n        self.subset_size = subset_size\n        self.batch_size = batch_size\n        \n        self.calls = 0\n        \n    def update_users(self):\n        '''\n        Selects users to use by cycling through the dataset\n        in chunks of size subset_size * n_users.\n        '''\n        start = int(self.n_users * ((self.subset_size * self.calls) % 1))\n        end = int(np.min([start + self.n_users * self.subset_size, self.n_users]))\n        \n        self.users = self.all_users[start:end]\n        self.calls += 1\n        \n    def random_update_users(self):\n        ''' \n        Randomly selects a subset of the users\n        of size = subset_size * n_users.\n        '''\n        size = int(self.n_users * self.subset_size)\n        users = np.random.choice(self.n_users, size)\n        self.users = users\n        \n    def set_batch_size(self, batch_size):\n        self.batch_size = batch_size\n        \n    def regenerate(self, random_choice = False):\n        '''prepare initial subset. must be called before the generator\n        can be used'''\n        \n        #choose a subset of users\n        if random_choice:\n            self.random_update_users()\n        else:\n            self.update_users()\n        \n        #generate the training datasets from the users\n        self.X_train, self.Y_train, self.weights = self.generate(self.users)\n        \n    def __len__(self):\n        return self.X_train.shape[0]//self.batch_size\n        \n    def __getitem__(self, idx):            \n        idx_l = idx * self.batch_size\n        idx_u = (idx + 1) * self.batch_size\n        return self.X_train[idx_l:idx_u,:,:], self.Y_train[idx_l:idx_u], self.weights[idx_l:idx_u]\n            \n    def generate(self, users):\n        ''' creates datasets X_train, y_train that are returned through __getitem__'''\n        \n        #get the features and targets as list of 2d np arrays per user\n        group = self.data.loc[self.data.user_id.isin(users)].groupby('user_id')\n        X = group[self.x_cols].aggregate(list).apply(lambda x: np.vstack(x).T, axis = 1).tolist()\n        Y = group[self.y_cols].aggregate(list).apply(lambda x: np.vstack(x).T, axis = 1).tolist()\n        \n        #pad so length at least window_size\n        X = [np.pad(\n            x, ((self.window_size - x.shape[0],0),(0,0))\n        ) if x.shape[0] < self.window_size else x for x in X]\n        \n        Y = [np.pad(\n            y, ((self.window_size - y.shape[0],0),(0,0))\n        ) if y.shape[0] < self.window_size else y for y in Y]\n        \n        #apply windowing\n        X = [strided_window(x, self.window_size, self.stride, True) for x in X]\n        X, ws = zip(*X) #unpack weights\n        Y = [strided_window(y, self.window_size, self.stride) for y in Y]\n        \n        #concatenate\n        X = np.concatenate(X, axis = 0) # (n_samples, window_size, features_x)\n        Y = np.concatenate(Y, axis = 0) # (n_samples, window_size, features_y)\n        ws = np.concatenate(ws, axis = 0) # (n_samples, window_size, 1)\n                \n        #shuffle\n        X, Y, ws = shuffle(X, Y, ws)\n        \n        #tensors\n        #X = tf.convert_to_tensor(X, dtype = tf.int64)\n        #Y = tf.convert_to_tensor(Y, dtype = tf.int64)\n        \n        return X, Y, ws","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataSampler(tf.keras.utils.Sequence):\n    '''\n    Sample users based on how much history they have. Select rows at random from each.\n    '''\n    \n    def __init__(self, data, cols, N, window):\n        '''\n        data : full dataset\n        x_cols : cols to use for model\n        N : number of users to sample\n        '''\n        \n        user_lengths = data.groupby('user_id').size() # number of rows for each user\n        users = user_lengths.index.to_numpy() # user id_s\n        self.user_lengths = user_lengths\n        self.user_dist = (user_lengths / user_lengths.sum()).to_numpy()\n        self.users = users\n        \n        self.data = data\n        self.cols = cols\n        self.N = N\n        self.w = window\n        \n    def sample_data(self):\n        '''\n        Generate input data as (x, y) from the data by sampling N rows.\n        '''\n        \n        # choose users and rows\n        users = np.random.choice(self.users, size = (self.N,), p = self.user_dist)\n        \n        # subset data\n        data = self.data.loc[self.data.user_id.isin(users)]\n                \n        # get users\n        X = [data.loc[data.user_id == user, cols].to_numpy() for user in users]\n        \n        # user lengths\n        lengths = np.array([x.shape[0] for x in X]) # (N,)\n        rows = np.random.randint(1, high = lengths)\n             \n        # extract rows and pad\n        X = [user_data[np.max([r-self.w,0]):r] for r, user_data in zip(rows, X)]\n        X = [np.pad(x, ((self.w - x.shape[0],0),(0,0))) if x.shape[0] < self.w else x for x in X]\n        X = np.array(X) # (n_users, window, n_feats)\n        \n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#save users for validation\nval_users = np.random.choice(train.user_id.unique(), int(0.1 * len(train.user_id.unique())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# samplers\nN = 20000\n\ntrain_sampler = DataSampler(train.loc[~train.user_id.isin(val_users)], cols, N, window_size)\nval_sampler = DataSampler(train.loc[train.user_id.isin(val_users)], cols, N, window_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = None","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Write TFRecords\nWrite the dataset into a TFRecord file."},{"metadata":{"trusted":true},"cell_type":"code","source":"def write_tfrec(X, filename):\n    \n    writer = tf.io.TFRecordWriter(filename)\n        \n    def _int_feature(array):\n        return tf.train.Feature(int64_list = tf.train.Int64List(value = array))\n    \n    def _float_feature(array):\n        return tf.train.Feature(float_list = tf.train.FloatList(value = array))\n    \n    for idx in range(X.shape[0]):\n        x = X[idx]\n        feature = {'content' : _int_feature(x[:,0]),\n                   'part' : _int_feature(x[:,1]),\n                   'container' : _int_feature(x[:,2]),\n                   'correct' : _int_feature(x[:,3]),\n                   'prior_q' : _int_feature(x[:,4]),\n                   'timestamp' : _int_feature(x[:,5])}\n        features = tf.train.Features(feature = feature)\n        example = tf.train.Example(features = features)\n        serialized = example.SerializeToString()\n        writer.write(serialized)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train\nfor i in range(60):\n    print(i)\n    filename = 'riiid_train_{}.tfrecords'.format(i)\n    write_tfrec(train_sampler.sample_data(), filename)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#val\nfor i in range(12):\n    print(i)\n    filename = 'riiid_val_{}.tfrecords'.format(i)\n    write_tfrec(val_sampler.sample_data(), filename)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read TFRecords\nHow to read the TFRecord files into tensors to input to the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = ['riiid_train_{}.tfrecords'.format(i) for i in range(125)]\ndataset = tf.data.TFRecordDataset(train_files)\nmaxlen = window_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _decode_features(example):\n    feature_desc = {'content' : tf.io.FixedLenFeature((maxlen,), tf.int64),\n                    'part' : tf.io.FixedLenFeature((maxlen,), tf.int64),\n                    'container' : tf.io.FixedLenFeature((maxlen,), tf.int64),\n                    'correct' : tf.io.FixedLenFeature((maxlen,), tf.int64),\n                    'prior_q' : tf.io.FixedLenFeature((maxlen,), tf.int64),\n                    'timestamp' : tf.io.FixedLenFeature((maxlen,), tf.int64)}\n    \n    example = tf.io.parse_single_example(example, feature_desc)\n    \n    x = tf.stack([example['content'], example['part'], example['container']], axis = 1)\n    y = tf.stack([example['correct'], example['prior_q'], example['timestamp']], axis = 1)\n    \n    return x, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = dataset.map(_decode_features)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}