{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.models import Model\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport time\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom pickle import dump, load\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n    # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 512\nWIN_SIZE = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_feather(file_name = \"train.feather\"):\n    data = pd.read_feather(file_name)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_dict = load(open('../input/processing-riiid-train-data/user_dict.pkl', 'rb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf = read_feather('../input/processing-riiid-train-data/all_train_dat_plus.feather')\ntdf = tdf[-10000:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf.numlect = tdf.numlect.astype(np.uint8)\ntdf.task_container_id[tdf.task_container_id > 2000] = 2000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf.drop(columns = ['timestamp', 'user_id'], inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rolling_window(a, w):\n    s0, s1 = a.strides\n    m, n = a.shape\n    return np.lib.stride_tricks.as_strided(a, \n                                           shape=(m-w+1, w, n), \n                                           strides=(s0, s0, s1))\n\ndef make_time_series(x, windows_size, pad_size=0):\n    x = np.pad(x, [[ windows_size-pad_size-1, 0], [0, 0]], constant_values=0)\n    x = rolling_window(x, windows_size)\n    return list(x)\n\ndef shift_answer(df):\n    # We add one to the column in order to have zeros as padding values\n    # Start Of Sentence (SOS) token will be 3. \n    df['answered_correctly'] = df['answered_correctly'].shift(fill_value=2)+1\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dat_generator(df, user_dict, for_train = True, val_p1 = None, val_p2 = None, win_size = 128, batch_size = 256):\n    pos_to_uid = list(user_dict.keys())\n    pos_to_uid.sort()\n    if val_p2 is None: val_p2 = int(len(user_dict) * 0.9)\n    if val_p1 is None: val_p1 = int(val_p2 * 0.5)\n    if for_train:\n        remain_uid = list(range(val_p2))\n        rem_uid = None\n        use_dat = None\n        rem_uidx = None\n        while True:\n            X = []\n            y = []\n            remain_batch = batch_size\n            if rem_uid is not None: \n                if uid < pos_to_uid[val_p1]: utotal_dat = min(3000, len(user_dict[rem_uid]))\n                else: utotal_dat = min(3000, int(len(user_dict[rem_uid])*0.8))\n                u_dat = utotal_dat - use_dat\n                if u_dat <= remain_batch:\n                    y.extend(df.answered_correctly[user_dict[rem_uid][use_dat]:user_dict[rem_uid][utotal_dat-1]+1])\n                    t_use_dat = max(0, use_dat - win_size + 1)\n                    t_use = max(0, t_use_dat - 1)\n                    pad_size = use_dat\n                    if t_use_dat != 0: pad_size = win_size - 1\n                    udf = df[user_dict[rem_uid][t_use]:user_dict[rem_uid][utotal_dat-1]+1].copy()\n                    udf = shift_answer(udf)\n                    if t_use_dat != 0: udf = udf[1:].copy()\n                    X.extend(make_time_series(udf, win_size, pad_size = pad_size))\n                    remain_batch -= u_dat\n                    remain_uid.pop(rem_uidx)\n                    rem_uid = None\n                    rem_uidx = None\n                    use_dat = None\n                else:\n                    y.extend(df.answered_correctly[user_dict[rem_uid][use_dat]:user_dict[rem_uid][use_dat+remain_batch]])\n                    t_use_dat = max(0, use_dat - win_size + 1)\n                    t_use = max(0, t_use_dat - 1)\n                    pad_size = use_dat\n                    if t_use_dat != 0: pad_size = win_size - 1\n                    udf = df[user_dict[rem_uid][t_use]:user_dict[rem_uid][use_dat+remain_batch]].copy()\n                    udf = shift_answer(udf)\n                    if t_use_dat != 0: udf = udf[1:].copy()\n                    X.extend(make_time_series(udf, win_size, pad_size))\n                    use_dat += remain_batch\n                    remain_batch = 0\n            while remain_batch > 0:\n                if len(remain_uid)==0: remain_uid = list(range(val_p2))\n                uidx = np.random.choice(len(remain_uid), 1)[0]\n                uid = pos_to_uid[remain_uid[uidx]]\n                u_dat = min(3000, len(user_dict[uid]))\n                if u_dat < 20:\n                    remain_uid.pop(uidx)\n                    continue\n                if uid > pos_to_uid[val_p1]: u_dat = min(3000, int(len(user_dict[uid])*0.8))\n                if u_dat <= remain_batch:\n                    y.extend(df.answered_correctly[user_dict[uid][0]:user_dict[uid][u_dat-1]+1])\n                    udf = df[user_dict[uid][0]:user_dict[uid][u_dat-1]+1].copy()\n                    udf = shift_answer(udf)\n                    X.extend(make_time_series(udf, win_size))\n                    remain_batch -= u_dat\n                    remain_uid.pop(uidx)\n                else:\n                    y.extend(df.answered_correctly[user_dict[uid][0]:user_dict[uid][remain_batch]])\n                    udf = df[user_dict[uid][0]:user_dict[uid][remain_batch]].copy()\n                    udf = shift_answer(udf)\n                    X.extend(make_time_series(udf, win_size))\n                    rem_uid = uid\n                    rem_uidx = uidx\n                    use_dat = remain_batch\n                    remain_batch = 0\n            yield np.asarray(X).astype(np.float32), np.asarray(y).astype(np.float32)\n    else:\n        remain_uid = []\n        rem_uid = None\n        use_dat = None\n        rem_uidx = None\n        while True:\n            X = []\n            y = []\n            if len(remain_uid) == 0: remain_uid = list(range(val_p1, len(user_dict)))\n            uidx = np.random.choice([0, len(remain_uid) - 1], 1)[0]\n            uid = pos_to_uid[remain_uid[uidx]]\n            if uid > pos_to_uid[val_p2]:\n                u_dat = len(user_dict[uid])\n                start_token = - u_dat\n                end_token = -1\n                if u_dat > BATCH_SIZE: end_token = -u_dat + BATCH_SIZE\n            else:\n                if len(user_dict[uid]) < 10:\n                    remain_uid.pop(uidx)\n                    continue \n                u_dat = int(len(user_dict[uid])*0.2)\n                start_token = - u_dat - 1\n                end_token = -1\n                if u_dat > BATCH_SIZE: end_token = -u_dat + BATCH_SIZE\n            y.extend(df.answered_correctly[user_dict[uid][-u_dat]:user_dict[uid][end_token]+1])\n            udf = df[user_dict[uid][start_token]:user_dict[uid][end_token]+1].copy()\n            udf = shift_answer(udf)\n            if udf.shape[0] != len(y): udf = udf[1:].copy()\n            X.extend(make_time_series(udf, win_size))\n            remain_uid.pop(uidx)\n            yield np.asarray(X).astype(np.float32), np.asarray(y).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadAttention(layers.Layer):\n    def __init__(self, embed_dim, num_heads=8, **kwargs):\n        super(MultiHeadAttention, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        if self.embed_dim % self.num_heads != 0:\n            raise ValueError(\n                f\"embedding dimension = {self.embed_dim} should be divisible by number of heads = {self.num_heads}\"\n            )\n        self.projection_dim = self.embed_dim // self.num_heads\n        self.query_dense = layers.Dense(embed_dim)\n        self.key_dense = layers.Dense(embed_dim)\n        self.value_dense = layers.Dense(embed_dim)\n        self.combine_heads = layers.Dense(embed_dim)\n    \n    def get_config(self):\n        cfg = super().get_config()\n        cfg.update({\n            'embed_dim': self.embed_dim,\n            'num_heads': self.num_heads,\n        })\n        return cfg\n\n    def attention(self, query, key, value, mask):\n        score = tf.matmul(query, key, transpose_b=True)\n        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n        scaled_score = score / tf.math.sqrt(dim_key)\n        if mask is not None:\n            scaled_score += (mask * -1e9)\n        weights = tf.nn.softmax(scaled_score, axis=-1)\n        output = tf.matmul(weights, value)\n        return output, weights\n\n    def separate_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, q , k ,v, mask):\n        batch_size = tf.shape(q)[0]\n        print(\"batch size\", batch_size)\n        query = self.query_dense(q)  # (batch_size, seq_len, embed_dim)\n        key = self.key_dense(k)  # (batch_size, seq_len, embed_dim)\n        value = self.value_dense(v)  # (batch_size, seq_len, embed_dim)\n        query = self.separate_heads(query, batch_size)  # (batch_size, num_heads, seq_len, projection_dim)\n        key = self.separate_heads(key, batch_size)  # (batch_size, num_heads, seq_len, projection_dim)\n        value = self.separate_heads(value, batch_size)  # (batch_size, num_heads, seq_len, projection_dim)\n        attention, weights = self.attention(query, key, value, mask)\n        attention = tf.transpose(attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len, num_heads, projection_dim)\n        concat_attention = tf.reshape(attention, (batch_size, -1, self.embed_dim))  # (batch_size, seq_len, embed_dim)\n        output = self.combine_heads(concat_attention)  # (batch_size, seq_len, embed_dim)\n        return output # can return weights\n\n\"\"\"\nEncoder block as a layer\n\"\"\"\n\nclass EncoderBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim = None, rate=0.1, **kwargs):\n        super(EncoderBlock, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        self.ff_dim = ff_dim\n        self.rate = rate\n        self.att = MultiHeadAttention(self.embed_dim, self.num_heads)\n        if self.ff_dim is None: self.ff_dim = 2*self.embed_dim\n        self.ffn = tf.keras.Sequential(\n            [layers.Dense(self.ff_dim, activation=\"relu\"), layers.Dense(self.embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(self.rate)\n        self.dropout2 = layers.Dropout(self.rate)\n        \n    def get_config(self):\n        cfg = super().get_config()\n        cfg.update({\n            'embed_dim': self.embed_dim,\n            'num_heads': self.num_heads,\n            'ff_dim': self.ff_dim,\n            'rate': self.rate\n        })\n        return cfg\n\n    def call(self, x, y, padding_mask, training):\n        attn_output = self.att(x, y, y, padding_mask)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n\n\"\"\"\nDecoder block as a layer\n\"\"\"\n\nclass DecoderBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim = None, rate = 0.1, **kwargs):\n        super(DecoderBlock, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        self.ff_dim = ff_dim\n        self.rate = rate\n        self.att1 = MultiHeadAttention(self.embed_dim, self.num_heads)\n        self.att2 = MultiHeadAttention(self.embed_dim, self.num_heads)\n        self.ffn = tf.keras.Sequential(\n            [layers.Dense(self.ff_dim, activation=\"relu\"), layers.Dense(self.embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(self.rate)\n        self.dropout2 = layers.Dropout(self.rate)\n        self.dropout3 = layers.Dropout(self.rate)\n        \n    def get_config(self):\n        cfg = super().get_config()\n        cfg.update({\n            'embed_dim': self.embed_dim,\n            'num_heads': self.num_heads,\n            'ff_dim': self.ff_dim,\n            'rate': self.rate\n        })\n        return cfg\n    \n    def call(self, x, enc_output, look_ahead_mask, padding_mask, training):\n        attn1 = self.att1(x, x, x, look_ahead_mask)\n        attn1 = self.dropout1(attn1, training = training)\n        out1 = self.layernorm1(attn1 + x)\n        \n        attn2 = self.att2(out1, enc_output, enc_output, padding_mask)\n        attn2 = self.dropout2(attn2, training = training)\n        out2 = self.layernorm2(attn2 + out1)\n        \n        ffn_output = self.ffn(out2)\n        ffn_output = self.dropout3(ffn_output, training = training)\n        return self.layernorm3(ffn_output + out2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_padding_mask(seqs):\n    mask = tf.cast(tf.reduce_all(tf.math.equal(seqs, 0), axis=-1), tf.float32)\n    return mask[:, tf.newaxis, tf.newaxis, :]\n\ndef create_look_ahead_mask(size):\n    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n    return mask  # (seq_len, seq_len)\n\ndef get_angles(pos, i, embed_dim):\n    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(embed_dim))\n    return pos * angle_rates\n\ndef positional_encoding(position, embed_dim):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                            np.arange(embed_dim)[np.newaxis, :],\n                            embed_dim)\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n    pos_encoding = angle_rads[np.newaxis, ...]\n    return tf.cast(pos_encoding, dtype=tf.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def custom_transformer_model(feature_dim, window_size, q_size=13524, embed_dim = 256, num_heads = 16, dense_dim = 1024):\n    inputs = layers.Input(shape=(window_size, feature_dim), name = \"enc_input\")\n    min_delta = inputs[...,0]\n    day_delta = inputs[...,1]\n    month_delta = inputs[...,2]\n    cid = inputs[...,3]\n    tid = inputs[...,4]\n    prior_elapsed = inputs[...,5]\n    prior_explained = inputs[...,6]\n    is_with = inputs[...,7]\n    num_lect = inputs[...,-1,8]\n    lec_type = inputs[...,-1,9:13]\n    lec_h_past = inputs[...,-1,13]\n    c_part = inputs[...,14:23]\n    tag1 = inputs[...,23]\n    tag2 = inputs[...,24]\n    tag3 = inputs[...,25]\n    tag4 = inputs[...,26]\n    tag5 = inputs[...,27]\n    tag6 = inputs[...,28]\n    prev_answered_correct = inputs[...,29]\n    \n    #====Excercise====\n    min_delta = layers.Embedding(input_dim=1443, output_dim=embed_dim//8, input_length=window_size,\n                                 embeddings_initializer = 'glorot_uniform')(min_delta)\n    day_delta = layers.Embedding(input_dim=33, output_dim=embed_dim//16, input_length=window_size,\n                                 embeddings_initializer = 'glorot_uniform')(day_delta)\n    month_delta = layers.Embedding(input_dim=9, output_dim=embed_dim//16, input_length=window_size,\n                                   embeddings_initializer = 'glorot_uniform')(month_delta)\n    cid = layers.Embedding(input_dim=q_size, output_dim=embed_dim, input_length=window_size,\n                           embeddings_initializer = 'glorot_uniform')(cid)\n    tid = layers.Embedding(input_dim=2001, output_dim=embed_dim//16, input_length=window_size,\n                           embeddings_initializer = 'glorot_uniform')(tid)\n    is_with = layers.Embedding(input_dim=3, output_dim=2, input_length=window_size,\n                               embeddings_initializer = 'glorot_uniform')(is_with)\n    c_part = layers.Dense(embed_dim//4, activation = 'relu', use_bias=False)(c_part)\n#     tag_emb = layers.Embedding(input_dim=189, output_dim=embed_dim//4)\n    tag1 = layers.Embedding(input_dim=189, output_dim=embed_dim//8, input_length=window_size,\n                            embeddings_initializer = 'glorot_uniform')(tag1)\n    tag2 = layers.Embedding(input_dim=179, output_dim=embed_dim//8, input_length=window_size,\n                            embeddings_initializer = 'glorot_uniform')(tag2)\n    tag3 = layers.Embedding(input_dim=162, output_dim=embed_dim//8, input_length=window_size,\n                            embeddings_initializer = 'glorot_uniform')(tag3)\n#     tag4 = tag_emb(tag4)\n#     tag5 = tag_emb(tag5)\n#     tag6 = tag_emb(tag6)\n    enc_ex = layers.Concatenate()([min_delta, day_delta, month_delta, tid, c_part,\n                                 tag1, tag2, tag3, is_with]) #tag4, tag5, tag6\n    enc_ex = layers.Dense(embed_dim, activation = 'relu')(enc_ex)\n    \n    #====Lecture====\n    num_lect = layers.Embedding(input_dim=160, output_dim=embed_dim//16,\n                                embeddings_initializer = 'glorot_uniform')(num_lect)\n    lec_type = layers.Dense(embed_dim//8, activation = 'relu', use_bias=False)(lec_type)\n    lec_h_past = layers.Embedding(input_dim=724, output_dim=embed_dim//8,\n                                  embeddings_initializer = 'glorot_uniform')(lec_h_past)\n    enc_lec = layers.Concatenate()([num_lect, lec_type, lec_h_past])\n    enc_lec = layers.Dense(embed_dim//2, activation = 'relu')(enc_lec)\n    enc_lec = layers.Dropout(0.1)(enc_lec)\n\n    #====Response====\n    prev_answered_correct = layers.Embedding(input_dim=4, output_dim=embed_dim, input_length=window_size,\n                                             embeddings_initializer = 'glorot_uniform')(prev_answered_correct)\n    prior_elapsed = layers.Embedding(input_dim=302, output_dim=embed_dim//4, input_length=window_size,\n                                     embeddings_initializer = 'glorot_uniform')(prior_elapsed)\n    prior_explained = layers.Embedding(input_dim=3, output_dim=embed_dim//4, input_length=window_size,\n                                       embeddings_initializer = 'glorot_uniform')(prior_explained)\n    prior_inter = layers.Concatenate()([prior_elapsed, prior_explained])\n    prior_inter = layers.Dense(embed_dim, activation = 'relu')(prior_inter)\n    \n    #====Mask====\n    padding_mask = create_padding_mask(inputs)\n    look_ahead_mask = create_look_ahead_mask(window_size)\n    dec_combined_mask = tf.maximum(padding_mask, look_ahead_mask)\n    pos_enc = positional_encoding(window_size, embed_dim)\n    \n    #++++Model++++\n    e_enc_input = layers.Add()([cid, pos_enc, enc_ex])\n    dec_input = layers.Add()([prev_answered_correct, pos_enc, prior_inter])\n    \n    x1 = EncoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(e_enc_input, e_enc_input, padding_mask)\n    x1 = EncoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(x1, x1, padding_mask)\n    x1 = layers.Add()([e_enc_input, x1])\n    x3 = DecoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(dec_input, x1,\n                                                                dec_combined_mask, padding_mask)\n    x3 = DecoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(x3, x1,\n                                                                dec_combined_mask, padding_mask)\n    x = x3[:, -1, :]\n    x = layers.Concatenate()([x, enc_lec])\n    x = layers.Dense(embed_dim, activation=\"relu\")(x)\n#     x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.2)(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\",\n                           kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02),\n                           name = \"output\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_step = int(tdf.shape[0]*0.2 / BATCH_SIZE)\ntrain_step = int(tdf.shape[0]*0.8 / BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = custom_transformer_model(30, WIN_SIZE, embed_dim = 128, dense_dim = 512, num_heads=8)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_name = \"SAINT_2\"\nmodel_folder = \"ckp_dir/\" + model_name\ntblogs = \"tbl_dir\"\nif not os.path.exists(model_folder):\n    os.makedirs(model_folder)\nif not os.path.exists(tblogs):\n    os.mkdir(tblogs)\n\n# model_checkpoints = ModelCheckpoint('{}/{}'.format(model_folder, model_name)+'-{epoch:02d}_{loss:.4f}_{auc:.4f}_{val_loss:.4f}_{val_auc:.4f}.h5', save_best_only=True, monitor='val_auc', mode='max')\nmodel_checkpoints = ModelCheckpoint('{}/{}'.format(model_folder, model_name)+'.{epoch:02d}_{loss:.4f}_{auc:.4f}_{val_loss:.4f}_{val_auc:.4f}.h5')\nlr_auto = ReduceLROnPlateau(monitor = \"val_loss\", factor = 0.1, patience = 7, mode = \"min\", min_delta = 0.0001, min_lr = 0.0000001, verbose = 1)\nlog_dir = \"{}/{}-{}\".format(tblogs, model_name, time.time())\ntensorboard = TensorBoard(log_dir=log_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_gen = dat_generator(tdf, user_dict, win_size = WIN_SIZE, batch_size=BATCH_SIZE)\nval_gen = dat_generator(tdf, user_dict, for_train = False, win_size = WIN_SIZE, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=tf.keras.losses.BinaryCrossentropy(), optimizer='adam',\n              run_eagerly=True, metrics=['AUC', 'acc'])\nmodel.fit(train_gen, epochs = 5, initial_epoch = 0,\n          callbacks=[model_checkpoints, lr_auto, tensorboard],\n          validation_data = val_gen, steps_per_epoch = train_step, validation_steps = val_step,\n          )#class_weight=class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}