{"cells":[{"metadata":{},"cell_type":"markdown","source":"Just A  simple tf2.0 training sub module for   https://www.kaggle.com/prokaj/bert-joint-baseline-notebook\n\n\n\nreference:\n\nhttps://tf.wiki/#english-version-in-progress\n\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"def train():\n    \n\n    #log\n    summary_writer = tf.summary.create_file_writer('./tensorboard') \n    #train    \n    raw_ds=tf.data.TFRecordDataset('../input/bert-joint-baseline/nq-train.tfrecords')\n    #token_map_ds = raw_ds.map(_decode_tokens)\n    decoded_ds = raw_ds.map(_decode_record_train)\n    #ds = decoded_ds.batch(batch_size=2,drop_remainder=False)\n    train_dataset = decoded_ds.shuffle(buffer_size=1000).batch(1)\n    optimizer = tf.keras.optimizers.Adam(learning_rate=5e-6)\n    manager = tf.train.CheckpointManager(cpkt, directory='./save_2', max_to_keep=2)\n    count_step=0\n    bar=tqdm.tqdm(cycle(train_dataset))\n    for a in bar:\n        count_step+=1\n        input_mask=a['input_mask']\n        input_ids=a['input_ids']\n        segment_ids=a['segment_ids']\n        start_positions=a['start_positions']\n        end_positions=a['end_positions']\n        inpanswer_typesut_ids=a['answer_types']\n        unique_id=tf.constant([0]*input_ids.shape[0])\n        \n        with tf.GradientTape() as tape:\n            _,start_logits,end_logits,ans_type=model([unique_id,input_ids,input_mask,segment_ids])\n            start_loss= compute_loss(start_logits,start_positions,bert_utils.FLAGS.max_seq_length)\n            end_loss=compute_loss(end_logits,end_positions,bert_utils.FLAGS.max_seq_length)\n            answer_type_loss=compute_label_loss(ans_type,inpanswer_typesut_ids,5)\n            total_loss = (start_loss + end_loss + answer_type_loss) / 3.0\n\n            bar.set_description(\"loss {}\".format(total_loss))\n\n            with summary_writer.as_default():                               \n                tf.summary.scalar(\"loss\", total_loss, step=count_step)\n        optimizer.apply_gradients(grads_and_vars=zip(tape.gradient(total_loss, model.trainable_variables), model.trainable_variables))  \n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}