2.6 INITIAL DATAFRAME INSTANTIATION
\n\n---\n","metadata":{"id":"8ChLuaTnUekk"}},{"cell_type":"code","source":"# !gsutil ls gs://kds-ebd140840cd463059a4465c0dddd014bba523491bf03cfa668287a31/","metadata":{"executionInfo":{"elapsed":1812,"status":"ok","timestamp":1621257408279,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"GIB3TnkdeHzB","outputId":"78427c38-4d00-43fd-9486-6ec6729bc355","execution":{"iopub.status.busy":"2021-08-27T05:26:34.770173Z","iopub.execute_input":"2021-08-27T05:26:34.770493Z","iopub.status.idle":"2021-08-27T05:26:34.775082Z","shell.execute_reply.started":"2021-08-27T05:26:34.770461Z","shell.execute_reply":"2021-08-27T05:26:34.77364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copy gcs csv files locally\n# !gsutil cp gs://kds-ebd140840cd463059a4465c0dddd014bba523491bf03cfa668287a31/train_labels_xtra.csv .\n# !gsutil cp gs://kds-ebd140840cd463059a4465c0dddd014bba523491bf03cfa668287a31/sample_submission_w_extra.csv .\n","metadata":{"executionInfo":{"elapsed":19256,"status":"ok","timestamp":1621257454213,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"z-0Sq5EjdKJl","outputId":"95ffb372-9eb5-4870-d2b2-632deec3f8e7","execution":{"iopub.status.busy":"2021-08-27T05:26:34.776824Z","iopub.execute_input":"2021-08-27T05:26:34.777248Z","iopub.status.idle":"2021-08-27T05:26:34.78977Z","shell.execute_reply.started":"2021-08-27T05:26:34.777204Z","shell.execute_reply":"2021-08-27T05:26:34.788349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... INITIAL DATAFRAME INSTANTIATION STARTING ...\\n\")\n\n# Load the train and submission dataframes\n# train_df = pd.read_csv(\"../input/bms-csvs-w-extra-metadata/train_labels_w_extra.csv\")\ntrain_df = pd.read_pickle(\"../input/train-labels-v5/train_labels_v5.pickle\")\nss_df = pd.read_csv(\"../input/bms-csvs-w-extra-metadata/sample_submission_w_extra.csv\")\n\n# --- Distribution Information ---\nN_EX = len(train_df)\nN_TEST = len(ss_df)\nN_VAL = 80_000 # Fixed from dataset creation information\nN_TRAIN = N_EX-N_VAL\n\n# --- Batching Information ---\nBATCH_SIZE_DEBUG = 2\nREPLICA_BATCH_SIZE = 128 # Could probably be 128\n\n# if DEBUG:\n# REPLICA_BATCH_SIZE = BATCH_SIZE_DEBUG\nOVERALL_BATCH_SIZE = REPLICA_BATCH_SIZE*N_REPLICAS\n\n\n# --- Input Image Information ---\nIMG_SHAPE = (192,384,3)\n\n# --- Autocalculate Training/Validation/Testing Information ---\nTRAIN_STEPS = N_TRAIN // OVERALL_BATCH_SIZE\nVAL_STEPS = N_VAL // OVERALL_BATCH_SIZE\nTEST_STEPS = int(np.ceil(N_TEST/OVERALL_BATCH_SIZE))\n\n# This is for padding our test dataset so we only have whole batches\nREQUIRED_DATASET_PAD = OVERALL_BATCH_SIZE-N_TEST%OVERALL_BATCH_SIZE\n\n# --- Modelling Information ---\nATTN_EMB_DIM = 192\nN_RNN_UNITS = 512\n\nprint(f\"\\n... # OF TRAIN+VAL EXAMPLES : {N_EX:<7} ...\")\nprint(f\"... # OF TRAIN EXAMPLES : {N_TRAIN:<7} ...\")\nprint(f\"... # OF VALIDATION EXAMPLES : {N_VAL:<7} ...\")\nprint(f\"... # OF TEST EXAMPLES : {N_TEST:<7} ...\\n\")\n\nprint(f\"\\n... REPLICA BATCH SIZE : {REPLICA_BATCH_SIZE} ...\")\nprint(f\"... OVERALL BATCH SIZE : {OVERALL_BATCH_SIZE} ...\\n\")\n\nprint(f\"\\n... IMAGE SHAPE : {IMG_SHAPE} ...\\n\")\n\nprint(f\"\\n... TRAIN STEPS PER EPOCH : {TRAIN_STEPS:<5} ...\")\nprint(f\"... VAL STEPS PER EPOCH : {VAL_STEPS:<5} ...\")\nprint(f\"... TEST STEPS PER EPOCH : {TEST_STEPS:<5} ...\\n\")\n\nprint(f\"\\n... ATTENTION EMBEDDING DIMENSION : {ATTN_EMB_DIM:<5} ...\")\nprint(f\"... NUMBER OF UNITS IN LSTM : {N_RNN_UNITS:<5} ...\\n\")\n\nprint(\"\\n... TRAIN DATAFRAME ...\\n\")\ndisplay(train_df.head(3))\n\nprint(\"\\n... SUBMISSION DATAFRAME ...\\n\")\ndisplay(ss_df.head(3))\n\nprint(\"\\n... INITIAL DATAFRAME INSTANTIATION COMPLETED...\\n\")","metadata":{"executionInfo":{"elapsed":21298,"status":"ok","timestamp":1621257535594,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"rgT8z9wEUekl","outputId":"cfbfdb6d-c289-4b6e-cbe2-01653191c475","execution":{"iopub.status.busy":"2021-08-27T05:26:34.791505Z","iopub.execute_input":"2021-08-27T05:26:34.79189Z","iopub.status.idle":"2021-08-27T05:27:13.273592Z","shell.execute_reply.started":"2021-08-27T05:26:34.791857Z","shell.execute_reply":"2021-08-27T05:27:13.272449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.7 USER INPUT VARIABLES
\n\n---\n","metadata":{"id":"uc5UzxzBUekl"}},{"cell_type":"code","source":"print(\"\\n... SPECIAL VARIABLE SETUP STARTING ...\\n\")\n\n\n# Whether to start training using previously checkpointed model\nLOAD_MODEL = False\nENCODER_CKPT_PATH = ''\nDECODER_CKPT_PATH = ''\n\nif LOAD_MODEL:\n print(f\"\\n... ENCODER MODEL TRAINING WILL RESUME FROM PREVIOUS CHECKPOINT:\\n\\t-->{ENCODER_CKPT_PATH}\\n\")\n print(f\"... DECODER MODEL TRAINING WILL RESUME FROM PREVIOUS CHECKPOINT:\\n\\t-->{DECODER_CKPT_PATH}\\n\")\nelse:\n print(f\"\\n... MODEL TRAINING WILL START FROM SCRATCH ...\\n\")\n\n \nprint(\"\\n... SPECIAL VARIABLE SETUP COMPLETED ...\\n\")","metadata":{"executionInfo":{"elapsed":21290,"status":"ok","timestamp":1621257535596,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"MXRcm7hfUekm","outputId":"3c4661cf-15da-4a15-fb22-c35ec37f44cb","execution":{"iopub.status.busy":"2021-08-27T05:27:13.274878Z","iopub.execute_input":"2021-08-27T05:27:13.275163Z","iopub.status.idle":"2021-08-27T05:27:13.282144Z","shell.execute_reply.started":"2021-08-27T05:27:13.275135Z","shell.execute_reply":"2021-08-27T05:27:13.280912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n\n\n\n\n
\n 3 HELPER FUNCTION & CLASSESS ⤒\n
","metadata":{"id":"91fOX6bpUekm"}},{"cell_type":"markdown","source":"3.1 GENERAL HELPER FUNCTIONS
\n\n---","metadata":{"id":"ipszRYU3Uekn"}},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\n\ndef tf_load_image(path, img_size=(192,384,3), invert=False):\n \"\"\" Load an image with the correct size and shape \n \n Args:\n path (tf.string): Path to the image to be loaded\n img_size (tuple, optional): Size to reshape image to (required for TPU)\n tile_to_3_channel (bool, optional): Whether to tile the single channel\n image to 3 channels which will be required for most off-the-shelf models\n invert (bool, optional): Whether or not to invert the background/foreground\n \n Returns:\n 3 channel tf.Constant image ready for training/inference\n \n \"\"\"\n img = decode_img(tf.io.read_file(path), img_size, n_channels=3, invert=invert) \n return img\n \n \ndef decode_image(image_data, resize_to=(192,384,3)):\n \"\"\" Function to decode the tf.string containing image information \n \n \n Args:\n image_data (tf.string): String containing encoded image data from tf.Example\n resize_to (tuple, optional): Size that we will reshape the tensor to (required for TPU)\n \n Returns:\n Tensor containing the resized single-channel image in the appropriate dtype\n \"\"\"\n image = tf.image.decode_png(image_data, channels=3)\n image = tf.reshape(image, resize_to)\n return tf.cast(image, TARGET_DTYPE)\n \n \n# sparse tensors are required to compute the Levenshtein distance\ndef dense_to_sparse(dense):\n \"\"\" Convert a dense tensor to a sparse tensor \n \n Args:\n dense (Tensor): TBD\n \n Returns:\n A sparse tensor \n \"\"\"\n indices = tf.where(tf.ones_like(dense))\n values = tf.reshape(dense, (MAX_LEN*OVERALL_BATCH_SIZE,))\n sparse = tf.SparseTensor(indices, values, dense.shape)\n return sparse\n\ndef get_levenshtein_distance(preds, lbls):\n \"\"\" Computes the Levenshtein distance between the predictions and labels \n \n Args:\n preds (tensor): Batch of predictions\n lbls (tensor): Batch of labels\n \n Returns:\n The mean Levenshtein distance calculated across the batch\n \"\"\"\n preds = tf.where(tf.not_equal(lbls, END_TOKEN) & tf.not_equal(lbls, PAD_TOKEN), preds, 0)\n lbls = tf.where(tf.not_equal(lbls, END_TOKEN), lbls, 0)\n\n preds_sparse = dense_to_sparse(preds)\n lbls_sparse = dense_to_sparse(lbls)\n\n batch_distance = tf.edit_distance(preds_sparse, lbls_sparse, normalize=False)\n mean_distance = tf.math.reduce_mean(batch_distance)\n \n return mean_distance","metadata":{"id":"tRHd39nmUekn","execution":{"iopub.status.busy":"2021-08-27T05:27:13.283673Z","iopub.execute_input":"2021-08-27T05:27:13.283967Z","iopub.status.idle":"2021-08-27T05:27:13.298255Z","shell.execute_reply.started":"2021-08-27T05:27:13.283938Z","shell.execute_reply":"2021-08-27T05:27:13.297277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n\n\n\n\n
4 PREPARE THE DATASET ⤒
\n\nIn this section we prepare the **`tf.data.Datasets`** we will use for training and validation","metadata":{"id":"KFbLZ1F9Ueko"}},{"cell_type":"code","source":"print(\"\\n\\n... STARTING PREPARING VARIABLES FOR DATASET ...\\n\")\n\n# tok_2_int = {c.strip(\"\\\\\"):i for i,c in enumerate(TOKEN_LIST)}\n# int_2_tok = {v:k for k,v in tok_2_int.items()}\ntok_2_int = {}\nfor i, c in enumerate(TOKEN_LIST):\n if not c == \"\\\\\\\\\":\n tok_2_int[c.replace(\"\\\\\", \"\")] = i\n else:\n tok_2_int[\"\\\\\"] = i\n\nint_2_tok = {v:k for k,v in tok_2_int.items()}\n\n# Max Length Was Determined Previously Using... \n# >>> MAX_LEN = train_df.smile.progress_apply(lambda x: len(re.findall(\"|\".join(TOKEN_LIST), x))).max()+1\n# MAX_LEN = ((train_df.smile_token_len.max()+1)//2) # //2 yields 138... which is half of max length (speeds up training)\nMAX_LEN = 141\nVOCAB_LEN = len(int_2_tok)\n\nprint(f\"\\t--> TOKEN TO INTEGER MAP : {tok_2_int}\")\nprint(f\"\\t--> INTEGER TO TOKEN MAP : {int_2_tok}\")\nprint(f\"\\t--> MAX # OF TOKENS IN smile : {MAX_LEN}\")\nprint(f\"\\t--> LENGTH OF VOCAB : {VOCAB_LEN}\")\n\nprint(f\"\\n\\n\\t--> CONVERTED smile STRINGS :\")\nfor i, row in train_df.iloc[:N_VAL].sample(3).iterrows():\n print(f\"\\n\\t\\t--> EXAMPLE #{i} FROM THE VALIDATION DATASET\")\n print(\"\\t\\t\\t--> RAW smile : \", row[\"smile\"])\n\nprint(\"\\n\\n... PREPARING VARIABLES FOR DATASET COMPLETED ...\\n\")","metadata":{"executionInfo":{"elapsed":21279,"status":"ok","timestamp":1621257535599,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"W2tNE5sZUeko","outputId":"aacb19b7-8d09-4bfb-f240-c88358954b1c","execution":{"iopub.status.busy":"2021-08-27T05:27:13.299829Z","iopub.execute_input":"2021-08-27T05:27:13.300199Z","iopub.status.idle":"2021-08-27T05:27:13.350283Z","shell.execute_reply.started":"2021-08-27T05:27:13.30017Z","shell.execute_reply":"2021-08-27T05:27:13.349311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.1 READ TFRECORD FILES - CREATE THE RAW DATASET(S)
\n\n---\n\nHere we will leverage **`tf.data.TFRecordDataset`** to read the TFRecord files.\n* The simplest way is to specify a list of filenames (paths) of TFRecord files.\n* It is a subclass of **`tf.data.Dataset`**.\n\nThis newly created raw dataset contains **`tf.train.Example`** messages, and when iterated over it, we get scalar string tensors.","metadata":{"id":"LxjV_QmNUekp"}},{"cell_type":"code","source":"print(\"\\n... CREATE TFRECORD RAW DATASETS STARTING ...\\n\")\n\n# Create tf.data.Dataset from filepaths for conversion later\nraw_train_ds = tf.data.TFRecordDataset(TRAIN_TFREC_PATHS, num_parallel_reads=None)\nraw_val_ds = tf.data.TFRecordDataset(VAL_TFREC_PATHS, num_parallel_reads=None)\nraw_test_ds = tf.data.TFRecordDataset(TEST_TFREC_PATHS, num_parallel_reads=None)\n\n# raw_test_ds = tf.data.TFRecordDataset(TEST_TFREC_PATHS, num_parallel_reads=None)\n\nprint(f\"\\n... THE RAW TF.DATA.TFRECORDDATASET OBJECT:\\n\\t--> {raw_train_ds}\\n\")\n\nprint(\"\\n... CREATE TFRECORD RAW DATASETS COMPLETED ...\\n\")","metadata":{"executionInfo":{"elapsed":21270,"status":"ok","timestamp":1621257535600,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"bZgcKtYCUekp","outputId":"229bcfb7-c45c-4963-dc36-866da9f522fb","execution":{"iopub.status.busy":"2021-08-27T05:27:13.351795Z","iopub.execute_input":"2021-08-27T05:27:13.352194Z","iopub.status.idle":"2021-08-27T05:27:13.387988Z","shell.execute_reply.started":"2021-08-27T05:27:13.35216Z","shell.execute_reply":"2021-08-27T05:27:13.386842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.2 WHAT TO DO IF YOU DON'T KNOW THE FEATURE DESCRIPTIONS OF THE DATASET?
\n\n---\n\nIf you are the author who created the TFRecord files, you definitely know how to define the feature description to parse the raw dataset.\n\nOtherwise, you can use like\n\n```python\nexample = tf.train.Example()\nexample.ParseFromString(serialized_example.numpy())\n```\n\nto check the information. You will get something like\n\n```python\nfeatures {\n feature {\n key: \"class\"\n value {\n int64_list {\n value: 57\n }\n }\n }\n feature {\n key: \"id\"\n value {\n bytes_list {\n value: \"338ab7bac\"\n }\n }\n }\n feature {\n key: \"image\"\n value {\n bytes_list {\n value: ...\n }\n }\n }\n ...\n}\n```\n\nThis should give you enough information to define the feature description.","metadata":{"id":"9XEcihC_Uekp"}},{"cell_type":"code","source":"print(\"\\n... RAW TFRECORD INVESTIGATION TO DETERMINE FEATURE DESCRIPTIONS STARTED ...\\n\")\n\nprint(\"\\n... EXAMPLE OF TRUNCATED RAW TFRECORD/TFEXAMPLE FROM TRAINING DATASET TO SHOW HOW TO FIND FEATURE DESCRIPTIONS:\\n\")\n# See an example\nfor raw in raw_train_ds.take(2):\n example = tf.train.Example()\n example.ParseFromString(raw.numpy())\n for i, (k,v) in enumerate(example.features.feature.items()):\n print(f\"\\tFEATURE #{i+1}\")\n print(f\"\\t\\t--> KEY = {k}\")\n if k!=\"image\":\n try:\n print('int')\n print(f\"\\t\\t\\t--> TRUNCATED-VALUE = {v.int64_list.value.__len__()} ...\\n\")\n print(f\"\\t\\t\\t--> TRUNCATED-VALUE = {v.int64_list.value[:15]} ...\\n\")\n except:\n print('byte')\n print(f\"\\t\\t\\t--> TRUNCATED-VALUE = {v.bytes_list.value.__len__()} ...\\n\")\n print(f\"\\t\\t\\t--> TRUNCATED-VALUE = {v.bytes_list.value[0][:25]} ...\\n\")\n else:\n print(f\"\\t\\t\\t--> TRUNCATED-VALUE = {str(v.bytes_list.value[0][:25])} ...\\n\") \n\nprint(\"\\n... RAW TFRECORD INVESTIGATION TO DETERMINE FEATURE DESCRIPTIONS COMPLETED ...\\n\")","metadata":{"executionInfo":{"elapsed":24290,"status":"ok","timestamp":1621257538624,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"W4vqT1TRUekq","outputId":"be2a673f-c4c0-4eab-a401-f4f529a26a69","execution":{"iopub.status.busy":"2021-08-27T05:27:13.389684Z","iopub.execute_input":"2021-08-27T05:27:13.390099Z","iopub.status.idle":"2021-08-27T05:27:15.114099Z","shell.execute_reply.started":"2021-08-27T05:27:13.390056Z","shell.execute_reply":"2021-08-27T05:27:15.112641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.3 PARSE THE RAW DATASET(S)
\n\n---\n\n\nThe general recipe to parse the string tensors in the raw dataset looks something like this:\n\n\n\n**STEP 1.** Create a description of the features. For example:\n\n```python\nfeature_description = { \n 'feature0': tf.io.FixedLenFeature([], tf.int64),\n 'feature1': tf.io.FixedLenFeature([], tf.string),\n 'feature2': tf.io.FixedLenFeature([], tf.float32),\n ...\n}\n```\n\n
\n\n**STEP 2.** Define a parsing function by using `tf.io.parse_single_example` and the defined feature description.\n```python\ndef _parse_function(example):\n \"\"\"\n Args:\n example: A string tensor representing a `tf.train.Example`.\n \"\"\"\n\n # Parse `example`.\n parsed_example = tf.io.parse_single_example(example, feature_description)\n\n return parsed_example\n```\n\n
\n\n**STEP 3.** Map the raw dataset by `_parse_function`.\n```python\ndataset = raw_dataset.map(_parse_function)\n```\n\n
\n\n---\n\n
\n\n**In the following cell, we apply the above recipe to our BMS tfrecord dataset.**\n\n\n
\n π NOTE:
- The parsed images are
- The smile strings and Image IDs will just be left as byte string tensors.\n
\n\n- The parsed images are
`tf.string`, which are then decoded with `tf.image.decode_png` which is an alias for `tf.io.decode_png`- The smile strings and Image IDs will just be left as byte string tensors.\n
\n π REFERENCE:
\n - Tutorial - TFRecord and tf.Example
\n - TFRecordDataset Documentation
\n - Decoding PNGs Documentation
\n
\n","metadata":{"id":"gMasPEdIUekq"}},{"cell_type":"code","source":"def decode(serialized_example, is_test=False, tokenized_smile=True):\n \"\"\" Parses a set of features and label from the given `serialized_example`.\n \n It is used as a map function for `dataset.map`\n\n Args:\n serialized_example (tf.Example): A serialized example containing the\n following features:\n β 'image'\n βΒ 'image_id'\n β 'smile'\n is_test (bool, optional): Whether to allow for the smile feature\n drop_id (bool, optional): Whether or not to drop the ID feature\n \n Returns:\n A decoded tf.data.Dataset object representing the tfrecord dataset\n \"\"\"\n feature_dict = {\n 'image': tf.io.FixedLenFeature(shape=[], dtype=tf.string, default_value=''),\n }\n \n if not is_test:\n if tokenized_smile:\n feature_dict[\"smile\"] = tf.io.FixedLenFeature(shape=[int(MAX_LEN)], dtype=tf.int64, default_value=[0]*int(MAX_LEN))\n else:\n feature_dict[\"smile\"] = tf.io.FixedLenFeature(shape=[], dtype=tf.string, default_value='')\n feature_dict[\"patterns\"] = tf.io.FixedLenFeature(shape=[591], dtype=tf.int64, default_value=[0]*591)\n else:\n feature_dict[\"image_id\"] = tf.io.FixedLenFeature(shape=[], dtype=tf.string, default_value='')\n \n # Define a parser\n features = tf.io.parse_single_example(serialized_example, features=feature_dict)\n \n # Decode the tf.string\n image = decode_image(features['image'], resize_to=IMG_SHAPE)\n \n # Figure out the correct information to return\n if is_test:\n image_id = features[\"image_id\"] \n return image, image_id\n else:\n# if tokenized_smile:\n smile = tf.cast(features[\"smile\"], tf.uint8)\n# else:\n# smile = features[\"smile\"]\n pattern = tf.cast(features[\"patterns\"], tf.uint8)\n return image, smile, pattern","metadata":{"id":"kPtkLlkEUekr","execution":{"iopub.status.busy":"2021-08-27T05:27:15.1157Z","iopub.execute_input":"2021-08-27T05:27:15.116091Z","iopub.status.idle":"2021-08-27T05:27:15.130543Z","shell.execute_reply.started":"2021-08-27T05:27:15.116054Z","shell.execute_reply":"2021-08-27T05:27:15.129243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... DECODING RAW TFRECORD DATASETS STARTING ...\\n\")\n\n# Decode the tfrecords completely ββ decode is our `_parse_function` (from recipe above)\ntrain_ds = raw_train_ds.map(lambda x: decode(x, is_test=False))\nval_ds = raw_val_ds.map(lambda x: decode(x, is_test=False))\ntest_ds = raw_test_ds.map(lambda x: decode(x, is_test=True))\n\nprint(f\"\\n... THE DECODED TF.DATA.TFRECORDDATASET OBJECT:\" \\\n f\"\\n\\t--> ((image), (image_id - optional), (smile))\" \\\n f\"\\n\\t--> {train_ds}\\n\")\n\nprint(\"\\n... 2 EXAMPLES OF IMAGES AND LABELS AFTER DECODING ...\")\nfor i, (img, smile, pattern) in enumerate(train_ds.take(2)):\n print(f\"\\nIMAGE SHAPE : {img.shape}\")\n print(f\"IMAGE smile : {[int_2_tok[x] for x in smile.numpy()]}\\n\")\n plt.figure(figsize=(10,10))\n plt.imshow(img.numpy().astype(np.int64), cmap=\"gray\")\n plt.title(f\"{''.join([int_2_tok[x] for x in smile.numpy() if x!=0][:50])} ... [truncated]\")\n plt.show()\n\nprint(\"\\n... DECODING RAW TFRECORD DATASETS COMPLETED ...\\n\")","metadata":{"executionInfo":{"elapsed":25649,"status":"ok","timestamp":1621257539992,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"E26ZA6f2Uekr","outputId":"c50f2685-1d28-434c-ba09-3be90ba62dee","execution":{"iopub.status.busy":"2021-08-27T05:27:15.132431Z","iopub.execute_input":"2021-08-27T05:27:15.132904Z","iopub.status.idle":"2021-08-27T05:27:16.032043Z","shell.execute_reply.started":"2021-08-27T05:27:15.132847Z","shell.execute_reply":"2021-08-27T05:27:16.030679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n - Tutorial - TFRecord and tf.Example
\n - TFRecordDataset Documentation
\n - Decoding PNGs Documentation
\n
4.4 WORKING WITH `TF.DATA.DATASET` OBJECTS
\n\n---\n\nWith the above parsing methods defined, we can define how to load the dataset with more options and further apply shuffling, bacthing, etc. In particular the following methods and attributes are of special interest to us:\n* Use **`num_parallel_reads`** in **`tf.data.TFRecordDataset`** to read files in parallel.\n* Set **`tf.data.Options.experimental_deterministic=False`** and use it to get a new dataset that ignores the order of elements.\n* Use **`num_parallel_calls`** in **`tf.data.Dataset.map()`** method to have parallel processing.\n* Use **`tf.data.Dataset.prefetch()`** to allow later batches to be prepared while the current batch is being processed.\n* Use **`tf.data.AUTOTUNE`** to automatically determine parallelization argument values\n\nThe parallel processing and prefetching are particular important when working with TPU:\n* This is because a TPU can process batches very quickly\n* The dataset pipeline should be able to provide data for TPU efficiently, otherwise the TPU will be idle.\n\n**In the cell below we will create the functions and configuration template which will later be used to create our respective datasets**\n\n\n π REFERENCE:
\n - Guide - tf.data: Build TensorFlow Input Pipelines
\n - Guide - Better Performance With the tf.data API
\n - tf.data.Dataset Documentation
\n
","metadata":{"id":"iEGRNrXoUeks"}},{"cell_type":"code","source":"# def load_dataset(filenames, is_test=False, ordered=False, tokenized_smile=True):\n# \"\"\"Read from TFRecords.\n \n# For optimal performance, reading from multiple files at once and disregarding data order (if `ordered=False`).\n# - If pulling smile from TFRecords than order does not matter since we will \n# be shuffling the data anyway (for training dataset).\n \n# Args:\n# filenames (list of strings): List of paths to that point to the respective TFRecord files\n# is_test (bool, optional): Whether or not to include the image ID or label in the returned dataset\n# ordered (bool, optional): Whether to ensured ordered results or maximize parallelization\n# tokenized_smile (bool, optional): Whether our dataset includes the tokenized smile or we will be \n# creating it from the caption numpy array\n \n# Returns:\n# Decoded tf.data.Dataset object\n# \"\"\"\n\n# options = tf.data.Options()\n# if not ordered:\n# # disable order, increase speed\n# options.experimental_deterministic = False\n# N_PARALLEL=tf.data.AUTOTUNE\n# else:\n# N_PARALLEL=None\n \n# # If not-ordered, this will read in by automatically interleaving multiple tfrecord files.\n# dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=N_PARALLEL)\n \n# # If not-ordered, this will ensure that we use data as soon as it \n# # streams in, rather than in its original order.\n# dataset = dataset.with_options(options) \n \n# # parse and return a dataset w/ the appropriate configuration\n# dataset = dataset.map(\n# lambda x: decode(x, is_test, tokenized_smile),\n# num_parallel_calls=N_PARALLEL,\n# )\n \n# return dataset\n\n# def get_dataset(filenames, batch_size, \n# is_test=False, \n# shuffle_buffer_size=1, \n# repeat_dataset=True, \n# preserve_file_order=False, \n# drop_remainder=True,\n# tokenized_smile=True,\n# external_smile_dataset=None):\n# \"\"\" Get a tf.data.Dataset w/ the appropriate configuration\n \n# Args:\n# TBD\n \n# Returns:\n# TBD\n \n# \"\"\"\n# # Load the dataset\n# dataset = load_dataset(filenames, is_test, preserve_file_order, tokenized_smile)\n \n# # If we are training than we will want to repeat the dataset. \n# # We will determine the number of steps (or updates) later for 1 training epoch.\n# if repeat_dataset:\n# dataset = dataset.repeat()\n \n# # If we need to add on manually the smile\n# if external_smile_dataset is not None:\n# # Zip the datasets and tile the 1 channel image to 3 channels & drop the old smile value\n# dataset = tf.data.Dataset.zip((dataset, external_smile_dataset))\n# dataset = dataset.map(lambda x,y: (tf.tile(tf.expand_dims(x[0], -1), tf.constant([1,1,3], tf.int32)), y))\n \n# # Shuffling\n# if shuffle_buffer_size!=1:\n# dataset = dataset.shuffle(shuffle_buffer_size)\n \n# # Batching\n# dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n \n# # prefetch next batch while training (autotune prefetch buffer size)\n# dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n \n# return dataset\n\n\ndef load_dataset(filenames, is_test=False, ordered=False, tokenized_smile=True):\n \"\"\"Read from TFRecords.\n \n For optimal performance, reading from multiple files at once and disregarding data order (if `ordered=False`).\n - If pulling smile from TFRecords than order does not matter since we will \n be shuffling the data anyway (for training dataset).\n \n Args:\n filenames (list of strings): List of paths to that point to the respective TFRecord files\n is_test (bool, optional): Whether or not to include the image ID or label in the returned dataset\n ordered (bool, optional): Whether to ensured ordered results or maximize parallelization\n tokenized_smile (bool, optional): Whether our dataset includes the tokenized smile or we will be \n creating it from the caption numpy array\n \n Returns:\n Decoded tf.data.Dataset object\n \"\"\"\n\n options = tf.data.Options()\n if not ordered:\n # disable order, increase speed\n options.experimental_deterministic = False\n N_PARALLEL=tf.data.AUTOTUNE\n else:\n N_PARALLEL=None\n \n # If not-ordered, this will read in by automatically interleaving multiple tfrecord files.\n dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=N_PARALLEL)\n \n # If not-ordered, this will ensure that we use data as soon as it \n # streams in, rather than in its original order.\n dataset = dataset.with_options(options) \n \n # parse and return a dataset w/ the appropriate configuration\n dataset = dataset.map(\n lambda x: decode(x, is_test, tokenized_smile),\n num_parallel_calls=N_PARALLEL,\n )\n \n return dataset\n\ndef get_dataset(filenames, batch_size, \n is_test=False, \n shuffle_buffer_size=1, \n repeat_dataset=True, \n preserve_file_order=False, \n drop_remainder=True,\n tokenized_smile=True,\n external_smile_dataset=None,\n test_padding=0):\n \"\"\" Get a tf.data.Dataset w/ the appropriate configuration\n \n Args:\n TBD\n test_padding (int, optional): Amount required to pad dataset to have only full batches\n \n Returns:\n TBD\n \n \"\"\"\n # Load the dataset\n dataset = load_dataset(filenames, is_test, preserve_file_order, tokenized_smile)\n \n if test_padding!=0:\n pad_dataset = tf.data.Dataset.from_tensor_slices((\n tf.zeros((test_padding, *IMG_SHAPE), dtype=TARGET_DTYPE), # Fake Images\n tf.constant([\"000000000000\",]*test_padding, dtype=tf.string)) # Fake IDs\n )\n dataset = dataset.concatenate(pad_dataset)\n \n # If we are training than we will want to repeat the dataset. \n # We will determine the number of steps (or updates) later for 1 training epoch.\n if repeat_dataset:\n dataset = dataset.repeat()\n \n # If we need to add on manually the smile\n if external_smile_dataset is not None:\n # Zip the datasets and tile the 1 channel image to 3 channels & drop the old smile value\n dataset = tf.data.Dataset.zip((dataset, external_smile_dataset))\n dataset = dataset.map(lambda x,y: (tf.tile(tf.expand_dims(x[0], -1), tf.constant([1,1,3], tf.int32)), y))\n \n # Shuffling\n if shuffle_buffer_size!=1:\n dataset = dataset.shuffle(shuffle_buffer_size)\n \n # Batching\n dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n \n # prefetch next batch while training (autotune prefetch buffer size)\n dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n \n return dataset","metadata":{"id":"oQ4noX9_Ueks","execution":{"iopub.status.busy":"2021-08-27T05:27:16.033356Z","iopub.execute_input":"2021-08-27T05:27:16.033665Z","iopub.status.idle":"2021-08-27T05:27:16.050603Z","shell.execute_reply.started":"2021-08-27T05:27:16.033634Z","shell.execute_reply":"2021-08-27T05:27:16.049427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####### ####### ####### ####### ####### ####### ####### #######\n\n# Template Configuration\nDS_TEMPLATE_CONFIG = dict(\n filenames=[],\n batch_size=1,\n is_test=False, \n shuffle_buffer_size=1, \n repeat_dataset=True, \n preserve_file_order=False, \n drop_remainder=True,\n tokenized_smile=True,\n external_smile_dataset=None,\n test_padding=0\n)\n\n####### ####### ####### ####### ####### ####### ####### #######\n\n# Individual Respective Configurations\nTRAIN_DS_CONFIG = DS_TEMPLATE_CONFIG.copy()\nTRAIN_DS_CONFIG.update(dict(\n filenames=TRAIN_TFREC_PATHS,\n batch_size=OVERALL_BATCH_SIZE,\n shuffle_buffer_size=OVERALL_BATCH_SIZE*6,\n))\n\nVAL_DS_CONFIG = DS_TEMPLATE_CONFIG.copy()\nVAL_DS_CONFIG.update(dict(\n filenames=VAL_TFREC_PATHS,\n batch_size=OVERALL_BATCH_SIZE,\n))\n\nTEST_DS_CONFIG = DS_TEMPLATE_CONFIG.copy()\nTEST_DS_CONFIG.update(dict(\n filenames=TEST_TFREC_PATHS,\n batch_size=OVERALL_BATCH_SIZE,\n is_test=True,\n repeat_dataset=False,\n drop_remainder=False,\n test_padding=REQUIRED_DATASET_PAD,\n))\n\n####### ####### ####### ####### ####### ####### ####### #######\n\ntrain_ds = get_dataset(**TRAIN_DS_CONFIG)\nval_ds = get_dataset(**VAL_DS_CONFIG)\ntest_ds = get_dataset(**TEST_DS_CONFIG)\n\nfor SPLIT, CONFIG in zip([\"TRAINING\", \"VALIDATION\", \"TESTING\"], [TRAIN_DS_CONFIG, VAL_DS_CONFIG, TEST_DS_CONFIG]): # , TEST_DS_CONFIG]\n print(f\"\\n... {SPLIT} CONFIGURATION:\")\n for k,v in CONFIG.items():\n if k==\"filenames\":\n print(f\"\\t--> {k:<23}: {[path.split('/', 4)[-1] for path in v[:2]]+['...']}\")\n else:\n print(f\"\\t--> {k:<23}: {v}\")\n\nprint(f\"\\n\\n... TRAINING DATASET : {train_ds} ...\")\nprint(f\"... VALIDATION DATASET : {val_ds} ...\")\nprint(f\"... TESTING DATASET : {test_ds} ...\\n\")\n\nprint(\"\\n\\n ... SOME VALIDATION EXAMPLES ... \\n\\n\")\nfor x,y,z in val_ds.take(1):\n for i in range(2):\n plt.figure(figsize=(12,12))\n plt.imshow(x[i].numpy().astype(np.int64))\n plt.title(f\"IMAGE smile : {''.join([int_2_tok[z] for z in y[i].numpy() if z not in [0,1,2]])}\\n\")\n plt.show()\n \nprint(\"\\n\\n ... SOME TESTING EXAMPLES ... \\n\\n\")\nfor x,y in test_ds.take(1):\n for i in range(2):\n plt.figure(figsize=(12,12))\n plt.imshow(x[i].numpy().astype(np.int64))\n plt.title(f\"{y[i].numpy().decode()}\")\n plt.show()","metadata":{"executionInfo":{"elapsed":40617,"status":"ok","timestamp":1621257554969,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"_x_EBOMyUekt","outputId":"3219b420-0e17-464f-ba8f-a2f51d229046","execution":{"iopub.status.busy":"2021-08-27T05:27:16.052013Z","iopub.execute_input":"2021-08-27T05:27:16.052357Z","iopub.status.idle":"2021-08-27T05:27:22.329427Z","shell.execute_reply.started":"2021-08-27T05:27:16.052326Z","shell.execute_reply":"2021-08-27T05:27:22.328323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n - Guide - tf.data: Build TensorFlow Input Pipelines
\n - Guide - Better Performance With the tf.data API
\n - tf.data.Dataset Documentation
\n
\n\n\n\n\n\n
5 MODEL PREPERATION ⤒
\n\nIn this section we prepare the models for training. We will be using a model architecture very similar to that found within the [**Show, Attend, and Tell Research Paper**](https://arxiv.org/pdf/1502.03044.pdf).\n\n\n\n
![](\"https://kelvinxu.github.io/projects/diags/model_diag.png\")
","metadata":{"id":"TTbfDAVyUeku"}},{"cell_type":"markdown","source":"
5.1 UNDERSTANDING THE MODELS - ENCODER
\n\n---\n\nWe will be leveraging an [**EfficientNetV2**](https://arxiv.org/pdf/2104.00298) model to act as the Encoder CNN in our network. \n* On **TPU/GPU/CPU** we will use an **EfficientNetV2-B2** model\n\n\n\n***Basic View of EfficientNetB0 Architecture w/ 380x380x3 Input ... this is very similar to EfficientNetV2***\n
\n\nOur encoder will create feature maps for each image which will in turn be passed to the decoder side of the network. \n\n
\n\n**In the following cell, we will create a function to generate our encoder model.**\n\n\n
\n π NOTE:
- For different encoder architectures we will have a different number of feature maps. i.e. If we utilized EfficientNetV2B7 we would have 2560 feature maps instead of the 1280 feature maps that EfficientNetV2B0 produces. \n
\n\n- For different encoder architectures we will have a different number of feature maps. i.e. If we utilized EfficientNetV2B7 we would have 2560 feature maps instead of the 1280 feature maps that EfficientNetV2B0 produces. \n
\n π REFERENCE:
\n - TF Tutorial β Transformer Model for Language Understanding
\n - TF Tutorial β Image Captioning
\n - TF Tutorial β Neural Machine Translation w/ Attention
\n
","metadata":{"id":"60DcSXjkUeku"}},{"cell_type":"code","source":"# Will yield, at the lowest feature level, (6,12,208) or (72,208)\ndef get_efficientnetv2_backbone(model_name, include_top=False, input_shape=(192,384,3), pooling=None, weights=None):\n # Catch unsupported arguments\n if pooling or weights or include_top:\n raise NotImplementedError(\"\\n...At this time we only want to use the raw \" \\\n \"(no pretraining), headless, features with no pooling ...\\n\")\n backbone = effnetv2_model.EffNetV2Model(model_name=model_name)\n backbone(tf.ones((BATCH_SIZE_DEBUG,*input_shape)), training=False, features_only=True)\n return backbone","metadata":{"id":"GKQ5l39sUekv","execution":{"iopub.status.busy":"2021-08-27T05:27:22.331036Z","iopub.execute_input":"2021-08-27T05:27:22.331657Z","iopub.status.idle":"2021-08-27T05:27:22.338975Z","shell.execute_reply.started":"2021-08-27T05:27:22.33161Z","shell.execute_reply":"2021-08-27T05:27:22.337946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Will yield, at the lowest feature level, (6,12,208) or (72,208)\ndef get_efficientnetv1_backbone(model_name, include_top=False, input_shape=(192,384,3), pooling=None, weights=None):\n# # Catch unsupported arguments\n# if pooling or weights or include_top:\n# raise NotImplementedError(\"\\n...At this time we only want to use the raw \" \\\n# \"(no pretraining), headless, features with no pooling ...\\n\")\n \n backbone = efn.EfficientNetB1(include_top=include_top, input_shape=input_shape, pooling=pooling, weights=weights)\n backbone(tf.ones((BATCH_SIZE_DEBUG,*input_shape)), training=False)\n return backbone","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:22.344115Z","iopub.execute_input":"2021-08-27T05:27:22.344434Z","iopub.status.idle":"2021-08-27T05:27:22.354686Z","shell.execute_reply.started":"2021-08-27T05:27:22.344405Z","shell.execute_reply":"2021-08-27T05:27:22.353291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_angles(pos, i, d_model):\n# angle_rates = tf.constant(1, TARGET_DTYPE) / tf.math.pow(tf.constant(10000, TARGET_DTYPE), (tf.constant(2, dtype=TARGET_DTYPE) * tf.cast((i//2), TARGET_DTYPE))/d_model)\n# return pos * angle_rates\n\ndef get_angles(pos, i, d_model):\n angle_rates = tf.constant(1, tf.bfloat16) / tf.math.pow(tf.constant(10000, tf.bfloat16), (tf.constant(2, dtype=tf.bfloat16) * tf.cast((i//2), tf.bfloat16))/d_model)\n return pos * angle_rates\n\ndef do_interleave(arr_a, arr_b):\n a_arr_tf_column = tf.range(arr_a.shape[1])*2 # [0 2 4 ...]\n b_arr_tf_column = tf.range(arr_b.shape[1])*2+1 # [1 3 5 ...]\n column_indices = tf.argsort(tf.concat([a_arr_tf_column,b_arr_tf_column],axis=-1))\n column, row = tf.meshgrid(column_indices,tf.range(arr_a.shape[0]))\n combine_indices = tf.stack([row,column],axis=-1)\n combine_value = tf.concat([arr_a,arr_b],axis=1)\n return tf.gather_nd(combine_value,combine_indices)\n\ndef positional_encoding_1d(position, d_model):\n angle_rads = get_angles(tf.cast(tf.range(position)[:, tf.newaxis], TARGET_DTYPE),\n tf.cast(tf.range(d_model)[tf.newaxis, :], TARGET_DTYPE),\n d_model)\n \n # apply sin to even indices in the array; 2i\n sin_angle_rads = tf.math.sin(angle_rads[:, ::2])\n cos_angle_rads = tf.math.cos(angle_rads[:, 1::2])\n angle_rads = do_interleave(sin_angle_rads, cos_angle_rads)\n pos_encoding = angle_rads[tf.newaxis, ...]\n return pos_encoding\n\ndef np_positional_encoding_2d(row,col,d_model):\n \"\"\" TBD \"\"\"\n assert d_model % 2 == 0\n row_pos = np.repeat(np.arange(row),col)[:,np.newaxis]\n col_pos = np.repeat(np.expand_dims(np.arange(col),0),row,axis=0).reshape(-1,1)\n\n angle_rads_row = get_angles(row_pos,np.arange(d_model//2)[np.newaxis,:],d_model//2).numpy()\n angle_rads_col = get_angles(col_pos,np.arange(d_model//2)[np.newaxis,:],d_model//2).numpy()\n \n angle_rads_row[:, 0::2] = np.sin(angle_rads_row[:, 0::2])\n angle_rads_row[:, 1::2] = np.cos(angle_rads_row[:, 1::2])\n angle_rads_col[:, 0::2] = np.sin(angle_rads_col[:, 0::2])\n angle_rads_col[:, 1::2] = np.cos(angle_rads_col[:, 1::2])\n pos_encoding = np.concatenate([angle_rads_row,angle_rads_col],axis=1)[np.newaxis, ...]\n return tf.cast(pos_encoding, dtype=TARGET_DTYPE)\n\ndef positional_encoding_2d(row,col,d_model):\n \"\"\" TBD \"\"\"\n row_pos = tf.repeat(tf.range(row), col)[:, tf.newaxis]\n col_pos = tf.reshape(tf.repeat(tf.expand_dims(tf.range(col),0), row, axis=0), (-1, 1))\n\n# angle_rads_row = get_angles(tf.cast(row_pos, tf.float32), tf.range(d_model//2)[tf.newaxis,:], d_model//2)\n# angle_rads_col = get_angles(tf.cast(col_pos, tf.float32), tf.range(d_model//2)[tf.newaxis,:], d_model//2)\n \n angle_rads_row = get_angles(tf.cast(row_pos, tf.bfloat16), tf.range(d_model//2)[tf.newaxis,:], d_model//2)\n angle_rads_col = get_angles(tf.cast(col_pos, tf.bfloat16), tf.range(d_model//2)[tf.newaxis,:], d_model//2)\n\n sin_angle_rads_row = tf.math.sin(angle_rads_row[:, ::2])\n cos_angle_rads_row = tf.math.cos(angle_rads_row[:, 1::2])\n angle_rads_row = do_interleave(sin_angle_rads_row, cos_angle_rads_row)\n\n sin_angle_rads_col = tf.math.sin(angle_rads_col[:, ::2])\n cos_angle_rads_col = tf.math.cos(angle_rads_col[:, 1::2])\n angle_rads_col = do_interleave(sin_angle_rads_col, cos_angle_rads_col)\n \n pos_encoding = tf.concat([angle_rads_row,angle_rads_col],axis=1)[tf.newaxis, ...]\n return pos_encoding\n\npos_encoding = positional_encoding_1d(256, 512)\n\nprint(pos_encoding.shape)\n\nplt.figure(figsize=(6,4))\nplt.pcolormesh(tf.cast(pos_encoding[0], tf.float32), cmap='RdBu')\nplt.xlim((0, 512))\nplt.ylim((0, 256))\nplt.xlabel('Depth', fontweight=\"bold\")\nplt.ylabel('Position', fontweight=\"bold\")\nplt.title(\"Visualization of Positional Encoding\", fontweight=\"bold\")\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:22.356974Z","iopub.execute_input":"2021-08-27T05:27:22.357291Z","iopub.status.idle":"2021-08-27T05:27:22.74093Z","shell.execute_reply.started":"2021-08-27T05:27:22.357261Z","shell.execute_reply":"2021-08-27T05:27:22.739925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... ENCODER MODEL CREATION STARTING ...\\n\")\n\n# SAMPLE IMAGES\nSAMPLE_IMGS, SAMPLE_LBLS, SAMPLE_PATTERNS = next(iter(val_ds.unbatch().batch(BATCH_SIZE_DEBUG)))\n\n# ENCODER_CONFIG\nPREPROCESSING_FN = tf.keras.applications.efficientnet.preprocess_input\nBB_FN = get_efficientnetv2_backbone\n# This will be the dimension the network outputs flattened\nIMG_EMB_DIM = (6,12,208)\nprint(f\"\\n... RAW BB OUTPUT: {IMG_EMB_DIM} ...\\n\")\nIMG_EMB_DIM = (IMG_EMB_DIM[0]*IMG_EMB_DIM[1], IMG_EMB_DIM[2])\n\n\nclass Encoder(tf.keras.Model):\n def __init__(self, image_embedding_dim, preprocessing_fn, backbone_fn, image_shape, do_permute=False, include_top=False, pretrained_weights=None, scale_factor=0):\n \"\"\" TODO\n \n Args:\n TODO \n \n Returns:\n TODO\n \"\"\"\n super(Encoder, self).__init__()\n \n self.image_embedding_dim = image_embedding_dim\n self.preprocessing_fn = preprocessing_fn\n self.encoder_backbone = backbone_fn(model_name=EV2_NAME, include_top=include_top, weights=pretrained_weights, input_shape=image_shape) \n self.reshape = tf.keras.layers.Reshape(self.image_embedding_dim, name='image_embedding')\n self.permute = tf.keras.layers.Permute([2, 1], name='permute_features')\n self.do_permute = do_permute\n self.include_top = include_top\n self.scale_factor = scale_factor\n self.pool = tf.keras.layers.GlobalAveragePooling2D()\n self.bn = tf.keras.layers.BatchNormalization()\n self.bn1 = tf.keras.layers.BatchNormalization()\n self.class_layer = tf.keras.layers.Dense(591, name='class_dense', activation=None)\n self.positional_encoding = positional_encoding_2d(6, 12, 208)\n self.logit_dense_1 = tf.keras.layers.Dense(208, name='logit_dense1', activation=None)\n self.logit_dense_2 = tf.keras.layers.Dense(72, name='logit_dense2', activation='sigmoid')\n \n def call(self, x, training):\n \"\"\" TODO\n \n Args:\n TODO \n \n Returns:\n TODO\n \"\"\"\n x = self.preprocessing_fn(x)\n x = self.encoder_backbone(x, training=training, features_only=not self.include_top)[self.scale_factor]\n # logit:[2, 591]\n # logit = self.class_layer(self.pool(x))\n logit = self.class_layer(self.pool(self.bn(x), training=training))\n \n # x:[2, 72, 208]\n x = self.reshape(self.bn(x, training=training), training=training)\n x = x + self.positional_encoding\n if self.do_permute:\n x = self.permute(x, training=training)\n \n \n img_logit= self.logit_dense_1(logit) \n \n img_logit=tf.expand_dims(img_logit,2)\n img_logit= self.logit_dense_2(img_logit)\n img_logit= tf.transpose(img_logit,perm=[0,2,1])\n# img_logit = self.bn1(img_logit, training=training)\n# x = tf.concat([x,img_logit],1)\n return x, logit, img_logit\n \n# Example enoder output\nwith tf.device('/CPU:0'):\n encoder = Encoder(IMG_EMB_DIM, PREPROCESSING_FN, BB_FN, IMG_SHAPE, do_permute=IMG_EMB_DIM[1]\n - TF Tutorial β Transformer Model for Language Understanding
\n - TF Tutorial β Image Captioning
\n - TF Tutorial β Neural Machine Translation w/ Attention
\n
5.2.0 TRANSFORMER - HYPERPARAMETERS
\n\n---\n","metadata":{}},{"cell_type":"code","source":"IMG_SEQ_LEN, IMG_EMB_DEPTH = IMG_EMB_DIM\nD_MODEL = IMG_EMB_DEPTH\nN_PE_POS = 72\n# D_FF = 1024\nD_FF = 256\n\nprint(f\"\\n... THE INPUT 'SEQUENCE LENGTH' IS {IMG_SEQ_LEN} (output of image encoder - shape flattened) ...\")\nprint(f\"... THE INPUT 'EMBEDDING DEPTH' IS {IMG_EMB_DEPTH} (output of image encoder - # of channels) ...\")\nprint(f\"... THE NUMBER OF POSITIONAL ENNCODING POSITIONS IS {N_PE_POS} (arbitray) ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:39.318186Z","iopub.execute_input":"2021-08-27T05:27:39.318593Z","iopub.status.idle":"2021-08-27T05:27:39.325363Z","shell.execute_reply.started":"2021-08-27T05:27:39.318547Z","shell.execute_reply":"2021-08-27T05:27:39.324102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2.1 TRANSFORMER - POSITIAL ENCODING
\n\n---\n\nSince this model doesn't contain any recurrence or convolution, positional encoding is added to give the model some information about the relative position of the words in the sentence. \n\nThe positional encoding vector is added to the embedding vector. \n* Embeddings represent a token in a **`d-dimensional`** space where tokens (encoded vectors) with similar meaning (feature representation) will be closer to each other. \n\nBut the embeddings do not encode the relative position of words in a sentence (or in our case the localization of features as encoded by our **efficientnetv2 encoder model**).\n* So after adding the positional encoding, words (feature representations) will be closer to each other based on the ***similarity of their meaning and their position in the sentence (feature vector)***, in the **`d-dimensional`** space.\n\nSee the notebook on **[positional encoding](https://www.tensorflow.org/tutorials/text/transformer#positional_encoding)** to learn more about it. The formula for calculating the positional encoding is as follows:\n\n---\n\n$$\\Large{PE_{(pos, 2i)} = sin(pos / 10000^{2i / d_{model}})} $$\n$$\\Large{PE_{(pos, 2i+1)} = cos(pos / 10000^{2i / d_{model}})} $$\n\n---","metadata":{}},{"cell_type":"markdown","source":"5.2.2 TRANSFORMER - MASKING
\n\n---\n\n\nMask all the pad tokens in the batch of sequence. It ensures that the model does not treat padding as the input. \n\nThe mask indicates where pad value **`0`** is present: \n* it outputs a **`1`** at those locations\n* it outputs a **`0`** otherwise.\n\n---\n\nThe **look-ahead mask** is used to mask the future tokens in a sequence. \n\nIn other words, the mask indicates which entries should not be used.\n* This means that to predict the third token, only the first and second tokens will be used. \n* Similarly to predict the fourth token, only the first, second and the third tokens will be used and so on.","metadata":{}},{"cell_type":"code","source":"def create_padding_mask(seq):\n seq = tf.cast(tf.math.equal(seq, 0), TARGET_DTYPE)\n\n # add extra dimensions to add the padding to the attention logits.\n # - (batch_size, 1, 1, seq_len)\n return seq[:, tf.newaxis, tf.newaxis, :]\n\ndef create_look_ahead_mask(size):\n mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n # (seq_len, seq_len)\n return tf.cast(mask, TARGET_DTYPE)\n\ndef create_mask(inp, tar):\n\n # Used in the 1st attention block in the decoder.\n # It is used to pad and mask future tokens in the input received by \n # the decoder.\n look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n dec_target_padding_mask = create_padding_mask(tar)\n combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n return tf.cast(combined_mask, TARGET_DTYPE)\n\nx = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\nprint(f\"x --\\n{x}\\n\\n\\nPADDING MASK\\n{create_padding_mask(x)}\\n\")\nprint(\"\")\n\nx = tf.random.uniform((1, 5))\nprint(f\"x.shape[1] -- {x.shape[1]}\")\nprint(f\"\\n\\nLOOK-AHEAD MASK\\n{create_look_ahead_mask(x.shape[1])}\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:39.326757Z","iopub.execute_input":"2021-08-27T05:27:39.327196Z","iopub.status.idle":"2021-08-27T05:27:39.367334Z","shell.execute_reply.started":"2021-08-27T05:27:39.327163Z","shell.execute_reply":"2021-08-27T05:27:39.366496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2.3 TRANSFORMER - SCALED DOT-PRODUCT ATTENTION
\n\n---\n\nScaled dot-product attention is an attention mechanism where the dot products are scaled down by $\\sqrt{d_k}$. \n\n---\n\n![\"scaled_dot_product_attention\"](\"https://www.tensorflow.org/images/tutorials/transformer/scaled_attention.png\")
5.2.4 TRANSFORMER - MULTI-HEAD ATTENTION
\n\n---\n\nThis is an implementation of multi-headed attention based on [**\"Attention is all you Need\"**](https://arxiv.org/abs/1706.03762). \n* If **`query`**, **`key`**, **`value`** are the same, then this is **self-attention**. \n* Each timestep in query attends to the corresponding sequence in key, and returns a fixed-width vector.\n\n\n\nThis layer (the MHA layer) first projects **`query`**, **`key`** and **`value`**. \n* These are (effectively) a list of tensors of length num_attention_heads, where the corresponding shapes are: \n * **`[batch_size, 1, key_dim]`**, **`[batch_size, 1, key_dim]`**, **`[batch_size, 1, value_dim]`**\n\nThen, the **`query`** and **`key`** tensors are **dot-producted and scaled** (see previous section). These values are softmaxed to obtain attention probabilities. The tensors are then interpolated by these probabilities, then concatenated back to a single tensor.\n\nFinally, the result tensor with the last dimension as value_dim can take an linear projection and return.\n\n---\n\n
\n\n
![\"multi-head](\"https://www.tensorflow.org/images/tutorials/transformer/multi_head_attention.png\")
\n\n\n**Multi-head attention consists of four parts:**\n* Linear layers and split into heads.\n* Scaled dot-product attention.\n* Concatenation of heads.\n* Final linear layer.\n\n---\n\nEach multi-head attention block gets three inputs;\n* **`Q` (query)**\n* **`K` (key)**\n* **`V` (value)**\n\nThese are put through linear (**`Dense`**) layers and split up into multiple heads. \n\nThe **`scaled_dot_product_attention`** defined above is applied to each head (broadcasted for efficiency). \n* An appropriate mask must be used in the attention step. \n* The attention output for each head is then concatenated (using **`tf.transpose`**, and **`tf.reshape`**) and put through a final **`Dense`** layer\n\nInstead of one single attention head, **`Q`**, **`K`**, and **`V`** are split into multiple heads because it allows the model to jointly attend to information at different positions from different representational spaces. \n\nAfter the split each head has a reduced dimensionality, so the total computation cost is the same as a single head attention with full dimensionality.\n\n---\n\nLet's create a **`MultiHeadAttention`** layer to try out. \n* At each location in the sequence, **`y`**, the **`MultiHeadAttention`** runs all **`8`** attention heads across all other locations in the sequence, returning a new vector of the same length at each location.","metadata":{}},{"cell_type":"code","source":"class MultiHeadAttention(tf.keras.layers.Layer):\n def __init__(self, d_model, num_heads):\n super(MultiHeadAttention, self).__init__()\n self.num_heads = num_heads\n self.d_model = d_model\n \n assert d_model % self.num_heads == 0\n \n self.depth = d_model // self.num_heads\n \n self.wq = tf.keras.layers.Dense(d_model)\n self.wk = tf.keras.layers.Dense(d_model)\n self.wv = tf.keras.layers.Dense(d_model)\n \n self.dense = tf.keras.layers.Dense(d_model)\n \n def split_heads(self, x, batch_size):\n \"\"\" Split the last dimension into (num_heads, depth).\n Then we transpose the result such that the shape is \n - (batch_size, num_heads, seq_len, depth)\n \"\"\"\n x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n return tf.transpose(x, perm=[0, 2, 1, 3])\n \n def call(self, v, k, q, mask):\n batch_size = tf.shape(q)[0]\n # (batch_size, seq_len, d_model)\n q = self.wq(q) \n # (batch_size, seq_len, d_model)\n k = self.wk(k) \n # (batch_size, seq_len, d_model)\n v = self.wv(v) \n\n # (batch_size, num_heads, seq_len_q, depth)\n q = self.split_heads(q, batch_size) \n # (batch_size, num_heads, seq_len_k, depth)\n k = self.split_heads(k, batch_size) \n # (batch_size, num_heads, seq_len_v, depth)\n v = self.split_heads(v, batch_size) \n \n # scaled_attention.shape β (batch_size, num_heads, seq_len_q, depth)\n # attention_weights.shape β (batch_size, num_heads, seq_len_q, seq_len_k)\n scaled_attention, attention_weights = \\\n scaled_dot_product_attention(q, k, v, mask)\n \n # (batch_size, seq_len_q, num_heads, depth)\n scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) \n\n # (batch_size, seq_len_q, d_model)\n concat_attention = \\\n tf.reshape(scaled_attention, (batch_size, -1, self.d_model)) \n\n # (batch_size, seq_len_q, d_model)\n output = self.dense(concat_attention) \n \n return output, attention_weights\n\n# CUSTOM\ntemp_mha = MultiHeadAttention(d_model=512, num_heads=8)\ny = tf.random.uniform((1, 60, 512), dtype=TARGET_DTYPE) # (batch_size, encoder_sequence, d_model)\nout, attn = temp_mha(y, y, y, mask=None)\nprint(f\"Custom MHA Layer:\\n\\t-->{out[0,:2]}\\n\\t-->{(out.shape, attn.shape)}\\n\")\n\n# TF NATIVE\ntemp_mha = tf.keras.layers.MultiHeadAttention(8, 512)\nout, attn = temp_mha(y, y, y, attention_mask=None, return_attention_scores=True)\nprint(f\"TF MHA Layer:\\n\\t-->{out[0,:2]}\\n\\t-->{(out.shape, attn.shape)}\\n\")\n\ndel temp_mha","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:39.428037Z","iopub.execute_input":"2021-08-27T05:27:39.428333Z","iopub.status.idle":"2021-08-27T05:27:39.730961Z","shell.execute_reply.started":"2021-08-27T05:27:39.428305Z","shell.execute_reply":"2021-08-27T05:27:39.730167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
5.2.5 TRANSFORMER - POINT-WISE FEED FORWARD NEURAL NETWORK
\n\n---\n\nPoint wise feed forward network consists of two fully-connected layers with a ReLU activation in-between.","metadata":{}},{"cell_type":"code","source":"def point_wise_feed_forward_network(d_model, dff):\n return tf.keras.Sequential([\n \n # INNER LAYER\n # β (batch_size, seq_len, dff)\n tf.keras.layers.Dense(dff, activation='relu'), \n \n # OUTPUT \n # β (batch_size, seq_len, d_model)\n tf.keras.layers.Dense(d_model) \n ])\n\nsample_ffn = point_wise_feed_forward_network(512, D_FF)\nprint(\"\\nFFN INPUT & OUTPUT SHAPE: \" \\\n f\"{sample_ffn(tf.random.uniform((64, 50, 512), dtype=TARGET_DTYPE)).shape}\" \\\n \"\\n\\nFFN SUMMARY:\")\nprint(sample_ffn.summary())\n\ndel sample_ffn","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:39.732114Z","iopub.execute_input":"2021-08-27T05:27:39.732553Z","iopub.status.idle":"2021-08-27T05:27:39.80284Z","shell.execute_reply.started":"2021-08-27T05:27:39.732522Z","shell.execute_reply":"2021-08-27T05:27:39.801647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2.6 TRANSFORMER - ENCODER-DECODER NETWORK ARCHITECTURE OVERVIEW
\n\n---\n\nThe transformer model follows the same general pattern as a standard [sequence to sequence with attention model](nmt_with_attention.ipynb). \n\n* The input sequennce ***(image embedding sequence in our case)*** is passed through **`N` encoder layers** that generates an output for each word/token in the sequence.\n* The **decoder** attends on the encoder's output and its own input (self-attention) to predict the next word/token. \n\n---\n\n![\"transformer\"](\"https://www.tensorflow.org/images/tutorials/transformer/transformer.png\")
\n","metadata":{}},{"cell_type":"markdown","source":"
5.2.7 TRANSFORMER - ENCODER
\n\n---\n\n**Each transformer encoder layer consists of sublayers:**\n\n1. **Multi-Head AAttention (with padding mask)**\n2. **Point-Wise Feed Forward Neural Networks**\n\nEach of these sublayers has a **residual connection** around it followed by a **layer normalization**. \n* Residual connections help in avoiding the vanishing gradient problem in deep networks.\n\nThe output of each sublayer is **`LayerNorm(x + Sublayer(x))`**. \n* The normalization is done on the **`d_model`** (last) axis. \n* There are **`N encoder` layers** in the ***transformer***.","metadata":{}},{"cell_type":"code","source":"class TransformerEncoderLayer(tf.keras.layers.Layer):\n def __init__(self, d_model, num_heads, dff, dropout_rate=0.1):\n \"\"\" Encoder Layer Component Of Transformer Encoder Block \n \n Args:\n d_model: the depth of the d-dimensional space used for positional encoding\n num_heads: The number of heads to use in the multi-head-attention block\n dff (int): Number of units to use in the feed-forward neural network\n drop_out_rate (float): Percentage of nodes to drop in a given layer\n \n Returns:\n none; This is an intiailization\n \"\"\"\n \n super(TransformerEncoderLayer, self).__init__()\n\n self.mha = tf.keras.layers.MultiHeadAttention(num_heads, key_dim=d_model,)\n self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n \n self.dropout1 = tf.keras.layers.Dropout(dropout_rate)\n self.dropout2 = tf.keras.layers.Dropout(dropout_rate)\n \n def call(self, x, training, mask=None):\n \"\"\" Call function for our encoder layer\n \n Args:\n self (class attributes): For the given layer, this exposes the attributes\n x (): TBD\n training (bool): Whether or not to apply certain operations (i.e. disable/enable dropout)\n mask (): tbd\n \n Returns:\n The encoded input sequence\n - shape --> (batch_size, input_seq_len, d_model)\n \"\"\"\n \n # returns --> (batch_size, input_seq_len, d_model)\n attn_output, _ = self.mha(x, x, x, mask, return_attention_scores=True) \n\n # Potentially unncessary by passing dropout1 to tf.keras.layers.MultiHeadAttention (if using tf MHA)\n attn_output = self.dropout1(attn_output, training=training)\n \n # Residual connection followed by layer normalization\n # β returns --> (batch_size, input_seq_len, d_model)\n out1 = self.layernorm1(x + attn_output, training=training)\n \n # Point-wise Feed Forward Step\n # β returns --> (batch_size, input_seq_len, d_model)\n ffn_output = self.ffn(out1, training=training) \n ffn_output = self.dropout2(ffn_output, training=training)\n \n\n # Residual connection followed by layer normalization\n # β returns --> (batch_size, input_seq_len, d_model)\n out2 = self.layernorm2(out1 + ffn_output, training=training) \n \n return out2\n\nsample_encoder_layer = TransformerEncoderLayer(D_MODEL, 8, D_FF)\nsample_encoder_layer_output = sample_encoder_layer(img_embedding_batch, training=False, mask=None)\n# sample_encoder_layer_output = sample_encoder_layer(img_logit, training=False, mask=None)\n\n\n\ndel sample_encoder_layer\n\n# (batch_size, input_seq_len, d_model)\nsample_encoder_layer_output.shape ","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:39.806083Z","iopub.execute_input":"2021-08-27T05:27:39.806389Z","iopub.status.idle":"2021-08-27T05:27:40.296145Z","shell.execute_reply.started":"2021-08-27T05:27:39.806359Z","shell.execute_reply":"2021-08-27T05:27:40.295145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2.8 TRANSFORMER - DECODER LAYER COMPONENT
\n\n---\n\n**Each transformer decoder layer consists of sublayers:**\n\n1. **Masked Multi-Head Attention (with look ahead mask and padding mask)**\n2. **Multi-Head Attention (with padding mask)** \n * **`V`** (value) and **`K`** (key) receive the ***encoder output*** as inputs. \n * **`Q`** (query) receives the ***output from the masked multi-head attention sublayer.***\n3. **Point-Wise Feed Forward Networks**\n\nEach of these sublayers has a **residual connection** around it followed by a **layer normalization**\n* The output of each sublayer is **`LayerNorm(x + Sublayer(x))`**\n* The normalization is done on the **`d_model`** (last) axis.\n\nThere are **`N` decoder layers** in the ***transformer***\n\nAs **`Q`** receives the output from decoder's first attention block, and **`K`** receives the encoder output, the attention weights represent the importance given to the decoder's input based on the encoder's output. \n* In other words, the decoder predicts the next word/token by looking at the encoder output and self-attending to its own output. \n* See the demonstration above in the scaled dot product attention section.","metadata":{}},{"cell_type":"code","source":"class TransformerDecoderLayer(tf.keras.layers.Layer):\n def __init__(self, d_model, num_heads, dff, dropout_rate=0.1):\n \"\"\" Decoder Layer Component Of Transformer Decoder Block \n \n Args:\n d_model: the depth of the d-dimensional space used for positional encoding of image embedding\n num_heads: The number of heads to use in the multi-head-attention block\n dff (int): Number of units to use in the feed-forward neural network\n dropout_rate (float): Percentage of nodes to drop in a given layer\n \n Returns:\n none; This is an intiailization\n \"\"\"\n super(TransformerDecoderLayer, self).__init__()\n\n # WE COULD USE A CUSTOM DEFINED MHA MODEL BUT WE WILL USE TFA INSTEAD\n self.mha1 = MultiHeadAttention(d_model, num_heads)\n self.mha2 = MultiHeadAttention(d_model, num_heads)\n self.mha3 = MultiHeadAttention(d_model, num_heads)\n #\n # # Multi Head Attention Layers\n # self.mha1 = tf.keras.layers.MultiHeadAttention(num_heads, key_dim=d_model,)\n # self.mha2 = tf.keras.layers.MultiHeadAttention(num_heads, key_dim=d_model,)\n\n # Feed Forward NN\n self.ffn = point_wise_feed_forward_network(d_model, dff)\n \n # Layer Normalization Layers\n self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n \n # Dropout Layers\n self.dropout1 = tf.keras.layers.Dropout(dropout_rate)\n self.dropout2 = tf.keras.layers.Dropout(dropout_rate)\n self.dropout3 = tf.keras.layers.Dropout(dropout_rate)\n \n # enc_output.shape == (batch_size, input_seq_len, d_model)\n def call(self, x, enc_output1, enc_output2, training, look_ahead_mask=None, padding_mask=None):\n# def call(self, x, enc_output1, training, look_ahead_mask=None, padding_mask=None):\n \"\"\" Call function for our encoder layer\n \n Args:\n self (class attributes): For the given layer, this exposes the attributes\n x (tf.float32 array): token embeddinng (batch_size, output_seq_len, embedding_dim)\n enc_output (tf.float32 array): token embeddinng (batch_size, output_seq_len, embedding_dim)\n training (bool): Whether or not to apply certain operations (i.e. disable/enable dropout)\n mask (): tbd\n \n Returns:\n The encoded input sequence\n - shape --> (batch_size, input_seq_len, d_model)\n \"\"\"\n \n attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)\n attn1 = self.dropout1(attn1, training=training)\n\n # Residual connection followed by layer normalization\n # β (batch_size, target_seq_len, d_model)\n out1 = self.layernorm1(attn1 + x, training=training)\n \n # Merging connection between encoder and decoder (MHA)\n # β (batch_size, target_seq_len, d_model)\n \n# attn2, attn_weights_block2 = self.mha2(enc_output1, enc_output2, out1, padding_mask) \n# enc_output1=enc_output1+enc_output2\n \n \n attn2, attn_weights_block2 = self.mha3(enc_output1, enc_output1, enc_output2, padding_mask) \n attn2, attn_weights_block2 = self.mha2(attn2, attn2, out1, padding_mask) \n\n \n# attn2, attn_weights_block2 = self.mha2(enc_output1, enc_output1, out1, padding_mask) \n \n attn2 = self.dropout2(attn2, training=training)\n \n # Residual connection followed by layer normalization\n # β (batch_size, target_seq_len, d_model)\n out2 = self.layernorm2(attn2 + out1, training=training) \n \n # (batch_size, target_seq_len, d_model)\n ffn_output = self.ffn(out2, training=training) \n ffn_output = self.dropout3(ffn_output, training=training)\n\n # Residual connection followed by layer normalization\n # β (batch_size, target_seq_len, d_model)\n out3 = self.layernorm3(ffn_output + out2, training=training) \n \n return out3, attn_weights_block1, attn_weights_block2\n\nsample_decoder_layer = TransformerDecoderLayer(D_MODEL, 8, D_FF)\nsample_decoder_layer_output, _, _ = sample_decoder_layer(tf.random.uniform((BATCH_SIZE_DEBUG,int(MAX_LEN), D_MODEL), dtype=TARGET_DTYPE), sample_encoder_layer_output,img_logit, False, None, None)\n# sample_decoder_layer_output, _, _ = sample_decoder_layer(tf.random.uniform((BATCH_SIZE_DEBUG, int(MAX_LEN), D_MODEL), dtype=TARGET_DTYPE), sample_encoder_layer_output, False, None, None)\ndel sample_decoder_layer\n\n# sample_decoder_layer_output.shape # (batch_size, target_seq_len, d_model)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:40.297488Z","iopub.execute_input":"2021-08-27T05:27:40.2978Z","iopub.status.idle":"2021-08-27T05:27:41.449005Z","shell.execute_reply.started":"2021-08-27T05:27:40.297772Z","shell.execute_reply":"2021-08-27T05:27:41.447744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2.9 TRANSFORMER - ENCODER COMPONENT
\n\n---\n\nThe **`TransformerEncoder`** consists of:\n1. Input Embedding\n2. Positional Encoding\n3. **`N`** encoder layers\n\n\n\nThe input is put through an embedding which is summed with the positional encoding. \n* The output of this summation is the input to the encoder layers. The output of the encoder is the input to the decoder.","metadata":{}},{"cell_type":"code","source":"class TransformerEncoder(tf.keras.layers.Layer):\n def __init__(self, num_layers, d_model, num_heads, dff,\n maximum_position_encoding, dropout_rate=0.1):\n super(TransformerEncoder, self).__init__()\n\n self.d_model = d_model\n self.num_layers = num_layers\n self.pos_encoding = positional_encoding_1d(maximum_position_encoding, self.d_model)\n self.enc_layers = [TransformerEncoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)]\n self.dropout = tf.keras.layers.Dropout(dropout_rate)\n \n def call(self, x, training, mask=None):\n \"\"\"\n Sequence of Operations:\n 1. Embed the input data as a fixed length vector\n 2. Scale the fixed length vector by the square root of the \n input/output dimensionality\n 3. Introduce the position encoding into the data\n 4. Perform some amount of dropout\n 5. Pass our preprocessed input data into a stack of encoding layers\n along with the input mask\n\n\n \"\"\"\n\n # adding embedding and position encoding.\n # β (batch_size, input_seq_len, d_model)\n x += self.pos_encoding\n x = self.dropout(x, training=training)\n \n for i in range(self.num_layers):\n x = self.enc_layers[i](x, training, mask)\n \n # β (batch_size, input_seq_len, d_model)\n return x \n\nsample_encoder = TransformerEncoder(num_layers=2, \n d_model=D_MODEL, \n num_heads=8, \n dff=D_FF,\n maximum_position_encoding=72)\n# maximum_position_encoding=73)\n \n\n# print(img_embedding_batch)\n# print(img_embedding_batch.shape)\nsample_encoder_output = sample_encoder(img_embedding_batch, training=False, mask=None)\n\n# sample_encoder_output = sample_encoder(img_logit, training=False, mask=None)\n\ndel sample_encoder\n\nprint(sample_encoder_output.shape) # (batch_size, input_seq_len, d_model)","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:41.450867Z","iopub.execute_input":"2021-08-27T05:27:41.451326Z","iopub.status.idle":"2021-08-27T05:27:41.914805Z","shell.execute_reply.started":"2021-08-27T05:27:41.451275Z","shell.execute_reply":"2021-08-27T05:27:41.913529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
5.2.10 TRANSFORMER - DECODER COMPONENT
\n\n---\n\n1. Output Embedding\n2. Positional Encoding\n3. **`N`** decoder layers\n\n\n\nThe target is put through an **embedding** which is **summed with the positional encoding.**\n* The output of this summation is the input to the decoder layers. \n* The output of the decoder is the input to the final linear layer.","metadata":{}},{"cell_type":"code","source":"class TransformerDecoder(tf.keras.layers.Layer):\n def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, dropout_rate=0.1):\n super(TransformerDecoder, self).__init__()\n self.d_model = d_model\n self.num_layers = num_layers\n \n self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n self.pos_encoding = positional_encoding_1d(maximum_position_encoding, d_model)\n \n self.dec_layers = [TransformerDecoderLayer(d_model, num_heads, dff, dropout_rate) for _ in range(num_layers)]\n self.dropout = tf.keras.layers.Dropout(dropout_rate)\n \n def call(self, x, enc_output1, enc_output2, training, look_ahead_mask=None, padding_mask=None):\n# def call(self, x, enc_output1, training, look_ahead_mask=None, padding_mask=None):\n\n seq_len = tf.shape(x)[1]\n attention_weights = {}\n \n # adding embedding and position encoding.\n # β (batch_size, target_seq_len, d_model)\n x = self.embedding(x) \n x *= tf.math.sqrt(tf.cast(self.d_model, TARGET_DTYPE))\n x += self.pos_encoding[:, :seq_len, :]\n \n x = self.dropout(x, training=training)\n\n for i in range(self.num_layers):\n x, block1, block2 = self.dec_layers[i](x, enc_output1, enc_output2, training, look_ahead_mask, padding_mask)\n# x, block1, block2 = self.dec_layers[i](x, enc_output1, training, look_ahead_mask, padding_mask)\n\n attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n \n # x.shape == (batch_size, target_seq_len, d_model)\n return x, attention_weights\n\n\nsample_decoder = TransformerDecoder(num_layers=2, d_model=D_MODEL, num_heads=8, \n dff=D_FF, target_vocab_size=VOCAB_LEN,\n maximum_position_encoding=MAX_LEN)\ntemp_input = tf.random.uniform((BATCH_SIZE_DEBUG, MAX_LEN), dtype=tf.int64, minval=0, maxval=VOCAB_LEN)\noutput, attn = sample_decoder(temp_input, \n enc_output1=sample_encoder_output, \n enc_output2=img_logit,\n training=False,\n look_ahead_mask=None, \n padding_mask=None)\ndel sample_decoder\noutput.shape, attn['decoder_layer2_block2'].shape","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:41.916467Z","iopub.execute_input":"2021-08-27T05:27:41.916972Z","iopub.status.idle":"2021-08-27T05:27:42.43958Z","shell.execute_reply.started":"2021-08-27T05:27:41.916914Z","shell.execute_reply":"2021-08-27T05:27:42.438596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
5.2.11 TRANSFORMER - PUT IT ALL TOGETHER
\n\n---\n\n\nOur Transformer consists of the **transformer encoder**, **transformer decoder** and a **final linear layer**. \n* The input to the encoder is the output of our image encoder (i.e. output of EfficientNetV2)\n* The output of the decoder is the input to the linear layer and its output is returned.","metadata":{}},{"cell_type":"code","source":"class Transformer(tf.keras.Model):\n def __init__(self, num_layers=2, d_model=D_MODEL, num_heads=8, dff=1024,\n target_vocab_size=VOCAB_LEN, pe_input=IMG_EMB_DIM[0], pe_target=MAX_LEN, dropout_rate=0.1):\n \"\"\"TBD\"\"\"\n super(Transformer, self).__init__()\n \n # self.t_encoder = TransformerEncoder(num_layers, d_model, num_heads, dff, pe_input, dropout_rate)\n self.t_decoder = TransformerDecoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, dropout_rate)\n self.t_final_layer = tf.keras.layers.Dense(target_vocab_size)\n \n def call(self, enc_output1, enc_output2, t_tar, \n# def call(self, enc_output1, t_tar, \n training, enc_padding_mask=None, \n look_ahead_mask=None, dec_padding_mask=None):\n \"\"\"TBD\"\"\"\n # Modified by zjh, bypass transformer encoder\n # (batch_size, inp_seq_len, d_model)\n # enc_output = self.t_encoder(t_inp, training, enc_padding_mask) \n \n # dec_output.shape == (batch_size, tar_seq_len, d_model)\n# dec_output, attention_weights = self.t_decoder(t_tar, \n# enc_output, \n# training, \n# look_ahead_mask, \n# dec_padding_mask)\n dec_output, attention_weights = self.t_decoder(t_tar, \n enc_output1,\n enc_output2, \n training, \n look_ahead_mask, \n dec_padding_mask)\n # (batch_size, tar_seq_len, target_vocab_size)\n final_output = self.t_final_layer(dec_output) \n \n return final_output, attention_weights\n\n\nsample_transformer = Transformer(num_layers=2, d_model=D_MODEL, \n num_heads=8, dff=1024, \n target_vocab_size=VOCAB_LEN, \n pe_input=IMG_EMB_DIM[0], pe_target=MAX_LEN)\nfn_out, _ = sample_transformer(img_embedding_batch,img_logit, SAMPLE_LBLS, training=False, \n# fn_out, _ = sample_transformer(img_embedding_batch, SAMPLE_LBLS, training=False, \n enc_padding_mask=None, \n look_ahead_mask=None,\n dec_padding_mask=None)\n\n# (batch_size, tar_seq_len, target_vocab_size)\nprint(fn_out.shape)\n\ndel sample_transformer","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:42.440953Z","iopub.execute_input":"2021-08-27T05:27:42.441268Z","iopub.status.idle":"2021-08-27T05:27:42.952953Z","shell.execute_reply.started":"2021-08-27T05:27:42.441237Z","shell.execute_reply":"2021-08-27T05:27:42.952084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.3 CREATE A LEARNING RATE SCHEDULER
\n\n---\n\nAdapting the learning rate for your stochastic gradient descent optimization procedure can increase performance and reduce training time. Sometimes this is called learning rate annealing or adaptive learning rates. Here we will call this approach a learning rate schedule. See [**this article**](https://machinelearningmastery.com/using-learning-rate-schedules-deep-learning-models-python-keras/) for a basic tutorial on learning rade schedules.\n\n\nWe will utilize a basic step function following a warmup phase. Warmup is commonly used in learning rate schedule where we start training a model with a much smaller learning rate and increase it during the first few epochs/steps until the initial learning rate is used.\n\nIntuitively, this method will allow a model to adjust itself less before it becomes more familiar with the dataset. This usually prevents breaking pretrained weights. For adaptive optimisers like Adam, warmup also allows the optimizers to compute bettere statistics of the gradients.","metadata":{"id":"1RpQcq0oUeky"}},{"cell_type":"code","source":"# print(\"\\n... LEARNING RATE SCHEDULE CREATION STARTING ...\\n\")\n\n# # Part of the Training Configuration\n# EPOCHS = 36\n# TOTAL_STEPS = TRAIN_STEPS*EPOCHS\n\n# # Learning Rate Scheduler Configuration\n# WARM_STEPS = 100\n# WARM_START_LR = 1e-5\n# PEAK_START_LR = 2e-3\n# FINAL_LR = 1e-4\n\n# def lr_schedule_fn(step, total_steps, warm_lr_start, warm_steps, peak_lr_start, lr_final, n_epochs):\n# \"\"\" Function to generate the learning rate for a given step based on parameters\n \n# Args:\n# step (int): The current step for which to calculate the respective learning rate\n# total_steps (int): The total number of steps for the entire training regime\n# warm_lr_start (float): The starting learning rate prior to warmup\n# warm_steps (int): The number of steps for which the learning rate will ramp up\n# to the desired peak learning rate value (more steps will result in less\n# dramatic changes to existing weights... better for pretrained models)\n# peark_lr_start (float): The starting learning rate after warmup (peak value)\n# lr_final (float): The final learning rate to step down to by the end of training\n# n_epochs (int): The total number of epochs for the training regime\n \n# Returns:\n# The learning rate (float) to be used for a given step\n# \"\"\"\n \n# # exponential warmup\n# if step < warm_steps:\n# warmup_factor = (step / warm_steps) ** 2\n# lr_rate = warm_lr_start + (peak_lr_start - warm_lr_start) * warmup_factor \n \n# # staircase decay\n# else:\n# power = (step - warm_steps) // ((total_steps - warm_steps) / (n_epochs + 1))\n# decay_factor = ((peak_lr_start / lr_final) ** (1 / n_epochs)) ** power\n# lr_rate = peak_lr_start / decay_factor\n \n# return round(lr_rate, 8)\n\n\n# def plot_lr_schedule(lr_schedule, name=\"\"):\n# \"\"\" Plot the learning rate schedule over the course of training\n \n# Args:\n# lr_schedule (list of floats): The values to use for the LR over the\n# course of training\n# name (str, optional): A name for the LR schedule\n \n# Returns:\n# None; A plot of the how the learning rate changes over time will be displayed\n \n# \"\"\"\n# schedule_info = f'start: {lr_schedule[0]:.6f}, max: {max(lr_schedule):.6f}, final: {lr_schedule[-1]:.6f}'\n# plt.figure(figsize=(18,6))\n# plt.plot(lr_schedule)\n# plt.title(f\"Step Learning Rate Schedule {name+', ' if name else name}{schedule_info}\", size=16, fontweight=\"bold\")\n# plt.grid()\n# plt.show()\n \n \n# class LRS():\n# \"\"\" LEARNING RATE SCHEDULER OBJECT\"\"\"\n# def __init__(self, optimizer, lr_schedule):\n# self.opt = optimizer\n# self.lr_schedule = lr_schedule\n \n# # assign initial learning rate\n# self.lr = lr_schedule[0]\n# self.opt.learning_rate.assign(self.lr)\n \n# def step(self, step):\n# self.lr = self.lr_schedule[step]\n# # assign learning rate to optimizer\n# self.opt.learning_rate.assign(self.lr)\n \n# def get_counter(self):\n# return self.c\n \n# def get_lr(self):\n# return self.lr\n\n# # Create the Schedule and Plot\n# lr_schedule = [\n# lr_schedule_fn(step, TOTAL_STEPS, WARM_START_LR, WARM_STEPS, PEAK_START_LR, FINAL_LR, EPOCHS) \\\n# for step in range(TOTAL_STEPS)\n# ]\n# plot_lr_schedule(lr_schedule)\n\n# print(\"\\n... LEARNING RATE SCHEDULE CREATION FINISHED ...\\n\")","metadata":{"executionInfo":{"elapsed":2721,"status":"ok","timestamp":1621257760600,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"FaGpq5KLUeky","outputId":"30d75d9e-fdd3-491a-8860-09a9e7ee5dfa","execution":{"iopub.status.busy":"2021-08-27T05:27:42.954316Z","iopub.execute_input":"2021-08-27T05:27:42.954758Z","iopub.status.idle":"2021-08-27T05:27:42.960527Z","shell.execute_reply.started":"2021-08-27T05:27:42.954726Z","shell.execute_reply":"2021-08-27T05:27:42.959205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... LEARNING RATE SCHEDULE CREATION STARTING ...\\n\")\n\n# Part of the Training Configuration\nEPOCHS = 100\nTOTAL_STEPS = TRAIN_STEPS*EPOCHS\n\n# Learning Rate Scheduler Configuration\nWARM_STEPS = (TRAIN_STEPS-1)*4 # Suuuuuper long ramp-up\n\nclass CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n def __init__(self, d_model, warmup_steps=4000):\n super(CustomSchedule, self).__init__()\n\n self.d_model = tf.cast(d_model, tf.float32)\n self.warmup_steps = warmup_steps\n\n def __call__(self, step):\n arg1 = tf.math.rsqrt(step)\n arg2 = step * (self.warmup_steps ** -1.5)\n\n return tf.math.rsqrt(self.d_model*1.75) * tf.math.minimum(arg1, arg2)\n \ntemp_learning_rate_schedule = CustomSchedule(D_MODEL, WARM_STEPS)\nplt.plot(temp_learning_rate_schedule(tf.range(TRAIN_STEPS*EPOCHS, dtype=tf.float32)))\nplt.ylabel(\"Learning Rate\")\nplt.xlabel(\"Train Step\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:42.961876Z","iopub.execute_input":"2021-08-27T05:27:42.962185Z","iopub.status.idle":"2021-08-27T05:27:43.147652Z","shell.execute_reply.started":"2021-08-27T05:27:42.962155Z","shell.execute_reply":"2021-08-27T05:27:43.146633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.4 WRAP THE CONFIGURATION DETAILS IN A CLASS OBJECT FOR EASY ACCESS
\n\n---\n","metadata":{"id":"DYWxCU9-Uekz"}},{"cell_type":"code","source":"class Config():\n def __init__(self,):\n self.encoder_config = {}\n self.decoder_config = {}\n self.lr_config = {}\n def initialize_encoder_config(self, image_embedding_dim, preprocessing_fn, backbone_fn, image_shape, do_permute=False, pretrained_weights=None):\n self.encoder_config = dict(\n image_embedding_dim=image_embedding_dim, \n preprocessing_fn=preprocessing_fn, \n backbone_fn=backbone_fn, \n image_shape=image_shape, \n do_permute=do_permute, \n pretrained_weights=pretrained_weights,\n )\n def initialize_decoder_config(self, vocab_len, attn_emb_depth, img_emb_dim, n_rnn_units, dropout_rate=0.05):\n self.decoder_config = dict(\n vocab_len=vocab_len, \n attn_emb_depth=attn_emb_depth, \n img_emb_dim=img_emb_dim, \n n_rnn_units=n_rnn_units, \n dropout_rate=dropout_rate,\n )\n# def initialize_lr_config(self, total_steps, warm_lr_start, warm_steps, peak_lr_start, lr_final, n_epochs):\n# self.lr_config = dict(\n# total_steps=total_steps, \n# warm_lr_start=warm_lr_start, \n# warm_steps=warm_steps, \n# peak_lr_start=peak_lr_start, \n# lr_final=lr_final, \n# n_epochs=n_epochs,\n# )\n def initialize_lr_config(self, warm_steps, n_epochs):\n self.lr_config = dict(\n warm_steps=warm_steps, \n n_epochs=n_epochs,\n )\n \ntraining_config = Config()\ntraining_config.initialize_encoder_config(image_embedding_dim=IMG_EMB_DIM, \n preprocessing_fn=PREPROCESSING_FN, \n backbone_fn=BB_FN, \n image_shape=IMG_SHAPE, \n do_permute=IMG_EMB_DIM[1]5.5 HOW TPU IMPACTS MODELS, METRICS, AND OPTIMIZERS
\n\nIn order to use TPU, or [**tensorflow distribute strategy**](https://www.tensorflow.org/api_docs/python/tf/distribute) in general, certain objects will have to be created inside the **strategy's scope**\n\n---\n\nHere is the rule of thumb:\n\n---\n\n* Anything that creates variables that will be used in a distributed way must be created inside **`strategy.scope()`**.\n* This includes, but is not limited to:\n - model creation\n - optimizer\n - metrics\n - sometimes, checkpoint restore\n - any custom code that creates distributed variables\n* Once a variable is created inside a strategy's scope, it captures the strategy's information, and **you can use it outside the strategy's scope.**\n* Unless using a high level API like **`model.fit()`**, defining something within the strategy's scope **WILL NOT automatically distribute the computation**. This will be discussed more in the section on training further down.\n\n---\n\nInside the scope, everything is defined in the same way it would be outside the distribution strategy. There is, however, a particularity about the loss function which we will discuss further down as well.\n\n**In the next cell, we instantiate the learning rate function, the loss object, and the model(s) inside the scope**\n\n","metadata":{"id":"TGbuWYKNUekz"}},{"cell_type":"code","source":"# @tf.function\ndef KerasFocalLoss(target, input):\n gamma = 2.\n target = tf.cast(target, tf.bfloat16)\n input = tf.cast(input, tf.bfloat16)\n max_val = K.relu(-input)\n loss = input - input * target + max_val + K.log(K.exp(-max_val) + K.exp(-input - max_val))\n invprobs = tf.math.log_sigmoid(-input * (target * 2.0 - 1.0))\n loss = K.exp(invprobs * gamma) * loss\n return K.mean(loss, axis=1)\n# return K.mean(K.sum(loss, axis=1))","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:27:43.163872Z","iopub.execute_input":"2021-08-27T05:27:43.164207Z","iopub.status.idle":"2021-08-27T05:27:43.178093Z","shell.execute_reply.started":"2021-08-27T05:27:43.164178Z","shell.execute_reply":"2021-08-27T05:27:43.177083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... TRAINING PREPERATION STARTING ...\\n\")\nimport tensorflow.keras.backend as K\n\ndef prepare_for_training(lr_config, encoder_config, decoder_config, encoder_wts=None, decoder_wts=None, verbose=0):\n \"\"\" Declare required objects under TPU session scope and return ready for training\n \n Args:\n lr_config (dict): Keyword arguments mapped to desired values for lr schedule function\n encoder_config (dict): Keyword arguments mapped to desired values for encoder model instantiation\n decoder_config (dict): Keyword arguments mapped to desired values for decoder model instantiation \n encoder_wts (str, optional): Path to pretrained model weights for encoder\n decoder_wts (str, optional): Path to pretrained model weights for decoder\n verbose (bool, optional): Whether or not to print model information and plot lr schedule\n \n Returns:\n loss_fn - TBD\n metrics - TBD\n optimizer - TBD\n lr_scheduler - TBD\n encoder - TBD\n decoder - TBD\n \n \"\"\"\n # Everything must be declared within the scope when leveraging the TPU strategy\n # - This will still function properly if scope is set to another type of accelerator\n with strategy.scope():\n \n print(\"\\t--> CREATING LOSS FUNCTION ...\")\n # Declare the loss object\n # - Sparse categorical cross entropy loss is used as root loss\n loss_func_seq = tf.keras.losses.SparseCategoricalCrossentropy(\n from_logits=True, reduction=tf.keras.losses.Reduction.NONE\n )\n loss_func_cls = tf.keras.losses.BinaryCrossentropy(\n from_logits=True, reduction=tf.keras.losses.Reduction.NONE\n )\n \n def loss_seq(real, pred):\n # Convert to uint8\n mask = tf.math.not_equal(real, 0)\n loss_ = loss_func_seq(real, pred)\n loss_ *= tf.cast(mask, dtype=loss_.dtype)\n\n # https://www.tensorflow.org/tutorials/distribute/custom_training#define_the_loss_function\n loss_ = tf.nn.compute_average_loss(loss_, global_batch_size=REPLICA_BATCH_SIZE)\n return loss_\n \n def loss_cls(real, pred):\n loss_ = KerasFocalLoss(real, pred)\n\n # https://www.tensorflow.org/tutorials/distribute/custom_training#define_the_loss_function\n loss_ = tf.nn.compute_average_loss(loss_, global_batch_size=REPLICA_BATCH_SIZE)\n return loss_*10\n \n \n # def loss_fn(real, pred):\n # per_example_loss = loss_object(real, pred)\n # return tf.nn.compute_average_loss(per_example_loss, global_batch_size=OVERALL_BATCH_SIZE)\n \n # Declare the metrics\n # - Loss (train only) and sparse categorical accuracy will be used\n print(\"\\t--> CREATING METRICS ...\")\n metrics = {\n 'batch_loss_seq':tf.keras.metrics.Mean(),\n 'batch_loss_cls':tf.keras.metrics.Mean(),\n 'train_loss_seq': tf.keras.metrics.Mean(),\n 'train_loss_cls': tf.keras.metrics.Mean(),\n 'train_acc_seq': tf.keras.metrics.SparseCategoricalAccuracy(),\n 'val_loss_seq': tf.keras.metrics.Mean(),\n 'val_loss_cls': tf.keras.metrics.Mean(),\n 'val_acc_seq': tf.keras.metrics.SparseCategoricalAccuracy(),\n 'val_lsd_seq': tf.keras.metrics.Mean(), \n }\n \n print(\"\\t--> CREATING OPTIMIZER ...\")\n \n lr_scheduler = CustomSchedule(D_MODEL, lr_config[\"warm_steps\"])\n \n # Instiate an optimizer\n optimizer = tf.keras.optimizers.Adam(lr_scheduler)\n \n print(\"\\t--> CREATING LEARNING RATE SCHEDULER ...\")\n \n# # Declare the learning rate schedule (try this as actual lr schedule and list...)\n# lr_schedule = [\n# lr_schedule_fn(step=step, **lr_config) \\\n# for step in range(lr_config[\"total_steps\"])\n# ]\n# lr_scheduler = LRS(optimizer, lr_schedule)\n \n # Instantiate the encoder model \n print(\"\\t--> CREATING ENCODER MODEL ARCHITECTURE ...\")\n encoder = Encoder(**encoder_config)\n initialization_batch,logit, initialization_logit = encoder(\n tf.ones(((REPLICA_BATCH_SIZE,)+encoder_config[\"image_shape\"]), dtype=TARGET_DTYPE), \n training=False,\n )\n # Instantiate the decoder model\n# print(\"\\t--> CREATING DECODER MODEL ARCHITECTURE...\")\n# decoder = Transformer(**decoder_config) \n \n decoder = Transformer(num_layers=2,\n d_model=D_MODEL,\n num_heads=8,\n dff=D_FF)\n# decoder(initialization_batch, tf.random.uniform((REPLICA_BATCH_SIZE, 1)), training=False)\n decoder(initialization_batch, initialization_logit, tf.random.uniform((REPLICA_BATCH_SIZE, 1)), training=False)\n \n \n # Load weights after variable initialization\n if encoder_wts is not None:\n print(\"\\t--> LOADING ENCODER MODEL WEIGHTS ...\")\n encoder.load_weights(encoder_wts)\n if decoder_wts is not None:\n print(\"\\t--> LOADING DECODER MODEL WEIGHTS ...\")\n decoder.load_weights(decoder_wts)\n \n # Show the model architectures and plot the learning rate\n if verbose:\n print(\"\\n\\n... ENCODER MODEL SUMMARY...\\n\")\n print(encoder.summary())\n\n print(\"\\n\\n... DECODER MODEL SUMMARY...\\n\")\n print(decoder.summary())\n\n# print(\"\\n\\n... LR SCHEDULE PLOT...\\n\")\n# plot_lr_schedule(lr_schedule)\n \n return loss_seq, loss_cls, metrics, optimizer, lr_scheduler, encoder, decoder\n \n \nprint(\"\\n... GENERATING THE FOLLOWING:\")\n# Instantiate our required training components in the correct scope\nloss_seq, loss_cls, metrics, optimizer, lr_scheduler, encoder, decoder = \\\n prepare_for_training(lr_config=training_config.lr_config,\n encoder_config=training_config.encoder_config,\n decoder_config=training_config.decoder_config,\n encoder_wts=(ENCODER_CKPT_PATH if ENCODER_CKPT_PATH!=\"\" else None),\n decoder_wts=(DECODER_CKPT_PATH if DECODER_CKPT_PATH!=\"\" else None),\n verbose=1,)\n\nprint(\"\\n... TRAINING PREPERATION FINISHED ...\\n\")","metadata":{"executionInfo":{"elapsed":33461,"status":"ok","timestamp":1621257791359,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"bbTV0zMVUek0","outputId":"ee4aba64-3dd4-4076-f832-b33f71994b16","execution":{"iopub.status.busy":"2021-08-27T05:27:43.179755Z","iopub.execute_input":"2021-08-27T05:27:43.180072Z","iopub.status.idle":"2021-08-27T05:28:42.097901Z","shell.execute_reply.started":"2021-08-27T05:27:43.180042Z","shell.execute_reply":"2021-08-27T05:28:42.095917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.6 LOSS CLASSES AND REDUCTION
\n\nIn order to accurately calculate loss when leveraging a TPU, we have to accumulate the losses that will be calculated across the individual replicas. Knowing this we are limited to using a **`reduction`** value of **`SUM`** or **`NONE`** as the default value and some of the other options will not work with TPU.\n\n---\n\nDuring training, when a batch is [**distributed to the replicas**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#run), each replica receives a part of the batch and\ncalculates the loss values separately. We **SHOULD NOT** calculate the average of the per-example losses on the (partial) batch the replica recieves. \n\n**The intuition behind this is as follows:**\n\n---\n* The gradients calculated on each replica will be synced across the replicas\n * Therefore, they are summed before the optimizer applies the gradients to update the model's parameters\n* If we use the averaged per examples loss to compute the graident on each replica, the final graident applied by the optimizer will correspond to the sum of these averaged per-examples losses for respective replicas.\n * This is incorrect. The optimizer should apply the gradient obtained from the averaged per-examples loss **over the whole distributed batch**\n * It's worth noting that each replica may infact receive different number of examples. \n * Therefore it is impossible, in general, to obtain the averaged per example loss over the whole distributed batch from by simply dividing it by the number of replicas.\n\n---\n\n**Therefore, we can see that for each replica, we calculate the sum of per examples losses divided by the batch size of the whole distributed batch, which will give the optimizer the correct gradients to apply.**\n\n**EDIT**\n* **In this notebook, we have the option to use [*gradient accumulation*](https://arxiv.org/pdf/1710.02368)**\n* In ***gradient accumulation***, each replica receives several batches before the optimizer applies the graidents\n * we divide the sum of per examples losses by the update size (i.e. the number of examples used for one parameter update) rather than by the size of a single distributed batch.\n\n**In the following cell we will demonstrate, using dummy values and pretending we are distributing them, how to deal with the accumulation of the loss values across replicas.**","metadata":{"id":"g8D_SWZ2Uek1"}},{"cell_type":"code","source":"# print(\"-\"*100) # Just for printing\n\n# accumulated_loss = np.zeros((BATCH_SIZE_DEBUG,))\n# for i in range(0, MAX_LEN, 1):\n# loss = loss_seq(SAMPLE_LBLS[:, i], pred_output).numpy()\n# accumulated_loss+=loss\n \n# if i%50==0:\n# print(f\"\\n\\nLABEL FOR STEP {i+1} OF {MAX_LEN} FOR THE {BATCH_SIZE_DEBUG} EXAMPLES IN OUR DEMO BATCH:\\n\\t--> {SAMPLE_LBLS[:, i]}\")\n# print(f\"\\nOUR PREDICTIONS:\\n\\tSHAPE: {pred_output.shape}\\n\\tARGMAX: {tf.argmax(pred_output, axis=1)}\")\n# print(f\"\\nTHE CALCULATED INDIVIDUAL STEP LOSS:\\n\\t--> {loss_seq(SAMPLE_LBLS[:, i], pred_output).numpy()}\\n\\n\")\n# print(\"-\"*100) # Just for printing\n \n# print(f\"\\n\\n... ACCUMULATED LOSS:\\n\\t--> {accumulated_loss}\")\n# print(f\"\\n\\n... AVERAGE LOSS BY EXAMPLE:\\n\\t--> {accumulated_loss/MAX_LEN}\")\n# print(f\"\\n\\n... AVERAGE LOSS ACROSS BATCH:\\n\\t--> {np.sum(accumulated_loss)/(MAX_LEN*BATCH_SIZE_DEBUG)}\\n\\n\")\n\n# print(\"-\"*100) # Just for printing","metadata":{"executionInfo":{"elapsed":34375,"status":"ok","timestamp":1621257792281,"user":{"displayName":"dragon zhang","photoUrl":"","userId":"17320165301386381342"},"user_tz":-480},"id":"i6JTTuBRUek1","outputId":"1194db30-adb6-48bd-8f20-16fea5c095f4","execution":{"iopub.status.busy":"2021-08-27T05:28:42.099683Z","iopub.execute_input":"2021-08-27T05:28:42.100125Z","iopub.status.idle":"2021-08-27T05:28:42.10593Z","shell.execute_reply.started":"2021-08-27T05:28:42.10008Z","shell.execute_reply":"2021-08-27T05:28:42.104727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.7 DISTRIBUTE THE DATASETS ACROSS REPLICAS
\n\nWith an input pipeline written using the [**tf.data.Dataset**](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) API, we can use [**strategy.experimental_distribute_dataset**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#experimental_distribute_dataset) to turn it into a ***distributed dataset***, which produces **`per-replica`** values (which are objects of type [**PerReplica**](https://github.com/tensorflow/tensorflow/blob/v2.3.0/tensorflow/python/distribute/values.py#L361)) when iterating over it. \n\nFor example, \n\n```python\n ds = (... something that is a `tf.data.Dataset` ...)\n dist_ds = strategy.experimental_distribute_dataset(ds)\n```\n\n**`dist_ds`** will now be distributed across all replicas.\n\n---\n\nThe distributed datasets (when working with TPU) contain objects of type [**tensorflow.python.distribute.values.PerReplica**](https://github.com/tensorflow/tensorflow/blob/v2.3.0/tensorflow/python/distribute/values.py#L361), which is a subclass of [**tf.distribute.DistributedValues**](https://www.tensorflow.org/api_docs/python/tf/distribute/DistributedValues) that is the base class for representing distributed values.\n\nWhen iterating over the dataset we will still get a tuple containing two values. However, the tuple now contains **`PerReplica`** objects wheras before that tuple contained tensors representing the image and the label/id respectively.","metadata":{"id":"bNjsPcluUek2"}},{"cell_type":"markdown","source":"5.8 DISTRIBUTED COMPUTATION & OPTIMIZING LOOPS
\n\nFor each distributed batch (which contains **`PerReplica`** objects as discussed previously) produced by a distributed dataset, we use [**`strategy.run`**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#run) to perform a distributed computation on different TPU replicas, each processes a part of the batch.\n\n\n---\n\nTo understand how **`strategy.run`** will execute across the replicas, we can look at an example:\n\n```python\n @tf.function\n def dist_step(dist_batch):\n strategy.run(replica_fn, args=dist_batch)\n \n for dist_batch in dist_ds:\n dist_step(dist_batch)\n```\n\nHere **`replica_fn`** is a function that is going to be run on each replica, and it should work with **tensors**, not with **`PerReplica`** objects.\n* You define the operations (for example, forward pass, compute loss values and gradients, etc.) to peform just like witout using TPU. \n\n---\n\nWhen working with **`TPU`**, either [**`strategy.run`**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#run) has to be called inside [**`tf.function`**](https://www.tensorflow.org/api_docs/python/tf/function) or the replica function has to be annotated with [**`tf.function`**](https://www.tensorflow.org/api_docs/python/tf/function). \n\nFor example:\n\n```python\n @tf.function\n def replica_fn(batch):\n \n model(batch)\n ...\n \n for dist_batch in dist_ds:\n strategy.run(replica_fn, args=dist_batch)\n```\n\nThe above code snippet is a high level concept, and **`replica_fn`** doesn't necessary receive a single argument. \n* In our case, the original dataset yields tuples of tensors\n* A distributed batch is also a tuple of **`PerReplica`** objects and the **`replica_fn`** is actually receiving the unpacked version of a tuple of tensors as arguments.\n\n---\n\nIf a dataset yield a single tensor, you can do things like \n\n```python\n @tf.function\n def replica_fn(batch):\n \n tensor0 (, ... tensorN) = batch\n model(tensor0, ... tensorN)\n\n strategy.run(replica_fn, args=(dist_batch,))\n```\n\nwhere **`replica_fn`** expects a single tensor as arugment. Even if a dataset yields tuples of tensors, the above code still works, but **`replica_fn`** expects a single tuple of tensors as argument.\n\n---\n\nWe also have to discuss how to collect the returned values from [**`strategy.run`**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#run).\n\nThe results of [**`strategy.run`**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#run) are also \ndistributed values, just like the distributed batches it takes as inputs. \n* For each return value, we can use [strategy.experimental_local_results](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#experimental_local_results) to obtain a tuple of tensors from all replicas, and we can use [**`tf.concat`**](https://www.tensorflow.org/api_docs/python/tf/concat) to aggregate them into a single tensor.\n* We will use this method to collect the labels and model predictions\n\n---\n\nWe will need to iterate over the dataset to perform inference/train on the whole (distributed) dataset. When leveraging a TPU this is a non-trivial task. An example of iterating over a distributed dataset is:\n\n```python\n for dist_batch in dist_ds:\n dist_step(dist_batch)\n```\n\nEvery step in the loop, which calls [**`strategy.run`**](https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy#run), will have a communication between the local VM (in our case, the Kaggle VM) and the remote TPU worker(s). \n\n**This is obviously not ideal.**\n\nHowever, you can iterate the distributed dataset inside a `tf.function` as shown by:\n\n``` python\n @tf.function\n def dist_run_on_dataset(dist_ds):\n \n for dist_batch in dist_ds:\n dist_step(dist_batch)\n \n dist_process_dataset(dist_ds)\n```\n\nThis way, all the operations conducted on the dataset are compiled into a graph which is sent to the remote TPU worker(s) for execution. This will vastly reduce the running time and limit the time TPUs will sit idle waiting for data from the local VM. See [**TPU: extreme optimizations**](https://www.kaggle.com/c/flower-classification-with-tpus/discussion/135443) for a good benchmark by [**Martin GΓΆrner**](https://www.kaggle.com/mgornergoogle).\n\nIn this notebook, we use a fixed number of training steps, so we can also use\n\n```python \n @tf.function\n def dist_process_dataset(dist_ds_iter):\n \n for _ in tf.range(n_stes):\n dist_step(next(dist_ds_iter))\n \n dist_ds_iter = iter(dist_ds)\n dist_process_dataset(dist_ds_iter)\n```\n\n---\n\n**With the above discussions, we are ready to define the routines used for training, validation and prediction. Let's get started!**\n\n\n π REFERENCE:
\n - Tutorial - Using Iterators
\n - Tutorial - Iterating Inside a
\n - Kaggle Discussion - TPU: Extreme Optimizations
\n - Kaggle Notebook - Custom Training Loop With 100+ Flowers on TPU
\n
\n","metadata":{"id":"8oc8c0jcUek2"}},{"cell_type":"markdown","source":"\n - Tutorial - Using Iterators
\n - Tutorial - Iterating Inside a
tf.function\n - Kaggle Discussion - TPU: Extreme Optimizations
\n - Kaggle Notebook - Custom Training Loop With 100+ Flowers on TPU
\n
\n\n\n\n\n\n
6 MODEL TRAINING ⤒
\n\nIn this section we will define the training and validation routines as well as the final custom training loop that will execute everything we have worked on up until this point.","metadata":{"id":"8d0xEFA4Uek3"}},{"cell_type":"markdown","source":"6.1 INDIVIDUAL TRAIN STEP
\n\n---\n\nINFORMATION","metadata":{"id":"QhW9xK9nUek3"}},{"cell_type":"code","source":"def train_step(_image_batch, _smile_batch, _pattern_batch):\n \"\"\" Forward pass (calculate gradients)\n \n Args:\n _image_batch (): TBD\n _smile_batch (): TBD\n \n Returns:\n tbd\n \"\"\"\n _smile_batch_input = _smile_batch[:, :-1]\n _smile_batch_target = _smile_batch[:, 1:]\n\n combined_mask = create_mask(_smile_batch_input, _smile_batch_target)\n\n with tf.GradientTape() as tape:\n _image_embedding, _pattern, _image_pattern = encoder(_image_batch, training=True) \n \n# prediction_batch, _ = decoder(_image_embedding, _smile_batch_input, training=True, look_ahead_mask=combined_mask)\n prediction_batch, _ = decoder(_image_embedding, _image_pattern, _smile_batch_input, training=True, look_ahead_mask=combined_mask)\n # Update Loss Accumulator\n batch_loss_seq = loss_seq(_smile_batch_target, prediction_batch)/(MAX_LEN-1)\n batch_loss_cls = loss_cls(_pattern_batch, _pattern)\n # Update Accuracy Metric\n metrics[\"train_acc_seq\"].update_state(_smile_batch_target, prediction_batch, \n sample_weight=tf.where(tf.not_equal(_smile_batch_target, PAD_TOKEN), 1.0, 0.0))\n\n\n # backpropagation using variables, gradients and loss\n # - split this into two seperate optimizers/lrs/etc in the future\n # - we use the batch loss accumulation to update gradients\n# gradients = tape.gradient(batch_loss_seq, encoder.trainable_variables + decoder.trainable_variables)\n gradients = tape.gradient([batch_loss_seq, batch_loss_cls], encoder.trainable_variables + decoder.trainable_variables)\n gradients, _ = tf.clip_by_global_norm(gradients, 10.0)\n optimizer.apply_gradients(zip(gradients, encoder.trainable_variables+decoder.trainable_variables))\n \n metrics[\"batch_loss_seq\"].update_state(batch_loss_seq)\n metrics[\"train_loss_seq\"].update_state(batch_loss_seq)\n metrics[\"batch_loss_cls\"].update_state(batch_loss_cls)\n metrics[\"train_loss_cls\"].update_state(batch_loss_cls)\n\n@tf.function\ndef dist_train_step(_image_batch, _smile_batch, _pattern_batch):\n strategy.run(train_step, args=(_image_batch, _smile_batch, _pattern_batch))","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:28:42.107599Z","iopub.execute_input":"2021-08-27T05:28:42.108111Z","iopub.status.idle":"2021-08-27T05:28:42.128221Z","shell.execute_reply.started":"2021-08-27T05:28:42.108076Z","shell.execute_reply":"2021-08-27T05:28:42.126905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.2 INDIVIDUAL VAL STEP
\n\n---\n\nINFORMATION","metadata":{"id":"_KSt9qngUek4"}},{"cell_type":"code","source":"def val_step(_image_batch, _smile_batch, _pattern_batch):\n \"\"\" Forward pass (calculate gradients)\n \n Args:\n image_batch (): TBD\n smile_batch (): TBD\n \n Returns:\n tbd\n \"\"\"\n # Initialize batch_loss\n batch_loss_seq = tf.constant(0.0, tf.bfloat16)\n batch_loss_cls = tf.constant(0.0, tf.bfloat16)\n decoder_pred_batch = tf.ones((REPLICA_BATCH_SIZE, 1), dtype=tf.uint8)\n \n # Get image embedding (once)\n _image_embedding, _logit, _image_logit = encoder(_image_batch, training=False) \n \n # Teacher forcing - feeding the target as the next input\n for c_idx in range(1, MAX_LEN):\n gt_batch_id = _smile_batch[:, c_idx]\n combined_mask = create_mask(_smile_batch, decoder_pred_batch)\n \n # predictions.shape == (batch_size, seq_len, vocab_size)\n# prediction_batch, attention_weights = decoder(_image_embedding, decoder_pred_batch, training=False, look_ahead_mask=combined_mask)\n prediction_batch, attention_weights = decoder(_image_embedding, _image_logit, decoder_pred_batch, training=False, look_ahead_mask=combined_mask)\n \n predicted_batch_id = prediction_batch[:, -1:, :]\n \n # Update Loss Accumulator\n batch_loss_seq += loss_seq(gt_batch_id, predicted_batch_id[:, -1])\n \n # Update Accuracy Metric\n metrics[\"val_acc_seq\"].update_state(gt_batch_id, predicted_batch_id[:, -1],\n sample_weight=tf.where(tf.not_equal(gt_batch_id, PAD_TOKEN), 1.0, 0.0))\n\n # no teacher forcing, predicted char is next transformer input\n decoder_pred_batch = tf.concat([decoder_pred_batch, tf.cast(tf.argmax(predicted_batch_id, axis=-1), tf.uint8)], axis=-1)\n \n batch_loss_cls = loss_cls(_pattern_batch, _logit)\n # Update Loss Metric\n metrics[\"val_loss_seq\"].update_state(batch_loss_seq)\n metrics[\"val_loss_cls\"].update_state(batch_loss_cls)\n return decoder_pred_batch\n\n \n@tf.function\ndef dist_val_step(_val_dist_ds):\n _val_image_batch, _val_smile_batch, _val_pattern_batch = next(_val_dist_ds)\n predictions_seq_batch_per_replica = strategy.run(val_step, args=(_val_image_batch, _val_smile_batch, _val_pattern_batch))\n predictions_seq_batch_accum = strategy.gather(predictions_seq_batch_per_replica, axis=0)\n _val_smile_batch_accum = strategy.gather(_val_smile_batch, axis=0)\n return predictions_seq_batch_accum, _val_smile_batch_accum","metadata":{"execution":{"iopub.status.busy":"2021-08-27T05:28:42.131644Z","iopub.execute_input":"2021-08-27T05:28:42.132005Z","iopub.status.idle":"2021-08-27T05:28:42.148562Z","shell.execute_reply.started":"2021-08-27T05:28:42.131964Z","shell.execute_reply":"2021-08-27T05:28:42.147106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.3 INITIALIZE LOGGER
\n\n---\n\nINFORMATION","metadata":{"id":"h-A7Zew0Uek7"}},{"cell_type":"code","source":"class StatLogger():\n def __init__(self, verbose_frequency=100, print_style=\"tight\"):\n self.train_loss = []\n self.train_acc = []\n self.val_loss = []\n self.val_acc = []\n self.val_lsd = []\n self.step = []\n self.epoch = []\n self.lr = []\n \n self.current_step = 0\n self.epoch_start_time = 0\n self.batch_start_time = 0\n self.verbose_frequency = verbose_frequency\n self.print_style = print_style\n \n def print_last_val(self, current_time):\n if self.print_style==\"tight\":\n print(f\"| VAL DATA | STEP {VAL_STEPS:>4}/{VAL_STEPS} | \" \\\n# f\"ACC: {str(self.val_acc[-1]*100)[:5]:<5} β \" \\\n# f\"LOSS: {str(self.val_loss[-1])[:5]:<5} β \" \\\n# f\"LSD: {str(self.val_lsd[-1]):<3} |\")\n f\"ACC: {str(self.val_acc*100)[:5]:<5} β \" \\\n f\"LOSS: {str(self.val_loss)[:5]:<5} β \" \\\n f\"LSD: {str(self.val_lsd):<3} |\")\n \n \n else:\n print(f'\\n\\n{\"-\"*100}\\n{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"VALIDATION ACCURACY : \"+str(self.val_acc[-1]*100): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n# f'{\"-\"*25:<25}{\"VALIDATION LOSS SEQ : \"+str(self.val_loss_seq[-1]): ^50}{\"-\"*25:>25}\\n' \\\n \n f'{\"-\"*25:<25}{\"VALIDATION LOSS SEQ : \"+str(self.val_loss_seq): ^50}{\"-\"*25:>25}\\n' \\\n\n f'{\"-\"*100}\\n' \\\n# f'{\"-\"*25:<25}{\"VALIDATION LOSS CLS : \"+str(self.val_loss_cls[-1]): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*25:<25}{\"VALIDATION LOSS CLS : \"+str(self.val_loss_cls): ^50}{\"-\"*25:>25}\\n' \\\n \n f'{\"-\"*100}\\n' \\\n# f'{\"-\"*25:<25}{\"VALIDATION LSD : \"+str(self.val_lsd[-1]): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*25:<25}{\"VALIDATION LSD : \"+str(self.val_lsd): ^50}{\"-\"*25:>25}\\n' \\\n \n f'{\"-\"*100}\\n{\"-\"*100}\\n\\n')\n \n def print_current_train(self, step, train_acc, train_loss_seq, train_loss_cls, batch_loss_seq, batch_loss_cls, current_time, current_lr):\n if self.print_style==\"tight\":\n print(f\"| TRAIN DATA | STEP {self.current_step:>4}/{TRAIN_STEPS} | \" \\\n f\"ACC: {str(train_acc*100)[:5]:<5} β \" \\\n f\"LOSS SEQ: {str(train_loss_seq)[:5]:<5} β \" \\\n f\"LOSS CLS: {str(train_loss_cls)[:5]:<5} β \" \\\n f\"LR: {current_lr:.2e} \" \\\n f\"| | TIME | EPOCH: {str(round((current_time-self.epoch_start_time)/3600,1))+'h':<5} β \" \\\n f\"SUBSET: {str(round((current_time-self.batch_start_time)*self.verbose_frequency,1))+'s':<6} β \" \\\n f\"BATCH: {str(round(current_time-self.batch_start_time,1))+'s':<5} |\")\n else:\n print(f'\\n\\n{\"-\"*100}\\n{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"CURRENT STEP : \"+str(step)+\" OF \"+str(TRAIN_STEPS): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"CURRENT TRAIN ACCURACY : \"+str(train_acc*100): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"CURRENT TRAIN LOSS SEQ : \"+str(train_loss_seq): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"CURRENT TRAIN LOSS CLS : \"+str(train_loss_cls): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"LAST BATCH LOSS SEQ : \"+str(batch_loss_seq): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"LAST BATCH LOSS CLS : \"+str(batch_loss_seq): ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"EPOCH ELAPSED TIME : \"+str(round(current_time-self.epoch_start_time,1))+\" SECONDS\": ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"LAST SET OF BATCHES TOOK : ~\"+str(round((current_time-self.batch_start_time)*self.verbose_frequency,1))+\" SECONDS\": ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n' \\\n f'{\"-\"*25:<25}{\"LAST SINGLE BATCH TOOK : \"+str(round(current_time-self.batch_start_time,1))+\" SECONDS\": ^50}{\"-\"*25:>25}\\n' \\\n f'{\"-\"*100}\\n{\"-\"*100}\\n\\n')","metadata":{"id":"olB5r_AVUek7","execution":{"iopub.status.busy":"2021-08-27T06:48:00.229312Z","iopub.execute_input":"2021-08-27T06:48:00.229985Z","iopub.status.idle":"2021-08-27T06:48:00.250345Z","shell.execute_reply.started":"2021-08-27T06:48:00.229934Z","shell.execute_reply":"2021-08-27T06:48:00.249339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.4 CUSTOM TRAIN LOOP
\n\n---\n\nINFORMATION","metadata":{"id":"RG-ODl4DUek8"}},{"cell_type":"code","source":"# Instantiate our tool for logging\nstat_logger = StatLogger()\n\nfor epoch in range(1,EPOCHS+1):\n print(f'\\n\\n{\"=\"*100}\\n{\"=\"*25:<25}{\"EPOCH #\"+str(epoch): ^50}{\"=\"*25:>25}\\n{\"=\"*100}\\n')\n \n stat_logger.current_step=0\n stat_logger.epoch_start_time = time.time() # to compute epoch duration\n \n # create distributed versions of dataset to run on TPU with 8 computation units\n train_dist_ds = strategy.experimental_distribute_dataset(train_ds)\n val_dist_ds = iter(strategy.experimental_distribute_dataset(val_ds))\n \n for image_batch, smile_batch, pattern_batch in train_dist_ds:\n \n # Update current step\n stat_logger.batch_start_time = time.time()\n \n # Update the current step\n stat_logger.current_step += 1\n # Calculate training step\n dist_train_step(image_batch, smile_batch, pattern_batch)\n \n # end of epoch validation step\n if stat_logger.current_step == TRAIN_STEPS:\n print(\"\\n... VALIDATION DATASET STATISTICS ... \\n\")\n for _ in range(VAL_STEPS):\n preds, lbls = dist_val_step(val_dist_ds)\n metrics[\"val_lsd_seq\"].update_state(get_levenshtein_distance(preds, lbls))\n \n # Record this epochs statistics\n# stat_logger.train_loss.append(metrics[\"train_loss_seq\"].result().numpy())\n# stat_logger.train_loss.append(metrics[\"train_loss_cls\"].result().numpy())\n# stat_logger.train_acc.append(metrics[\"train_acc_seq\"].result().numpy())\n# stat_logger.val_loss.append(metrics[\"val_loss_seq\"].result().numpy())\n# stat_logger.val_loss.append(metrics[\"val_loss_cls\"].result().numpy())\n# stat_logger.val_acc.append(metrics[\"val_acc_seq\"].result().numpy())\n# stat_logger.val_lsd.append(metrics[\"val_lsd_seq\"].result().numpy())\n# \n stat_logger.train_loss = metrics[\"train_loss_seq\"].result().numpy()\n stat_logger.train_loss= metrics[\"train_loss_cls\"].result().numpy()\n stat_logger.train_acc=metrics[\"train_acc_seq\"].result().numpy()\n stat_logger.val_loss=metrics[\"val_loss_seq\"].result().numpy()\n stat_logger.val_loss=metrics[\"val_loss_cls\"].result().numpy()\n stat_logger.val_acc=metrics[\"val_acc_seq\"].result().numpy()\n stat_logger.val_lsd=metrics[\"val_lsd_seq\"].result().numpy()\n \n stat_logger.step.append(stat_logger.current_step)\n stat_logger.epoch.append(epoch)\n # stat_logger.lr.append(lr_scheduler.lr)\n stat_logger.lr.append(lr_scheduler(tf.cast(stat_logger.current_step+TRAIN_STEPS*(epoch-1), tf.float32)))\n \n # Reset the validation metrics as one epoch should not effect the next\n metrics[\"val_lsd_seq\"].reset_states()\n metrics[\"val_acc_seq\"].reset_states()\n metrics[\"val_loss_seq\"].reset_states()\n metrics[\"val_loss_cls\"].reset_states()\n metrics[\"train_acc_seq\"].reset_states()\n metrics[\"train_loss_seq\"].reset_states()\n metrics[\"train_loss_cls\"].reset_states()\n metrics[\"batch_loss_seq\"].reset_states()\n metrics[\"batch_loss_cls\"].reset_states()\n \n # Print validation scores\n stat_logger.print_last_val(current_time=time.time())\n \n # verbose logging step\n if stat_logger.current_step % stat_logger.verbose_frequency == 0: \n stat_logger.print_current_train(\n stat_logger.current_step,\n metrics[\"train_acc_seq\"].result().numpy(), \n metrics[\"train_loss_seq\"].result().numpy(), \n metrics[\"train_loss_cls\"].result().numpy(), \n metrics[\"batch_loss_seq\"].result().numpy(), \n metrics[\"batch_loss_cls\"].result().numpy(), \n \n \n current_time=time.time(),\n current_lr=lr_scheduler(tf.cast(stat_logger.current_step+TRAIN_STEPS*(epoch-1), tf.float32))\n# current_lr=lr_scheduler.lr\n )\n metrics[\"train_acc_seq\"].reset_states()\n metrics[\"train_loss_seq\"].reset_states()\n metrics[\"train_loss_cls\"].reset_states()\n metrics[\"batch_loss_seq\"].reset_states()\n metrics[\"batch_loss_cls\"].reset_states()\n # stop training when NaN loss is detected\n if stat_logger.current_step == TRAIN_STEPS:\n break\n \n # update learning rate\n # lr_scheduler.step(stat_logger.current_step+((epoch-1)*TRAIN_STEPS))\n \n # Save every other epoch (starting with first epoch)\n # Save after last epoch too...\n # if epoch%2==1 or epoch==EPOCHS:\n # save weights\n print(\"\\n...SAVING MODELS TO DISK ... \\n\")\n encoder.save_weights(f'./encoder_epoch_{epoch}.h5')\n decoder.save_weights(f'./decoder_epoch_{epoch}.h5')","metadata":{"id":"c18utTJUUek8","outputId":"0e3c9209-7c0b-4ad9-cb9f-bdf6efb61184","execution":{"iopub.status.busy":"2021-08-27T06:48:03.582985Z","iopub.execute_input":"2021-08-27T06:48:03.58341Z","iopub.status.idle":"2021-08-27T07:05:30.256709Z","shell.execute_reply.started":"2021-08-27T06:48:03.583374Z","shell.execute_reply":"2021-08-27T07:05:30.253897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##6.5 JUST-IN-CASE SAVE
\n\n---\n\nINFORMATION","metadata":{"id":"7PYZD70bUek9"}},{"cell_type":"code","source":"encoder.save_weights(f'./encoder_epoch_safety_save.h5')\ndecoder.save_weights(f'./decoder_epoch_safety_save.h5')","metadata":{"id":"TSDFNnjCUek9","execution":{"iopub.status.busy":"2021-08-27T06:08:33.005224Z","iopub.status.idle":"2021-08-27T06:08:33.005691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.6 VIEW PREDICTIONS & DISTRIBUTION OF LEVENSHTEIN DISTANCE FOR VAL DATASET
\n\n---\n\nINFORMATION","metadata":{"id":"szrOMXESUek-"}},{"cell_type":"code","source":"# pred_s = []\n# lbls_s = []\n# LDS = []\n\n# for _ in tqdm(range(VAL_STEPS), total=VAL_STEPS):\n# preds, lbls, patterns = dist_val_step(val_dist_ds)\n# for i, (p, l) in enumerate(zip(preds.numpy(), lbls.numpy())):\n# try:\n# p_s = \"\".join([int_2_tok[x] for x in p[:np.where(p==2)[0][0]]])\n# except:\n# p_s = \"\".join([int_2_tok[x] for x in p])\n# l_s = \"\".join([int_2_tok[x] for x in l[:np.where(l==2)[0][0]]])\n# LD = Levenshtein.distance(p_s,l_s)\n \n# if i % 1000==0:\n# print(f\"Levenshtein Distance: {LD}\")\n# print(f\"\\t--> {p_s}\")\n# print(f\"\\t--> {l_s}\\n\")\n\n# pred_s.append(p_s)\n# lbls_s.append(l_s)\n# LDS.append(LD)\n \n# ld_df = pd.DataFrame({\"lbl_smile\":lbls_s, \"pred_smile\":pred_s, \"lev_dist\":LDS})\n# ld_df.to_csv(\"./levenshtein_on_val.csv\")\n\n# print(ld_df.describe())\n# display(ld_df)\n\n# px.histogram(ld_df.lev_dist, log_y=True)","metadata":{"id":"K63PrhHPUek-","execution":{"iopub.status.busy":"2021-08-27T06:08:33.006994Z","iopub.status.idle":"2021-08-27T06:08:33.007424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n\n\n\n\n