Below is the code and the error stack trace I am getting while trying to build the Keras model with the BERT layer on TPU everything works just fine with GPU. I suspect that TPU doesn't allow using layers that are not locally saved yet I don't see anything on this topic in the documentation.
\n\nmax_seq_length = 256 # Your choice here.\ninput_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_word_ids\")\ninput_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_mask\")\nsegment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"segment_ids\")
\n\nbert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",trainable=True)
\n\npooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\noutput = tf.keras.layers.Dense(32, activation='relu')(pooled_output)\noutput = tf.keras.layers.Dense(1, activation='sigmoid', name='labels')(output)
\n\nInvalidArgumentError Traceback (most recent call last)\n in \n 8 name=\"segment_ids\")\n 9 # BERT layer from pretrained model\n---> 10 bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",trainable=True)\n 11 # Dense Layers\n 12 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_hub/keras_layer.py in init(self, handle, trainable, arguments, _sentinel, tags, signature, signature_outputs_as_dict, output_key, output_shape, **kwargs)\n 144 _convert_nest_to_shapes(output_shape))\n 145 \n--> 146 self._func = load_module(handle, tags)\n 147 self._has_training_argument = func_has_training_argument(self._func)\n 148 self._is_hub_module_v1 = getattr(self._func, \"_is_hub_module_v1\", False)
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_hub/keras_layer.py in load_module(handle, tags)\n 396 return handle\n 397 else:\n--> 398 return module_v2.load(handle, tags=tags)\n 399 \n 400
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_hub/module_v2.py in load(handle, tags)\n 100 if tags is None and is_hub_module_v1:\n 101 tags = []\n--> 102 obj = tf_v1.saved_model.load_v2(module_path, tags=tags)\n 103 obj._is_hub_module_v1 = is_hub_module_v1 # pylint: disable=protected-access\n 104 return obj
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in load(export_dir, tags)\n 526 ValueError: If tags don't match a MetaGraph in the SavedModel.\n 527 \"\"\"\n--> 528 return load_internal(export_dir, tags)\n 529 \n 530
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in load_internal(export_dir, tags, loader_cls)\n 550 loader = loader_cls(object_graph_proto,\n 551 saved_model_proto,\n--> 552 export_dir)\n 553 root = loader.get(0)\n 554 root.tensorflow_version = meta_graph_def.meta_info_def.tensorflow_version
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in init(self, object_graph_proto, saved_model_proto, export_dir)\n 126 self._setup_functions_structures()\n 127 self._setup_functions_captures()\n--> 128 self._restore_checkpoint()\n 129 \n 130 for node in self._nodes:
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in _restore_checkpoint(self)\n 278 with ops.device(\"CPU\"):\n 279 saver._file_prefix_placeholder = constant_op.constant(variables_path)\n--> 280 load_status = saver.restore(variables_path)\n 281 load_status.assert_existing_objects_matched()\n 282 checkpoint = load_status._checkpoint
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/util.py in restore(self, save_path)\n 1281 graph_view=self._graph_view)\n 1282 base.CheckpointPosition(\n-> 1283 checkpoint=checkpoint, proto_id=0).restore(self._graph_view.root)\n 1284 load_status = CheckpointLoadStatus(\n 1285 checkpoint,
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py in restore(self, trackable)\n 207 # This object's correspondence with a checkpointed object is new, so\n 208 # process deferred restorations for it and its dependencies.\n--> 209 restore_ops = trackable._restore_from_checkpoint_position(self) # pylint: disable=protected-access\n 210 if restore_ops:\n 211 self._checkpoint.new_restore_ops(restore_ops)
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py in _restore_from_checkpoint_position(self, checkpoint_position)\n 906 restore_ops.extend(\n 907 current_position.checkpoint.restore_saveables(\n--> 908 tensor_saveables, python_saveables))\n 909 return restore_ops\n 910
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/util.py in restore_saveables(self, tensor_saveables, python_saveables)\n 287 \"expecting %s\") % (tensor_saveables.keys(), validated_names))\n 288 new_restore_ops = functional_saver.MultiDeviceSaver(\n--> 289 validated_saveables).restore(self.save_path_tensor)\n 290 if not context.executing_eagerly():\n 291 for name, restore_op in sorted(new_restore_ops.items()):
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/saving/functional_saver.py in restore(self, file_prefix)\n 253 for device, saver in sorted(self._single_device_savers.items()):\n 254 with ops.device(device):\n--> 255 restore_ops.update(saver.restore(file_prefix))\n 256 return restore_ops
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/saving/functional_saver.py in restore(self, file_prefix)\n 100 structured_restored_tensors):\n 101 restore_ops[saveable.name] = saveable.restore(\n--> 102 restored_tensors, restored_shapes=None)\n 103 return restore_ops\n 104
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/saving/saveable_object_util.py in restore(self, restored_tensors, restored_shapes)\n 112 # Copy the restored tensor to the variable's device.\n 113 with ops.device(self._var_device):\n--> 114 restored_tensor = array_ops.identity(restored_tensor)\n 115 return resource_variable_ops.shape_safe_assign_variable_handle(\n 116 self.handle_op, self._var_shape, restored_tensor)
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/util/dispatch.py in wrapper(*args, **kwargs)\n 178 \"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\n 179 try:\n--> 180 return target(*args, **kwargs)\n 181 except (TypeError, ValueError):\n 182 # Note: convert_to_eager_tensor currently raises a ValueError, not a
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/array_ops.py in identity(input, name)\n 265 # variables. Variables have correct handle data when graph building.\n 266 input = ops.convert_to_tensor(input)\n--> 267 ret = gen_array_ops.identity(input, name=name)\n 268 # Propagate handle data for happier shape inference for resource variables.\n 269 if hasattr(input, \"_handle_data\"):
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_array_ops.py in identity(input, name)\n 3824 pass # Add nodes to the TensorFlow graph.\n 3825 except _core._NotOkStatusException as e:\n-> 3826 _ops.raise_from_not_ok_status(e, name)\n 3827 # Add nodes to the TensorFlow graph.\n 3828 _, _, _op, _outputs = _op_def_library._apply_op_helper(
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in raise_from_not_ok_status(e, name)\n 6604 message = e.message + (\" name: \" + name if name is not None else \"\")\n 6605 # pylint: disable=protected-access\n-> 6606 six.raise_from(core._status_to_exception(e.code, message), None)\n 6607 # pylint: enable=protected-access\n 6608
\n\n/opt/conda/lib/python3.7/site-packages/six.py in raise_from(value, from_value)
\n\nInvalidArgumentError: Unsuccessful TensorSliceReader constructor: Failed to get matching files on /tmp/tfhub_modules/3e9209b9f2a53dfa4e6d93250dfceb5e64d73b66/variables/variables: Unimplemented: File system scheme '[local]' not implemented (file: '/tmp/tfhub_modules/3e9209b9f2a53dfa4e6d93250dfceb5e64d73b66/variables/variables') [Op:Identity]
", "messages": [ { "id": "874088", "postDate": "06/04/2020 16:23:41", "content": "Below is the code and the error stack trace I am getting while trying to build the Keras model with the BERT layer on TPU everything works just fine with GPU. I suspect that TPU doesn't allow using layers that are not locally saved yet I don't see anything on this topic in the documentation.
\n\nmax_seq_length = 256 # Your choice here.\ninput_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_word_ids\")\ninput_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_mask\")\nsegment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"segment_ids\")
\n\nbert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",trainable=True)
\n\npooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\noutput = tf.keras.layers.Dense(32, activation='relu')(pooled_output)\noutput = tf.keras.layers.Dense(1, activation='sigmoid', name='labels')(output)
\n\nInvalidArgumentError Traceback (most recent call last)\n in \n 8 name=\"segment_ids\")\n 9 # BERT layer from pretrained model\n---> 10 bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",trainable=True)\n 11 # Dense Layers\n 12 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_hub/keras_layer.py in init(self, handle, trainable, arguments, _sentinel, tags, signature, signature_outputs_as_dict, output_key, output_shape, **kwargs)\n 144 _convert_nest_to_shapes(output_shape))\n 145 \n--> 146 self._func = load_module(handle, tags)\n 147 self._has_training_argument = func_has_training_argument(self._func)\n 148 self._is_hub_module_v1 = getattr(self._func, \"_is_hub_module_v1\", False)
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_hub/keras_layer.py in load_module(handle, tags)\n 396 return handle\n 397 else:\n--> 398 return module_v2.load(handle, tags=tags)\n 399 \n 400
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_hub/module_v2.py in load(handle, tags)\n 100 if tags is None and is_hub_module_v1:\n 101 tags = []\n--> 102 obj = tf_v1.saved_model.load_v2(module_path, tags=tags)\n 103 obj._is_hub_module_v1 = is_hub_module_v1 # pylint: disable=protected-access\n 104 return obj
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in load(export_dir, tags)\n 526 ValueError: If tags don't match a MetaGraph in the SavedModel.\n 527 \"\"\"\n--> 528 return load_internal(export_dir, tags)\n 529 \n 530
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in load_internal(export_dir, tags, loader_cls)\n 550 loader = loader_cls(object_graph_proto,\n 551 saved_model_proto,\n--> 552 export_dir)\n 553 root = loader.get(0)\n 554 root.tensorflow_version = meta_graph_def.meta_info_def.tensorflow_version
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in init(self, object_graph_proto, saved_model_proto, export_dir)\n 126 self._setup_functions_structures()\n 127 self._setup_functions_captures()\n--> 128 self._restore_checkpoint()\n 129 \n 130 for node in self._nodes:
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py in _restore_checkpoint(self)\n 278 with ops.device(\"CPU\"):\n 279 saver._file_prefix_placeholder = constant_op.constant(variables_path)\n--> 280 load_status = saver.restore(variables_path)\n 281 load_status.assert_existing_objects_matched()\n 282 checkpoint = load_status._checkpoint
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/util.py in restore(self, save_path)\n 1281 graph_view=self._graph_view)\n 1282 base.CheckpointPosition(\n-> 1283 checkpoint=checkpoint, proto_id=0).restore(self._graph_view.root)\n 1284 load_status = CheckpointLoadStatus(\n 1285 checkpoint,
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py in restore(self, trackable)\n 207 # This object's correspondence with a checkpointed object is new, so\n 208 # process deferred restorations for it and its dependencies.\n--> 209 restore_ops = trackable._restore_from_checkpoint_position(self) # pylint: disable=protected-access\n 210 if restore_ops:\n 211 self._checkpoint.new_restore_ops(restore_ops)
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py in _restore_from_checkpoint_position(self, checkpoint_position)\n 906 restore_ops.extend(\n 907 current_position.checkpoint.restore_saveables(\n--> 908 tensor_saveables, python_saveables))\n 909 return restore_ops\n 910
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/util.py in restore_saveables(self, tensor_saveables, python_saveables)\n 287 \"expecting %s\") % (tensor_saveables.keys(), validated_names))\n 288 new_restore_ops = functional_saver.MultiDeviceSaver(\n--> 289 validated_saveables).restore(self.save_path_tensor)\n 290 if not context.executing_eagerly():\n 291 for name, restore_op in sorted(new_restore_ops.items()):
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/saving/functional_saver.py in restore(self, file_prefix)\n 253 for device, saver in sorted(self._single_device_savers.items()):\n 254 with ops.device(device):\n--> 255 restore_ops.update(saver.restore(file_prefix))\n 256 return restore_ops
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/saving/functional_saver.py in restore(self, file_prefix)\n 100 structured_restored_tensors):\n 101 restore_ops[saveable.name] = saveable.restore(\n--> 102 restored_tensors, restored_shapes=None)\n 103 return restore_ops\n 104
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/saving/saveable_object_util.py in restore(self, restored_tensors, restored_shapes)\n 112 # Copy the restored tensor to the variable's device.\n 113 with ops.device(self._var_device):\n--> 114 restored_tensor = array_ops.identity(restored_tensor)\n 115 return resource_variable_ops.shape_safe_assign_variable_handle(\n 116 self.handle_op, self._var_shape, restored_tensor)
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/util/dispatch.py in wrapper(*args, **kwargs)\n 178 \"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\n 179 try:\n--> 180 return target(*args, **kwargs)\n 181 except (TypeError, ValueError):\n 182 # Note: convert_to_eager_tensor currently raises a ValueError, not a
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/array_ops.py in identity(input, name)\n 265 # variables. Variables have correct handle data when graph building.\n 266 input = ops.convert_to_tensor(input)\n--> 267 ret = gen_array_ops.identity(input, name=name)\n 268 # Propagate handle data for happier shape inference for resource variables.\n 269 if hasattr(input, \"_handle_data\"):
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_array_ops.py in identity(input, name)\n 3824 pass # Add nodes to the TensorFlow graph.\n 3825 except _core._NotOkStatusException as e:\n-> 3826 _ops.raise_from_not_ok_status(e, name)\n 3827 # Add nodes to the TensorFlow graph.\n 3828 _, _, _op, _outputs = _op_def_library._apply_op_helper(
\n\n/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in raise_from_not_ok_status(e, name)\n 6604 message = e.message + (\" name: \" + name if name is not None else \"\")\n 6605 # pylint: disable=protected-access\n-> 6606 six.raise_from(core._status_to_exception(e.code, message), None)\n 6607 # pylint: enable=protected-access\n 6608
\n\n/opt/conda/lib/python3.7/site-packages/six.py in raise_from(value, from_value)
\n\nInvalidArgumentError: Unsuccessful TensorSliceReader constructor: Failed to get matching files on /tmp/tfhub_modules/3e9209b9f2a53dfa4e6d93250dfceb5e64d73b66/variables/variables: Unimplemented: File system scheme '[local]' not implemented (file: '/tmp/tfhub_modules/3e9209b9f2a53dfa4e6d93250dfceb5e64d73b66/variables/variables') [Op:Identity]
", "rawMarkdown": "Below is the code and the error stack trace I am getting while trying to build the Keras model with the BERT layer on TPU everything works just fine with GPU. I suspect that TPU doesn't allow using layers that are not locally saved yet I don't see anything on this topic in the documentation.\n\n#Building the model \nmax_seq_length = 256 # Your choice here.\ninput_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_word_ids\")\ninput_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_mask\")\nsegment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"segment_ids\")\n# BERT layer from pretrained model\nbert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",trainable=True)\n# Dense Layers\npooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\noutput = tf.keras.layers.Dense(32, activation='relu')(pooled_output)\noutput = tf.keras.layers.Dense(1, activation='sigmoid', name='labels')(output)\n\n_____________________________________________________________________________________________________\n\nInvalidArgumentError Traceback (most recent call last)", "votes": null }, { "id": "877658", "postDate": "06/07/2020 20:17:33", "content": "I am also facing the same error, did you find any solution.
", "rawMarkdown": "I am also facing the same error, did you find any solution.", "votes": null }, { "id": "877681", "postDate": "06/07/2020 21:06:19", "content": "No, my intuition is that TPU doesn't work with remote models. One has to have the model saved locally. Everything works fine on GRU yet it runs out of memory on large training files
", "rawMarkdown": "No, my intuition is that TPU doesn't work with remote models. One has to have the model saved locally. Everything works fine on GRU yet it runs out of memory on large training files", "votes": null }, { "id": "880000", "postDate": "06/09/2020 22:38:40", "content": "To load models from TF Hub, follow the example in the getting started notebook: https://www.kaggle.com/kivlichangoogle/jigsaw-multilingual-getting-started\nWhen using TPUs, you have to copy the model to a Kaggle Dataset, get the GCS path to the dataset and load from there.
", "rawMarkdown": "To load models from TF Hub, follow the example in the getting started notebook: https://www.kaggle.com/kivlichangoogle/jigsaw-multilingual-getting-started\nWhen using TPUs, you have to copy the model to a Kaggle Dataset, get the GCS path to the dataset and load from there.", "votes": null } ], "comments": [ { "id": 877658, "author_name": "keenborder", "author_url": "", "post_date": "06/07/2020 20:17:33", "content": "I am also facing the same error, did you find any solution.
", "votes": null, "replies": [ { "id": 877681, "author_name": "pastrop", "author_url": "", "post_date": "06/07/2020 21:06:19", "content": "No, my intuition is that TPU doesn't work with remote models. One has to have the model saved locally. Everything works fine on GRU yet it runs out of memory on large training files
", "votes": null, "replies": [] } ] }, { "id": 880000, "author_name": "mgorner", "author_url": "", "post_date": "06/09/2020 22:38:40", "content": "To load models from TF Hub, follow the example in the getting started notebook: https://www.kaggle.com/kivlichangoogle/jigsaw-multilingual-getting-started\nWhen using TPUs, you have to copy the model to a Kaggle Dataset, get the GCS path to the dataset and load from there.
", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "874088": "Below is the code and the error stack trace I am getting while trying to build the Keras model with the BERT layer on TPU everything works just fine with GPU. I suspect that TPU doesn't allow using layers that are not locally saved yet I don't see anything on this topic in the documentation.\n\n#Building the model \nmax_seq_length = 256 # Your choice here.\ninput_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_word_ids\")\ninput_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"input_mask\")\nsegment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n name=\"segment_ids\")\n# BERT layer from pretrained model\nbert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",trainable=True)\n# Dense Layers\npooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\noutput = tf.keras.layers.Dense(32, activation='relu')(pooled_output)\noutput = tf.keras.layers.Dense(1, activation='sigmoid', name='labels')(output)\n\n_____________________________________________________________________________________________________\n\nInvalidArgumentError Traceback (most recent call last)", "877658": "I am also facing the same error, did you find any solution.", "877681": "No, my intuition is that TPU doesn't work with remote models. One has to have the model saved locally. Everything works fine on GRU yet it runs out of memory on large training files", "880000": "To load models from TF Hub, follow the example in the getting started notebook: https://www.kaggle.com/kivlichangoogle/jigsaw-multilingual-getting-started\nWhen using TPUs, you have to copy the model to a Kaggle Dataset, get the GCS path to the dataset and load from there." }, "source": "meta" }