{
  "id": 212058,
  "title": "ValueError: None values not supported. Error on TPU, working fine on GPU",
  "url": "/competitions/cassava-leaf-disease-classification/discussion/212058",
  "author_name": "",
  "post_date": "2021-01-17T10:19:15.744980900Z",
  "votes": null,
  "comment_count": 1,
  "views": 0,
  "content": "<p>ValueError                                Traceback (most recent call last)<br>\n in <br>\n      5                     batch_size=BATCH_SIZE,<br>\n      6                     callbacks=[tf.keras.callbacks.ModelCheckpoint('model_v0.25.h5', save_best_only=True),<br>\n----&gt; 7                                tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),<br>\n      8                     ]<br>\n      9          )</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)<br>\n     64   def _method_wrapper(self, *args, **kwargs):<br>\n     65     if not self._in_multi_worker_mode():  # pylint: disable=protected-access<br>\n---&gt; 66       return method(self, *args, **kwargs)<br>\n     67 <br>\n     68     # Running inside <code>run_distribute_coordinator</code> already.</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)<br>\n    846                 batch_size=batch_size):<br>\n    847               callbacks.on_train_batch_begin(step)<br>\n--&gt; 848               tmp_logs = train_function(iterator)<br>\n    849               # Catch OutOfRangeError for Datasets of unknown size.<br>\n    850               # This blocks until the batch has finished executing.</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in <strong>call</strong>(self, *args, *<em>kwds)\n    578         xla_context.Exit()\n    579     else:\n--&gt; 580       result = self._call(</em>args, **kwds)<br>\n    581 <br>\n    582     if tracing_count == self._get_tracing_count():</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)<br>\n    625       # This is the first call of <strong>call</strong>, so we have to initialize.<br>\n    626       initializers = []<br>\n--&gt; 627       self._initialize(args, kwds, add_initializers_to=initializers)<br>\n    628     finally:<br>\n    629       # At this point we know that the initialization is complete (or less</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)<br>\n    504     self._concrete_stateful_fn = (<br>\n    505         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access<br>\n--&gt; 506             *args, *<em>kwds))\n    507 \n    508     def invalid_creator_scope(</em>unused_args, **unused_kwds):</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in <em>get_concrete_function_internal_garbage_collected(self, *args, **kwargs)\n   2444       args, kwargs = None, None\n   2445     with self._lock:\n-&gt; 2446       graph_function, </em>, _ = self._maybe_define_function(args, kwargs)<br>\n   2447     return graph_function<br>\n   2448 </p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)<br>\n   2775 <br>\n   2776       self._function_cache.missed.add(call_context_key)<br>\n-&gt; 2777       graph_function = self._create_graph_function(args, kwargs)<br>\n   2778       self._function_cache.primary[cache_key] = graph_function<br>\n   2779       return graph_function, args, kwargs</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)<br>\n   2665             arg_names=arg_names,<br>\n   2666             override_flat_arg_shapes=override_flat_arg_shapes,<br>\n-&gt; 2667             capture_by_value=self._capture_by_value),<br>\n   2668         self._function_attributes,<br>\n   2669         # Tell the ConcreteFunction to clean up its graph once it goes out of</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)<br>\n    979         _, original_func = tf_decorator.unwrap(python_func)<br>\n    980 <br>\n--&gt; 981       func_outputs = python_func(*func_args, **func_kwargs)<br>\n    982 <br>\n    983       # invariant: <code>func_outputs</code> contains only Tensors, CompositeTensors,</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, *<em>kwds)\n    439         # <strong>wrapped</strong> allows AutoGraph to swap in a converted function. We give\n    440         # the function a weak reference to itself to avoid a reference cycle.\n--&gt; 441         return weak_wrapped_fn().<strong>wrapped</strong>(</em>args, **kwds)<br>\n    442     weak_wrapped_fn = weakref.ref(wrapped_fn)<br>\n    443 </p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)<br>\n    966           except Exception as e:  # pylint:disable=broad-except<br>\n    967             if hasattr(e, \"ag_error_metadata\"):<br>\n--&gt; 968               raise e.ag_error_metadata.to_exception(e)<br>\n    969             else:<br>\n    970               raise</p>\n<p>ValueError: in user code:</p>\n<pre><code>/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:571 train_function  *\n    outputs = self.distribute_strategy.run(\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:174 run  **\n    return self.extended.tpu_run(fn, args, kwargs, options)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:867 tpu_run\n    return func(args, kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:934 tpu_function\n    padding_spec=padding_spec)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:893 replicate\n    padding_spec=padding_spec)[1]\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:1280 split_compile_and_replicate\n    outputs = computation(*computation_inputs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:896 replicated_fn\n    result[0] = fn(*replica_args, **replica_kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:531 train_step  **\n    y_pred = self(x, training=True)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n    outputs = call_fn(cast_inputs, *args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:719 call\n    convert_kwargs_to_constants=base_layer_utils.call_context().saving)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:888 _run_internal_graph\n    output_tensors = layer(computed_tensors, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:531 __call__\n    return super(Bidirectional, self).__call__(inputs, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n    outputs = call_fn(cast_inputs, *args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:645 call\n    initial_state=forward_state, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent.py:654 __call__\n    return super(RNN, self).__call__(inputs, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n    outputs = call_fn(cast_inputs, *args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1187 call\n    runtime) = lstm_with_backend_selection(**normal_lstm_kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1566 lstm_with_backend_selection\n    **params)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2419 __call__\n    graph_function, args, kwargs = self._maybe_define_function(args, kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2777 _maybe_define_function\n    graph_function = self._create_graph_function(args, kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2667 _create_graph_function\n    capture_by_value=self._capture_by_value),\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py:981 func_graph_from_py_func\n    func_outputs = python_func(*func_args, **func_kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1320 standard_lstm\n    zero_output_for_mask=zero_output_for_mask)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:4107 rnn\n    max_iterations = math_ops.reduce_max(input_length)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n    return target(*args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:2420 reduce_max\n    _ReductionDims(input_tensor, axis))\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:1627 _ReductionDims\n    return range(0, array_ops.rank(x))\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n    return target(*args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:787 rank\n    return rank_internal(input, name, optimize=True)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:807 rank_internal\n    input = ops.convert_to_tensor(input)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py:1341 convert_to_tensor\n    ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:321 _constant_tensor_conversion_function\n    return constant(v, dtype=dtype, name=name)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:262 constant\n    allow_broadcast=True)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:300 _constant_impl\n    allow_broadcast=allow_broadcast))\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py:439 make_tensor_proto\n    raise ValueError(\"None values not supported.\")\n\nValueError: None values not supported.\n</code></pre>",
  "messages": [
    {
      "id": "1156667",
      "postDate": "01/17/2021 10:19:15",
      "content": "<p>ValueError                                Traceback (most recent call last)<br>\n in <br>\n      5                     batch_size=BATCH_SIZE,<br>\n      6                     callbacks=[tf.keras.callbacks.ModelCheckpoint('model_v0.25.h5', save_best_only=True),<br>\n----&gt; 7                                tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),<br>\n      8                     ]<br>\n      9          )</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)<br>\n     64   def _method_wrapper(self, *args, **kwargs):<br>\n     65     if not self._in_multi_worker_mode():  # pylint: disable=protected-access<br>\n---&gt; 66       return method(self, *args, **kwargs)<br>\n     67 <br>\n     68     # Running inside <code>run_distribute_coordinator</code> already.</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)<br>\n    846                 batch_size=batch_size):<br>\n    847               callbacks.on_train_batch_begin(step)<br>\n--&gt; 848               tmp_logs = train_function(iterator)<br>\n    849               # Catch OutOfRangeError for Datasets of unknown size.<br>\n    850               # This blocks until the batch has finished executing.</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in <strong>call</strong>(self, *args, *<em>kwds)\n    578         xla_context.Exit()\n    579     else:\n--&gt; 580       result = self._call(</em>args, **kwds)<br>\n    581 <br>\n    582     if tracing_count == self._get_tracing_count():</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)<br>\n    625       # This is the first call of <strong>call</strong>, so we have to initialize.<br>\n    626       initializers = []<br>\n--&gt; 627       self._initialize(args, kwds, add_initializers_to=initializers)<br>\n    628     finally:<br>\n    629       # At this point we know that the initialization is complete (or less</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)<br>\n    504     self._concrete_stateful_fn = (<br>\n    505         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access<br>\n--&gt; 506             *args, *<em>kwds))\n    507 \n    508     def invalid_creator_scope(</em>unused_args, **unused_kwds):</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in <em>get_concrete_function_internal_garbage_collected(self, *args, **kwargs)\n   2444       args, kwargs = None, None\n   2445     with self._lock:\n-&gt; 2446       graph_function, </em>, _ = self._maybe_define_function(args, kwargs)<br>\n   2447     return graph_function<br>\n   2448 </p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)<br>\n   2775 <br>\n   2776       self._function_cache.missed.add(call_context_key)<br>\n-&gt; 2777       graph_function = self._create_graph_function(args, kwargs)<br>\n   2778       self._function_cache.primary[cache_key] = graph_function<br>\n   2779       return graph_function, args, kwargs</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)<br>\n   2665             arg_names=arg_names,<br>\n   2666             override_flat_arg_shapes=override_flat_arg_shapes,<br>\n-&gt; 2667             capture_by_value=self._capture_by_value),<br>\n   2668         self._function_attributes,<br>\n   2669         # Tell the ConcreteFunction to clean up its graph once it goes out of</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)<br>\n    979         _, original_func = tf_decorator.unwrap(python_func)<br>\n    980 <br>\n--&gt; 981       func_outputs = python_func(*func_args, **func_kwargs)<br>\n    982 <br>\n    983       # invariant: <code>func_outputs</code> contains only Tensors, CompositeTensors,</p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, *<em>kwds)\n    439         # <strong>wrapped</strong> allows AutoGraph to swap in a converted function. We give\n    440         # the function a weak reference to itself to avoid a reference cycle.\n--&gt; 441         return weak_wrapped_fn().<strong>wrapped</strong>(</em>args, **kwds)<br>\n    442     weak_wrapped_fn = weakref.ref(wrapped_fn)<br>\n    443 </p>\n<p>/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)<br>\n    966           except Exception as e:  # pylint:disable=broad-except<br>\n    967             if hasattr(e, \"ag_error_metadata\"):<br>\n--&gt; 968               raise e.ag_error_metadata.to_exception(e)<br>\n    969             else:<br>\n    970               raise</p>\n<p>ValueError: in user code:</p>\n<pre><code>/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:571 train_function  *\n    outputs = self.distribute_strategy.run(\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:174 run  **\n    return self.extended.tpu_run(fn, args, kwargs, options)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:867 tpu_run\n    return func(args, kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:934 tpu_function\n    padding_spec=padding_spec)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:893 replicate\n    padding_spec=padding_spec)[1]\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:1280 split_compile_and_replicate\n    outputs = computation(*computation_inputs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:896 replicated_fn\n    result[0] = fn(*replica_args, **replica_kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:531 train_step  **\n    y_pred = self(x, training=True)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n    outputs = call_fn(cast_inputs, *args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:719 call\n    convert_kwargs_to_constants=base_layer_utils.call_context().saving)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:888 _run_internal_graph\n    output_tensors = layer(computed_tensors, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:531 __call__\n    return super(Bidirectional, self).__call__(inputs, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n    outputs = call_fn(cast_inputs, *args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:645 call\n    initial_state=forward_state, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent.py:654 __call__\n    return super(RNN, self).__call__(inputs, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n    outputs = call_fn(cast_inputs, *args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1187 call\n    runtime) = lstm_with_backend_selection(**normal_lstm_kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1566 lstm_with_backend_selection\n    **params)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2419 __call__\n    graph_function, args, kwargs = self._maybe_define_function(args, kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2777 _maybe_define_function\n    graph_function = self._create_graph_function(args, kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2667 _create_graph_function\n    capture_by_value=self._capture_by_value),\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py:981 func_graph_from_py_func\n    func_outputs = python_func(*func_args, **func_kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1320 standard_lstm\n    zero_output_for_mask=zero_output_for_mask)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:4107 rnn\n    max_iterations = math_ops.reduce_max(input_length)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n    return target(*args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:2420 reduce_max\n    _ReductionDims(input_tensor, axis))\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:1627 _ReductionDims\n    return range(0, array_ops.rank(x))\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n    return target(*args, **kwargs)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:787 rank\n    return rank_internal(input, name, optimize=True)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:807 rank_internal\n    input = ops.convert_to_tensor(input)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py:1341 convert_to_tensor\n    ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:321 _constant_tensor_conversion_function\n    return constant(v, dtype=dtype, name=name)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:262 constant\n    allow_broadcast=True)\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:300 _constant_impl\n    allow_broadcast=allow_broadcast))\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py:439 make_tensor_proto\n    raise ValueError(\"None values not supported.\")\n\nValueError: None values not supported.\n</code></pre>",
      "rawMarkdown": "ValueError                                Traceback (most recent call last)\n<ipython-input-17-43be058b1460> in <module>\n      5                     batch_size=BATCH_SIZE,\n      6                     callbacks=[tf.keras.callbacks.ModelCheckpoint('model_v0.25.h5', save_best_only=True),\n----> 7                                tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),\n      8                     ]\n      9          )\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)\n     64   def _method_wrapper(self, *args, **kwargs):\n     65     if not self._in_multi_worker_mode():  # pylint: disable=protected-access\n---> 66       return method(self, *args, **kwargs)\n     67 \n     68     # Running inside `run_distribute_coordinator` already.\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\n    846                 batch_size=batch_size):\n    847               callbacks.on_train_batch_begin(step)\n--> 848               tmp_logs = train_function(iterator)\n    849               # Catch OutOfRangeError for Datasets of unknown size.\n    850               # This blocks until the batch has finished executing.\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)\n    578         xla_context.Exit()\n    579     else:\n--> 580       result = self._call(*args, **kwds)\n    581 \n    582     if tracing_count == self._get_tracing_count():\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)\n    625       # This is the first call of __call__, so we have to initialize.\n    626       initializers = []\n--> 627       self._initialize(args, kwds, add_initializers_to=initializers)\n    628     finally:\n    629       # At this point we know that the initialization is complete (or less\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)\n    504     self._concrete_stateful_fn = (\n    505         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access\n--> 506             *args, **kwds))\n    507 \n    508     def invalid_creator_scope(*unused_args, **unused_kwds):\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)\n   2444       args, kwargs = None, None\n   2445     with self._lock:\n-> 2446       graph_function, _, _ = self._maybe_define_function(args, kwargs)\n   2447     return graph_function\n   2448 \n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)\n   2775 \n   2776       self._function_cache.missed.add(call_context_key)\n-> 2777       graph_function = self._create_graph_function(args, kwargs)\n   2778       self._function_cache.primary[cache_key] = graph_function\n   2779       return graph_function, args, kwargs\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)\n   2665             arg_names=arg_names,\n   2666             override_flat_arg_shapes=override_flat_arg_shapes,\n-> 2667             capture_by_value=self._capture_by_value),\n   2668         self._function_attributes,\n   2669         # Tell the ConcreteFunction to clean up its graph once it goes out of\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)\n    979         _, original_func = tf_decorator.unwrap(python_func)\n    980 \n--> 981       func_outputs = python_func(*func_args, **func_kwargs)\n    982 \n    983       # invariant: `func_outputs` contains only Tensors, CompositeTensors,\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)\n    439         # __wrapped__ allows AutoGraph to swap in a converted function. We give\n    440         # the function a weak reference to itself to avoid a reference cycle.\n--> 441         return weak_wrapped_fn().__wrapped__(*args, **kwds)\n    442     weak_wrapped_fn = weakref.ref(wrapped_fn)\n    443 \n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)\n    966           except Exception as e:  # pylint:disable=broad-except\n    967             if hasattr(e, \"ag_error_metadata\"):\n--> 968               raise e.ag_error_metadata.to_exception(e)\n    969             else:\n    970               raise\n\nValueError: in user code:\n\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:571 train_function  *\n        outputs = self.distribute_strategy.run(\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:174 run  **\n        return self.extended.tpu_run(fn, args, kwargs, options)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:867 tpu_run\n        return func(args, kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:934 tpu_function\n        padding_spec=padding_spec)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:893 replicate\n        padding_spec=padding_spec)[1]\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:1280 split_compile_and_replicate\n        outputs = computation(*computation_inputs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:896 replicated_fn\n        result[0] = fn(*replica_args, **replica_kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:531 train_step  **\n        y_pred = self(x, training=True)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n        outputs = call_fn(cast_inputs, *args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:719 call\n        convert_kwargs_to_constants=base_layer_utils.call_context().saving)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:888 _run_internal_graph\n        output_tensors = layer(computed_tensors, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:531 __call__\n        return super(Bidirectional, self).__call__(inputs, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n        outputs = call_fn(cast_inputs, *args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:645 call\n        initial_state=forward_state, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent.py:654 __call__\n        return super(RNN, self).__call__(inputs, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n        outputs = call_fn(cast_inputs, *args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1187 call\n        runtime) = lstm_with_backend_selection(**normal_lstm_kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1566 lstm_with_backend_selection\n        **params)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2419 __call__\n        graph_function, args, kwargs = self._maybe_define_function(args, kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2777 _maybe_define_function\n        graph_function = self._create_graph_function(args, kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2667 _create_graph_function\n        capture_by_value=self._capture_by_value),\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py:981 func_graph_from_py_func\n        func_outputs = python_func(*func_args, **func_kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1320 standard_lstm\n        zero_output_for_mask=zero_output_for_mask)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:4107 rnn\n        max_iterations = math_ops.reduce_max(input_length)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n        return target(*args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:2420 reduce_max\n        _ReductionDims(input_tensor, axis))\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:1627 _ReductionDims\n        return range(0, array_ops.rank(x))\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n        return target(*args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:787 rank\n        return rank_internal(input, name, optimize=True)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:807 rank_internal\n        input = ops.convert_to_tensor(input)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py:1341 convert_to_tensor\n        ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:321 _constant_tensor_conversion_function\n        return constant(v, dtype=dtype, name=name)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:262 constant\n        allow_broadcast=True)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:300 _constant_impl\n        allow_broadcast=allow_broadcast))\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py:439 make_tensor_proto\n        raise ValueError(\"None values not supported.\")\n\n    ValueError: None values not supported.",
      "votes": null
    },
    {
      "id": "1156671",
      "postDate": "01/17/2021 10:20:56",
      "content": "<p>Please help if possible the code is working fine on GPU but when training on TPU throwing error. I am trying to ensemble the model.</p>",
      "rawMarkdown": "Please help if possible the code is working fine on GPU but when training on TPU throwing error. I am trying to ensemble the model.",
      "votes": null
    }
  ],
  "comments": [
    {
      "id": 1156671,
      "author_name": "shubham219",
      "author_url": "",
      "post_date": "01/17/2021 10:20:56",
      "content": "<p>Please help if possible the code is working fine on GPU but when training on TPU throwing error. I am trying to ensemble the model.</p>",
      "votes": null,
      "replies": []
    }
  ],
  "raw_markdown_by_id": {
    "1156667": "ValueError                                Traceback (most recent call last)\n<ipython-input-17-43be058b1460> in <module>\n      5                     batch_size=BATCH_SIZE,\n      6                     callbacks=[tf.keras.callbacks.ModelCheckpoint('model_v0.25.h5', save_best_only=True),\n----> 7                                tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),\n      8                     ]\n      9          )\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)\n     64   def _method_wrapper(self, *args, **kwargs):\n     65     if not self._in_multi_worker_mode():  # pylint: disable=protected-access\n---> 66       return method(self, *args, **kwargs)\n     67 \n     68     # Running inside `run_distribute_coordinator` already.\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\n    846                 batch_size=batch_size):\n    847               callbacks.on_train_batch_begin(step)\n--> 848               tmp_logs = train_function(iterator)\n    849               # Catch OutOfRangeError for Datasets of unknown size.\n    850               # This blocks until the batch has finished executing.\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)\n    578         xla_context.Exit()\n    579     else:\n--> 580       result = self._call(*args, **kwds)\n    581 \n    582     if tracing_count == self._get_tracing_count():\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)\n    625       # This is the first call of __call__, so we have to initialize.\n    626       initializers = []\n--> 627       self._initialize(args, kwds, add_initializers_to=initializers)\n    628     finally:\n    629       # At this point we know that the initialization is complete (or less\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)\n    504     self._concrete_stateful_fn = (\n    505         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access\n--> 506             *args, **kwds))\n    507 \n    508     def invalid_creator_scope(*unused_args, **unused_kwds):\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)\n   2444       args, kwargs = None, None\n   2445     with self._lock:\n-> 2446       graph_function, _, _ = self._maybe_define_function(args, kwargs)\n   2447     return graph_function\n   2448 \n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)\n   2775 \n   2776       self._function_cache.missed.add(call_context_key)\n-> 2777       graph_function = self._create_graph_function(args, kwargs)\n   2778       self._function_cache.primary[cache_key] = graph_function\n   2779       return graph_function, args, kwargs\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)\n   2665             arg_names=arg_names,\n   2666             override_flat_arg_shapes=override_flat_arg_shapes,\n-> 2667             capture_by_value=self._capture_by_value),\n   2668         self._function_attributes,\n   2669         # Tell the ConcreteFunction to clean up its graph once it goes out of\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)\n    979         _, original_func = tf_decorator.unwrap(python_func)\n    980 \n--> 981       func_outputs = python_func(*func_args, **func_kwargs)\n    982 \n    983       # invariant: `func_outputs` contains only Tensors, CompositeTensors,\n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)\n    439         # __wrapped__ allows AutoGraph to swap in a converted function. We give\n    440         # the function a weak reference to itself to avoid a reference cycle.\n--> 441         return weak_wrapped_fn().__wrapped__(*args, **kwds)\n    442     weak_wrapped_fn = weakref.ref(wrapped_fn)\n    443 \n\n/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)\n    966           except Exception as e:  # pylint:disable=broad-except\n    967             if hasattr(e, \"ag_error_metadata\"):\n--> 968               raise e.ag_error_metadata.to_exception(e)\n    969             else:\n    970               raise\n\nValueError: in user code:\n\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:571 train_function  *\n        outputs = self.distribute_strategy.run(\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:174 run  **\n        return self.extended.tpu_run(fn, args, kwargs, options)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:867 tpu_run\n        return func(args, kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:934 tpu_function\n        padding_spec=padding_spec)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:893 replicate\n        padding_spec=padding_spec)[1]\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py:1280 split_compile_and_replicate\n        outputs = computation(*computation_inputs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py:896 replicated_fn\n        result[0] = fn(*replica_args, **replica_kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:531 train_step  **\n        y_pred = self(x, training=True)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n        outputs = call_fn(cast_inputs, *args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:719 call\n        convert_kwargs_to_constants=base_layer_utils.call_context().saving)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/network.py:888 _run_internal_graph\n        output_tensors = layer(computed_tensors, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:531 __call__\n        return super(Bidirectional, self).__call__(inputs, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n        outputs = call_fn(cast_inputs, *args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/wrappers.py:645 call\n        initial_state=forward_state, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent.py:654 __call__\n        return super(RNN, self).__call__(inputs, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py:927 __call__\n        outputs = call_fn(cast_inputs, *args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1187 call\n        runtime) = lstm_with_backend_selection(**normal_lstm_kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1566 lstm_with_backend_selection\n        **params)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2419 __call__\n        graph_function, args, kwargs = self._maybe_define_function(args, kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2777 _maybe_define_function\n        graph_function = self._create_graph_function(args, kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py:2667 _create_graph_function\n        capture_by_value=self._capture_by_value),\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py:981 func_graph_from_py_func\n        func_outputs = python_func(*func_args, **func_kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/recurrent_v2.py:1320 standard_lstm\n        zero_output_for_mask=zero_output_for_mask)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:4107 rnn\n        max_iterations = math_ops.reduce_max(input_length)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n        return target(*args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:2420 reduce_max\n        _ReductionDims(input_tensor, axis))\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:1627 _ReductionDims\n        return range(0, array_ops.rank(x))\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:180 wrapper\n        return target(*args, **kwargs)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:787 rank\n        return rank_internal(input, name, optimize=True)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py:807 rank_internal\n        input = ops.convert_to_tensor(input)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py:1341 convert_to_tensor\n        ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:321 _constant_tensor_conversion_function\n        return constant(v, dtype=dtype, name=name)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:262 constant\n        allow_broadcast=True)\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py:300 _constant_impl\n        allow_broadcast=allow_broadcast))\n    /opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py:439 make_tensor_proto\n        raise ValueError(\"None values not supported.\")\n\n    ValueError: None values not supported.",
    "1156671": "Please help if possible the code is working fine on GPU but when training on TPU throwing error. I am trying to ensemble the model."
  },
  "source": "meta"
}