InvalidSubmission:
Traceback (most recent call last):
File "experiment_manager.py", line 540, in run_episode
r: MsgReceived = await loop.run_in_executor(executor, f)
File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 133, in write_topic_and_expect
ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 216, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 299, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 327, in read_until_over
raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "ego" aborted:
error in ego |Exception while handling a message on topic "get_commands".
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node conv2d/Conv2D}}]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node conv2d/Conv2D}}]]
|| [[Tanh/_47]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 331, in loop
|| handle_message_node(parsed, receiver0, context0)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 486, in handle_message_node
|| call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 66, in on_received_get_commands
|| pwm_left, pwm_right = self.compute_action(self.current_image)
|| File "solution.py", line 61, in compute_action
|| action = self.model.predict(observation)
|| File "/submission/model.py", line 64, in predict
|| action = self.tf_session.run(self._computation_graph, feed_dict={
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node conv2d/Conv2D (defined at /submission/model.py:28) ]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node conv2d/Conv2D (defined at /submission/model.py:28) ]]
|| [[Tanh/_47]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Errors may have originated from an input operation.
|| Input Source operations connected to node conv2d/Conv2D:
|| map_1/TensorArrayV2Stack/TensorListStack (defined at /submission/model.py:91)
||
|| Input Source operations connected to node conv2d/Conv2D:
|| map_1/TensorArrayV2Stack/TensorListStack (defined at /submission/model.py:91)
||
|| Original stack trace for 'conv2d/Conv2D':
|| File "solution.py", line 101, in <module>
|| main()
|| File "solution.py", line 97, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 244, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 314, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 42, in init
|| self.model = TfInference(observation_shape=(1,) + expect_shape,
|| File "/submission/model.py", line 61, in __init__
|| self._initialize(observation_shape, action_shape, graph_location)
|| File "/submission/model.py", line 86, in _initialize
|| self._create(input_shape, action_shape)
|| File "/submission/model.py", line 99, in _create
|| self._computation_graph = self.computation_graph()
|| File "/submission/model.py", line 70, in computation_graph
|| model = one_residual(self._preprocessed_state, seed=self.seed)
|| File "/submission/model.py", line 28, in one_residual
|| nn = tf.compat.v1.layers.conv2d(x, filters=32, kernel_size=5, strides=2, padding='same',
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/deprecation.py", line 324, in new_func
|| return func(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/layers/convolutional.py", line 424, in conv2d
|| return layer.apply(inputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/deprecation.py", line 324, in new_func
|| return func(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 1695, in apply
|| return self.__call__(inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/layers/base.py", line 547, in __call__
|| outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 778, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/autograph/impl/api.py", line 262, in wrapper
|| return converted_call(f, args, kwargs, options=options)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/autograph/impl/api.py", line 492, in converted_call
|| return _call_unconverted(f, args, kwargs, options)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/autograph/impl/api.py", line 346, in _call_unconverted
|| return f(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 207, in call
|| outputs = self._convolution_op(inputs, self.kernel)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/nn_ops.py", line 1106, in __call__
|| return self.conv_op(inp, filter)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/nn_ops.py", line 638, in __call__
|| return self.call(inp, filter)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/nn_ops.py", line 231, in __call__
|| return self.conv_op(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/nn_ops.py", line 2006, in conv2d
|| return gen_nn_ops.conv2d(input, # pylint: disable=redefined-builtin
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 965, in conv2d
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3319, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1791, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 678, in scoring_context
yield cie
File "experiment_manager.py", line 683, in go
wrap(cie)
File "experiment_manager.py", line 668, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "experiment_manager.py", line 281, in main
length_s = await run_episode(
File "experiment_manager.py", line 544, in run_episode
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "experiment_manager.py", line 540, in run_episode
r: MsgReceived = await loop.run_in_executor(executor, f)
File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 133, in write_topic_and_expect
ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 216, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 299, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 327, in read_until_over
raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "ego" aborted:
error in ego |Exception while handling a message on topic "get_commands".
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.InternalError: 2 root error(s) found.
|| (0) Internal: Dst tensor is not initialized.
|| [[{{node _arg_state_0_0}}]]
|| [[Tanh/_47]]
|| (1) Internal: Dst tensor is not initialized.
|| [[{{node _arg_state_0_0}}]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 331, in loop
|| handle_message_node(parsed, receiver0, context0)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 486, in handle_message_node
|| call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 66, in on_received_get_commands
|| pwm_left, pwm_right = self.compute_action(self.current_image)
|| File "solution.py", line 61, in compute_action
|| action = self.model.predict(observation)
|| File "/submission/model.py", line 64, in predict
|| action = self.tf_session.run(self._computation_graph, feed_dict={
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.InternalError: 2 root error(s) found.
|| (0) Internal: Dst tensor is not initialized.
|| [[{{node _arg_state_0_0}}]]
|| [[Tanh/_47]]
|| (1) Internal: Dst tensor is not initialized.
|| [[{{node _arg_state_0_0}}]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 678, in scoring_context
yield cie
File "experiment_manager.py", line 683, in go
wrap(cie)
File "experiment_manager.py", line 668, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "experiment_manager.py", line 281, in main
length_s = await run_episode(
File "experiment_manager.py", line 544, in run_episode
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "experiment_manager.py", line 540, in run_episode
r: MsgReceived = await loop.run_in_executor(executor, f)
File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 133, in write_topic_and_expect
ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 216, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 299, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 334, in read_until_over
raise RemoteNodeAborted(msg) from None
zuper_nodes.structures.RemoteNodeAborted: External node "ego" closed communication.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 678, in scoring_context
yield cie
File "experiment_manager.py", line 683, in go
wrap(cie)
File "experiment_manager.py", line 668, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "experiment_manager.py", line 281, in main
length_s = await run_episode(
File "experiment_manager.py", line 544, in run_episode
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "experiment_manager.py", line 540, in run_episode
r: MsgReceived = await loop.run_in_executor(executor, f)
File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 133, in write_topic_and_expect
ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 216, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 299, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 334, in read_until_over
raise RemoteNodeAborted(msg) from None
zuper_nodes.structures.RemoteNodeAborted: External node "ego" closed communication.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 678, in scoring_context
yield cie
File "experiment_manager.py", line 683, in go
wrap(cie)
File "experiment_manager.py", line 668, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "experiment_manager.py", line 281, in main
length_s = await run_episode(
File "experiment_manager.py", line 544, in run_episode
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "experiment_manager.py", line 540, in run_episode
r: MsgReceived = await loop.run_in_executor(executor, f)
File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 133, in write_topic_and_expect
ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 216, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 299, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 334, in read_until_over
raise RemoteNodeAborted(msg) from None
zuper_nodes.structures.RemoteNodeAborted: External node "ego" closed communication.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 678, in scoring_context
yield cie
File "experiment_manager.py", line 683, in go
wrap(cie)
File "experiment_manager.py", line 668, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "experiment_manager.py", line 281, in main
length_s = await run_episode(
File "experiment_manager.py", line 544, in run_episode
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
Frank (Chude) Qian 🇨🇦