Job 24422

InvalidSubmission:
Traceback (most recent call last):
  File "experiment_manager.py", line 299, in run_episode
    r: MsgReceived = agent.write_topic_and_expect('get_commands', expect='commands')
  File "/usr/local/lib/python3.7/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 96, in write_topic_and_expect
    ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
  File "/usr/local/lib/python3.7/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 177, in read_one
    nickname=self.nickname)
  File "/usr/local/lib/python3.7/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 258, in read_reply
    others = read_until_over(fpout, timeout=timeout, nickname=nickname)
  File "/usr/local/lib/python3.7/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 286, in read_until_over
    raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "agent" aborted:

error in agent |Exception while handling a message on topic "get_commands".
               |
               || Traceback (most recent call last):
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 1334, in _do_call
               ||     return fn(*args)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
               ||     options, feed_dict, fetch_list, target_list, run_metadata)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
               ||     run_metadata)
               || tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value prefix/ConvNet/fc_layer_2/kernel
               || 	 [[{{node prefix/ConvNet/fc_layer_2/kernel/read}}]]
               ||
               || During handling of the above exception, another exception occurred:
               ||
               || Traceback (most recent call last):
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/wrapper.py", line 277, in loop
               ||     handle_message_node(parsed, receiver0, context0)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/wrapper.py", line 431, in handle_message_node
               ||     call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/utils.py", line 19, in call_if_fun_exists
               ||     f(**kwargs)
               ||   File "imitation_agent.py", line 71, in on_received_get_commands
               ||     pwm_left, pwm_right = self.compute_action(self.config.current_image)
               ||   File "imitation_agent.py", line 60, in compute_action
               ||     x: observation
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 929, in run
               ||     run_metadata_ptr)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 1152, in _run
               ||     feed_dict_tensor, options, run_metadata)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 1328, in _do_run
               ||     run_metadata)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py", line 1348, in _do_call
               ||     raise type(e)(node_def, op, message)
               || tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value prefix/ConvNet/fc_layer_2/kernel
               || 	 [[node prefix/ConvNet/fc_layer_2/kernel/read (defined at /workspace/graph_utils.py:14) ]]
               ||
               || Caused by op 'prefix/ConvNet/fc_layer_2/kernel/read', defined at:
               ||   File "imitation_agent.py", line 136, in <module>
               ||     main()
               ||   File "imitation_agent.py", line 132, in main
               ||     wrap_direct(node=node, protocol=protocol)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/interface.py", line 22, in wrap_direct
               ||     run_loop(node, protocol, args)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/wrapper.py", line 196, in run_loop
               ||     config=config)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/wrapper.py", line 277, in loop
               ||     handle_message_node(parsed, receiver0, context0)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/wrapper.py", line 431, in handle_message_node
               ||     call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
               ||   File "/usr/local/lib/python3.7/dist-packages/zuper_nodes_wrapper/utils.py", line 19, in call_if_fun_exists
               ||     f(**kwargs)
               ||   File "imitation_agent.py", line 71, in on_received_get_commands
               ||     pwm_left, pwm_right = self.compute_action(self.config.current_image)
               ||   File "imitation_agent.py", line 43, in compute_action
               ||     graph = load_graph(frozen_model_filename)
               ||   File "/workspace/graph_utils.py", line 14, in load_graph
               ||     tf.import_graph_def(graph_def, name="prefix")
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
               ||     return func(*args, **kwargs)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/importer.py", line 442, in import_graph_def
               ||     _ProcessNewOps(graph)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/importer.py", line 235, in _ProcessNewOps
               ||     for new_op in graph._add_new_tf_operations(compute_devices=False):  # pylint: disable=protected-access
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py", line 3433, in _add_new_tf_operations
               ||     for c_op in c_api_util.new_tf_operations(self)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py", line 3433, in <listcomp>
               ||     for c_op in c_api_util.new_tf_operations(self)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py", line 3325, in _create_op_from_tf_operation
               ||     ret = Operation(c_op, self)
               ||   File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py", line 1801, in __init__
               ||     self._traceback = tf_stack.extract_stack()
               ||
               || FailedPreconditionError (see above for traceback): Attempting to use uninitialized value prefix/ConvNet/fc_layer_2/kernel
               || 	 [[node prefix/ConvNet/fc_layer_2/kernel/read (defined at /workspace/graph_utils.py:14) ]]
               ||
               ||

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/duckietown_challenges/cie_concrete.py", line 675, in scoring_context
    yield cie
  File "experiment_manager.py", line 444, in <module>
    wrap(cie)
  File "experiment_manager.py", line 432, in wrap
    main(cie, logdir, attempts)
  File "experiment_manager.py", line 136, in main
    physics_dt=config.physics_dt)
  File "experiment_manager.py", line 303, in run_episode
    raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.