InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 307, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 307, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego3" aborted with the following error:
error in ego3 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 307, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 307, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 307, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego2" aborted with the following error:
error in ego2 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 307, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 242, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego2" aborted with the following error:
error in ego2 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| return session_or_none.run(symbolic_out[0], feed_dict)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| [[default_policy/strided_slice_1/_3]]
|| (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| 0 successful operations.
|| 0 derived errors ignored.
||
|| Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| File "solution.py", line 126, in <module>
|| main()
|| File "solution.py", line 122, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| postprocessed_batch = self.postprocess_trajectory(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| return postprocess_fn(self, sample_batch, other_agent_batches,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| symbolic_out[0] = fn(*placeholders)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| model_out, _ = self.model({
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| res = self.forward(restored, state or [], seq_lens)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| model_out, self._value_out = self.base_model(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| return self._run_internal_graph(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| outputs = node.layer(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| outputs = call_fn(cast_inputs, *args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| return self.activation(outputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[{{node default_policy/functional_1_1/conv_value_1/Relu}}]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 86, in call
|| | return session_or_none.run(symbolic_out[0], feed_dict)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.UnknownError: 2 root error(s) found.
|| | (0) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | [[default_policy/strided_slice_1/_3]]
|| | (1) Unknown: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
|| | [[node default_policy/functional_1_1/conv_value_1/Relu (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:103) ]]
|| | 0 successful operations.
|| | 0 derived errors ignored.
|| |
|| | Original stack trace for 'default_policy/functional_1_1/conv_value_1/Relu':
|| | File "solution.py", line 126, in <module>
|| | main()
|| | File "solution.py", line 122, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 324, in _initialize_loss
|| | postprocessed_batch = self.postprocess_trajectory(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 155, in postprocess_trajectory
|| | return postprocess_fn(self, sample_batch, other_agent_batches,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 182, in postprocess_ppo_gae
|| | last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1],
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/utils/tf_ops.py", line 84, in call
|| | symbolic_out[0] = fn(*placeholders)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/ppo/ppo_tf_policy.py", line 235, in value
|| | model_out, _ = self.model({
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/modelv2.py", line 150, in __call__
|| | res = self.forward(restored, state or [], seq_lens)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 103, in forward
|| | model_out, self._value_out = self.base_model(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 385, in call
|| | return self._run_internal_graph(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/functional.py", line 508, in _run_internal_graph
|| | outputs = node.layer(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
|| | outputs = call_fn(cast_inputs, *args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 269, in call
|| | return self.activation(outputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 10435, in relu
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 681, in scoring_context
yield cie
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 68, in go
wrap(cie)
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 34, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 249, in main
raise InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Getting agent protocol
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
András Kalapos 🇭🇺