| message | InvalidEnvironment:
[...]InvalidEnvironment:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 271, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego1" aborted with the following error:
error in ego1 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "submission.py", line 60, in init
|| self.agent = self.create_agent(self.env)
|| File "submission.py", line 162, in create_agent_dagger
|| agent = DaggerAgent.load_from_checkpoint(
|| File "/usr/local/lib/python3.8/dist-packages/pytorch_lightning/utilities/device_dtype_mixin.py", line 124, in cuda
|| return super().cuda(device=device)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in cuda
|| return self._apply(lambda t: t.cuda(device))
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
|| param_applied = fn(param)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in <lambda>
|| return self._apply(lambda t: t.cuda(device))
|| RuntimeError: CUDA error: out of memory
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 681, in scoring_context
yield cie
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 277, in main
raise InvalidEnvironment(msg) from e
duckietown_challenges.exceptions.InvalidEnvironment: Detected out of CUDA memory:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 271, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego1" aborted with the following error:
error in ego1 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "submission.py", line 60, in init
|| self.agent = self.create_agent(self.env)
|| File "submission.py", line 162, in create_agent_dagger
|| agent = DaggerAgent.load_from_checkpoint(
|| File "/usr/local/lib/python3.8/dist-packages/pytorch_lightning/utilities/device_dtype_mixin.py", line 124, in cuda
|| return super().cuda(device=device)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in cuda
|| return self._apply(lambda t: t.cuda(device))
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
|| param_applied = fn(param)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in <lambda>
|| return self._apply(lambda t: t.cuda(device))
|| RuntimeError: CUDA error: out of memory
||
|
Andras Beres