InvalidEnvironment:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(0, experiment_idx=0, checkpoint_idx=0, logger=context)
|| File "/submission/model.py", line 42, in __init__
|| dummy_env = wrap_env(config["env_config"], extra_config={
|| File "/submission/duckietown_utils/env.py", line 46, in wrap_env
|| env = SegmentObsWrapper(env, model=extra_config['model'])
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 43, in __init__
|| self.model.cuda()
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in cuda
|| return self._apply(lambda t: t.cuda(device))
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
|| param_applied = fn(param)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in <lambda>
|| return self._apply(lambda t: t.cuda(device))
|| RuntimeError: CUDA error: out of memory
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 306, in main
raise InvalidEnvironment(msg) from e
duckietown_challenges.exceptions.InvalidEnvironment: Detected out of CUDA memory:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(0, experiment_idx=0, checkpoint_idx=0, logger=context)
|| File "/submission/model.py", line 42, in __init__
|| dummy_env = wrap_env(config["env_config"], extra_config={
|| File "/submission/duckietown_utils/env.py", line 46, in wrap_env
|| env = SegmentObsWrapper(env, model=extra_config['model'])
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 43, in __init__
|| self.model.cuda()
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in cuda
|| return self._apply(lambda t: t.cuda(device))
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
|| param_applied = fn(param)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in <lambda>
|| return self._apply(lambda t: t.cuda(device))
|| RuntimeError: CUDA error: out of memory
||
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 343, in main
length_s = await run_episode(
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 720, in run_episode
msg = await loop.run_in_executor(executor, f)
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 243, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 333, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 360, in read_until_over
raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "ego0" aborted:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 355, in loop
|| handle_message_node(parsed, receiver0, context0)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 531, in handle_message_node
|| call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 90, in on_received_get_commands
|| pwm_left, pwm_right = self.compute_action(self.current_image)
|| File "solution.py", line 84, in compute_action
|| action = self.model.predict(observation)
|| File "/submission/model.py", line 71, in predict
|| observation = wrapper.observation(observation)
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 56, in observation
|| _, pred = torch.max(self.model.forward(obs_prep), 1)
|| File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 145, in forward
|| x = self.featureExtractor(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 102, in forward
|| out = self.denseBlocksUp[i](out)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 38, in forward
|| out = layer(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 15, in forward
|| return super().forward(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py", line 117, in forward
|| input = module(input)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/batchnorm.py", line 131, in forward
|| return F.batch_norm(
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/functional.py", line 2056, in batch_norm
|| return torch.batch_norm(
|| RuntimeError: CUDA out of memory. Tried to allocate 12.00 MiB (GPU 0; 14.76 GiB total capacity; 231.75 MiB already allocated; 25.56 MiB free; 240.00 MiB reserved in total by PyTorch)
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 423, in main
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: external protocol violation
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 343, in main
length_s = await run_episode(
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 720, in run_episode
msg = await loop.run_in_executor(executor, f)
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 243, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 333, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 360, in read_until_over
raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "ego0" aborted:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 355, in loop
|| handle_message_node(parsed, receiver0, context0)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 531, in handle_message_node
|| call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 90, in on_received_get_commands
|| pwm_left, pwm_right = self.compute_action(self.current_image)
|| File "solution.py", line 84, in compute_action
|| action = self.model.predict(observation)
|| File "/submission/model.py", line 71, in predict
|| observation = wrapper.observation(observation)
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 56, in observation
|| _, pred = torch.max(self.model.forward(obs_prep), 1)
|| File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 145, in forward
|| x = self.featureExtractor(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 102, in forward
|| out = self.denseBlocksUp[i](out)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 38, in forward
|| out = layer(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 15, in forward
|| return super().forward(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py", line 117, in forward
|| input = module(input)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/batchnorm.py", line 131, in forward
|| return F.batch_norm(
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/functional.py", line 2056, in batch_norm
|| return torch.batch_norm(
|| RuntimeError: CUDA out of memory. Tried to allocate 14.00 MiB (GPU 0; 14.76 GiB total capacity; 280.34 MiB already allocated; 29.31 MiB free; 340.00 MiB reserved in total by PyTorch)
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 423, in main
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: external protocol violation
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidEnvironment:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(0, experiment_idx=0, checkpoint_idx=0, logger=context)
|| File "/submission/model.py", line 42, in __init__
|| dummy_env = wrap_env(config["env_config"], extra_config={
|| File "/submission/duckietown_utils/env.py", line 46, in wrap_env
|| env = SegmentObsWrapper(env, model=extra_config['model'])
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 43, in __init__
|| self.model.cuda()
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in cuda
|| return self._apply(lambda t: t.cuda(device))
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
|| param_applied = fn(param)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in <lambda>
|| return self._apply(lambda t: t.cuda(device))
|| RuntimeError: CUDA error: out of memory
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 306, in main
raise InvalidEnvironment(msg) from e
duckietown_challenges.exceptions.InvalidEnvironment: Detected out of CUDA memory:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 298, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 156, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 338, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(0, experiment_idx=0, checkpoint_idx=0, logger=context)
|| File "/submission/model.py", line 42, in __init__
|| dummy_env = wrap_env(config["env_config"], extra_config={
|| File "/submission/duckietown_utils/env.py", line 46, in wrap_env
|| env = SegmentObsWrapper(env, model=extra_config['model'])
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 43, in __init__
|| self.model.cuda()
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in cuda
|| return self._apply(lambda t: t.cuda(device))
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
|| module._apply(fn)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
|| param_applied = fn(param)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 463, in <lambda>
|| return self._apply(lambda t: t.cuda(device))
|| RuntimeError: CUDA error: out of memory
||
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
InvalidSubmission:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 343, in main
length_s = await run_episode(
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 720, in run_episode
msg = await loop.run_in_executor(executor, f)
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 243, in read_one
msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 333, in read_reply
others = read_until_over(fpout, timeout=timeout, nickname=nickname)
File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper_outside.py", line 360, in read_until_over
raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "ego0" aborted:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 355, in loop
|| handle_message_node(parsed, receiver0, context0)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 531, in handle_message_node
|| call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 90, in on_received_get_commands
|| pwm_left, pwm_right = self.compute_action(self.current_image)
|| File "solution.py", line 84, in compute_action
|| action = self.model.predict(observation)
|| File "/submission/model.py", line 71, in predict
|| observation = wrapper.observation(observation)
|| File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 56, in observation
|| _, pred = torch.max(self.model.forward(obs_prep), 1)
|| File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 145, in forward
|| x = self.featureExtractor(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 102, in forward
|| out = self.denseBlocksUp[i](out)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 38, in forward
|| out = layer(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 15, in forward
|| return super().forward(x)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py", line 117, in forward
|| input = module(input)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
|| result = self.forward(*input, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/batchnorm.py", line 131, in forward
|| return F.batch_norm(
|| File "/usr/local/lib/python3.8/dist-packages/torch/nn/functional.py", line 2056, in batch_norm
|| return torch.batch_norm(
|| RuntimeError: CUDA out of memory. Tried to allocate 14.00 MiB (GPU 0; 14.76 GiB total capacity; 280.34 MiB already allocated; 26.25 MiB free; 340.00 MiB reserved in total by PyTorch)
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges/cie_concrete.py", line 691, in scoring_context
yield cie
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/dist-packages/duckietown_experiment_manager/code.py", line 423, in main
raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: external protocol violation
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.