Job 62347

InvalidSubmission:
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 563, in run_episode
    r: MsgReceived = await loop.run_in_executor(executor, f)
  File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 135, in write_topic_and_expect
    ob: MsgReceived = self.read_one(expect_topic=expect, timeout=timeout)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 218, in read_one
    msgs = read_reply(self.fpout, timeout=timeout, waiting_for=waiting_for, nickname=self.nickname,)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 304, in read_reply
    others = read_until_over(fpout, timeout=timeout, nickname=nickname)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 331, in read_until_over
    raise RemoteNodeAborted(m)
zuper_nodes.structures.RemoteNodeAborted: External node "ego0" aborted:

error in ego0 |Unexpected error:
              |
              || Traceback (most recent call last):
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 355, in loop
              ||     handle_message_node(parsed, receiver0, context0)
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 531, in handle_message_node
              ||     call_if_fun_exists(agent, expect_fn, data=ob, context=context, timing=timing)
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
              ||     f(**kwargs)
              ||   File "solution.py", line 90, in on_received_get_commands
              ||     pwm_left, pwm_right = self.compute_action(self.current_image)
              ||   File "solution.py", line 84, in compute_action
              ||     action = self.model.predict(observation)
              ||   File "/submission/model.py", line 71, in predict
              ||     observation = wrapper.observation(observation)
              ||   File "/submission/duckietown_utils/wrappers/SegmentObsWrapper.py", line 56, in observation
              ||     _, pred = torch.max(self.model.forward(obs_prep), 1)
              ||   File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 145, in forward
              ||     x = self.featureExtractor(x)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
              ||     result = self.forward(*input, **kwargs)
              ||   File "/submission/duckietown_utils/wrappers/my_models/tiramisu.py", line 102, in forward
              ||     out = self.denseBlocksUp[i](out)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
              ||     result = self.forward(*input, **kwargs)
              ||   File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 38, in forward
              ||     out = layer(x)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
              ||     result = self.forward(*input, **kwargs)
              ||   File "/submission/duckietown_utils/wrappers/my_models/layers.py", line 15, in forward
              ||     return super().forward(x)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py", line 117, in forward
              ||     input = module(input)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
              ||     result = self.forward(*input, **kwargs)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/batchnorm.py", line 131, in forward
              ||     return F.batch_norm(
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/functional.py", line 2056, in batch_norm
              ||     return torch.batch_norm(
              || RuntimeError: CUDA out of memory. Tried to allocate 14.00 MiB (GPU 0; 10.76 GiB total capacity; 280.34 MiB already allocated; 12.12 MiB free; 340.00 MiB reserved in total by PyTorch)
              ||

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 681, in scoring_context
    yield cie
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 68, in go
    wrap(cie)
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 34, in wrap
    asyncio.run(main(cie, logdir, attempts), debug=True)
  File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
    return future.result()
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 286, in main
    length_s = await run_episode(
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 567, in run_episode
    raise dc.InvalidSubmission(msg) from e
duckietown_challenges.exceptions.InvalidSubmission: Trouble with communication to the agent.