Evaluator 5798

Uncaught exception:
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges_runner/uploading.py", line 293, in upload
    aws_object.load()
  File "/usr/local/lib/python3.8/dist-packages/boto3/resources/factory.py", line 565, in do_action
    response = action(self, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/boto3/resources/action.py", line 88, in __call__
    response = getattr(parent.meta.client, operation_name)(*args, **params)
  File "/usr/local/lib/python3.8/dist-packages/botocore/client.py", line 553, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.8/dist-packages/botocore/client.py", line 1009, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (404) when calling the HeadObject operation: Not Found

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/boto3/s3/transfer.py", line 371, in upload_file
    future.result()
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/futures.py", line 103, in result
    return self._coordinator.result()
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/futures.py", line 266, in result
    raise self._exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 715, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connectionpool.py", line 416, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 96, in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/urllib3/connection.py", line 244, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/usr/lib/python3.8/http/client.py", line 1256, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 149, in _send_output
    self.send(message_body)
  File "/usr/local/lib/python3.8/dist-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
  File "/usr/lib/python3.8/http/client.py", line 964, in send
    datablock = data.read(self.blocksize)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/utils.py", line 515, in read
    data = self._fileobj.read(amount_to_read)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 90, in read
    raise self._transfer_coordinator.exception
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/tasks.py", line 139, in __call__
    return self._execute_main(kwargs)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/tasks.py", line 162, in _execute_main
    return_value = self._main(**kwargs)
  File "/usr/local/lib/python3.8/dist-packages/s3transfer/upload.py", line 793, in _main
    response = client.upload_part(
  File "/usr/local/lib/python3.8/dist-packages/botocore/client.py", line 553, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.8/dist-packages/botocore/client.py", line 1009, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (InternalError) when calling the UploadPart operation (reached max retries: 4): We encountered an internal error. Please try again.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges_runner/runner.py", line 936, in get_cr
    uploaded = upload_files(wd, aws_config, copy_to_machine_cache=copy_to_machine_cache)
  File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges_runner/uploading.py", line 50, in upload_files
    uploaded = upload(aws_config, toupload, copy_to_machine_cache=copy_to_machine_cache, quiet=True)
  File "/usr/local/lib/python3.8/dist-packages/duckietown_challenges_runner/uploading.py", line 305, in upload
    aws_object.upload_file(realfile, ExtraArgs={"ContentType": mime_type})
  File "/usr/local/lib/python3.8/dist-packages/boto3/s3/inject.py", line 320, in object_upload_file
    return self.meta.client.upload_file(
  File "/usr/local/lib/python3.8/dist-packages/boto3/s3/inject.py", line 145, in upload_file
    return transfer.upload_file(
  File "/usr/local/lib/python3.8/dist-packages/boto3/s3/transfer.py", line 377, in upload_file
    raise S3UploadFailedError(
boto3.exceptions.S3UploadFailedError: Failed to upload /tmp/tmpj0f5l54dlog.gs2.cbor to duckietown-ai-driving-olympics-1/v3/frankfurt/by-value/sha256/659952554d2207cbcf6cbcefefe0ada000d4bedba6c6a11063cf2c610a357f34: An error occurred (InternalError) when calling the UploadPart operation (reached max retries: 4): We encountered an internal error. Please try again.