Duckietown Challenges Home Challenges Submissions

Submission 2866

Submission2866
Competingyes
Challengeaido2-LF-sim-testing
UserKonstantin Chaika
Date submitted
Last status update
Completecomplete
DetailsEvaluation is complete.
Sisters
Result💚
Jobsstep1-simulation: 22732
Next
User labelchallenge-aido_LF-baseline-duckietown
Admin priority50
Blessingn/a
User priority50

The highlights are available only to the owner and the admins.

Evaluation jobs for this submission

Show only up-to-date jobs
Job IDstepstatusup to datedate starteddate completeddurationmessage
22732step1-simulationsuccessyes0:31:18
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
driven_lanedir_consec_median0.24671783642101008
survival_time_median4.449999999999992
deviation-center-line_median0.21740686391939587
in-drivable-lane_median0.14999999999999947


other stats
agent_compute-ego_max0.11395562949933503
agent_compute-ego_mean0.10646578257951216
agent_compute-ego_median0.10604851979475756
agent_compute-ego_min0.0992668634173514
deviation-center-line_max0.5410051280540246
deviation-center-line_mean0.2322045937351865
deviation-center-line_min0.09709119920093633
deviation-heading_max2.335064223187794
deviation-heading_mean1.639071667033012
deviation-heading_median1.6217424294464378
deviation-heading_min1.0045413067653672
driven_any_max1.1745507824729131
driven_any_mean0.4178376009913278
driven_any_median0.33075696258931925
driven_any_min0.15799583171243467
driven_lanedir_consec_max0.5812652382396115
driven_lanedir_consec_mean0.31451619839906175
driven_lanedir_consec_min0.13536267353535347
driven_lanedir_max0.5812652382396115
driven_lanedir_mean0.31451619839906175
driven_lanedir_median0.24671783642101008
driven_lanedir_min0.13536267353535347
in-drivable-lane_max7.700000000000095
in-drivable-lane_mean0.7366666666666721
in-drivable-lane_min0
per-episodes
details{"ETHZ_autolab_technical_track-0-0": {"driven_any": 0.4274058805625903, "sim_physics": 0.12199818042286656, "survival_time": 5.699999999999988, "driven_lanedir": 0.3797453961530901, "sim_render-ego": 0.07008045598080284, "in-drivable-lane": 0, "agent_compute-ego": 0.11395562949933503, "deviation-heading": 1.9922709411396304, "set_robot_commands": 0.11529943817540218, "deviation-center-line": 0.3176699139496802, "driven_lanedir_consec": 0.3797453961530901, "sim_compute_sim_state": 0.04263411906727573, "sim_compute_performance-ego": 0.07980118090646308, "sim_compute_robot_state-ego": 0.09271154069063955}, "ETHZ_autolab_technical_track-1-0": {"driven_any": 0.20620509555673744, "sim_physics": 0.11560838797996784, "survival_time": 2.8999999999999977, "driven_lanedir": 0.14709274835295216, "sim_render-ego": 0.06654161009295233, "in-drivable-lane": 0.14999999999999947, "agent_compute-ego": 0.1087527768365268, "deviation-heading": 1.4814381778227086, "set_robot_commands": 0.10536793593702644, "deviation-center-line": 0.11406534579599092, "driven_lanedir_consec": 0.14709274835295216, "sim_compute_sim_state": 0.044122679480190934, "sim_compute_performance-ego": 0.07764923983606799, "sim_compute_robot_state-ego": 0.08308242929392848}, "ETHZ_autolab_technical_track-2-0": {"driven_any": 0.15799583171243467, "sim_physics": 0.1182698737020078, "survival_time": 2.3, "driven_lanedir": 0.13536267353535347, "sim_render-ego": 0.0631628243819527, "in-drivable-lane": 0, "agent_compute-ego": 0.10280407511669656, "deviation-heading": 1.0045413067653672, "set_robot_commands": 0.1072214012560637, "deviation-center-line": 0.22007221066592267, "driven_lanedir_consec": 0.13536267353535347, "sim_compute_sim_state": 0.042782498442608376, "sim_compute_performance-ego": 0.07197926873746126, "sim_compute_robot_state-ego": 0.07625272481337837}, "ETHZ_autolab_technical_track-3-0": {"driven_any": 0.5437260479173953, "sim_physics": 0.1233572221138108, "survival_time": 7.099999999999983, "driven_lanedir": 0.4805089801131692, "sim_render-ego": 0.06696581840515137, "in-drivable-lane": 0.1999999999999993, "agent_compute-ego": 0.1040911338698696, "deviation-heading": 1.9653899722963435, "set_robot_commands": 0.10368442031699168, "deviation-center-line": 0.21740686391939587, "driven_lanedir_consec": 0.4805089801131692, "sim_compute_sim_state": 0.041490793228149414, "sim_compute_performance-ego": 0.07430774729016801, "sim_compute_robot_state-ego": 0.08054928712441888}, "ETHZ_autolab_technical_track-4-0": {"driven_any": 0.5638086188799438, "sim_physics": 0.12490794123435509, "survival_time": 7.349999999999982, "driven_lanedir": 0.46303670019861887, "sim_render-ego": 0.06707178654313899, "in-drivable-lane": 0.6499999999999977, "agent_compute-ego": 0.10601490208891784, "deviation-heading": 1.7096441617409257, "set_robot_commands": 0.10808877555691468, "deviation-center-line": 0.16805948426737172, "driven_lanedir_consec": 0.46303670019861887, "sim_compute_sim_state": 0.041697182622896574, "sim_compute_performance-ego": 0.07485677109283655, "sim_compute_robot_state-ego": 0.08011532965160552}, "ETHZ_autolab_technical_track-5-0": {"driven_any": 0.3669189283422206, "sim_physics": 0.12820505366033438, "survival_time": 4.899999999999991, "driven_lanedir": 0.2509974226134841, "sim_render-ego": 0.06936177185603551, "in-drivable-lane": 0.7999999999999972, "agent_compute-ego": 0.11031027959317576, "deviation-heading": 1.761710510902259, "set_robot_commands": 0.11102322412996876, "deviation-center-line": 0.2078683190205053, "driven_lanedir_consec": 0.2509974226134841, "sim_compute_sim_state": 0.04129396166120257, "sim_compute_performance-ego": 0.077971414643891, "sim_compute_robot_state-ego": 0.08980623070074588}, "ETHZ_autolab_technical_track-6-0": {"driven_any": 1.1745507824729131, "sim_physics": 0.11004300355911256, "survival_time": 14.950000000000076, "driven_lanedir": 0.48465777620308703, "sim_render-ego": 0.06173054218292237, "in-drivable-lane": 7.700000000000095, "agent_compute-ego": 0.10414315064748128, "deviation-heading": 2.335064223187794, "set_robot_commands": 0.10062631050745646, "deviation-center-line": 0.2811154974207896, "driven_lanedir_consec": 0.48465777620308703, "sim_compute_sim_state": 0.038680925369262695, "sim_compute_performance-ego": 0.07064624865849813, "sim_compute_robot_state-ego": 0.07848153114318848}, "ETHZ_autolab_technical_track-7-0": {"driven_any": 0.1821042406785927, "sim_physics": 0.11682304052206186, "survival_time": 2.5999999999999988, "driven_lanedir": 0.15793000174726934, "sim_render-ego": 0.06357078827344455, "in-drivable-lane": 0, "agent_compute-ego": 0.10604851979475756, "deviation-heading": 1.0712968188829397, "set_robot_commands": 0.10852131018271811, "deviation-center-line": 0.24221339574719655, "driven_lanedir_consec": 0.15793000174726934, "sim_compute_sim_state": 0.04105632580243624, "sim_compute_performance-ego": 0.07483163246741661, "sim_compute_robot_state-ego": 0.08789157408934373}, "ETHZ_autolab_technical_track-8-0": {"driven_any": 0.33075696258931925, "sim_physics": 0.1117258607671502, "survival_time": 4.449999999999992, "driven_lanedir": 0.23258452715322697, "sim_render-ego": 0.06621848867180642, "in-drivable-lane": 0.5999999999999979, "agent_compute-ego": 0.10287774278876488, "deviation-heading": 1.60808845658844, "set_robot_commands": 0.10827553406190336, "deviation-center-line": 0.09709119920093633, "driven_lanedir_consec": 0.23258452715322697, "sim_compute_sim_state": 0.04181873932313383, "sim_compute_performance-ego": 0.07619554808970248, "sim_compute_robot_state-ego": 0.08180150021328016}, "ETHZ_autolab_technical_track-9-0": {"driven_any": 0.2825515076677565, "sim_physics": 0.1344953543179995, "survival_time": 3.8499999999999943, "driven_lanedir": 0.2397283166675086, "sim_render-ego": 0.06687375787016633, "in-drivable-lane": 0, "agent_compute-ego": 0.10908771799756337, "deviation-heading": 1.481796881343552, "set_robot_commands": 0.10788253065827604, "deviation-center-line": 0.1910915202831569, "driven_lanedir_consec": 0.2397283166675086, "sim_compute_sim_state": 0.04223867205830364, "sim_compute_performance-ego": 0.07545216981466714, "sim_compute_robot_state-ego": 0.08576008561369661}, "ETHZ_autolab_technical_track-10-0": {"driven_any": 0.5156037129034067, "sim_physics": 0.12454442271479854, "survival_time": 6.749999999999984, "driven_lanedir": 0.48090252571958, "sim_render-ego": 0.06723249046890824, "in-drivable-lane": 0, "agent_compute-ego": 0.10784333723562736, "deviation-heading": 1.6443263024592747, "set_robot_commands": 0.10838997275741012, "deviation-center-line": 0.5410051280540246, "driven_lanedir_consec": 0.48090252571958, "sim_compute_sim_state": 0.04178886766786929, "sim_compute_performance-ego": 0.07406593252111364, "sim_compute_robot_state-ego": 0.08382867177327474}, "ETHZ_autolab_technical_track-11-0": {"driven_any": 0.6120478889954961, "sim_physics": 0.12055632603243462, "survival_time": 7.94999999999998, "driven_lanedir": 0.5812652382396115, "sim_render-ego": 0.06599080337668364, "in-drivable-lane": 0, "agent_compute-ego": 0.10892086208991283, "deviation-heading": 1.7525412375760343, "set_robot_commands": 0.10755092093029862, "deviation-center-line": 0.3660595386254619, "driven_lanedir_consec": 0.5812652382396115, "sim_compute_sim_state": 0.04142953614768742, "sim_compute_performance-ego": 0.07471144124396942, "sim_compute_robot_state-ego": 0.08716841163875172}, "ETHZ_autolab_technical_track-12-0": {"driven_any": 0.32272253633077014, "sim_physics": 0.10959420807060154, "survival_time": 4.3499999999999925, "driven_lanedir": 0.21564222896748397, "sim_render-ego": 0.06487908034489073, "in-drivable-lane": 0.6999999999999975, "agent_compute-ego": 0.0992668634173514, "deviation-heading": 1.6217424294464378, "set_robot_commands": 0.1043833754528528, "deviation-center-line": 0.17154367212186306, "driven_lanedir_consec": 0.21564222896748397, "sim_compute_sim_state": 0.04080619209114163, "sim_compute_performance-ego": 0.07208354719753923, "sim_compute_robot_state-ego": 0.07926469287653079}, "ETHZ_autolab_technical_track-13-0": {"driven_any": 0.3146898079848945, "sim_physics": 0.11322177999159869, "survival_time": 4.249999999999993, "driven_lanedir": 0.24671783642101008, "sim_render-ego": 0.06668231627520393, "in-drivable-lane": 0.2499999999999991, "agent_compute-ego": 0.1094050267163445, "deviation-heading": 1.5795620194924105, "set_robot_commands": 0.11365328676560346, "deviation-center-line": 0.09881646657807308, "driven_lanedir_consec": 0.24671783642101008, "sim_compute_sim_state": 0.042490645015940944, "sim_compute_performance-ego": 0.07727251052856446, "sim_compute_robot_state-ego": 0.0902558495016659}, "ETHZ_autolab_technical_track-14-0": {"driven_any": 0.26647617227544407, "sim_physics": 0.11912110407058508, "survival_time": 3.649999999999995, "driven_lanedir": 0.22157060390048056, "sim_render-ego": 0.06488250706293812, "in-drivable-lane": 0, "agent_compute-ego": 0.10346472100035783, "deviation-heading": 1.576661565851058, "set_robot_commands": 0.10358775478519804, "deviation-center-line": 0.2489903503774295, "driven_lanedir_consec": 0.22157060390048056, "sim_compute_sim_state": 0.04065038080084814, "sim_compute_performance-ego": 0.07320197314432222, "sim_compute_robot_state-ego": 0.0772975993483034}}
set_robot_commands_max0.11529943817540218
set_robot_commands_mean0.10757041276493896
set_robot_commands_median0.10788253065827604
set_robot_commands_min0.10062631050745646
sim_compute_performance-ego_max0.07980118090646308
sim_compute_performance-ego_mean0.07500177507817878
sim_compute_performance-ego_median0.07483163246741661
sim_compute_performance-ego_min0.07064624865849813
sim_compute_robot_state-ego_max0.09271154069063955
sim_compute_robot_state-ego_mean0.08361783056485014
sim_compute_robot_state-ego_median0.08308242929392848
sim_compute_robot_state-ego_min0.07625272481337837
sim_compute_sim_state_max0.044122679480190934
sim_compute_sim_state_mean0.041665434585263166
sim_compute_sim_state_median0.041697182622896574
sim_compute_sim_state_min0.038680925369262695
sim_physics_max0.1344953543179995
sim_physics_mean0.11949811727731234
sim_physics_median0.11912110407058508
sim_physics_min0.10959420807060154
sim_render-ego_max0.07008045598080284
sim_render-ego_mean0.06608300278579987
sim_render-ego_median0.06654161009295233
sim_render-ego_min0.06173054218292237
simulation-passed1
survival_time_max14.950000000000076
survival_time_mean5.536666666666663
survival_time_min2.3
No reset possible
20957step1-simulationsuccessno0:37:05
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
20273step1-simulationsuccessno0:15:22
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
19336step1-simulationsuccessno0:17:17
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
19301step1-simulationhost-errorno0:08:33
Uncaught exception: [...]
Uncaught exception:
Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 990, in run_docker
    return subprocess.check_output(cmd0, cwd=cwd, stderr=sys.stderr)
  File "/usr/lib/python3.6/subprocess.py", line 336, in check_output
    **kwargs).stdout
  File "/usr/lib/python3.6/subprocess.py", line 418, in run
    output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['docker-compose', '-p', 'job19301-84122', 'ps', '-q', 'solution']' returned non-zero exit status 255.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 478, in get_cr
    submission_id=submission_id, timeout_sec=timeout_sec)
  File "/project/src/duckietown_challenges_runner/runner.py", line 573, in run_single
    write_logs(wd, project, services=config['services'])
  File "/project/src/duckietown_challenges_runner/runner.py", line 899, in write_logs
    container_id = get_id_for_service(wd, project, service)
  File "/project/src/duckietown_challenges_runner/runner.py", line 585, in get_id_for_service
    o = run_docker(wd, project, cmd, get_output=True)
  File "/project/src/duckietown_challenges_runner/runner.py", line 998, in run_docker
    raise DockerComposeFail(msg) from e
duckietown_challenges_runner.runner.DockerComposeFail: Could not run ['docker-compose', '-p', 'job19301-84122', 'ps', '-q', 'solution']:

   >  Command '['docker-compose', '-p', 'job19301-84122', 'ps', '-q', 'solution']' returned non-zero exit status 255.

stdout | 

stderr | 
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
19188step1-simulationerrorno0:05:16
InvalidEvaluator: Tr [...]
InvalidEvaluator:
Traceback (most recent call last):
  File "experiment_manager.py", line 173, in main
    make_video1(fn, out_video)
  File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 56, in make_video1
    pg('video_aido', dict(filename=fn, output=output))
  File "/project/src/procgraph/src/procgraph/scripts/pgmain.py", line 244, in pg
    model.init()
  File "/project/src/procgraph/src/procgraph/core/model.py", line 218, in init
    block.init()
  File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 16, in init
    self.log = read_simulator_log_cbor(self.get_config('filename'))
  File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 179, in read_simulator_log_cbor
    render_time = read_perfomance(ld)
  File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 74, in read_perfomance
    for i, ob in enumerate(read_topic2(ld, 'timing_information')):
  File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 49, in read_topic2
    for ob in ld.objects:
AttributeError: 'str' object has no attribute 'objects'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/project/src/duckietown-challenges/src/duckietown_challenges/cie_concrete.py", line 610, in scoring_context
    yield cie
  File "experiment_manager.py", line 435, in <module>
    wrap(cie)
  File "experiment_manager.py", line 423, in wrap
    main(cie, logdir, attempts)
  File "experiment_manager.py", line 186, in main
    raise dc.InvalidEvaluator(msg) from e
duckietown_challenges.exceptions.InvalidEvaluator: Anomalous error while running episodes:

 > Traceback (most recent call last):
 >   File "experiment_manager.py", line 173, in main
 >     make_video1(fn, out_video)
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 56, in make_video1
 >     pg('video_aido', dict(filename=fn, output=output))
 >   File "/project/src/procgraph/src/procgraph/scripts/pgmain.py", line 244, in pg
 >     model.init()
 >   File "/project/src/procgraph/src/procgraph/core/model.py", line 218, in init
 >     block.init()
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 16, in init
 >     self.log = read_simulator_log_cbor(self.get_config('filename'))
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 179, in read_simulator_log_cbor
 >     render_time = read_perfomance(ld)
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 74, in read_perfomance
 >     for i, ob in enumerate(read_topic2(ld, 'timing_information')):
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 49, in read_topic2
 >     for ob in ld.objects:
 > AttributeError: 'str' object has no attribute 'objects'
 > 
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
18586step1-simulationsuccessno0:12:24
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
18569step1-simulationsuccessno0:11:25
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible