From fe683c1e1bc7c28dd4c882be27fe20800490075c Mon Sep 17 00:00:00 2001 From: Bug Date: Thu, 30 Apr 2026 16:51:06 +0800 Subject: [PATCH] updated --- .DS_Store | Bin 0 -> 6148 bytes README.md | 61 +- dataset/.DS_Store | Bin 0 -> 6148 bytes pyproject.toml | 10 +- rsl_rl/.DS_Store | Bin 0 -> 6148 bytes rsl_rl/__pycache__/__init__.cpython-312.pyc | Bin 201 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 381 -> 0 bytes .../__pycache__/amp_ppo.cpython-312.pyc | Bin 24423 -> 0 bytes .../__pycache__/distillation.cpython-312.pyc | Bin 8185 -> 0 bytes .../__pycache__/ppo.cpython-312.pyc | Bin 19202 -> 0 bytes rsl_rl/build/lib/rsl_rl/__init__.py | 6 - .../build/lib/rsl_rl/algorithms/__init__.py | 11 - rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py | 571 ------------------ .../lib/rsl_rl/algorithms/distillation.py | 185 ------ rsl_rl/build/lib/rsl_rl/algorithms/ppo.py | 469 -------------- .../lib/rsl_rl/build/lib/rsl_rl/__init__.py | 6 - .../build/lib/rsl_rl/algorithms/__init__.py | 11 - .../build/lib/rsl_rl/algorithms/amp_ppo.py | 571 ------------------ .../lib/rsl_rl/algorithms/distillation.py | 185 ------ .../rsl_rl/build/lib/rsl_rl/algorithms/ppo.py | 469 -------------- .../lib/rsl_rl/build/lib/rsl_rl/__init__.py | 6 - .../build/lib/rsl_rl/algorithms/__init__.py | 11 - .../build/lib/rsl_rl/algorithms/amp_ppo.py | 571 ------------------ .../lib/rsl_rl/algorithms/distillation.py | 185 ------ .../rsl_rl/build/lib/rsl_rl/algorithms/ppo.py | 469 -------------- .../rsl_rl/build/lib/rsl_rl/env/__init__.py | 10 - .../rsl_rl/build/lib/rsl_rl/env/vec_env.py | 113 ---- .../build/lib/rsl_rl/modules/__init__.py | 21 - .../build/lib/rsl_rl/modules/actor_critic.py | 195 ------ .../rsl_rl/modules/actor_critic_recurrent.py | 218 ------- .../lib/rsl_rl/modules/discriminator_multi.py | 102 ---- .../rsl_rl/build/lib/rsl_rl/modules/rnd.py | 209 ------- .../lib/rsl_rl/modules/student_teacher.py | 206 ------- .../modules/student_teacher_recurrent.py | 249 -------- .../build/lib/rsl_rl/modules/symmetry.py | 24 - .../build/lib/rsl_rl/networks/__init__.py | 10 - .../build/lib/rsl_rl/networks/memory.py | 70 --- .../rsl_rl/build/lib/rsl_rl/networks/mlp.py | 120 ---- .../lib/rsl_rl/networks/normalization.py | 130 ---- .../build/lib/rsl_rl/runners/__init__.py | 12 - .../rsl_rl/runners/amp_on_policy_runner.py | 521 ---------------- .../lib/rsl_rl/runners/distillation_runner.py | 179 ------ .../lib/rsl_rl/runners/on_policy_runner.py | 460 -------------- .../build/lib/rsl_rl/storage/__init__.py | 10 - .../lib/rsl_rl/storage/replay_buffer_multi.py | 38 -- .../lib/rsl_rl/storage/rollout_storage.py | 260 -------- .../rsl_rl/build/lib/rsl_rl/utils/__init__.py | 13 - .../lib/rsl_rl/utils/motion_loader_g1.py | 388 ------------ .../build/lib/rsl_rl/utils/motion_util.py | 97 --- .../build/lib/rsl_rl/utils/neptune_utils.py | 94 --- .../rsl_rl/build/lib/rsl_rl/utils/pose3d.py | 283 --------- .../rsl_rl/build/lib/rsl_rl/utils/utils.py | 360 ----------- .../build/lib/rsl_rl/utils/wandb_utils.py | 87 --- .../rsl_rl/build/lib/rsl_rl/env/__init__.py | 10 - .../rsl_rl/build/lib/rsl_rl/env/vec_env.py | 113 ---- .../build/lib/rsl_rl/modules/__init__.py | 21 - .../build/lib/rsl_rl/modules/actor_critic.py | 195 ------ .../rsl_rl/modules/actor_critic_recurrent.py | 218 ------- .../lib/rsl_rl/modules/discriminator_multi.py | 102 ---- .../rsl_rl/build/lib/rsl_rl/modules/rnd.py | 209 ------- .../lib/rsl_rl/modules/student_teacher.py | 206 ------- .../modules/student_teacher_recurrent.py | 249 -------- .../build/lib/rsl_rl/modules/symmetry.py | 24 - .../build/lib/rsl_rl/networks/__init__.py | 10 - .../build/lib/rsl_rl/networks/memory.py | 70 --- .../rsl_rl/build/lib/rsl_rl/networks/mlp.py | 120 ---- .../lib/rsl_rl/networks/normalization.py | 130 ---- .../build/lib/rsl_rl/runners/__init__.py | 12 - .../rsl_rl/runners/amp_on_policy_runner.py | 521 ---------------- .../lib/rsl_rl/runners/distillation_runner.py | 179 ------ .../lib/rsl_rl/runners/on_policy_runner.py | 460 -------------- .../build/lib/rsl_rl/storage/__init__.py | 10 - .../lib/rsl_rl/storage/replay_buffer_multi.py | 38 -- .../lib/rsl_rl/storage/rollout_storage.py | 260 -------- .../rsl_rl/build/lib/rsl_rl/utils/__init__.py | 13 - .../lib/rsl_rl/utils/motion_loader_g1.py | 388 ------------ .../build/lib/rsl_rl/utils/motion_util.py | 97 --- .../build/lib/rsl_rl/utils/neptune_utils.py | 94 --- .../rsl_rl/build/lib/rsl_rl/utils/pose3d.py | 283 --------- .../rsl_rl/build/lib/rsl_rl/utils/utils.py | 360 ----------- .../build/lib/rsl_rl/utils/wandb_utils.py | 87 --- rsl_rl/build/lib/rsl_rl/env/__init__.py | 10 - rsl_rl/build/lib/rsl_rl/env/vec_env.py | 113 ---- rsl_rl/build/lib/rsl_rl/modules/__init__.py | 21 - .../build/lib/rsl_rl/modules/actor_critic.py | 195 ------ .../rsl_rl/modules/actor_critic_recurrent.py | 218 ------- .../lib/rsl_rl/modules/discriminator_multi.py | 102 ---- rsl_rl/build/lib/rsl_rl/modules/rnd.py | 209 ------- .../lib/rsl_rl/modules/student_teacher.py | 206 ------- .../modules/student_teacher_recurrent.py | 249 -------- rsl_rl/build/lib/rsl_rl/modules/symmetry.py | 24 - rsl_rl/build/lib/rsl_rl/networks/__init__.py | 10 - rsl_rl/build/lib/rsl_rl/networks/memory.py | 70 --- rsl_rl/build/lib/rsl_rl/networks/mlp.py | 120 ---- .../lib/rsl_rl/networks/normalization.py | 130 ---- rsl_rl/build/lib/rsl_rl/runners/__init__.py | 12 - .../rsl_rl/runners/amp_on_policy_runner.py | 521 ---------------- .../lib/rsl_rl/runners/distillation_runner.py | 179 ------ .../lib/rsl_rl/runners/on_policy_runner.py | 460 -------------- rsl_rl/build/lib/rsl_rl/storage/__init__.py | 10 - .../lib/rsl_rl/storage/replay_buffer_multi.py | 38 -- .../lib/rsl_rl/storage/rollout_storage.py | 260 -------- rsl_rl/build/lib/rsl_rl/utils/__init__.py | 13 - .../lib/rsl_rl/utils/motion_loader_g1.py | 388 ------------ rsl_rl/build/lib/rsl_rl/utils/motion_util.py | 97 --- .../build/lib/rsl_rl/utils/neptune_utils.py | 94 --- rsl_rl/build/lib/rsl_rl/utils/pose3d.py | 283 --------- rsl_rl/build/lib/rsl_rl/utils/utils.py | 360 ----------- rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py | 87 --- .../env/__pycache__/__init__.cpython-312.pyc | Bin 288 -> 0 bytes .../env/__pycache__/vec_env.cpython-312.pyc | Bin 4395 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 641 -> 0 bytes .../__pycache__/actor_critic.cpython-312.pyc | Bin 9987 -> 0 bytes .../actor_critic_recurrent.cpython-312.pyc | Bin 11682 -> 0 bytes .../discriminator_multi.cpython-312.pyc | Bin 5983 -> 0 bytes .../modules/__pycache__/rnd.cpython-312.pyc | Bin 10119 -> 0 bytes .../student_teacher.cpython-312.pyc | Bin 10497 -> 0 bytes .../student_teacher_recurrent.cpython-312.pyc | Bin 12937 -> 0 bytes .../__pycache__/symmetry.cpython-312.pyc | Bin 642 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 404 -> 0 bytes .../__pycache__/memory.cpython-312.pyc | Bin 3639 -> 0 bytes .../networks/__pycache__/mlp.cpython-312.pyc | Bin 5308 -> 0 bytes .../__pycache__/normalization.cpython-312.pyc | Bin 6838 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 461 -> 0 bytes .../amp_on_policy_runner.cpython-312.pyc | Bin 30533 -> 0 bytes .../distillation_runner.cpython-312.pyc | Bin 8627 -> 0 bytes .../on_policy_runner.cpython-312.pyc | Bin 25885 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 385 -> 0 bytes .../replay_buffer_multi.cpython-312.pyc | Bin 2588 -> 0 bytes .../rollout_storage.cpython-312.pyc | Bin 13863 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 305 -> 0 bytes .../motion_loader_g1.cpython-312.pyc | Bin 24487 -> 0 bytes .../__pycache__/motion_util.cpython-312.pyc | Bin 2645 -> 0 bytes .../utils/__pycache__/pose3d.cpython-312.pyc | Bin 9852 -> 0 bytes .../utils/__pycache__/utils.cpython-312.pyc | Bin 19641 -> 0 bytes src/.DS_Store | Bin 0 -> 6148 bytes src/mjlab_husky/.DS_Store | Bin 0 -> 6148 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 158 -> 0 bytes .../__pycache__/lerobot_numpy.cpython-312.pyc | Bin 1995 -> 0 bytes .../__pycache__/mujoco_gl.cpython-312.pyc | Bin 1070 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 228 -> 0 bytes .../g1_skater_constants.cpython-312.pyc | Bin 8894 -> 0 bytes .../envs/__pycache__/__init__.cpython-312.pyc | Bin 509 -> 0 bytes .../g1_skate_rl_env.cpython-312.pyc | Bin 39742 -> 0 bytes src/mjlab_husky/lerobot_numpy.py | 14 + src/mjlab_husky/mujoco_gl.py | 20 +- .../rl/__pycache__/__init__.cpython-312.pyc | Bin 503 -> 0 bytes .../rl/__pycache__/config.cpython-312.pyc | Bin 4778 -> 0 bytes .../vecenv_wrapper.cpython-312.pyc | Bin 7798 -> 0 bytes .../scripts/__pycache__/play.cpython-312.pyc | Bin 11791 -> 0 bytes .../scripts/export_lerobot_qpos.py | 17 +- src/mjlab_husky/scripts/play.py | 88 ++- .../__pycache__/__init__.cpython-312.pyc | Bin 327 -> 0 bytes .../__pycache__/registry.cpython-312.pyc | Bin 3058 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 225 -> 0 bytes .../skater_env_cfg.cpython-312.pyc | Bin 11599 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 178 -> 0 bytes .../g1/__pycache__/__init__.cpython-312.pyc | Bin 645 -> 0 bytes .../g1/__pycache__/env_cfgs.cpython-312.pyc | Bin 5488 -> 0 bytes .../g1/__pycache__/rl_cfg.cpython-312.pyc | Bin 1357 -> 0 bytes .../mdp/__pycache__/__init__.cpython-312.pyc | Bin 323 -> 0 bytes .../__pycache__/observations.cpython-312.pyc | Bin 6365 -> 0 bytes .../mdp/__pycache__/rewards.cpython-312.pyc | Bin 14796 -> 0 bytes .../__pycache__/terminations.cpython-312.pyc | Bin 1197 -> 0 bytes .../velocity_command.cpython-312.pyc | Bin 11600 -> 0 bytes .../rl/__pycache__/__init__.cpython-312.pyc | Bin 258 -> 0 bytes .../rl/__pycache__/exporter.cpython-312.pyc | Bin 1658 -> 0 bytes .../rl/__pycache__/runner.cpython-312.pyc | Bin 2094 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 311 -> 0 bytes .../rerun_play_viewer.cpython-312.pyc | Bin 20452 -> 0 bytes .../viewer/rerun_native_play_viewer.py | 54 ++ src/mjlab_husky/viewer/rerun_play_viewer.py | 18 +- .../viewer/rerun_viser_play_viewer.py | 65 ++ 173 files changed, 311 insertions(+), 17199 deletions(-) create mode 100644 .DS_Store create mode 100644 dataset/.DS_Store create mode 100644 rsl_rl/.DS_Store delete mode 100644 rsl_rl/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/algorithms/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/algorithms/__pycache__/amp_ppo.cpython-312.pyc delete mode 100644 rsl_rl/algorithms/__pycache__/distillation.cpython-312.pyc delete mode 100644 rsl_rl/algorithms/__pycache__/ppo.cpython-312.pyc delete mode 100644 rsl_rl/build/lib/rsl_rl/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/algorithms/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py delete mode 100644 rsl_rl/build/lib/rsl_rl/algorithms/distillation.py delete mode 100644 rsl_rl/build/lib/rsl_rl/algorithms/ppo.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/env/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/env/vec_env.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/actor_critic.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/rnd.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/student_teacher.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py delete mode 100644 rsl_rl/build/lib/rsl_rl/modules/symmetry.py delete mode 100644 rsl_rl/build/lib/rsl_rl/networks/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/networks/memory.py delete mode 100644 rsl_rl/build/lib/rsl_rl/networks/mlp.py delete mode 100644 rsl_rl/build/lib/rsl_rl/networks/normalization.py delete mode 100644 rsl_rl/build/lib/rsl_rl/runners/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py delete mode 100644 rsl_rl/build/lib/rsl_rl/storage/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py delete mode 100644 rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/__init__.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/motion_util.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/pose3d.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/utils.py delete mode 100644 rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py delete mode 100644 rsl_rl/env/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/env/__pycache__/vec_env.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/actor_critic.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/actor_critic_recurrent.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/discriminator_multi.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/rnd.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/student_teacher.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/student_teacher_recurrent.cpython-312.pyc delete mode 100644 rsl_rl/modules/__pycache__/symmetry.cpython-312.pyc delete mode 100644 rsl_rl/networks/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/networks/__pycache__/memory.cpython-312.pyc delete mode 100644 rsl_rl/networks/__pycache__/mlp.cpython-312.pyc delete mode 100644 rsl_rl/networks/__pycache__/normalization.cpython-312.pyc delete mode 100644 rsl_rl/runners/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/runners/__pycache__/amp_on_policy_runner.cpython-312.pyc delete mode 100644 rsl_rl/runners/__pycache__/distillation_runner.cpython-312.pyc delete mode 100644 rsl_rl/runners/__pycache__/on_policy_runner.cpython-312.pyc delete mode 100644 rsl_rl/storage/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/storage/__pycache__/replay_buffer_multi.cpython-312.pyc delete mode 100644 rsl_rl/storage/__pycache__/rollout_storage.cpython-312.pyc delete mode 100644 rsl_rl/utils/__pycache__/__init__.cpython-312.pyc delete mode 100644 rsl_rl/utils/__pycache__/motion_loader_g1.cpython-312.pyc delete mode 100644 rsl_rl/utils/__pycache__/motion_util.cpython-312.pyc delete mode 100644 rsl_rl/utils/__pycache__/pose3d.cpython-312.pyc delete mode 100644 rsl_rl/utils/__pycache__/utils.cpython-312.pyc create mode 100644 src/.DS_Store create mode 100644 src/mjlab_husky/.DS_Store delete mode 100644 src/mjlab_husky/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/__pycache__/lerobot_numpy.cpython-312.pyc delete mode 100644 src/mjlab_husky/__pycache__/mujoco_gl.cpython-312.pyc delete mode 100644 src/mjlab_husky/asset_zoo/robots/skateboard/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/asset_zoo/robots/skateboard/__pycache__/g1_skater_constants.cpython-312.pyc delete mode 100644 src/mjlab_husky/envs/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/envs/__pycache__/g1_skate_rl_env.cpython-312.pyc delete mode 100644 src/mjlab_husky/rl/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/rl/__pycache__/config.cpython-312.pyc delete mode 100644 src/mjlab_husky/rl/__pycache__/vecenv_wrapper.cpython-312.pyc delete mode 100644 src/mjlab_husky/scripts/__pycache__/play.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/__pycache__/registry.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/__pycache__/skater_env_cfg.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/config/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/config/g1/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/config/g1/__pycache__/env_cfgs.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/config/g1/__pycache__/rl_cfg.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/mdp/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/mdp/__pycache__/observations.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/mdp/__pycache__/rewards.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/mdp/__pycache__/terminations.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/mdp/__pycache__/velocity_command.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/rl/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/rl/__pycache__/exporter.cpython-312.pyc delete mode 100644 src/mjlab_husky/tasks/skater/rl/__pycache__/runner.cpython-312.pyc delete mode 100644 src/mjlab_husky/viewer/__pycache__/__init__.cpython-312.pyc delete mode 100644 src/mjlab_husky/viewer/__pycache__/rerun_play_viewer.cpython-312.pyc create mode 100644 src/mjlab_husky/viewer/rerun_native_play_viewer.py create mode 100644 src/mjlab_husky/viewer/rerun_viser_play_viewer.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..6a88466a78d6e2c881e9f21548db41fcb1f17964 GIT binary patch literal 6148 zcmeHKJx@Y06utGK5?qLJaC~uhV0C&DoE#Z+Q@)~65CXWEn7r9Ppbiek!N}}iFb+=s z1_vh-Ru|&AZIPA-VK6bq+?%wg@AkI$^zdFwiAd#7t2v@95v5TWQyEkfjQhDHY|eUC zfJ)ENsUPHum1d)rh!uw@APW3F1^C;YrwScYi@M~0zgL5IzqipW6w38x8Q%2H+DH5D z<#}^FqB?P_@jk?|*qTU{p5iXBju0P~4Hcz8D;xawT<1?rE zEKnOv>a+)}du)P-b+GaJT`n8jy^H%-t;tx-h{?1Vk4LJ{x&&a+ri#lTxH)Q%dfeP+ z=YydRcAUHZWUreqxaD+|!EaDQ&MsEo!8=i-6RNq{iJo?+#kpKQFPA&^-IzG1%WPJx z%acg!qRC42V+C+#v!#~|N-YYA0;0gT0=z$jD2$QC*r47zQ0XfGFa@_ZwDlPZ4A}vU zEXD@mfhjW-Xof0##ZYEA{GpADEXD@Sa8f!m=CPTTy`djodit0>yg Y7jgp_S&R+B1Cu`jS_Y{^fnQbN3-U*}9RL6T literal 0 HcmV?d00001 diff --git a/README.md b/README.md index 7bd2d30..212efed 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ cd humanoid_skateboarding uv sync && uv pip install -e . ``` -**（可选）LeRobot v3 导出 / 边播边录** 需要额外安装 `lerobot`（不在默认 `pyproject` 依赖里）： +**（可选）LeRobot v3 导出 / 边播边录** 需要额外安装 `lerobot`（不在默认依赖里，以免与现有 PyTorch/CUDA 栈冲突）： ```bash uv pip install lerobot ``` -若安装后出现 `import torch` 报 NCCL 符号错误，可尝试： +若安装 `lerobot` 等包后出现 `import torch` 报 **NCCL 符号错误**（例如 `undefined symbol: ncclDevCommDestroy`）：多为 **`nvidia-nccl-cu12` 与 `torch`（cu13）并存**，二者都往 `site-packages/nvidia/nccl/lib/` 装 `libnccl.so.2`，旧库覆盖了新库。可卸载 cu12 并重装 cu13 的 NCCL： ```bash uv pip uninstall nvidia-nccl-cu12 @@ -58,7 +58,46 @@ uv run play Mjlab-Skater-Flat-Unitree-G1 --checkpoint_file ckpts/test.pt - **`--viewer auto`**（默认）：有 `DISPLAY` / `WAYLAND_DISPLAY` 时用 **native**，否则 **rerun**。 - **`--viewer native`**：本机有图形界面时使用 MuJoCo 原生窗口。 - **`--viewer rerun`**：Rerun Web Viewer（无头服务器常用）。 -- **`--viewer viser`**：Viser。 +- **`--viewer rerun_native`**：同一套 mjlab 仿真与策略步进，**同时**打开仓库自带的 **MuJoCo 原生 viewer**（`NativeMujocoViewer` / GLFW）并把离屏相机 / qpos **推到 Rerun**。与「两个进程各跑一套仿真」无关：仍是 **单一 `env`/单一仿真循环**。**不能**与 `--lerobot-record` 共用。 +- **`--viewer viser`**：Viser（浏览器三维面板，另一种自带前端）。 +- **`--viewer rerun_viser`**：**同一进程、单一仿真**，同时在浏览器里打开 **Rerun**（`--rerun-web-port` / `--rerun-grpc-port`）与 **Viser mjlab 面板**（`--viser-port`）。三者各占不同端口；适合 RoboHub 左 Rerun、右 Mujoco 双 iframe。 + - **注意**：`rerun_native` 里的 **native 是 GLFW 桌面窗口，不占用 HTTP 端口**；若你要「两个端口都是网页服务」，用 **`rerun_viser`**，不要用 `rerun_native` 来凑端口。 + +`rerun_native` 示例（端口与 `rerun` 相同，见下节 SSH 转发）： + +```bash +uv run play Mjlab-Skater-Flat-Unitree-G1 --checkpoint_file ckpts/test.pt \ + --viewer rerun_native --rerun-web-port 18080 --rerun-grpc-port 19876 +``` + +`rerun_viser` 示例（**三个端口**：Rerun Web、Rerun gRPC、Viser，互不重复）： + +```bash +uv run play Mjlab-Skater-Flat-Unitree-G1 --checkpoint_file ckpts/test.pt \ + --viewer rerun_viser \ + --rerun-web-port 18080 \ + --rerun-grpc-port 19876 \ + --viser-port 19090 +``` + +远程浏览器需 **三个** 本地转发（把示例端口换成你实际用的）： + +```bash +ssh -N \ + -L 18080:127.0.0.1:18080 \ + -L 19876:127.0.0.1:19876 \ + -L 19090:127.0.0.1:19090 \ + user@云主机 +``` + +**SSH / 无桌面 / RoboHub 技能里 `RuntimeError: … DISPLAY`：** +`rerun_native` 里的「自带 viewer」是 **本机 X11/Wayland 上的 GLFW 窗口**，不是 Rerun 网页。若 shell 里 **没有** `DISPLAY` 或 `WAYLAND_DISPLAY`（很多容器/编排默认不传），会报错。处理方式： + +- **只想要浏览器里看 Rerun**（单后端、无 MuJoCo 小窗）：用 `--viewer rerun`。 +- **仍要 `rerun_native` 但机器无物理桌面**：可装 `xvfb` 用虚拟显示，例如： + `xvfb-run -a uv run play Mjlab-Skater-Flat-Unitree-G1 ... --viewer rerun_native` + （具体以你镜像是否已含 `xvfb` 为准。） +- **RoboHub 侧**：需在技能/容器环境注入 `DISPLAY` 或把启动命令包在 `xvfb-run` 里，否则与本地终端直跑表现一致。 完整参数： @@ -68,14 +107,22 @@ uv run play Mjlab-Skater-Flat-Unitree-G1 --help ### 无头 OpenGL（MuJoCo 离屏相机） -在无 `DISPLAY` 的 Linux 上，`play` 会在导入 MuJoCo 前尽量设置 **`MUJOCO_GL=egl`**（见 `mjlab_husky/mujoco_gl.py`）。若仍失败可手动指定： +在无 `DISPLAY` / `WAYLAND_DISPLAY` 的 Linux 上，`play` 等在 **`import mujoco` 之前** 调用 `mujlab_husky/mujoco_gl.py`：未设置 `MUJOCO_GL` 时默认 **`osmesa`**。仅设 `MUJOCO_GL` 不够：无头时 PyOpenGL 仍可能按 `linux` 选 **GLX**，导致 `glGetError` / `eglQueryString`；因此脚本会同步设置 **`PYOPENGL_PLATFORM=osmesa`**（或在你使用 `MUJOCO_GL=egl` 时为 **`egl`**）。可按需手动指定： ```bash -export MUJOCO_GL=egl # GPU 无头（常见） -# 或 -export MUJOCO_GL=osmesa # 纯 CPU 软件光栅（更慢） +export MUJOCO_GL=egl # GPU + 可用 NVIDIA EGL 时（更快） +# 未设置时由 mujoco_gl 默认 osmesa；或显式 CPU 光栅： +export MUJOCO_GL=osmesa # 需系统已装 libosmesa6（见下） ``` +**Ubuntu（OSMesa）**：若仍报 OpenGL / `glGetError`，请先安装运行时： + +```bash +sudo apt-get update && sudo apt-get install -y libosmesa6 +``` + +若报错 **`mjENBL_MULTICCD`**：来自 **MuJoCo Python 枚举与 `mujoco-warp` Git 修订不一致**。本项目用 PyPI `mujoco==3.8.x` 时，`uv.lock` 已将 **`mujoco-warp` 固定为上游标签 `v3.8.0`**；若在别处自行 `uv lock --upgrade-package mjlab`，需再次确认锁里两处一致。 + ### Rerun：端口与远程浏览器 Rerun 需要 **两个端口**：**Web**（默认 `8080`）+ **gRPC**（默认多为 `9876`，以终端打印为准）。 diff --git a/dataset/.DS_Store b/dataset/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..fcfb195138971f2e85186e9db50f63d62071a877 GIT binary patch literal 6148 zcmeHKK~BR!475un3S4^RxIaMoL8!tD@&KT;P^qmF6?)GT_yb?y2}m5c@Dx6P#CUC` z4M_zj1jv?lXX733I+G|H6A_QrlZ0qUL<2Oz-T}G}lX+<^>zPL;Ij&JsLANwr+$4F6 z?T+85fIi$6P3fA>Y3uo270Xf+-oFP9=o4pIWZ8I8jAiceSKLOP{3?06xdN_$EAZD8 zz?m&JI8yZ96>tSyfm#9iJ_Izu)UZ;FpAK}f1OWEoj)t++5)u;(Q^QITGceXrpoX%Q z7_8y&2jiuNm7<0dTk*lRGh2niad)gABAqx@^xhS41)2)%>E%$)|Fhloe>2FhTme_$ zUn#)-EYC)`B|BSNHz#LpK)XN_6JDiQhhPwoVsPasK1PcMe~l)*b! I;71ks02*ytga7~l literal 0 HcmV?d00001 diff --git a/pyproject.toml b/pyproject.toml index 558084c..5fc73a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,12 @@ play-lerobot-rerun = "mjlab_husky.scripts.play_lerobot_rerun:main" src = ["src"] # Helpful for recognizing first-party imports. indent-width = 4 +[tool.uv] +# py.mujoco.org 上的 3.7.0.dev* 预发布包会被撤下，wheel 404；强制使用 PyPI 稳定版。 +override-dependencies = ["mujoco>=3.8.0,<3.9"] +# `mjlab` 声明的 mujoco-warp git rev（1dc288c）依赖 mjENBL_MULTICCD，PyPI 的 mujoco 3.8.x 尚无该枚举。 +# `uv.lock` 将 mujoco-warp 固定为上游标签 v3.8.0（与 mujoco 3.8.0 对齐）；升级 mjlab / 跑 `uv lock` 后若冲突请复查该包。 + [tool.uv.sources] -mjlab = { path = "/opt/vendor/mjlab" } -rsl-rl-lib = { path = "rsl_rl" } \ No newline at end of file +mjlab = { git = "https://github.com/mujocolab/mjlab.git", rev = "13212ad" } +rsl-rl-lib = { path = "rsl_rl" } diff --git a/rsl_rl/.DS_Store b/rsl_rl/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ba25014a79851ecc2af24f7ce46efe35d42948a2 GIT binary patch literal 6148 zcmeHKJx{|h5IxhPR%Iz014G2bz|aotp(xuTE4YqB0u$Vg+4>ahzMqmh{LvypVHD z=#*wOpbi31^C;oP=|)pr;v>Aci6ADkHgVe5q@gt;YMMz*&T*m#LMH_ z$K>YuaVHy@yMh18=t z)Cp*6*S+0HBh31DdvX25JSLBPl*I?DE3AyW53s&o$|rS}WVH33g)uq3wqj1HxlC4H z;VfR%VqPV@H*;`fc9SHlm-{rr^?K8Q3r2zU_ZC*1Ifoi zn0!w92A>3K@&4qaEIvw~WvDa5&NxhJ^WcMv5jA;Cu3{b*V!IR(xPJ#cd0Iv@o8hc;+NO`$*pp!=c_#%d-;hN79oa1TlYac0(2u!F{c$F&j5kshS zjC&gAYac1E(n-k5m`7I@`h+56b&PuwPQq8-^-ck&fUQ75{jBo-zng#lx0Bp2r+`!7 zPbnaZ&33baThe=L>gIT__0W#c*ccZnFH_Lz?O0y$R=kEL4Ka@|;C<~Q)2$EV~c$H%W^_zW`Xmx+FPeo=NYP)VwO zMrm$hUVdgud~tSSNorDlVo^$FUb;TS6#e-4%)HE!_;|g7%3B;Z5S#3Z*nozB+*Ay5 T@&{%{M#fJ(42)VuEI=**x#~BA diff --git a/rsl_rl/algorithms/__pycache__/__init__.cpython-312.pyc b/rsl_rl/algorithms/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 6d2ed609a1a73436caaf00f54f48042f7695ea6d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 381 zcmYk1KT88K7{=dRuDw#OP7V$oZYmVCH|Qve;3D|vaJ(hN=6VfHE=O`j>gJcw-ObP9 zHwbhR2c48|P9}$f@9^aJB+rwOm%*S9ZXSjod>;Y4dh<8IFKiARJOYCp7-E4yGROnu z84E1KJZw1>9ofyh2ufg`CzMC4?l`&|oyl4hvZxx`a8)I0mM}h>72542*XIc>9hk|x zdmF~c#;4pgT!=R8JkGK!w?row8S#j9=zI?zjKy`W+KP>+ ztciEN<~vjfLY`scEPNlQw@NQeO{YaVUrJgj&WKsirnpg5GhUUcHiBr8Qc)_+o4GV8 xA-v)ZA(MJ#dxui7r6`W{mb=gY#_Txx#0a6U7|`AujMlKdhN126-bXrihd<8`XjT9K diff --git a/rsl_rl/algorithms/__pycache__/amp_ppo.cpython-312.pyc b/rsl_rl/algorithms/__pycache__/amp_ppo.cpython-312.pyc deleted file mode 100644 index a36b38a856919bac70c970d5396fe98bdb1c6d77..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24423 zcmdUXZA@EPmf$nkVDK|GU&dez2J_|cnUDkm;VTgGL4c%^OuozI_&LPJAJnr+0%cFC zd%I^zPkI8{oegaFD4>~DD0S@=HR@5Nq>(y3+pA9Rtft?MDLwbww3eQ!)u{HzZmO$B zHQgF%_ni0qYzLd9x@xr2KFoXfe2R+-1-i4mS-$Lx*=g=-I&hct}1s4lMcdl3G(_hq!Znd}2S9GyRbmw`CeI*x5e5DslMSs56 z;48aWrl7K@BNU_g7RAs%$dahOXq;2ow4b2IrVum^zdsOigxmpt5WG2OT>fBy1y}Zv ze-1us#^suE24?0az@u_rcewrF&-JCCLjk+f0co## zT~3Ky>76Zo6rcS8IPOy}>Y{?7E@qvDn!lLMWVw_~_Cbir`;vc*HVMfTFsg4syD&N5 zqAunzx$w?qbd3623TTx)@Z~Z2@XpT@>+zz7p}|MbDQ$WocWC^i{p88X`Qs!Hw0u&2GnTXOWtXQ=b|t?*h&vzO3)+v60u>!H{4+ZlZSP7OA&+t48DByMJmD-K}`sq;=^cu z5U+g&6*9@808JxEs8SA*Xwr-h2{igmZwE_C%@sG zuqs^qgE&J48p`CjF!skp?+;^tOw@B2drw6IlJe}8-O2A)p;oMuM7xC!$k;qRvMc$O zdmT_xC5L|L6v>o5(z#zdSJ zd`$^oKKK&7J*)xWu0)&;e92y|2VbIJhl{}1nt&_>Ut7Xg0KWEwuLOJ@vTu&El}-qn zDR1Da!)s?9{%Ii}W>PO>54z`FEOPEokg*6_ugk&uaiU}$A=f|r^E;nD9lv^VK*(WS zH{4DaOr%*}Rw$hfy6lW2hO~_A*Jl0B`GX@sfF+dx zi_4GmI%Byy8?uD1yDSKYf``Q&6e*Br{vHdfgVKWRkTc_89loHT^gDbmAmfViaysT%p+I=W-*r#S@ z1s%NXsdZj}0m*t7vS$J_EKYhtHtT1EycyQTxKZ#C@pF}rBdCM%MiI1(JLqIV zjPN@!&n$T1oFwD{rVunRWU9e32IGR2SYT`op#X#Z0Tu)jBtxuCAvX@kqFgKFpMh=` z1o=DA)j#t-Oxn#X)n|wWWvmPT`UR$LT#cl><;FJ14hJ@%Rb{` zQ%JIz1ZB|Wy(U%`yBkum`|!0N(@PZ!MYsmt5r{WnHL#)_YriTAveHN$#1zPs*#UeZ z8)OgT3)wF_j4z~z9mUrXd>zBrQG8(!Wsk#4$i+_Nyf!7!a$gXA+CqLeQ;@Gh4{QS1R9p&`kjKk?_zdq}8_ycZWHPg_mt_B<|1LM1$4FV(bwkJf_U^^Ch zW+u=!GbgC+c4*3w-9E36OTcY%h#9gD^1!j)rk)?9%FGMOyP8;~ZKH&%+_#|IRF`jQ zjGU$}X6@K86YIdj$R=%x)wXZYq_%HivLSX2KM)ntIHs3UdP5tXrl=u4#an2A`Uok56!?u96Yw z!iirMS3lF2a)wKr`pX~d%9mTX#sN|>_!H}fP+~Ju_IIXDLdJF=HK9-$#u7T-m_$Y(?pi;t4TBBv{kBo#q^gKJO zD1Ra6xthw;-P7LD#tap_p@|rpctaa8wDE=m#Bd;L=#5p@#0=G+<|@nbKc|#=8YpmD zf=*)SuAniV3Qc2|h2NDykn;tW>Oyu17Y@`+&p(pxsoDw8pIakwi<((t%%d zthx=qm9feu{FcW`D*!A%R&0QeI%eL5p%t-)gZQ<^D(XZe@miB(_ncW!exz>y4Ei_a zOso{ZgX=X|DJed~F@B#4;Y2MyAf;x36srUYm-65ozU}!z91BPSiP$;n9mNF9Zpfo0 zj?e`zTMj!0A3+O@83rD%ljmk!f*P5s%YP#%l%XrhwSyvWgBT$@a5X4s9pW?_vn9X~as4V>v!M%$LX`IyH0!;DRVty-j^wRv)9wV{}Q3w#I0k=+@sGyEFEg zHrJTHp!_b)o&`WBR7v7Nw^&7c;SCGdj0?j|Rm9qMfQ1F{_9rQjds3Sx4rW)%@H=3{ zsdd{ER>{#62mW6o`x%GS&WRiaiG? zY=x2_M}rj0x}38t3lfu1IOPi2ue%w>sCSgLYR;ZJOGyoA&2w2kQ-+R zg4%`h6C@IHL_sPjlps2pUw8vk_8B&CRnVt}XrvIk&*cCCGVTe67=f1E?kSMpXy89C z_J*{-$l@{@nA;e9*h@%cE(#R@5=0IGJV|6o>{am0Yh*qI`2Gw6f=XblZ&NXi?q2Vm zUS4B;sgp-_bMw=PFd73m&N1J^zbW^Nh*TdN)=FOlFxZJIud{yfWu|OX0x@dD{c9`fWkrorkHy znUyRHqX;W+Q0zB{NfGsk-p`$96FqzEDAo{8pHrAeYnxIBqHrbG}G zPT9wUy{7C5ArH1JTy{8>39`8BYconyOUdjEY#_K2dj}%lX6pe0&VIZBze}21a#`0+ zSSP~X2Loa8Ndq=IV6!9WcEaR@u|nbYkf69AhOLAEi<40B%~_WVwkNJ3`n*}c?B4@^ zN%qBUT_FdMy4a97d9YrLLy=g}IT$8YL>(K| z2GT&%t$N$T^K0k%`a`7t(BsZ%{SjQT)^0&8On5wPeM(zbL(<$r=)R9=T*%R7%MC}cph!f*eaD26i(UX{z07d+fr?Cjic!?=V zj-{wf2W6r>n0XRO$gW8YfHxs)CPQiJU4?v9qM%?PXW^l2afZDun_Ak&azSF&i7#o) zI0L?!Ss34}D>TdcrST|jP~pbY3!<6E>j;LV?X_*<**5`nzDyno+faT4k-?MjdYjrZ zRNS9ln&u6=h+)@yXVlQn8}<{!{-~jQL4(bgX!aO%)e>JYsf(Z|6=P3u<;A0@QR26{Tl~2mW_BiewQ^?ayMU z2pkP$Tz?`&o(KU43z-C@OzqF^2H1p10aV5x4^q7C0mv`i{_B1rU>o@~HyESXpCqd>?h4i{%SmX%@Vfi{3>mhmOO zG6a)74}tR*uw!N%I!zL_;EH*;zmu3{*Dv}~~q%>UgN|++3!Av4xDxTtS z$tz*tfHEV$!;_J157ZE-R!IE>j%DM$Pi&D4OG;1Yw&dUq7bfb#yhFbGGNh976_r_% z`9`XCzK#&aZvzK!JcVV46u85&?TM$TffT6EvF%yR`bHDP^lu9W zFOxS73zEew)|fKF2Gil>v4JBj1@cfjWJ>HvdTiJdF2f$lG7WN^sun?6=^2Y(|AxW+V;Gwlgk0)1?hWj8vuCcfrpEO9V5tn8A<<+hSBa(wdZ!4>efXN0-}vRtHohf>uurP$bai%X5)PPg6=Rzn6|H+!$_>+w^5pN(9fOX42EM^Pv7y6)r-WJh;KI?a5pMsI818 zlPN)-7uf}POGLc5i?Qut5!nESOXA$BSU=6%kYY6p)De9kUGxMQgJLQ#LevtF_LT;Je)Zl^MY)MDubD@ zGnr+>_7`NsTdDL&?zp<{cG)nqEcRA7g<&^4#7zP9DFJj&Mg@X4#!Z-L3>T zu4gmVQ>HSuNA{#})VI^~^ZcvyFqytcCrI3#;f}D0p(?2Go|lzHf|2`aWbY0%8bvwZ z5Z<8|Fy$B@-YfPNE%p|_KD_s3`HDHE7lC4boHD8!(gEZ$iYuYaC*a>sPZicnZINi5 zeNfJHFv|0<(*wi%lBu@O5j$YHlaA$f63hM_QvG3is$||Tt;?>j(O0^muXKm^hr1Ge z;6PXx=?ND^dVi;uxSLMB=U#ePxW}`UAvB|WA=f+9R|e@LL2nPlf7#r7UxBMPt%dbW z1@>(W|8^E*x+boZA_v0<83W3Uk-l)B=ij84NbaY=9~&UgKTd}T_j>N9hbE7X=S$`~kka~Zr_=c|{u543yPTdjJmC3WdZ=7unOi6+YbM%#@M}=m&uCLp zXQHwH>(GdNFhS=KLx%^$LpTFtc;pb2@ZYAF-}5KwVc{O8@?CQR)3EAod=E3#xT^vX8CE+0kp~bKgt&e9G-aiB zQ$fYeECqD|c3u^5)^hjIt?UcbO+{La`tE<(Ax3rgKkpEuy!$ID##S>SS^$ZP8dE)Y zJO1vE<_`C>Mj(T&fR~^|{RB9>#T_>mhrE!D0a=2|=Lk**S=_+_YZ%4>b}BbqQ=eY{ z`-Q*!r;q!mAQ+#)xMOSJlm7z`oG2?Dx7@*Zz?LX56Oa3+0U4V5hqp@p{K&87zcpY} zh)Nm&mmOrCY#E^X2N5CleBcwTKNh7=p)kebNHp!ZCggxs3m6W6^3RanCm>g#9da%h zdR%23c3ZAcAlY=t)w)Au&MIV_`cmzspItbWm~-w1hE*?@y_f|h}xbcb9%7SBKgS~Mbp zVg+Vrg3ys~&9FBy9JG6&eI>J+BO>aVz|Evtji3^B+eKA(uuX}VMpAuv6T!3mej#`K z^x+d=_$Fvk$ra6KM1@&FFBw!N^e9*sm+wHWg9Sjz0!WvL1sioxUt;%&L^%9&qOFjS zdkqZUp?HE))QD1|;hvzEl_L_JG)Gdf60)6OfTjYieXyp<10AS%L#nSjoYUB6*j}(^zvv@)SR!pF&#NyVFZB#NF&|&vTwUjCr0}{hZ zG{O>SF;E_0TG59?3QYxtBGD@sLkiQ2VW}fd3P~AqV7i4_iNB=!}pL`K{~PoO0)Mxm-;sxN^eKN%`0MI;J)ywAyHmFSHt zqKov}v4Vs$89?l5*tOynsrG?MoJ7!z9Zr`R`EkhNxM3%-Y?tqB0fR>rr;B_Gn+gfs}&@p5Af;MKZ;=r=XRLOxEn)u#|>q1tV?}*lI;~3EQoxWI2 zEdh(XntDz$#M7r}`5P-A&?_`=945x$$0Pj61u_D^YwdKQj>rlF;duWl#R?R@npsU8&-7V8LIhl-O23H<1Ts4IEm z&Iz8bA#}~^$iowBC-}Of1b+0-FSV76wddk|TP@YR!~dh%ZY zh}=$Bo%J$1H#F2BEtbM!AxDuCe>Mkt!_vv`nKdh|#5@-Boprt`r>&G1?fT zO`^99sDcJK0tL3V@U{C%?f#87N$mknw58OS@Vfm^b^D){R@}d`bmfojv8HaWcZ}~n zNqSF4dtW0>uW{$5_;b_b+;sGukDCdRb0N|=%azo9kxd!uLETedaxe0o$ZE+$^O~7! zAAMSLWXsn2p7|X!*ELCOCpmq?ubPKq#u_mDj6u6-SG23wh_UtaTxClU(KjvTE%$-> zo~dft%~$OqReLseld64O_ZX=fi&mUi%mu>Rjw<-p3DP?8~$x-b^2v^*GF8RIM4No70NF~aR0;VKU=pm8VEC|Gz9-LCb}dy#h{kMpDL$Dx7v zQX46?eN@`~ti0+$`AYdSqxnJJO5SSohK|$@JZ>PhN2A7L&rFsF2UiZR>!POazuNbs zgC8DzLUZTrwCrh}hmXO$m>tW984nO$B5 z&01yoT$58_SUd!cQ(gDKv*KAV`ksFg`;oDdFKc>Q)&vCbMjJ8Oc;jATgev|?DL?cY z8G4N$x52rJ0m9ufO@UtRrTsm6Uxq zf~3CZsj27L?!6ml9_{6O$4T!vck(ppJssVBW_jeJ%C?vp#^}S_YquX2!!UhnJ_tRa z^RLZ2rP!=&OQpHahQoNHgx@_(b`OIY&)Cn#HtWtW=cSaUlNdWUjy>MZ89O(Phhwe# z-+TR?*Q2d{%gRmTEZ9exGkVwApxQSp#NAd`mxR>0j2YjyyHmKFy|@h8157 zDJVnB=SLM346S2J$GED!Kb-v_@_vLnY>#%mL8{(}mN*s;V_TY6%v@tvw7i=$c0bxj zjDriu&^%cjPXp1KLC!MxlYKw!{c&$hU%D7vKJ&o7Vkc!S>&}gl4<_HAByGc>hdrV| z9qgI{jy8V|_QBQ%7gjE;&T)H&qvjFbe2kcnMa^Ris9}v`e%Djou9&5B<1Oy+dD3;B z-)|@T?J=;GwlCRXY_Bp818V`m+Es)|i(uH~P&F;UnAq3Bic4e%{iMGC(HuW8 zNd_jP1E=_bb7bILbl^g?9+*)rbmzML-2UT`ma~ol-}y!BFfco=VgI9%AB}%F{-l8% zJoR~YRvU1kmPjh|QsMl=D_};xi&X52RUc{hv2^m5VPYA6JkF1tAtPtFvlsZYZ;-QZL`NK)bBcHRiPImo1QxYv3Aoee z_|vbG)2~M@_C+m_UQ`2>P}{uTO=|Z3#i>VG8)r#PF9;binqp! zsy0&97Om>oFal@p`LO5l^`H8F?E6mv?)9tu>u&P8d$ZRQt(socJWEUPXwOenKh6Jf z{*&&X_x-HzZw3EvI9VP<}sqH1EgNte$vh}=W53%f#>kmveILmO< za%D;TaeSW01|~B0{@A8za8dotQvZ-%qt{PAD*2J=L(`_^@S+w;@veB+J2yffy!HND zn^nUSdE3LlxMT7Bs+N?t61r_+L==9i30=+84TNr3jqr`#q_I2N*uystlE%R(Fi(2; z%Rxbl!IV95XM(3uK-;82zR}+szcYS!ato{>p+AG|3a_sv`dVJ!Li8=0`nH(f$m?r} zzGhR8#xKb8EKhY75fBYrHuX)P16rMAf+Na(XQYwdOX_>0^?i>*{J<$PaB7o&4RaH7 z$MS5_HIONfnHpog_xjg^i_R8&ObqAq47C;QFUhS8rzNobu1UQhq zKt5X@=PTPtWm~kegRk62D)&Vzy9nL2Z~`Q_vmt#sw8hPPl3jAH9{5ldIpa)8hv)`1B4lE^JxC8ukO>CLM~oM8U6n&>+gwOd%6 z)|kF_)yFsBfQ~lw5Pi>6;B1q*RNqa-^E0_rnMt~~0q4$90zbnj8g7&=_Y!pj2>WW& z@>@i`3nhMa$+Cv1tswTROP9e;rw(L)b;a^DQQL3EYd?eP0lr!LK3 z$a`KyLBpY?Rdvh4u|*dcBcbUPt}g2KLx2Z_dKm0L#TV{e;AuM4mVKN!2WVWmY^~i)BsCcL(qe%H?x(_T220XZFoXkCQX~b(GGSo@N zQi2-g_6PFGeledpC0yvJnr}QL?@3y4Bh+h2-t%M$1>hn_ujR3C-bG9L|lJMJIuWZg1d?0jxGBi!6lR?Ha@}h-Q$A&0Wa*U3OX-rhe{hm;sV~5 zCGFnPA|=uuU@(Xq>v21Ip(KeYN#}f}yhP6wTLkHYxRC?b^W;$F<_DS;4R35C#~B^1P`##>XO%u;>5EzhHtE5S)mmPSws*X`ov7ggpT{+vx_whU zlCXcY0z+6mu~E1-xvAcpKpVG#tb)}(Pq)F;u^xUj^v>;%Xt)K3qD#`vymEnBaST0% z_g5K1Bh)>GGiwU0vai6dDvT#Zrt$Q);;ue(Or}wmRCZa)y4#|pB3a@%&2T`)o)S&} zGQ?!Ep_j+s{|lO(K(ly6n3cS@Gt(FkCv3ta*(uy0!vSH&G5xAEMbN(6o=mrQzMGwL zw2k)z7`7z>uXq57ap_Ui%W69Uw@YT?9c*3lZcn^{G84?p@X8c}F`_b}3M)aWn|);u zG^Gfe9JY<4dEWKZ`RB^w0sd}~Oqry6F&+!Enr*|coc_FM3ZoPkthZS_!@y)Ii z(b6f>bz?!)1AXeeZkS?{xFc{iL}++B_I7A6hsLX^M&$a{gOs^}^wp zuH@eQoq3|GkHIDzOp4#rE~>WlrT6of@+BkrdZMpib#LT6GCZ#Tu!?Iq$~T-O4JWx% zzsH}tL{44ePrX4-y%BA2Z0fIW=?%;EOS+|;Pqmc6@|$1RP=(hNzxlh;sUR|~KPfz7 zP`+n|*N?jjkC<})I?oJmTd^ogxL5x`L9wWG`3VX*@#`#qHZX;U?j?WzzXztSiO>HU zU>6~t{Vu*r@rB+q>_Q^&3AjclCMMwWunBPf7EW(r(1k4%c#Xp(f;B=nU{5Ao=fGvt z;^lsRxa81o7xL|R91NEzf}6J6;Uw4_kIA*$nSc|5b#{Bo1>$x)izZ6!6?|RA7wVg{ z-^3RlAFxKu5ql_W!Pi-QoyHeVCgKa#2!d8MI}rxnrx^x>eT%7biULnp3FGDxSAd zMds)2l6wzTR{OkDa_^^%*3TrEU*F2kQ|15KK&u)*tE^Gg{CZTO0)N4bQdRz($;(oy zf34Q21{I%GRH~%w21J@|m6C49SB-ayAqQc!Lr@DUhycY?GQODBT-g^kQHe zoK#6Sn99(pmw-ywiPG^T;=!_bGpW=X79=gvLW26BluUs5Lc{ST_)msIzErw;y-YW^aJQ7bIJrtmG|{=Y;z6wCks diff --git a/rsl_rl/algorithms/__pycache__/distillation.cpython-312.pyc b/rsl_rl/algorithms/__pycache__/distillation.cpython-312.pyc deleted file mode 100644 index 145edc81485e2e8934eba6357352567b8ef7ee1d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8185 zcmbt3ZEPDycC-A_l3a{5;x3YWH5Kn7e`0*aG@G>DA?wSxpW8ExV8Uw*dek8(}=mR%%(fd(j2pnzfrC|VS_ zzS-q2DVk31u48NF&Ac~nW@p~KFaC35qYFX#Rrh~H|GopEej*3bY{X9wP{w$@Iq3$B5;u<0juqM` zMSa579-lFme0Tmtrh;BZNSlQp&Xq(VGKfnWyXhcw%sK}bHMfH#;Q@w#EGElV% zA_%HQN~sN^u5d6BJ=K-wFqaS{0gI~B)H~R&;jX}^h*9$3RTsQ!18#|PqQovmdC+vM zC~=Y?s;$XPf{hCtCNG~A(y7Rj2%Ewj9|d&>nB=0gq&Auq7-qq1=iX%r4Vz5iglb8` zIt!U(gsjNLRT@r-C8wggw4>7D1~S4f0O`-Tct$vbaSE%pIA|x;wk)iQs+(*jljh03 z!fw?j3h@O@J|723wU1Q$Nfm-hb!zO$v11}MxP??i@2E_9A(oFuB#}JuP^J9O=}|b` zWiib~gwdr;f=i~NaNWzW=zNO9JejoO3jwp;E}cX-E_@19(@73q#A?m*t%SL!{o6O?;TtWOo0P^o8h zhmm`ZuTMQ@_CM+ED-ZUPcK@!!;6gLE-8bF0n{GC3NL%l1zE`AsOYL2-00T6g$BG?; z_g^V=Oz!Xxmme%E$1cdnE)xzF^_78t+w+3A6%r1=> z|28AT$8{aN@07cau20>%0HfV-2#@K&cXreo%VVd2YEd zmnd9K=C7pk!E~YJojmi-GdM;RF;y*ehzi$?$teJ~kW?`n6ev1t&Xxlu7Bh_o5S7cP zWiFuNOzbN9gqjIkwMV1NPdiYD9hj&pwb8JLED3wB`z(9WLaJM@G#-ZXmV42%Gr)R(o>B(2gD*#!SAE5+xtsQM~U&3nBEfO z(O~cG^Eb~wYixicxBUf!iB^o5-vu3VM%ph(c^hE-VT>YhW3W1_7^VewP@wXhMZ2fq_UaDiqS>0lxxp z*1JaxfWcKl#zXZ!deSp+Z|LsOz0td)JH3UTV@l6)P!ffn6YI9yo|_(cx255+Utjy= znlgM+9zOX+Z(;bS%J8&2JY5*RXwVEkIS4OVr|&CyT5td2<}Vb_fb1E_(*wHnRNuBK zJxBjw>7fvDJYqKZhMH34pn)?|m%^L{c%ss4O3ZX_Dv0`=3|WTV)LkiNxf~H^QojDVxoOi56AP> z+C{ZTi1AhJ5M&5a*j|w%d=lvKDKg0+ybB6R6v-uO+D2D4-ZV}{1hBfGoxtD=>1=S1 zq}8IY!*p>0DiAx}&M5RDnLc#CwMY+xdFVBTXn!fxZwOS7&@wj}h3=E-zB^a$v3J=b zJz0WL5Kx5<%5;#3sll;BraQL#AJZXS@JtmB>yqcHsVxLwLe$o~R1Ah{3r}T11R|D7 z9Kl=GDEPC$pH(u|TK!8mu4)QxP}lzxp*8S=>nV?z>4R6+2g*7E`B~NVzkpXAk=-L` zriWfxtDc}4bF{tRLg+1cDr@#N$6CW0#8q{_qP4=&fO2-@4Az`5%ZxA#Jc+g-mMI$) z@X#$eR~C?K7ErXV7?IPOV(g3D%Nmul>Nm7z1(S?off{1yW55Gbg zBTipUF;*eUB}QDqno@;Fu}}>Xy4VYA^*%qoXI|g)czi(`Fh=V|KCvMKTWMh>(>zs1 zV}~oS0maOi#u!1!*b#84vmfJTG!<3v?973 z4C(uQ_LBRZd2Ud*Zrk8(-`ceykXLIK1=(HTs(ti_=af;R4n+`joO6qTMAmbjH@3WvLK7-+#vaLGkM zkC?RB=?vk}O;A4FiP;+f>PE-8C?o|4?I0o}{woAG3j(COA_DFwGi^~xNaz6=iM3QG z1ji{Y0E7*2k!51VFwrw46~<>G0$YhrwP+0=AV_-#5>XIcM&i+QC46K_WUD3LFi{i` z_kp2OZ7WgX2ATzN#=Z0Ocv1@)NhU?JqUB0ArVibZXIuA}P<=28GeisM%l8qOIfV{9q66C}!R|rC zQwr|OLkQ*ZuG>mYo^|Wj{?6?QrE6U78dv-i>t{-g=FBThP-cSnr65PE&15rzWY+X=aSXv0?W_S{J*124-1FBb-0mA$X#y*(vw z>&82q^oA7(eeK&mrDIrz|Ee?TjNBXZ!#j_+aDgDz!kN*O(}y8mQIQ^mJ+y`L2gdWhiFM}_PasbR zo_A_&ARL-g^Em*&Pb_9Al|^cek_Z>A)HN&hlrY6ppIP>p3WG&eTR$e#)2|gZ4Vh$H z_TVX;;`j*Ukji=C$|EN!VlEyJz4pdcam3J9RuM;LAett7Q(aoVNlVG7cF;4CCDqMC zRs~1rGmtA&{qqKMHZ>m;KnH1=Ax|8Z)N@tZ+;SYHGFoO@S2JRDK&ARBj4J!f9@z6} z>ydl}yA;nt1t}W`>Ii=9{=luvaUpOd-#)g^d`&khG;v-PdQ_$%S@Q+R&1jLHGJWM< zneM%FVW;`-#mDr-^MYV{x7(m8{t!W!Lbs`iWf45z>w}t2qw0lE?^CzDxw@E`84gz@ zfVUXeylJ_0*akix2%9NR^}OAjIRkucl916#%*;OqIrqFf#gJ=9bp70u2@_M&Z&l`@Byh2UU zl39?>B7S!#s^<;=?);4#Hd;6_!kPH$wp*ZCRza#Lkg7H61`6A=cTdAFT<`=#MjBWQ zE#OqbeBmK3$%nWVa2OVakhCNuMqDn#US3&>0(ChT90E83dI56b+ybPk331497(!_X zSc&2gSCf!hNhKlWt`SsLjU;!%C z8sQ|oKp+>eVs#fqr7vr)%Eh#5oxM7ZNf4ykA+;yORU71>IS9N{=~eJfh$q5wqLCN0 zYidQclSLr+PB`kIBek#qAvy-JLA7~5JPU)_eS5UfbFR|gkcq#A3RDn-#$S3k^k7Ii zdR9Jqws`bh!7Io|&)xU`qwp`We~QU7{Dyz4bF*`6|K|QXEsvY~@<-3D+w)%GOJB#< z>zl7{3k6@_y0c1WX~Fr=|87zlpOVL?l<{eKd|Da5B#&Rp(|twya*1x){W!+KocWhnTyr%evWdG1_{KMd`czn0BH?y**zZB4vrr{h}uT}`>6*tFHj;0u|H})(_s5`5Gu&T5jXf7qD&V5_mgiQP0zn^Vg?gEq1rf} zOTY!EGc#ciNy{hUM-i4)T`UV0A`>St!?N#WxOf@Uz_NTQ!m^l{WK688>S0+fnM}bC zBJlf#2s3QN4a6gafcW}ak`BK{AdXZqQjrV&1ZhfM&*TwJh)G2@D-v=0bM&vaH=i(_ z`QU`iOcZPrC0pOF3kCO;0-dEc?SD&qsU-kM`lLVnZTo~hxZ6gYGk){EXQcNuPC4za zr)>`V!KYrA{lL?}h`nR?ZOUS&pVAHX$!9ICc0G%tEgW|1@=j~^btIY8-R>s+_lyz! zK*XvKH!#L8OC)qrnKHTNtZP3uY0UAT0kZPBD*Yp!-;eP+AV?BN?15_6N>S7^22qV) aA^*RlQ(vL*S7`9R9Lp9e^c0a=>0{UOWCoUY2C$88Oe>2l;s66gnGLX=-H+5Y)mu!h8&HxSB)uQAcDrYL zCKzD%oGKQ{mP&Rf0k$mPx{q_ux%b?A&pr3tQ~d8mMMesq|4{qCg3m`O>Nl8>AFUKo zy?7TQcPT&RSB0sF>Y_@8adlXIQ7yhT5$#27CQlbBxL6>@HQ~aD{-R!tYr}?!@uE?T z>%yi8eUTR91>vGd@x@{>UKln*N=+T1{Q7TGe#84}sRb8HZm8YH&oJRu2_|ne z8jE}5!B~_Xg|yB$=?zA);D9e4V-7OGc+dyQ>eJq+KNcAc#IMDesi7bn4~D~HH7G1T z9SetJGx0M}!#fdxoKiN<1fvt4c+BJT0^sFvz$cNHLXGs2xe=f)s{GVN^&7yIiyFT= zp!I9^0Epph{PP#(p!~&t-M4iYb^d~HQx^;Tg%B2E?=R~81_%v)BZNi+Oze6v;te}a#==404aeAYJQxYiirsX0!xJ&!#$?3NIvI~ov%9;y zz0CFC)vg#b(e1s=c5mC&v#YD8uXo2TcNe3B)(ZvGBA!uSnbch|#1iPf>O}!eiMv#y zI6--`#fefsHKT$Ym7J9*eXjy?Rn?FI`I$k4c&T5V$;&X4qN2rqO+JiY3#n{r!t!1$ zVF?*z%(=H87$#4TWLexKmjF8p6I4h}=DmZegynBl-@}#`$k_l%%kkVhRGf>;u~3N| z%e~*vM3d$5EVn}CxpFy{C{L7qpqE<<^s}vn-Xn*y?GaPi(nQI7!95mYvf#jrV%7wE9b~b`g?i*z?k&$3=y7)B5;)&&GE71ZsoePC z%E*pg0_L^Ho`uvysxF(-K`J+*1(0%N%Z!l9ji3oqjoGpyNM%PbK|`u3TV{Y%b2ep$ zRCa7`P;T?6V44WWE_=fs#v7dyj38vgeh(X*4KTRkKf?(r6oms`CW>N}@x}wc{kKQY zpNw2SIVcqP16PAS6uas$BUH?=0gvAs_j(hoM<9y3fZuU>Chmw&1{?^7+m0j1iY<`$iQSGzl)Ht&NiEPp))Sct$Ag}U z=^4QcfhV6$#h8&}m7tl9O*6Ru1q~DR3x;VX;1A+9CK#u^j2C(vU|2yHL$NMs5BR+i zp$GsyVg~YB&`ra(6&H*_7Q5(;`T~LmwlYSKNcD0P1>-lo;hDf8WTv2JGkj!F)(Gl& zOfcjS7P8Ww$P{9s@hl843tMU|Dj4H=abGw%?Gf83ltpJE9(fczf$5lUk`*kN83{&% zp3C02Z!*9N=BwBQaVR~$Sm264!%#D^=^Ii?H{p#$yn-f-eT;aodnOpK-xG~75up-i z0ur1K_&qr+g3%vfgTOJ*R2Vu1MEo=10As;+H)2I^7+B=-`mcInPE7aF!%XDI}9fMvBa3wR_FhIo24h(i;unU7e3~=O`eh7p@930=334xY* zET)RGQ$kD@AQVje0+D~LKx6;p3M`5fhvM`5-QB3mu+v^&pnGyA;*G|Fu;!*00+nIaj-LPP?YBUN>1eQ&Za6vtlRC!MUL|+L3PTUZF|j-npTV>H2kNQzp}i@A_v2 z6kW!d>eef2?~X5wFV(L!r7CvK9sabchg6L{*8arwV-r_3wt5Xxb3?a}u2)pwJ-=|C zuV^6^EqujxQn5W%v19J=(+1ZA+p_K9;gzdv4f~K(-Fj6G)<`!rBc(c|1Z{J!*R*i0 zgQRBvlN#UZ3GUQoGUS^(@vE}>rNR$r=IlF(rElF_!`1daG4J^M^Q&R*%oI5sUfUaaTIqPER#jNj)+VlbFR||9 zss^8FR22tQFEpB}(r1*Wr1Y7Qvb*PozB{sB>wp%HK+7_m)YErNw@i16ZxzqS@4mC} z&YIr&X?5Lw+dbRT(1YRSVZLcMY1*A?+Ouk3s~!X97q7Rp!;5}uYvpad#MYa#!8qJD z1In6ajHI1?I4+K~qX*yibbS}TYtyx@;&`PiYyf6VmsP?`pSHJQt}X4_gKuZr)+8da zI^b25E!}>0PW!RG`vpt}%9qtgAc0~8^fA?o*CBG3if8l;&<|9wTWMhbg1sk%5`V+< zeg+Hd8`;tu)FaiXTO&#qY0ioBvwW;Yv9Dp}=Gra@7E1eNTO2NiN9-g(S2e&uo{ z-kzbDA*kiHNO}gUMohpr!!WS#3ndeQxMwox_XnaL&|BgGmgxXsq3A1lf(BMMs*7>2 zZ&E1CEOkL2KrH|&2?e4$zzXGvPL>4W*o0@AiCq>fn{rH2jwcfEfHmS?T`Fxd-x;_yz?+;; zOwM#i&xga0hWU-Pe5r=re8U0Ka3IxiP=={Xw|9Ry@MwTX3(5-1#+k>g)Dc-f0xc2g{A>tZJxem+4&U8!rgLjtS z;am2Smc4w-5NR1owPcH%)>~jRk=oXjHkaMGdFv)`b`i6S)4L?um@mC>-HNjD^vh%; zlx=)+(gKdWR46Ooa&SzIa(?Jl*wmq{^vYq&lXtk?ZFoG{KEZZJg_Iz&QdzlG%rWKs zPJ? zmA@g>TsJ(*_(KY^F`<=AMxpEhIETugH7JC#N8ns3(8wNzbGfo)pM~**`k7rDQyX~! z^&Po-axCPOW4X6-Z)8g1Y|V~Yu2!fy7nfu5oD=KFnR)L}OI{)qO=xnI;0Uf66%24J z40ynf$uc;X+~ty_EuWc&Q*J;!JmDEg6k`mU62Mv-jzt5Ks4Zjy*WlCu=Sx52Wn?HE zCgDKI27Mr_u~aD8n8S*PBS8npM3l+w+cSXxoJ7wcdc#aqPPaljr{oD{uiu}q#9>Ro zI3Vbl02_$AbvY5sd=qP;g;6L7TnD|6WpN3LN}Z(BSwOh^0xXN_A#5Ij>Dh7=!W(nH z2esIC&{*H4)+Ifh*S8XV>%+1&eJ9H3{_pgsY0I6Xw~q3(o6zou&NaI0nVxdCJ?LES z{F`oB*wv>Upn6M!&yJ@K3szlkc0V}3e4cN9oix9`+M8-Vf~S(kbtnY^&(qE)v~wvg z$x=e^{g`%Q)v84oF*%TI9?*l?<=Gz|`q9V_M$(q@yXFNmZ*dWe>tW@JiMaRh?!&}= zc=hd+`xx&&LEI-kI{S<9pO2^97l`EoH-71bM!mE6Ii+sJpr{xL87V{29r_l{>zjza zY3b|(&$4GtzcUSNfUSks+lk(e8;s1BYND@RZ2nm9*sS8Ba09WbdhrAzTdKG(6}%xF zzYUMP0Fq$yQc^gStiEU;+d|}J)ubF5fX%8kxm=q1K=poROHomS6-%873;N2FX-_CIC3z;Auuw76en+%f=;(1M9GmTcQE=)Lm8U?nNk~!(#U4dZ$QBW|HY9#4OVNo;87)2@Lvu8aifF6 zjSr)NGHn#DeUf0eR6N!ogieIObD_f4tu)rmHh@(UoY=w*4?uqHv6h8?vLM>C1qG$U z${$4`Q7}Z!>)-q}MO}hTIa!$0Ck;ts(v+l=Makl%Icf2$lO>75kir%!ZG6d6XrsdY z0W%b@l9`~iamp)cCk2~{F=15NKxLR&Xd6H%Y@#xRUo#8DW`XlbBV6ynKbec-R+WGF z%`}SnD`>YuzF&*BCFZ+Zln!tTJ)R1X(-V5$6vg$#SP8ePZV*n1m3Xd^idL<0l;uTkjGp})(C~(M4LH4fGX!%wo;@6d@Op8>6B^u*0m+2z6?OJ+ zt`mBFbKVDLxgW4zuzf?DVP@e#BDVeT<}$wuX9;sftgA4;{3bk9OaWH7D?KLH_rtwg z=(wE9z5NzZCS~Mh{vBI%=SwKS*i1nk#+k7BOGH|A@evuTA9oIATF^4N_xZH87$tAs zIFHKMb8lrTkc;}thUbGanQpS?72}8}hYe4n#;^LIT*d(7Rug(-0|+GLb-AHu-`ETp zI=?w@h(|JG#PijLCt>%agnny7HY7z^uvHn9%X^X7p87m0M>pe2)FtZWF@2eq zl5{{@E^n^S%q9$pii82%6v6}IhDW&vz%9j|$t^*dmuvvMCJ}F715!x(vvM-om}vM~ zaL!k7qA}r=>A#c{P0_}LDN*t2mc1NH0y%nRbI%e@qCo@-pnOuH-1;)SVqF-EWHZ#x zNVI$$^D5C$r%)p4hU_8bdj7BIhUjCVgx#;K8@j%t8;0`RBgHeGNgH)TWsN4XxjyLd zvCX-%EDF87b(y?=lP!ssjLy=krf!x&95oarJ}LFlm2~GxRAw{1(9GsMxqZnt(74+Y zE$FvpqZ00y^+ke_=V`Kii#A$CJ-;%s#VqhwV|k)ooGoC6Og2X?iT0P#6)A1*0>%6^ zZ&ksLz44LzD4v8APq3kD8|y&J&?gCW)a>YVKtDHwQK1`~GZP)TS{wI>EwH?^8OzNa zmd-6|B{$c~(S9i}yZ(f^(hYN^JJFfw%FY2iM=R~GH#fIW9u3&Nn?vutv|N$jOdHVp z*5-Plm(se~^GaGu`l$tI{rP5Exjw|R>n!x|Hp7K(ZO#LXN@ugb*bFICff~0JN;7gn zJnbp*{*FSOOqN(jfrkw0!3z1Im*Y|;nqH=dpd!=LqD06vJLh zF>I|4yd|U*vLtMO}cCt_ti~w|{?KbAh_1+Ek*u{jDuZ)VGsTiQ7IZ zx&n$<6BD~{_x$>MHxBJ%(89&wVNlSb{T}S4s3|h1Fh_$KYC#wAvQsRBClEpJ^?Skl z?P_4+`Q&fr{`0?lx^Duq@!6k=tPg(nf8hZuuGV`!$UXw6!q{}C>|X|DXzI7`l>gn4 zU(J4d(5(`$B>+ytGCrmZQ2ka!NIe_;47tSMjh0Z7=Xxl5s9kYugks72O!fp63@lt{ z$0qP5x-jF-Cf>6!Er7y(EEktd$tMtd8)C1^-h}6QqPh)`6ggJ)+ z%2dgj4c*#=;$ZYj0Pg91;DH(Qqf6mT1YGlC6G8#p?|7L2gWIoQ@`GDF69fmzfFEwt z!{B-bK6pNG-Z?G4$EF3%nKwol8)8Q-K+u3^pX8yZfqn@E@j#S??v`Si;W88R`h8$B z7qr26Ai}sYmlmDmpj)w-X%=S4^=Zb3`EY;ajZ914dOJnb)3IwgZ#_XL`U8pXLE!k7 z>5XK^e-pvg(Wp>3a^}zpa4i*z(0u^C@`|#yBjf2QRWM}d7dU)gW+l(eoCB3OrGzr6 zlbKtK42&h~M47FXx&Fui*ka)QE4fgxKuq$#$du7o7MaPU%$Q=)%RMOkW7wA zT>qIwA$os-U#?IFA2Wb2FL)g1&3#djRBl1oB&}AdU;1ndO!?G{WG7S!N)4{0g;Qb4 zKg1AbJkkUeOnI{q?kw>}mH9Jq#B+m^@lX|tb6M!OEEGx2kc(+4iA8wnnxB%LTm@q; zSI~+`RG!R`%6*mEoAGcF+v`CF2~{#cb^>G=sgHn{snj6Z*HlWHvlBl9S(b4_2G80+ z0Sy`H()t+n40-7!ZnWAAtNrW+awxR5q9bCTz% zCC^32T6>?qNAuQ$#CmXbh#$H@hTvk=!;iCMoK0Edw~npLAg>YYYmb}w-D70;74=0iLSGja^Bqc#N79^!glw~g*X4bC*9i54Gi-GC&|Fc)WB)ddYU^o!JnHV z=cZEUBHVPGoQso|8LqtPOAS@o47b;o@;k}zB$vt`*q7~G_u(fEN7mh)AKD+;xm{z# zeUh`de${>;ZEXNA-!$~9=w8v%6=LoDqEOpWN-VAOhQ+;K6jL>Ii$T6_JE_~gvW?X3 z-gQ*PcaDFLIW)b=PaWYd>-AU#}&3S)6HS zFYnw88>=R6e2NQ4$as{q zb){^vxnXFewH6(2`Km5b)x~um&P`6#(+Om!cJ0pcs%q{97iMf+3m=&in2 z{nTo|Z@6bzYF{yv#=%t=X*`;;9(!8jxWDJ#o`>dCP5R)%v8W6&k=>-NfqVtviSnSp2gJ{=jK+;52_=oE#YE4|vD{ z4`GdtzX)~JNpRyS^E|XpV1;@a-p6J?WZ8$!lPRCC$5^)a-t`ZO6*l z$2<6e5i&5sojgMZ&ZM@TT^#zjwkvIKT8cloxqS0+8EM(`#J&e+K<_`;x2myL+m+wu zCijCA%O_UK`E3Wuwu9ioHvIG9wWjlnhP>YN5^L|uvDIyywRg>WDBaoj;Wr{~P~4Lz~CpKGYrgQ_nNsHn=0FOI4x zSX##xj&XH6emL`^%dP3(w2&OcJb_e&pi*R>Uii|8T!%K55`E>L2$_!QlUFWLor9& zzbK&UocAx>yRdYF+kP-*AL8xDi2Yc~K0G%Ht;aRr_Qc$lcJ!{i!yP(LcAe+@JfzQ) z2G@Acf(O?2lK(+$IR;pFl_JtoST+SzL&w7iaSZVF`$_%&)sj^G!Ff8}y7O@nX+6ZZ z949TuKiZaR8R0BVPwkB&`2@EUs2=C-W5hm|vY%QouGh6coFa9v@%DYhzVGqdDfx%wUt{bc0FBOkfQo>O0F z)LkI)+U8YUS=$#zsN|Gj>`rk&KZr)oOq^}lqqrpsMi%Wke^fGgj_?R$gYcZuw~ zl-l=A&ckpXmfIJHF%gGDPnBEm?p@f+HEe%U)|)Q3E)Lxvxi`YK?IpGQR_&yAl$4Jy z46Qe9llYlx+C4uaN$cz1xeha=vL++&p@);ple}vWaqW40fja^Mah%(GDdl=|ehmFL z#lk`28eBD}TqozpzSK!_AAC2(qJHA)e_Wq(9TO#R%Y**q{s)80gDZn6=Rw|ioH&oC zoFJVYpW5p0x88%V4BCjTEoJMNFZ^XqBj@bn`j4h+j?L?#aUhsSZ;kSF6QP^dXxPOp zcSdfF+#Xv8=TB%gI4kj%Mq+8?Egi(tv1aK?TdcgLfmj;WEa>foGu!dR><|IbqiN03 z`URjhOFmbK7Nm?go&%(LAl1D0ahxAKMFvl;(WjA`NWGrW^=otk5arDcPs|N@WLt=( zWzFISQbo0kv-fY_yP2x*A#~5&SemY1+9wIPl(V1E{jk5nF1$Fx*LIQGu2gLgU%Qjk z?o8G0BJ{4g6YJ($&fJ=|*q5xh(Ep~MSbFA*){$Um+S0fb;a$kslxsJ!?0y0>>}?%v z_s_2CsHz&ivWrx9p$I!n;IHy9y603a4iLQyR1B zv2{|d`)_4O1}4qAPg-_Q*6Mzz)=koe-Ly(VU&eg^;3*0XTi}GCda(q# ze~43Y3?-h@iYOII0M(QuM+j}Tj$31qsmxfU++jdHn?Nk`c?7cM2U zt|FpoO-7OTR{B&p0ghF8{LN8U?v40WhyqU(dkL498y}#f`6`_oTKHOAH{0^Me6)7p zc|PnAk2B%{)e(=um*Qc^k&|z*U6R4(I^Y$pHluEXKD(& zqV0h}%cW3}J;HlpmqP(x+!F>1fnW~9!CpG!iRXcgM?IWiu`$wVnay{t7s_*pa!sAJ z;t4>h>{)PzG5q&s_pkp_e{LCi&(o-)*h~E+Z>(s_q6gf3X~=71O2j|cWft) z?Yv_zaqLYw2G{8QpX!TvJ-S2k`fj3!@7z`!IDPk;ekkig>Vy@%bYi7sd2CI;Ba1fU zMp_4F44&>Hbl1bg;{%UweoVv1ViaAz+54^#S}QK0RS5r>B{W3cQTfyp;MV;nn3P~S zsg#z}yQs(8zZyAW+cn5-l1V9?`12g-_6-f`~qE3yu3_^W^BnxtzVl2D>HafFt40}3U2-KtVkAYF+*!NFanaH`ptR3 zH?(9I;7z`b3ZW%#!}Im=%OYBS!>Ax;yxGv*j%%qKu;x^-=8~#w5XvzSvC9YHCq+Qw zInc%z1gjG3xWdFDS=s7 zP>5(eUe;YG*)w&b3>S1r1MEU*D3+q4%~&jCg@V|XEAS}*^J~D$qIAq~M>GO4{}D1~ zD^UJ#GO_*`h5rl$ApB{nch`^Ff6&fvKT5VAUE6*`&U#MRy{kWuw+kSjb$5{edL0+;F|IzgyT<77`vFrHSt`n(>39{?N!vJ^k4gTas zf`3=$19w9Up}Vn#*wW>XE8X0#69k~>2{=BMSL5MwF_0>6nlpe}Zzz_x6VVg@!Vh=y zy+fpTi0>UCy(4_@Dbjn2(>JZ@Pp9=I^Jn>rRsw(eR#5OjYqQ}e)OI!Rn7xl`ZZPmPmPtd zF7KM<^17vRv3bF~Q1HBns&xG8=M7ZJ71h7~x?)27Uj462j#O$tv_tUIT_s0q3jV=h zhtOT}8F(pCpW!^2_{)3v{Csc%Z(+*+?tc$XToIpt8f0+YGI%r3piUzu&oUrUq)b8M zo1Qs5Iywp;3ggx7@8Q+~mRZ6vieIt}AlN@)fVWs;%w2>ZyGP;I6FeTl=)oHd_+0>q z(;g38UW79xg&vPT=7Vgr$CLM)LLLuu05KiMU<`vG24M{T2L_*DfYXtw!r*lb_G5qw zr5NCq68o1B{5kbY?b&s@Dy6MTYa2mhx7VesZSZM!y0{dMH)(4P9P!hYHR;OgbiFfO zQj;#Lf?-Ry_pH}D*PB}56O(71<0{=7s%K49sr^~A6mO-f8lSnPcn4*5zK~!(pH&-l z#?LEFx|YvvHM)k+ck3Wo9M|ZopM?w6I{oK*lWxE2g{@X6edi%I)?JD`x-tVFGXRem z6#LR;l3x74rz;v|z7KhVNlJ=86v3bn77TDn!^C5;Fw2ONpji6tfi8*Ptn`aQrC-hL z0#mW{gBLRNSAK4S&OnQICgN{cNL>9Yrv?2UiN-$W-=HC|@+lVY%HXqnmFk~qO6B|w mMSnt7d_vX0-#=2WPpJM+DEB8+` self.desired_kl * 2.0: - self.learning_rate = max(1e-5, self.learning_rate / 1.5) - elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0: - self.learning_rate = min(1e-2, self.learning_rate * 1.5) - - # Update the learning rate for all GPUs - if self.is_multi_gpu: - lr_tensor = torch.tensor(self.learning_rate, device=self.device) - torch.distributed.broadcast(lr_tensor, src=0) - self.learning_rate = lr_tensor.item() - - # Update the learning rate for all parameter groups - for param_group in self.optimizer.param_groups: - param_group["lr"] = self.learning_rate - - # Surrogate loss - ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) - surrogate = -torch.squeeze(advantages_batch) * ratio - surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( - ratio, 1.0 - self.clip_param, 1.0 + self.clip_param - ) - surrogate_loss = torch.max(surrogate, surrogate_clipped).mean() - - # Value function loss - if self.use_clipped_value_loss: - value_clipped = target_values_batch + (value_batch - target_values_batch).clamp( - -self.clip_param, self.clip_param - ) - value_losses = (value_batch - returns_batch).pow(2) - value_losses_clipped = (value_clipped - returns_batch).pow(2) - value_loss = torch.max(value_losses, value_losses_clipped).mean() - else: - value_loss = (returns_batch - value_batch).pow(2).mean() - - loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy_batch.mean() - - # Symmetry loss - if self.symmetry: - # obtain the symmetric actions - # if we did augmentation before then we don't need to augment again - if not self.symmetry["use_data_augmentation"]: - data_augmentation_func = self.symmetry["data_augmentation_func"] - obs_batch, _ = data_augmentation_func(obs=obs_batch, actions=None, env=self.symmetry["_env"]) - # compute number of augmentations per sample - num_aug = int(obs_batch.shape[0] / original_batch_size) - - # actions predicted by the actor for symmetrically-augmented observations - mean_actions_batch = self.policy.act_inference(obs_batch.detach().clone()) - - # compute the symmetrically augmented actions - # note: we are assuming the first augmentation is the original one. - # We do not use the action_batch from earlier since that action was sampled from the distribution. - # However, the symmetry loss is computed using the mean of the distribution. - action_mean_orig = mean_actions_batch[:original_batch_size] - _, actions_mean_symm_batch = data_augmentation_func( - obs=None, actions=action_mean_orig, env=self.symmetry["_env"] - ) - - # compute the loss (we skip the first augmentation as it is the original one) - mse_loss = torch.nn.MSELoss() - symmetry_loss = mse_loss( - mean_actions_batch[original_batch_size:], actions_mean_symm_batch.detach()[original_batch_size:] - ) - # add the loss to the total loss - if self.symmetry["use_mirror_loss"]: - loss += self.symmetry["mirror_loss_coeff"] * symmetry_loss - else: - symmetry_loss = symmetry_loss.detach() - - # Random Network Distillation loss - # TODO: Move this processing to inside RND module. - if self.rnd: - # extract the rnd_state - # TODO: Check if we still need torch no grad. It is just an affine transformation. - with torch.no_grad(): - rnd_state_batch = self.rnd.get_rnd_state(obs_batch[:original_batch_size]) - rnd_state_batch = self.rnd.state_normalizer(rnd_state_batch) - # predict the embedding and the target - predicted_embedding = self.rnd.predictor(rnd_state_batch) - target_embedding = self.rnd.target(rnd_state_batch).detach() - # compute the loss as the mean squared error - mseloss = torch.nn.MSELoss() - rnd_loss = mseloss(predicted_embedding, target_embedding) - - expert_states = sample_amp_expert - policy_states = sample_amp_policy - - with torch.no_grad(): - expert_states = self.amp_normalizer.normalize_torch(expert_states.to(self.device), self.device) - policy_states = self.amp_normalizer.normalize_torch(policy_states, self.device) - - contact_phase_push = obs_batch['critic'][:, -4] - mask_push = contact_phase_push == 1. - - if mask_push.any(): - policy_d = self.discriminator(policy_states.flatten(1)) - expert_states = expert_states.to(self.device) - expert_d = self.discriminator(expert_states.flatten(1)) - - expert_loss = torch.nn.MSELoss()(expert_d, torch.ones(expert_d.size(), device=self.device)) - policy_loss = torch.nn.MSELoss()(policy_d, -1 * torch.ones(policy_d.size(), device=self.device)) - amp_loss = 0.5 * (expert_loss + policy_loss) - - # grad penalty - grad_pen_loss = self.discriminator.compute_grad_pen(expert_states, lambda_=5) - else: - amp_loss = torch.tensor(0.0, device=self.device) - grad_pen_loss = torch.tensor(0.0, device=self.device) - expert_loss = torch.tensor(0.0, device=self.device) - policy_loss = torch.tensor(0.0, device=self.device) - - loss += (amp_loss + grad_pen_loss) - self.amp_normalizer.update(policy_states.cpu().numpy()) - self.amp_normalizer.update(expert_states.cpu().numpy()) - - # Compute the gradients - # -- For PPO - self.optimizer.zero_grad() - loss.backward() - # -- For RND - if self.rnd: - self.rnd_optimizer.zero_grad() # type: ignore - rnd_loss.backward() - - # Collect gradients from all GPUs - if self.is_multi_gpu: - self.reduce_parameters() - - # Apply the gradients - # -- For PPO - nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) - self.optimizer.step() - # -- For RND - if self.rnd_optimizer: - self.rnd_optimizer.step() - - # Store the losses - mean_value_loss += value_loss.item() - mean_surrogate_loss += surrogate_loss.item() - mean_entropy += entropy_batch.mean().item() - mean_amp_loss += amp_loss.item() - mean_grad_pen_loss += grad_pen_loss.item() - mean_policy_pred += policy_loss.mean().item() - mean_expert_pred += expert_loss.mean().item() - # -- RND loss - if mean_rnd_loss is not None: - mean_rnd_loss += rnd_loss.item() - # -- Symmetry loss - if mean_symmetry_loss is not None: - mean_symmetry_loss += symmetry_loss.item() - - # -- For PPO - num_updates = self.num_learning_epochs * self.num_mini_batches - mean_value_loss /= num_updates - mean_surrogate_loss /= num_updates - mean_entropy /= num_updates - mean_amp_loss /= num_updates - mean_grad_pen_loss /= num_updates - mean_policy_pred /= num_updates - mean_expert_pred /= num_updates - - # -- For RND - if mean_rnd_loss is not None: - mean_rnd_loss /= num_updates - # -- For Symmetry - if mean_symmetry_loss is not None: - mean_symmetry_loss /= num_updates - # -- Clear the storage - self.storage.clear() - - # construct the loss dictionary - loss_dict = { - "value_function": mean_value_loss, - "surrogate": mean_surrogate_loss, - "entropy": mean_entropy, - "amp": mean_amp_loss, - "amp_grad_pen": mean_grad_pen_loss, - "amp_policy_pred": mean_policy_pred, - "amp_expert_pred": mean_expert_pred, - } - if self.rnd: - loss_dict["rnd"] = mean_rnd_loss - if self.symmetry: - loss_dict["symmetry"] = mean_symmetry_loss - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - if self.rnd: - model_params.append(self.rnd.predictor.state_dict()) - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - if self.rnd: - self.rnd.predictor.load_state_dict(model_params[1]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - if self.rnd: - grads += [param.grad.view(-1) for param in self.rnd.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - - # Get all parameters - all_params = self.policy.parameters() - if self.rnd: - all_params = chain(all_params, self.rnd.parameters()) - - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in all_params: - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py b/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py deleted file mode 100644 index 3a86e00..0000000 --- a/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -import torch -import torch.nn as nn - -from rsl_rl.modules import StudentTeacher, StudentTeacherRecurrent -from rsl_rl.storage import RolloutStorage -from rsl_rl.utils import resolve_optimizer - - -class Distillation: - """Distillation algorithm for training a student model to mimic a teacher model.""" - - policy: StudentTeacher | StudentTeacherRecurrent - """The student teacher model.""" - - def __init__( - self, - policy, - num_learning_epochs=1, - gradient_length=15, - learning_rate=1e-3, - max_grad_norm=None, - loss_type="mse", - optimizer="adam", - device="cpu", - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # distillation components - self.policy = policy - self.policy.to(self.device) - self.storage = None # initialized later - - # initialize the optimizer - self.optimizer = resolve_optimizer(optimizer)(self.policy.parameters(), lr=learning_rate) - - # initialize the transition - self.transition = RolloutStorage.Transition() - self.last_hidden_states = None - - # distillation parameters - self.num_learning_epochs = num_learning_epochs - self.gradient_length = gradient_length - self.learning_rate = learning_rate - self.max_grad_norm = max_grad_norm - - # initialize the loss function - loss_fn_dict = { - "mse": nn.functional.mse_loss, - "huber": nn.functional.huber_loss, - } - if loss_type in loss_fn_dict: - self.loss_fn = loss_fn_dict[loss_type] - else: - raise ValueError(f"Unknown loss type: {loss_type}. Supported types are: {list(loss_fn_dict.keys())}") - - self.num_updates = 0 - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs): - # compute the actions - self.transition.actions = self.policy.act(obs).detach() - self.transition.privileged_actions = self.policy.evaluate(obs).detach() - # record the observations - self.transition.observations = obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras): - # update the normalizers - self.policy.update_normalization(obs) - - # record the rewards and dones - self.transition.rewards = rewards - self.transition.dones = dones - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.policy.reset(dones) - - def update(self): - self.num_updates += 1 - mean_behavior_loss = 0 - loss = 0 - cnt = 0 - - for epoch in range(self.num_learning_epochs): - self.policy.reset(hidden_states=self.last_hidden_states) - self.policy.detach_hidden_states() - for obs, _, privileged_actions, dones in self.storage.generator(): - - # inference the student for gradient computation - actions = self.policy.act_inference(obs) - - # behavior cloning loss - behavior_loss = self.loss_fn(actions, privileged_actions) - - # total loss - loss = loss + behavior_loss - mean_behavior_loss += behavior_loss.item() - cnt += 1 - - # gradient step - if cnt % self.gradient_length == 0: - self.optimizer.zero_grad() - loss.backward() - if self.is_multi_gpu: - self.reduce_parameters() - if self.max_grad_norm: - nn.utils.clip_grad_norm_(self.policy.student.parameters(), self.max_grad_norm) - self.optimizer.step() - self.policy.detach_hidden_states() - loss = 0 - - # reset dones - self.policy.reset(dones.view(-1)) - self.policy.detach_hidden_states(dones.view(-1)) - - mean_behavior_loss /= cnt - self.storage.clear() - self.last_hidden_states = self.policy.get_hidden_states() - self.policy.detach_hidden_states() - - # construct the loss dictionary - loss_dict = {"behavior": mean_behavior_loss} - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in self.policy.parameters(): - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py b/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py deleted file mode 100644 index 6c21fc5..0000000 --- a/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py +++ /dev/null @@ -1,469 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import torch.optim as optim -from itertools import chain - -from rsl_rl.modules import ActorCritic -from rsl_rl.modules.rnd import RandomNetworkDistillation -from rsl_rl.storage import RolloutStorage -from rsl_rl.utils import string_to_callable - - -class PPO: - """Proximal Policy Optimization algorithm (https://arxiv.org/abs/1707.06347).""" - - policy: ActorCritic - """The actor critic module.""" - - def __init__( - self, - policy, - num_learning_epochs=5, - num_mini_batches=4, - clip_param=0.2, - gamma=0.99, - lam=0.95, - value_loss_coef=1.0, - entropy_coef=0.01, - learning_rate=0.001, - max_grad_norm=1.0, - use_clipped_value_loss=True, - schedule="adaptive", - desired_kl=0.01, - device="cpu", - normalize_advantage_per_mini_batch=False, - # RND parameters - rnd_cfg: dict | None = None, - # Symmetry parameters - symmetry_cfg: dict | None = None, - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # RND components - if rnd_cfg is not None: - # Extract parameters used in ppo - rnd_lr = rnd_cfg.pop("learning_rate", 1e-3) - # Create RND module - self.rnd = RandomNetworkDistillation(device=self.device, **rnd_cfg) - # Create RND optimizer - params = self.rnd.predictor.parameters() - self.rnd_optimizer = optim.Adam(params, lr=rnd_lr) - else: - self.rnd = None - self.rnd_optimizer = None - - # Symmetry components - if symmetry_cfg is not None: - # Check if symmetry is enabled - use_symmetry = symmetry_cfg["use_data_augmentation"] or symmetry_cfg["use_mirror_loss"] - # Print that we are not using symmetry - if not use_symmetry: - print("Symmetry not used for learning. We will use it for logging instead.") - # If function is a string then resolve it to a function - if isinstance(symmetry_cfg["data_augmentation_func"], str): - symmetry_cfg["data_augmentation_func"] = string_to_callable(symmetry_cfg["data_augmentation_func"]) - # Check valid configuration - if symmetry_cfg["use_data_augmentation"] and not callable(symmetry_cfg["data_augmentation_func"]): - raise ValueError( - "Data augmentation enabled but the function is not callable:" - f" {symmetry_cfg['data_augmentation_func']}" - ) - # Store symmetry configuration - self.symmetry = symmetry_cfg - else: - self.symmetry = None - - # PPO components - self.policy = policy - self.policy.to(self.device) - # Create optimizer - self.optimizer = optim.Adam(self.policy.parameters(), lr=learning_rate) - # Create rollout storage - self.storage: RolloutStorage = None # type: ignore - self.transition = RolloutStorage.Transition() - - # PPO parameters - self.clip_param = clip_param - self.num_learning_epochs = num_learning_epochs - self.num_mini_batches = num_mini_batches - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - self.gamma = gamma - self.lam = lam - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.desired_kl = desired_kl - self.schedule = schedule - self.learning_rate = learning_rate - self.normalize_advantage_per_mini_batch = normalize_advantage_per_mini_batch - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs): - if self.policy.is_recurrent: - self.transition.hidden_states = self.policy.get_hidden_states() - # compute the actions and values - self.transition.actions = self.policy.act(obs).detach() - self.transition.values = self.policy.evaluate(obs).detach() - self.transition.actions_log_prob = self.policy.get_actions_log_prob(self.transition.actions).detach() - self.transition.action_mean = self.policy.action_mean.detach() - self.transition.action_sigma = self.policy.action_std.detach() - # need to record obs before env.step() - self.transition.observations = obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras): - # update the normalizers - self.policy.update_normalization(obs) - if self.rnd: - self.rnd.update_normalization(obs) - - # Record the rewards and dones - # Note: we clone here because later on we bootstrap the rewards based on timeouts - self.transition.rewards = rewards.clone() - self.transition.dones = dones - - # Compute the intrinsic rewards and add to extrinsic rewards - if self.rnd: - # Compute the intrinsic rewards - self.intrinsic_rewards = self.rnd.get_intrinsic_reward(obs) - # Add intrinsic rewards to extrinsic rewards - self.transition.rewards += self.intrinsic_rewards - - # Bootstrapping on time outs - if "time_outs" in extras: - self.transition.rewards += self.gamma * torch.squeeze( - self.transition.values * extras["time_outs"].unsqueeze(1).to(self.device), 1 - ) - - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.policy.reset(dones) - - def compute_returns(self, obs): - # compute value for the last step - last_values = self.policy.evaluate(obs).detach() - self.storage.compute_returns( - last_values, self.gamma, self.lam, normalize_advantage=not self.normalize_advantage_per_mini_batch - ) - - def update(self): # noqa: C901 - mean_value_loss = 0 - mean_surrogate_loss = 0 - mean_entropy = 0 - # -- RND loss - if self.rnd: - mean_rnd_loss = 0 - else: - mean_rnd_loss = None - # -- Symmetry loss - if self.symmetry: - mean_symmetry_loss = 0 - else: - mean_symmetry_loss = None - - # generator for mini batches - if self.policy.is_recurrent: - generator = self.storage.recurrent_mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - else: - generator = self.storage.mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - - # iterate over batches - for ( - obs_batch, - actions_batch, - target_values_batch, - advantages_batch, - returns_batch, - old_actions_log_prob_batch, - old_mu_batch, - old_sigma_batch, - hid_states_batch, - masks_batch, - ) in generator: - - # number of augmentations per sample - # we start with 1 and increase it if we use symmetry augmentation - num_aug = 1 - # original batch size - # we assume policy group is always there and needs augmentation - original_batch_size = obs_batch.batch_size[0] - - # check if we should normalize advantages per mini batch - if self.normalize_advantage_per_mini_batch: - with torch.no_grad(): - advantages_batch = (advantages_batch - advantages_batch.mean()) / (advantages_batch.std() + 1e-8) - - # Perform symmetric augmentation - if self.symmetry and self.symmetry["use_data_augmentation"]: - # augmentation using symmetry - data_augmentation_func = self.symmetry["data_augmentation_func"] - # returned shape: [batch_size * num_aug, ...] - obs_batch, actions_batch = data_augmentation_func( - obs=obs_batch, - actions=actions_batch, - env=self.symmetry["_env"], - ) - # compute number of augmentations per sample - # we assume policy group is always there and needs augmentation - num_aug = int(obs_batch.batch_size[0] / original_batch_size) - # repeat the rest of the batch - # -- actor - old_actions_log_prob_batch = old_actions_log_prob_batch.repeat(num_aug, 1) - # -- critic - target_values_batch = target_values_batch.repeat(num_aug, 1) - advantages_batch = advantages_batch.repeat(num_aug, 1) - returns_batch = returns_batch.repeat(num_aug, 1) - - # Recompute actions log prob and entropy for current batch of transitions - # Note: we need to do this because we updated the policy with the new parameters - # -- actor - self.policy.act(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[0]) - actions_log_prob_batch = self.policy.get_actions_log_prob(actions_batch) - # -- critic - value_batch = self.policy.evaluate(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[1]) - # -- entropy - # we only keep the entropy of the first augmentation (the original one) - mu_batch = self.policy.action_mean[:original_batch_size] - sigma_batch = self.policy.action_std[:original_batch_size] - entropy_batch = self.policy.entropy[:original_batch_size] - - # KL - if self.desired_kl is not None and self.schedule == "adaptive": - with torch.inference_mode(): - kl = torch.sum( - torch.log(sigma_batch / old_sigma_batch + 1.0e-5) - + (torch.square(old_sigma_batch) + torch.square(old_mu_batch - mu_batch)) - / (2.0 * torch.square(sigma_batch)) - - 0.5, - axis=-1, - ) - kl_mean = torch.mean(kl) - - # Reduce the KL divergence across all GPUs - if self.is_multi_gpu: - torch.distributed.all_reduce(kl_mean, op=torch.distributed.ReduceOp.SUM) - kl_mean /= self.gpu_world_size - - # Update the learning rate - # Perform this adaptation only on the main process - # TODO: Is this needed? If KL-divergence is the "same" across all GPUs, - # then the learning rate should be the same across all GPUs. - if self.gpu_global_rank == 0: - if kl_mean > self.desired_kl * 2.0: - self.learning_rate = max(1e-5, self.learning_rate / 1.5) - elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0: - self.learning_rate = min(1e-2, self.learning_rate * 1.5) - - # Update the learning rate for all GPUs - if self.is_multi_gpu: - lr_tensor = torch.tensor(self.learning_rate, device=self.device) - torch.distributed.broadcast(lr_tensor, src=0) - self.learning_rate = lr_tensor.item() - - # Update the learning rate for all parameter groups - for param_group in self.optimizer.param_groups: - param_group["lr"] = self.learning_rate - - # Surrogate loss - ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) - surrogate = -torch.squeeze(advantages_batch) * ratio - surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( - ratio, 1.0 - self.clip_param, 1.0 + self.clip_param - ) - surrogate_loss = torch.max(surrogate, surrogate_clipped).mean() - - # Value function loss - if self.use_clipped_value_loss: - value_clipped = target_values_batch + (value_batch - target_values_batch).clamp( - -self.clip_param, self.clip_param - ) - value_losses = (value_batch - returns_batch).pow(2) - value_losses_clipped = (value_clipped - returns_batch).pow(2) - value_loss = torch.max(value_losses, value_losses_clipped).mean() - else: - value_loss = (returns_batch - value_batch).pow(2).mean() - - loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy_batch.mean() - - # Symmetry loss - if self.symmetry: - # obtain the symmetric actions - # if we did augmentation before then we don't need to augment again - if not self.symmetry["use_data_augmentation"]: - data_augmentation_func = self.symmetry["data_augmentation_func"] - obs_batch, _ = data_augmentation_func(obs=obs_batch, actions=None, env=self.symmetry["_env"]) - # compute number of augmentations per sample - num_aug = int(obs_batch.shape[0] / original_batch_size) - - # actions predicted by the actor for symmetrically-augmented observations - mean_actions_batch = self.policy.act_inference(obs_batch.detach().clone()) - - # compute the symmetrically augmented actions - # note: we are assuming the first augmentation is the original one. - # We do not use the action_batch from earlier since that action was sampled from the distribution. - # However, the symmetry loss is computed using the mean of the distribution. - action_mean_orig = mean_actions_batch[:original_batch_size] - _, actions_mean_symm_batch = data_augmentation_func( - obs=None, actions=action_mean_orig, env=self.symmetry["_env"] - ) - - # compute the loss (we skip the first augmentation as it is the original one) - mse_loss = torch.nn.MSELoss() - symmetry_loss = mse_loss( - mean_actions_batch[original_batch_size:], actions_mean_symm_batch.detach()[original_batch_size:] - ) - # add the loss to the total loss - if self.symmetry["use_mirror_loss"]: - loss += self.symmetry["mirror_loss_coeff"] * symmetry_loss - else: - symmetry_loss = symmetry_loss.detach() - - # Random Network Distillation loss - # TODO: Move this processing to inside RND module. - if self.rnd: - # extract the rnd_state - # TODO: Check if we still need torch no grad. It is just an affine transformation. - with torch.no_grad(): - rnd_state_batch = self.rnd.get_rnd_state(obs_batch[:original_batch_size]) - rnd_state_batch = self.rnd.state_normalizer(rnd_state_batch) - # predict the embedding and the target - predicted_embedding = self.rnd.predictor(rnd_state_batch) - target_embedding = self.rnd.target(rnd_state_batch).detach() - # compute the loss as the mean squared error - mseloss = torch.nn.MSELoss() - rnd_loss = mseloss(predicted_embedding, target_embedding) - - # Compute the gradients - # -- For PPO - self.optimizer.zero_grad() - loss.backward() - # -- For RND - if self.rnd: - self.rnd_optimizer.zero_grad() # type: ignore - rnd_loss.backward() - - # Collect gradients from all GPUs - if self.is_multi_gpu: - self.reduce_parameters() - - # Apply the gradients - # -- For PPO - nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) - self.optimizer.step() - # -- For RND - if self.rnd_optimizer: - self.rnd_optimizer.step() - - # Store the losses - mean_value_loss += value_loss.item() - mean_surrogate_loss += surrogate_loss.item() - mean_entropy += entropy_batch.mean().item() - # -- RND loss - if mean_rnd_loss is not None: - mean_rnd_loss += rnd_loss.item() - # -- Symmetry loss - if mean_symmetry_loss is not None: - mean_symmetry_loss += symmetry_loss.item() - - # -- For PPO - num_updates = self.num_learning_epochs * self.num_mini_batches - mean_value_loss /= num_updates - mean_surrogate_loss /= num_updates - mean_entropy /= num_updates - # -- For RND - if mean_rnd_loss is not None: - mean_rnd_loss /= num_updates - # -- For Symmetry - if mean_symmetry_loss is not None: - mean_symmetry_loss /= num_updates - # -- Clear the storage - self.storage.clear() - - # construct the loss dictionary - loss_dict = { - "value_function": mean_value_loss, - "surrogate": mean_surrogate_loss, - "entropy": mean_entropy, - } - if self.rnd: - loss_dict["rnd"] = mean_rnd_loss - if self.symmetry: - loss_dict["symmetry"] = mean_symmetry_loss - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - if self.rnd: - model_params.append(self.rnd.predictor.state_dict()) - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - if self.rnd: - self.rnd.predictor.load_state_dict(model_params[1]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - if self.rnd: - grads += [param.grad.view(-1) for param in self.rnd.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - - # Get all parameters - all_params = self.policy.parameters() - if self.rnd: - all_params = chain(all_params, self.rnd.parameters()) - - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in all_params: - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py deleted file mode 100644 index ebc2e20..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Main module for the rsl_rl package.""" diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py deleted file mode 100644 index 0ef9163..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of different RL agents.""" - -from .distillation import Distillation -from .ppo import PPO -from .amp_ppo import AMP_PPO -__all__ = ["PPO", "Distillation", "AMP_PPO"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py deleted file mode 100644 index a2164eb..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations -from torch._tensor import Tensor -from torch._tensor import Tensor -from typing import Any -from copy import deepcopy - -import torch -import torch.nn as nn -import torch.optim as optim -from itertools import chain - -from rsl_rl.modules import ActorCritic -from rsl_rl.modules.rnd import RandomNetworkDistillation -from rsl_rl.storage import RolloutStorage, ReplayBufferMulti -from rsl_rl.utils import string_to_callable - - -class AMP_PPO: - """Proximal Policy Optimization algorithm (https://arxiv.org/abs/1707.06347).""" - - policy: ActorCritic - """The actor critic module.""" - - def __init__( - self, - policy, - discriminator, - amp_data, - amp_normalizer, - amp_num_frames=1, - amp_replay_buffer_size=100000, - num_learning_epochs=5, - num_mini_batches=4, - clip_param=0.2, - gamma=0.99, - lam=0.95, - value_loss_coef=1.0, - entropy_coef=0.01, - learning_rate=0.001, - max_grad_norm=1.0, - use_clipped_value_loss=True, - schedule="adaptive", - desired_kl=0.01, - device="cpu", - normalize_advantage_per_mini_batch=False, - # RND parameters - rnd_cfg: dict | None = None, - # Symmetry parameters - symmetry_cfg: dict | None = None, - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # RND components - if rnd_cfg is not None: - # Extract parameters used in ppo - rnd_lr = rnd_cfg.pop("learning_rate", 1e-3) - # Create RND module - self.rnd = RandomNetworkDistillation(device=self.device, **rnd_cfg) - # Create RND optimizer - params = self.rnd.predictor.parameters() - self.rnd_optimizer = optim.Adam(params, lr=rnd_lr) - else: - self.rnd = None - self.rnd_optimizer = None - - # Symmetry components - if symmetry_cfg is not None: - # Check if symmetry is enabled - use_symmetry = symmetry_cfg["use_data_augmentation"] or symmetry_cfg["use_mirror_loss"] - # Print that we are not using symmetry - if not use_symmetry: - print("Symmetry not used for learning. We will use it for logging instead.") - # If function is a string then resolve it to a function - if isinstance(symmetry_cfg["data_augmentation_func"], str): - symmetry_cfg["data_augmentation_func"] = string_to_callable(symmetry_cfg["data_augmentation_func"]) - # Check valid configuration - if symmetry_cfg["use_data_augmentation"] and not callable(symmetry_cfg["data_augmentation_func"]): - raise ValueError( - "Data augmentation enabled but the function is not callable:" - f" {symmetry_cfg['data_augmentation_func']}" - ) - # Store symmetry configuration - self.symmetry = symmetry_cfg - else: - self.symmetry = None - - ## AMP components - self.discriminator = discriminator - self.discriminator.to(self.device) - - self.amp_storage = ReplayBufferMulti(discriminator.state_dim, amp_replay_buffer_size, amp_num_frames, device) - self.amp_data = amp_data - self.amp_normalizer = amp_normalizer - - # PPO components - self.policy = policy - self.policy.to(self.device) - - # Create rollout storage - self.storage: RolloutStorage = None # type: ignore - self.transition = RolloutStorage.Transition() - self.amp_transition = RolloutStorage.Transition() - params = [ - {'params': self.policy.parameters(), 'name': 'policy'}, - ] - - params.append({ - 'params': self.discriminator.trunk.parameters(), - 'weight_decay': 10e-4, - 'name': f'amp_trunk' - }) - params.append({ - 'params': self.discriminator.amp_linear.parameters(), - 'weight_decay': 10e-2, - 'name': f'amp_head' - }) - - # Create optimizer - self.optimizer = optim.Adam(params, lr=learning_rate) - - # PPO parameters - self.clip_param = clip_param - self.num_learning_epochs = num_learning_epochs - self.num_mini_batches = num_mini_batches - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - self.gamma = gamma - self.lam = lam - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.desired_kl = desired_kl - self.schedule = schedule - self.learning_rate = learning_rate - self.normalize_advantage_per_mini_batch = normalize_advantage_per_mini_batch - - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs, amp_obs): - if self.policy.is_recurrent: - self.transition.hidden_states = self.policy.get_hidden_states() - # compute the actions and values - self.transition.actions = self.policy.act(obs).detach() - self.transition.values = self.policy.evaluate(obs).detach() - self.transition.actions_log_prob = self.policy.get_actions_log_prob(self.transition.actions).detach() - self.transition.action_mean = self.policy.action_mean.detach() - self.transition.action_sigma = self.policy.action_std.detach() - # need to record obs before env.step() - self.transition.observations = obs - self.amp_transition.observations = amp_obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras,amp_obs, amp_obs_frames=None): - # update the normalizers - self.policy.update_normalization(obs) - if self.rnd: - self.rnd.update_normalization(obs) - - # Record the rewards and dones - # Note: we clone here because later on we bootstrap the rewards based on timeouts - self.transition.rewards = rewards.clone() - self.transition.dones = dones - - # Compute the intrinsic rewards and add to extrinsic rewards - if self.rnd: - # Compute the intrinsic rewards - self.intrinsic_rewards = self.rnd.get_intrinsic_reward(obs) - # Add intrinsic rewards to extrinsic rewards - self.transition.rewards += self.intrinsic_rewards - - # Bootstrapping on time outs - if "time_outs" in extras: - self.transition.rewards += self.gamma * torch.squeeze( - self.transition.values * extras["time_outs"].unsqueeze(1).to(self.device), 1 - ) - - if amp_obs_frames is not None: - self.amp_storage.insert(amp_obs_frames) - else: - self.amp_storage.insert(self.amp_transition.observations, amp_obs) - - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.amp_transition.clear() - self.policy.reset(dones) - - def compute_returns(self, obs): - # compute value for the last step - last_values = self.policy.evaluate(obs).detach() - self.storage.compute_returns( - last_values, self.gamma, self.lam, normalize_advantage=not self.normalize_advantage_per_mini_batch - ) - - def update(self): # noqa: C901 - mean_value_loss = 0 - mean_surrogate_loss = 0 - mean_entropy = 0 - mean_amp_loss = 0 - mean_grad_pen_loss = 0 - mean_policy_pred = 0 - mean_expert_pred = 0 - # -- RND loss - if self.rnd: - mean_rnd_loss = 0 - else: - mean_rnd_loss = None - # -- Symmetry loss - if self.symmetry: - mean_symmetry_loss = 0 - else: - mean_symmetry_loss = None - - # generator for mini batches - if self.policy.is_recurrent: - generator = self.storage.recurrent_mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - else: - generator = self.storage.mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - - - amp_policy_generator = self.amp_storage.feed_forward_generator( - self.num_learning_epochs * self.num_mini_batches, - self.storage.num_envs * self.storage.num_transitions_per_env // self.num_mini_batches, - ) - - amp_expert_generator = self.amp_data.feed_forward_generator_23dof_multi( - self.num_learning_epochs * self.num_mini_batches, - self.storage.num_envs * self.storage.num_transitions_per_env // self.num_mini_batches, - ) - - # iterate over batches - for sample, sample_amp_policy, sample_amp_expert in zip(generator, amp_policy_generator, amp_expert_generator): - ( - obs_batch, - actions_batch, - target_values_batch, - advantages_batch, - returns_batch, - old_actions_log_prob_batch, - old_mu_batch, - old_sigma_batch, - hid_states_batch, - masks_batch, - ) = sample - - # number of augmentations per sample - # we start with 1 and increase it if we use symmetry augmentation - num_aug = 1 - # original batch size - # we assume policy group is always there and needs augmentation - original_batch_size = obs_batch.batch_size[0] - - # check if we should normalize advantages per mini batch - if self.normalize_advantage_per_mini_batch: - with torch.no_grad(): - advantages_batch = (advantages_batch - advantages_batch.mean()) / (advantages_batch.std() + 1e-8) - - # Perform symmetric augmentation - if self.symmetry and self.symmetry["use_data_augmentation"]: - # augmentation using symmetry - data_augmentation_func = self.symmetry["data_augmentation_func"] - # returned shape: [batch_size * num_aug, ...] - obs_batch, actions_batch = data_augmentation_func( - obs=obs_batch, - actions=actions_batch, - env=self.symmetry["_env"], - ) - # compute number of augmentations per sample - # we assume policy group is always there and needs augmentation - num_aug = int(obs_batch.batch_size[0] / original_batch_size) - # repeat the rest of the batch - # -- actor - old_actions_log_prob_batch = old_actions_log_prob_batch.repeat(num_aug, 1) - # -- critic - target_values_batch = target_values_batch.repeat(num_aug, 1) - advantages_batch = advantages_batch.repeat(num_aug, 1) - returns_batch = returns_batch.repeat(num_aug, 1) - - # Recompute actions log prob and entropy for current batch of transitions - # Note: we need to do this because we updated the policy with the new parameters - # -- actor - self.policy.act(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[0]) - actions_log_prob_batch = self.policy.get_actions_log_prob(actions_batch) - # -- critic - value_batch = self.policy.evaluate(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[1]) - # -- entropy - # we only keep the entropy of the first augmentation (the original one) - mu_batch = self.policy.action_mean[:original_batch_size] - sigma_batch = self.policy.action_std[:original_batch_size] - entropy_batch = self.policy.entropy[:original_batch_size] - - # KL - if self.desired_kl is not None and self.schedule == "adaptive": - with torch.inference_mode(): - kl = torch.sum( - torch.log(sigma_batch / old_sigma_batch + 1.0e-5) - + (torch.square(old_sigma_batch) + torch.square(old_mu_batch - mu_batch)) - / (2.0 * torch.square(sigma_batch)) - - 0.5, - axis=-1, - ) - kl_mean = torch.mean(kl) - - # Reduce the KL divergence across all GPUs - if self.is_multi_gpu: - torch.distributed.all_reduce(kl_mean, op=torch.distributed.ReduceOp.SUM) - kl_mean /= self.gpu_world_size - - # Update the learning rate - # Perform this adaptation only on the main process - # TODO: Is this needed? If KL-divergence is the "same" across all GPUs, - # then the learning rate should be the same across all GPUs. - if self.gpu_global_rank == 0: - if kl_mean > self.desired_kl * 2.0: - self.learning_rate = max(1e-5, self.learning_rate / 1.5) - elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0: - self.learning_rate = min(1e-2, self.learning_rate * 1.5) - - # Update the learning rate for all GPUs - if self.is_multi_gpu: - lr_tensor = torch.tensor(self.learning_rate, device=self.device) - torch.distributed.broadcast(lr_tensor, src=0) - self.learning_rate = lr_tensor.item() - - # Update the learning rate for all parameter groups - for param_group in self.optimizer.param_groups: - param_group["lr"] = self.learning_rate - - # Surrogate loss - ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) - surrogate = -torch.squeeze(advantages_batch) * ratio - surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( - ratio, 1.0 - self.clip_param, 1.0 + self.clip_param - ) - surrogate_loss = torch.max(surrogate, surrogate_clipped).mean() - - # Value function loss - if self.use_clipped_value_loss: - value_clipped = target_values_batch + (value_batch - target_values_batch).clamp( - -self.clip_param, self.clip_param - ) - value_losses = (value_batch - returns_batch).pow(2) - value_losses_clipped = (value_clipped - returns_batch).pow(2) - value_loss = torch.max(value_losses, value_losses_clipped).mean() - else: - value_loss = (returns_batch - value_batch).pow(2).mean() - - loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy_batch.mean() - - # Symmetry loss - if self.symmetry: - # obtain the symmetric actions - # if we did augmentation before then we don't need to augment again - if not self.symmetry["use_data_augmentation"]: - data_augmentation_func = self.symmetry["data_augmentation_func"] - obs_batch, _ = data_augmentation_func(obs=obs_batch, actions=None, env=self.symmetry["_env"]) - # compute number of augmentations per sample - num_aug = int(obs_batch.shape[0] / original_batch_size) - - # actions predicted by the actor for symmetrically-augmented observations - mean_actions_batch = self.policy.act_inference(obs_batch.detach().clone()) - - # compute the symmetrically augmented actions - # note: we are assuming the first augmentation is the original one. - # We do not use the action_batch from earlier since that action was sampled from the distribution. - # However, the symmetry loss is computed using the mean of the distribution. - action_mean_orig = mean_actions_batch[:original_batch_size] - _, actions_mean_symm_batch = data_augmentation_func( - obs=None, actions=action_mean_orig, env=self.symmetry["_env"] - ) - - # compute the loss (we skip the first augmentation as it is the original one) - mse_loss = torch.nn.MSELoss() - symmetry_loss = mse_loss( - mean_actions_batch[original_batch_size:], actions_mean_symm_batch.detach()[original_batch_size:] - ) - # add the loss to the total loss - if self.symmetry["use_mirror_loss"]: - loss += self.symmetry["mirror_loss_coeff"] * symmetry_loss - else: - symmetry_loss = symmetry_loss.detach() - - # Random Network Distillation loss - # TODO: Move this processing to inside RND module. - if self.rnd: - # extract the rnd_state - # TODO: Check if we still need torch no grad. It is just an affine transformation. - with torch.no_grad(): - rnd_state_batch = self.rnd.get_rnd_state(obs_batch[:original_batch_size]) - rnd_state_batch = self.rnd.state_normalizer(rnd_state_batch) - # predict the embedding and the target - predicted_embedding = self.rnd.predictor(rnd_state_batch) - target_embedding = self.rnd.target(rnd_state_batch).detach() - # compute the loss as the mean squared error - mseloss = torch.nn.MSELoss() - rnd_loss = mseloss(predicted_embedding, target_embedding) - - expert_states = sample_amp_expert - policy_states = sample_amp_policy - - with torch.no_grad(): - expert_states = self.amp_normalizer.normalize_torch(expert_states.to(self.device), self.device) - policy_states = self.amp_normalizer.normalize_torch(policy_states, self.device) - - contact_phase_push = obs_batch['critic'][:, -4] - mask_push = contact_phase_push == 1. - - if mask_push.any(): - policy_d = self.discriminator(policy_states.flatten(1)) - expert_states = expert_states.to(self.device) - expert_d = self.discriminator(expert_states.flatten(1)) - - expert_loss = torch.nn.MSELoss()(expert_d, torch.ones(expert_d.size(), device=self.device)) - policy_loss = torch.nn.MSELoss()(policy_d, -1 * torch.ones(policy_d.size(), device=self.device)) - amp_loss = 0.5 * (expert_loss + policy_loss) - - # grad penalty - grad_pen_loss = self.discriminator.compute_grad_pen(expert_states, lambda_=5) - else: - amp_loss = torch.tensor(0.0, device=self.device) - grad_pen_loss = torch.tensor(0.0, device=self.device) - expert_loss = torch.tensor(0.0, device=self.device) - policy_loss = torch.tensor(0.0, device=self.device) - - loss += (amp_loss + grad_pen_loss) - self.amp_normalizer.update(policy_states.cpu().numpy()) - self.amp_normalizer.update(expert_states.cpu().numpy()) - - # Compute the gradients - # -- For PPO - self.optimizer.zero_grad() - loss.backward() - # -- For RND - if self.rnd: - self.rnd_optimizer.zero_grad() # type: ignore - rnd_loss.backward() - - # Collect gradients from all GPUs - if self.is_multi_gpu: - self.reduce_parameters() - - # Apply the gradients - # -- For PPO - nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) - self.optimizer.step() - # -- For RND - if self.rnd_optimizer: - self.rnd_optimizer.step() - - # Store the losses - mean_value_loss += value_loss.item() - mean_surrogate_loss += surrogate_loss.item() - mean_entropy += entropy_batch.mean().item() - mean_amp_loss += amp_loss.item() - mean_grad_pen_loss += grad_pen_loss.item() - mean_policy_pred += policy_loss.mean().item() - mean_expert_pred += expert_loss.mean().item() - # -- RND loss - if mean_rnd_loss is not None: - mean_rnd_loss += rnd_loss.item() - # -- Symmetry loss - if mean_symmetry_loss is not None: - mean_symmetry_loss += symmetry_loss.item() - - # -- For PPO - num_updates = self.num_learning_epochs * self.num_mini_batches - mean_value_loss /= num_updates - mean_surrogate_loss /= num_updates - mean_entropy /= num_updates - mean_amp_loss /= num_updates - mean_grad_pen_loss /= num_updates - mean_policy_pred /= num_updates - mean_expert_pred /= num_updates - - # -- For RND - if mean_rnd_loss is not None: - mean_rnd_loss /= num_updates - # -- For Symmetry - if mean_symmetry_loss is not None: - mean_symmetry_loss /= num_updates - # -- Clear the storage - self.storage.clear() - - # construct the loss dictionary - loss_dict = { - "value_function": mean_value_loss, - "surrogate": mean_surrogate_loss, - "entropy": mean_entropy, - "amp": mean_amp_loss, - "amp_grad_pen": mean_grad_pen_loss, - "amp_policy_pred": mean_policy_pred, - "amp_expert_pred": mean_expert_pred, - } - if self.rnd: - loss_dict["rnd"] = mean_rnd_loss - if self.symmetry: - loss_dict["symmetry"] = mean_symmetry_loss - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - if self.rnd: - model_params.append(self.rnd.predictor.state_dict()) - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - if self.rnd: - self.rnd.predictor.load_state_dict(model_params[1]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - if self.rnd: - grads += [param.grad.view(-1) for param in self.rnd.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - - # Get all parameters - all_params = self.policy.parameters() - if self.rnd: - all_params = chain(all_params, self.rnd.parameters()) - - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in all_params: - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py deleted file mode 100644 index 3a86e00..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -import torch -import torch.nn as nn - -from rsl_rl.modules import StudentTeacher, StudentTeacherRecurrent -from rsl_rl.storage import RolloutStorage -from rsl_rl.utils import resolve_optimizer - - -class Distillation: - """Distillation algorithm for training a student model to mimic a teacher model.""" - - policy: StudentTeacher | StudentTeacherRecurrent - """The student teacher model.""" - - def __init__( - self, - policy, - num_learning_epochs=1, - gradient_length=15, - learning_rate=1e-3, - max_grad_norm=None, - loss_type="mse", - optimizer="adam", - device="cpu", - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # distillation components - self.policy = policy - self.policy.to(self.device) - self.storage = None # initialized later - - # initialize the optimizer - self.optimizer = resolve_optimizer(optimizer)(self.policy.parameters(), lr=learning_rate) - - # initialize the transition - self.transition = RolloutStorage.Transition() - self.last_hidden_states = None - - # distillation parameters - self.num_learning_epochs = num_learning_epochs - self.gradient_length = gradient_length - self.learning_rate = learning_rate - self.max_grad_norm = max_grad_norm - - # initialize the loss function - loss_fn_dict = { - "mse": nn.functional.mse_loss, - "huber": nn.functional.huber_loss, - } - if loss_type in loss_fn_dict: - self.loss_fn = loss_fn_dict[loss_type] - else: - raise ValueError(f"Unknown loss type: {loss_type}. Supported types are: {list(loss_fn_dict.keys())}") - - self.num_updates = 0 - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs): - # compute the actions - self.transition.actions = self.policy.act(obs).detach() - self.transition.privileged_actions = self.policy.evaluate(obs).detach() - # record the observations - self.transition.observations = obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras): - # update the normalizers - self.policy.update_normalization(obs) - - # record the rewards and dones - self.transition.rewards = rewards - self.transition.dones = dones - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.policy.reset(dones) - - def update(self): - self.num_updates += 1 - mean_behavior_loss = 0 - loss = 0 - cnt = 0 - - for epoch in range(self.num_learning_epochs): - self.policy.reset(hidden_states=self.last_hidden_states) - self.policy.detach_hidden_states() - for obs, _, privileged_actions, dones in self.storage.generator(): - - # inference the student for gradient computation - actions = self.policy.act_inference(obs) - - # behavior cloning loss - behavior_loss = self.loss_fn(actions, privileged_actions) - - # total loss - loss = loss + behavior_loss - mean_behavior_loss += behavior_loss.item() - cnt += 1 - - # gradient step - if cnt % self.gradient_length == 0: - self.optimizer.zero_grad() - loss.backward() - if self.is_multi_gpu: - self.reduce_parameters() - if self.max_grad_norm: - nn.utils.clip_grad_norm_(self.policy.student.parameters(), self.max_grad_norm) - self.optimizer.step() - self.policy.detach_hidden_states() - loss = 0 - - # reset dones - self.policy.reset(dones.view(-1)) - self.policy.detach_hidden_states(dones.view(-1)) - - mean_behavior_loss /= cnt - self.storage.clear() - self.last_hidden_states = self.policy.get_hidden_states() - self.policy.detach_hidden_states() - - # construct the loss dictionary - loss_dict = {"behavior": mean_behavior_loss} - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in self.policy.parameters(): - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py deleted file mode 100644 index 6c21fc5..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py +++ /dev/null @@ -1,469 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import torch.optim as optim -from itertools import chain - -from rsl_rl.modules import ActorCritic -from rsl_rl.modules.rnd import RandomNetworkDistillation -from rsl_rl.storage import RolloutStorage -from rsl_rl.utils import string_to_callable - - -class PPO: - """Proximal Policy Optimization algorithm (https://arxiv.org/abs/1707.06347).""" - - policy: ActorCritic - """The actor critic module.""" - - def __init__( - self, - policy, - num_learning_epochs=5, - num_mini_batches=4, - clip_param=0.2, - gamma=0.99, - lam=0.95, - value_loss_coef=1.0, - entropy_coef=0.01, - learning_rate=0.001, - max_grad_norm=1.0, - use_clipped_value_loss=True, - schedule="adaptive", - desired_kl=0.01, - device="cpu", - normalize_advantage_per_mini_batch=False, - # RND parameters - rnd_cfg: dict | None = None, - # Symmetry parameters - symmetry_cfg: dict | None = None, - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # RND components - if rnd_cfg is not None: - # Extract parameters used in ppo - rnd_lr = rnd_cfg.pop("learning_rate", 1e-3) - # Create RND module - self.rnd = RandomNetworkDistillation(device=self.device, **rnd_cfg) - # Create RND optimizer - params = self.rnd.predictor.parameters() - self.rnd_optimizer = optim.Adam(params, lr=rnd_lr) - else: - self.rnd = None - self.rnd_optimizer = None - - # Symmetry components - if symmetry_cfg is not None: - # Check if symmetry is enabled - use_symmetry = symmetry_cfg["use_data_augmentation"] or symmetry_cfg["use_mirror_loss"] - # Print that we are not using symmetry - if not use_symmetry: - print("Symmetry not used for learning. We will use it for logging instead.") - # If function is a string then resolve it to a function - if isinstance(symmetry_cfg["data_augmentation_func"], str): - symmetry_cfg["data_augmentation_func"] = string_to_callable(symmetry_cfg["data_augmentation_func"]) - # Check valid configuration - if symmetry_cfg["use_data_augmentation"] and not callable(symmetry_cfg["data_augmentation_func"]): - raise ValueError( - "Data augmentation enabled but the function is not callable:" - f" {symmetry_cfg['data_augmentation_func']}" - ) - # Store symmetry configuration - self.symmetry = symmetry_cfg - else: - self.symmetry = None - - # PPO components - self.policy = policy - self.policy.to(self.device) - # Create optimizer - self.optimizer = optim.Adam(self.policy.parameters(), lr=learning_rate) - # Create rollout storage - self.storage: RolloutStorage = None # type: ignore - self.transition = RolloutStorage.Transition() - - # PPO parameters - self.clip_param = clip_param - self.num_learning_epochs = num_learning_epochs - self.num_mini_batches = num_mini_batches - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - self.gamma = gamma - self.lam = lam - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.desired_kl = desired_kl - self.schedule = schedule - self.learning_rate = learning_rate - self.normalize_advantage_per_mini_batch = normalize_advantage_per_mini_batch - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs): - if self.policy.is_recurrent: - self.transition.hidden_states = self.policy.get_hidden_states() - # compute the actions and values - self.transition.actions = self.policy.act(obs).detach() - self.transition.values = self.policy.evaluate(obs).detach() - self.transition.actions_log_prob = self.policy.get_actions_log_prob(self.transition.actions).detach() - self.transition.action_mean = self.policy.action_mean.detach() - self.transition.action_sigma = self.policy.action_std.detach() - # need to record obs before env.step() - self.transition.observations = obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras): - # update the normalizers - self.policy.update_normalization(obs) - if self.rnd: - self.rnd.update_normalization(obs) - - # Record the rewards and dones - # Note: we clone here because later on we bootstrap the rewards based on timeouts - self.transition.rewards = rewards.clone() - self.transition.dones = dones - - # Compute the intrinsic rewards and add to extrinsic rewards - if self.rnd: - # Compute the intrinsic rewards - self.intrinsic_rewards = self.rnd.get_intrinsic_reward(obs) - # Add intrinsic rewards to extrinsic rewards - self.transition.rewards += self.intrinsic_rewards - - # Bootstrapping on time outs - if "time_outs" in extras: - self.transition.rewards += self.gamma * torch.squeeze( - self.transition.values * extras["time_outs"].unsqueeze(1).to(self.device), 1 - ) - - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.policy.reset(dones) - - def compute_returns(self, obs): - # compute value for the last step - last_values = self.policy.evaluate(obs).detach() - self.storage.compute_returns( - last_values, self.gamma, self.lam, normalize_advantage=not self.normalize_advantage_per_mini_batch - ) - - def update(self): # noqa: C901 - mean_value_loss = 0 - mean_surrogate_loss = 0 - mean_entropy = 0 - # -- RND loss - if self.rnd: - mean_rnd_loss = 0 - else: - mean_rnd_loss = None - # -- Symmetry loss - if self.symmetry: - mean_symmetry_loss = 0 - else: - mean_symmetry_loss = None - - # generator for mini batches - if self.policy.is_recurrent: - generator = self.storage.recurrent_mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - else: - generator = self.storage.mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - - # iterate over batches - for ( - obs_batch, - actions_batch, - target_values_batch, - advantages_batch, - returns_batch, - old_actions_log_prob_batch, - old_mu_batch, - old_sigma_batch, - hid_states_batch, - masks_batch, - ) in generator: - - # number of augmentations per sample - # we start with 1 and increase it if we use symmetry augmentation - num_aug = 1 - # original batch size - # we assume policy group is always there and needs augmentation - original_batch_size = obs_batch.batch_size[0] - - # check if we should normalize advantages per mini batch - if self.normalize_advantage_per_mini_batch: - with torch.no_grad(): - advantages_batch = (advantages_batch - advantages_batch.mean()) / (advantages_batch.std() + 1e-8) - - # Perform symmetric augmentation - if self.symmetry and self.symmetry["use_data_augmentation"]: - # augmentation using symmetry - data_augmentation_func = self.symmetry["data_augmentation_func"] - # returned shape: [batch_size * num_aug, ...] - obs_batch, actions_batch = data_augmentation_func( - obs=obs_batch, - actions=actions_batch, - env=self.symmetry["_env"], - ) - # compute number of augmentations per sample - # we assume policy group is always there and needs augmentation - num_aug = int(obs_batch.batch_size[0] / original_batch_size) - # repeat the rest of the batch - # -- actor - old_actions_log_prob_batch = old_actions_log_prob_batch.repeat(num_aug, 1) - # -- critic - target_values_batch = target_values_batch.repeat(num_aug, 1) - advantages_batch = advantages_batch.repeat(num_aug, 1) - returns_batch = returns_batch.repeat(num_aug, 1) - - # Recompute actions log prob and entropy for current batch of transitions - # Note: we need to do this because we updated the policy with the new parameters - # -- actor - self.policy.act(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[0]) - actions_log_prob_batch = self.policy.get_actions_log_prob(actions_batch) - # -- critic - value_batch = self.policy.evaluate(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[1]) - # -- entropy - # we only keep the entropy of the first augmentation (the original one) - mu_batch = self.policy.action_mean[:original_batch_size] - sigma_batch = self.policy.action_std[:original_batch_size] - entropy_batch = self.policy.entropy[:original_batch_size] - - # KL - if self.desired_kl is not None and self.schedule == "adaptive": - with torch.inference_mode(): - kl = torch.sum( - torch.log(sigma_batch / old_sigma_batch + 1.0e-5) - + (torch.square(old_sigma_batch) + torch.square(old_mu_batch - mu_batch)) - / (2.0 * torch.square(sigma_batch)) - - 0.5, - axis=-1, - ) - kl_mean = torch.mean(kl) - - # Reduce the KL divergence across all GPUs - if self.is_multi_gpu: - torch.distributed.all_reduce(kl_mean, op=torch.distributed.ReduceOp.SUM) - kl_mean /= self.gpu_world_size - - # Update the learning rate - # Perform this adaptation only on the main process - # TODO: Is this needed? If KL-divergence is the "same" across all GPUs, - # then the learning rate should be the same across all GPUs. - if self.gpu_global_rank == 0: - if kl_mean > self.desired_kl * 2.0: - self.learning_rate = max(1e-5, self.learning_rate / 1.5) - elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0: - self.learning_rate = min(1e-2, self.learning_rate * 1.5) - - # Update the learning rate for all GPUs - if self.is_multi_gpu: - lr_tensor = torch.tensor(self.learning_rate, device=self.device) - torch.distributed.broadcast(lr_tensor, src=0) - self.learning_rate = lr_tensor.item() - - # Update the learning rate for all parameter groups - for param_group in self.optimizer.param_groups: - param_group["lr"] = self.learning_rate - - # Surrogate loss - ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) - surrogate = -torch.squeeze(advantages_batch) * ratio - surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( - ratio, 1.0 - self.clip_param, 1.0 + self.clip_param - ) - surrogate_loss = torch.max(surrogate, surrogate_clipped).mean() - - # Value function loss - if self.use_clipped_value_loss: - value_clipped = target_values_batch + (value_batch - target_values_batch).clamp( - -self.clip_param, self.clip_param - ) - value_losses = (value_batch - returns_batch).pow(2) - value_losses_clipped = (value_clipped - returns_batch).pow(2) - value_loss = torch.max(value_losses, value_losses_clipped).mean() - else: - value_loss = (returns_batch - value_batch).pow(2).mean() - - loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy_batch.mean() - - # Symmetry loss - if self.symmetry: - # obtain the symmetric actions - # if we did augmentation before then we don't need to augment again - if not self.symmetry["use_data_augmentation"]: - data_augmentation_func = self.symmetry["data_augmentation_func"] - obs_batch, _ = data_augmentation_func(obs=obs_batch, actions=None, env=self.symmetry["_env"]) - # compute number of augmentations per sample - num_aug = int(obs_batch.shape[0] / original_batch_size) - - # actions predicted by the actor for symmetrically-augmented observations - mean_actions_batch = self.policy.act_inference(obs_batch.detach().clone()) - - # compute the symmetrically augmented actions - # note: we are assuming the first augmentation is the original one. - # We do not use the action_batch from earlier since that action was sampled from the distribution. - # However, the symmetry loss is computed using the mean of the distribution. - action_mean_orig = mean_actions_batch[:original_batch_size] - _, actions_mean_symm_batch = data_augmentation_func( - obs=None, actions=action_mean_orig, env=self.symmetry["_env"] - ) - - # compute the loss (we skip the first augmentation as it is the original one) - mse_loss = torch.nn.MSELoss() - symmetry_loss = mse_loss( - mean_actions_batch[original_batch_size:], actions_mean_symm_batch.detach()[original_batch_size:] - ) - # add the loss to the total loss - if self.symmetry["use_mirror_loss"]: - loss += self.symmetry["mirror_loss_coeff"] * symmetry_loss - else: - symmetry_loss = symmetry_loss.detach() - - # Random Network Distillation loss - # TODO: Move this processing to inside RND module. - if self.rnd: - # extract the rnd_state - # TODO: Check if we still need torch no grad. It is just an affine transformation. - with torch.no_grad(): - rnd_state_batch = self.rnd.get_rnd_state(obs_batch[:original_batch_size]) - rnd_state_batch = self.rnd.state_normalizer(rnd_state_batch) - # predict the embedding and the target - predicted_embedding = self.rnd.predictor(rnd_state_batch) - target_embedding = self.rnd.target(rnd_state_batch).detach() - # compute the loss as the mean squared error - mseloss = torch.nn.MSELoss() - rnd_loss = mseloss(predicted_embedding, target_embedding) - - # Compute the gradients - # -- For PPO - self.optimizer.zero_grad() - loss.backward() - # -- For RND - if self.rnd: - self.rnd_optimizer.zero_grad() # type: ignore - rnd_loss.backward() - - # Collect gradients from all GPUs - if self.is_multi_gpu: - self.reduce_parameters() - - # Apply the gradients - # -- For PPO - nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) - self.optimizer.step() - # -- For RND - if self.rnd_optimizer: - self.rnd_optimizer.step() - - # Store the losses - mean_value_loss += value_loss.item() - mean_surrogate_loss += surrogate_loss.item() - mean_entropy += entropy_batch.mean().item() - # -- RND loss - if mean_rnd_loss is not None: - mean_rnd_loss += rnd_loss.item() - # -- Symmetry loss - if mean_symmetry_loss is not None: - mean_symmetry_loss += symmetry_loss.item() - - # -- For PPO - num_updates = self.num_learning_epochs * self.num_mini_batches - mean_value_loss /= num_updates - mean_surrogate_loss /= num_updates - mean_entropy /= num_updates - # -- For RND - if mean_rnd_loss is not None: - mean_rnd_loss /= num_updates - # -- For Symmetry - if mean_symmetry_loss is not None: - mean_symmetry_loss /= num_updates - # -- Clear the storage - self.storage.clear() - - # construct the loss dictionary - loss_dict = { - "value_function": mean_value_loss, - "surrogate": mean_surrogate_loss, - "entropy": mean_entropy, - } - if self.rnd: - loss_dict["rnd"] = mean_rnd_loss - if self.symmetry: - loss_dict["symmetry"] = mean_symmetry_loss - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - if self.rnd: - model_params.append(self.rnd.predictor.state_dict()) - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - if self.rnd: - self.rnd.predictor.load_state_dict(model_params[1]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - if self.rnd: - grads += [param.grad.view(-1) for param in self.rnd.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - - # Get all parameters - all_params = self.policy.parameters() - if self.rnd: - all_params = chain(all_params, self.rnd.parameters()) - - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in all_params: - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py deleted file mode 100644 index ebc2e20..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Main module for the rsl_rl package.""" diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py deleted file mode 100644 index 0ef9163..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of different RL agents.""" - -from .distillation import Distillation -from .ppo import PPO -from .amp_ppo import AMP_PPO -__all__ = ["PPO", "Distillation", "AMP_PPO"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py deleted file mode 100644 index a2164eb..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/amp_ppo.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations -from torch._tensor import Tensor -from torch._tensor import Tensor -from typing import Any -from copy import deepcopy - -import torch -import torch.nn as nn -import torch.optim as optim -from itertools import chain - -from rsl_rl.modules import ActorCritic -from rsl_rl.modules.rnd import RandomNetworkDistillation -from rsl_rl.storage import RolloutStorage, ReplayBufferMulti -from rsl_rl.utils import string_to_callable - - -class AMP_PPO: - """Proximal Policy Optimization algorithm (https://arxiv.org/abs/1707.06347).""" - - policy: ActorCritic - """The actor critic module.""" - - def __init__( - self, - policy, - discriminator, - amp_data, - amp_normalizer, - amp_num_frames=1, - amp_replay_buffer_size=100000, - num_learning_epochs=5, - num_mini_batches=4, - clip_param=0.2, - gamma=0.99, - lam=0.95, - value_loss_coef=1.0, - entropy_coef=0.01, - learning_rate=0.001, - max_grad_norm=1.0, - use_clipped_value_loss=True, - schedule="adaptive", - desired_kl=0.01, - device="cpu", - normalize_advantage_per_mini_batch=False, - # RND parameters - rnd_cfg: dict | None = None, - # Symmetry parameters - symmetry_cfg: dict | None = None, - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # RND components - if rnd_cfg is not None: - # Extract parameters used in ppo - rnd_lr = rnd_cfg.pop("learning_rate", 1e-3) - # Create RND module - self.rnd = RandomNetworkDistillation(device=self.device, **rnd_cfg) - # Create RND optimizer - params = self.rnd.predictor.parameters() - self.rnd_optimizer = optim.Adam(params, lr=rnd_lr) - else: - self.rnd = None - self.rnd_optimizer = None - - # Symmetry components - if symmetry_cfg is not None: - # Check if symmetry is enabled - use_symmetry = symmetry_cfg["use_data_augmentation"] or symmetry_cfg["use_mirror_loss"] - # Print that we are not using symmetry - if not use_symmetry: - print("Symmetry not used for learning. We will use it for logging instead.") - # If function is a string then resolve it to a function - if isinstance(symmetry_cfg["data_augmentation_func"], str): - symmetry_cfg["data_augmentation_func"] = string_to_callable(symmetry_cfg["data_augmentation_func"]) - # Check valid configuration - if symmetry_cfg["use_data_augmentation"] and not callable(symmetry_cfg["data_augmentation_func"]): - raise ValueError( - "Data augmentation enabled but the function is not callable:" - f" {symmetry_cfg['data_augmentation_func']}" - ) - # Store symmetry configuration - self.symmetry = symmetry_cfg - else: - self.symmetry = None - - ## AMP components - self.discriminator = discriminator - self.discriminator.to(self.device) - - self.amp_storage = ReplayBufferMulti(discriminator.state_dim, amp_replay_buffer_size, amp_num_frames, device) - self.amp_data = amp_data - self.amp_normalizer = amp_normalizer - - # PPO components - self.policy = policy - self.policy.to(self.device) - - # Create rollout storage - self.storage: RolloutStorage = None # type: ignore - self.transition = RolloutStorage.Transition() - self.amp_transition = RolloutStorage.Transition() - params = [ - {'params': self.policy.parameters(), 'name': 'policy'}, - ] - - params.append({ - 'params': self.discriminator.trunk.parameters(), - 'weight_decay': 10e-4, - 'name': f'amp_trunk' - }) - params.append({ - 'params': self.discriminator.amp_linear.parameters(), - 'weight_decay': 10e-2, - 'name': f'amp_head' - }) - - # Create optimizer - self.optimizer = optim.Adam(params, lr=learning_rate) - - # PPO parameters - self.clip_param = clip_param - self.num_learning_epochs = num_learning_epochs - self.num_mini_batches = num_mini_batches - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - self.gamma = gamma - self.lam = lam - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.desired_kl = desired_kl - self.schedule = schedule - self.learning_rate = learning_rate - self.normalize_advantage_per_mini_batch = normalize_advantage_per_mini_batch - - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs, amp_obs): - if self.policy.is_recurrent: - self.transition.hidden_states = self.policy.get_hidden_states() - # compute the actions and values - self.transition.actions = self.policy.act(obs).detach() - self.transition.values = self.policy.evaluate(obs).detach() - self.transition.actions_log_prob = self.policy.get_actions_log_prob(self.transition.actions).detach() - self.transition.action_mean = self.policy.action_mean.detach() - self.transition.action_sigma = self.policy.action_std.detach() - # need to record obs before env.step() - self.transition.observations = obs - self.amp_transition.observations = amp_obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras,amp_obs, amp_obs_frames=None): - # update the normalizers - self.policy.update_normalization(obs) - if self.rnd: - self.rnd.update_normalization(obs) - - # Record the rewards and dones - # Note: we clone here because later on we bootstrap the rewards based on timeouts - self.transition.rewards = rewards.clone() - self.transition.dones = dones - - # Compute the intrinsic rewards and add to extrinsic rewards - if self.rnd: - # Compute the intrinsic rewards - self.intrinsic_rewards = self.rnd.get_intrinsic_reward(obs) - # Add intrinsic rewards to extrinsic rewards - self.transition.rewards += self.intrinsic_rewards - - # Bootstrapping on time outs - if "time_outs" in extras: - self.transition.rewards += self.gamma * torch.squeeze( - self.transition.values * extras["time_outs"].unsqueeze(1).to(self.device), 1 - ) - - if amp_obs_frames is not None: - self.amp_storage.insert(amp_obs_frames) - else: - self.amp_storage.insert(self.amp_transition.observations, amp_obs) - - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.amp_transition.clear() - self.policy.reset(dones) - - def compute_returns(self, obs): - # compute value for the last step - last_values = self.policy.evaluate(obs).detach() - self.storage.compute_returns( - last_values, self.gamma, self.lam, normalize_advantage=not self.normalize_advantage_per_mini_batch - ) - - def update(self): # noqa: C901 - mean_value_loss = 0 - mean_surrogate_loss = 0 - mean_entropy = 0 - mean_amp_loss = 0 - mean_grad_pen_loss = 0 - mean_policy_pred = 0 - mean_expert_pred = 0 - # -- RND loss - if self.rnd: - mean_rnd_loss = 0 - else: - mean_rnd_loss = None - # -- Symmetry loss - if self.symmetry: - mean_symmetry_loss = 0 - else: - mean_symmetry_loss = None - - # generator for mini batches - if self.policy.is_recurrent: - generator = self.storage.recurrent_mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - else: - generator = self.storage.mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - - - amp_policy_generator = self.amp_storage.feed_forward_generator( - self.num_learning_epochs * self.num_mini_batches, - self.storage.num_envs * self.storage.num_transitions_per_env // self.num_mini_batches, - ) - - amp_expert_generator = self.amp_data.feed_forward_generator_23dof_multi( - self.num_learning_epochs * self.num_mini_batches, - self.storage.num_envs * self.storage.num_transitions_per_env // self.num_mini_batches, - ) - - # iterate over batches - for sample, sample_amp_policy, sample_amp_expert in zip(generator, amp_policy_generator, amp_expert_generator): - ( - obs_batch, - actions_batch, - target_values_batch, - advantages_batch, - returns_batch, - old_actions_log_prob_batch, - old_mu_batch, - old_sigma_batch, - hid_states_batch, - masks_batch, - ) = sample - - # number of augmentations per sample - # we start with 1 and increase it if we use symmetry augmentation - num_aug = 1 - # original batch size - # we assume policy group is always there and needs augmentation - original_batch_size = obs_batch.batch_size[0] - - # check if we should normalize advantages per mini batch - if self.normalize_advantage_per_mini_batch: - with torch.no_grad(): - advantages_batch = (advantages_batch - advantages_batch.mean()) / (advantages_batch.std() + 1e-8) - - # Perform symmetric augmentation - if self.symmetry and self.symmetry["use_data_augmentation"]: - # augmentation using symmetry - data_augmentation_func = self.symmetry["data_augmentation_func"] - # returned shape: [batch_size * num_aug, ...] - obs_batch, actions_batch = data_augmentation_func( - obs=obs_batch, - actions=actions_batch, - env=self.symmetry["_env"], - ) - # compute number of augmentations per sample - # we assume policy group is always there and needs augmentation - num_aug = int(obs_batch.batch_size[0] / original_batch_size) - # repeat the rest of the batch - # -- actor - old_actions_log_prob_batch = old_actions_log_prob_batch.repeat(num_aug, 1) - # -- critic - target_values_batch = target_values_batch.repeat(num_aug, 1) - advantages_batch = advantages_batch.repeat(num_aug, 1) - returns_batch = returns_batch.repeat(num_aug, 1) - - # Recompute actions log prob and entropy for current batch of transitions - # Note: we need to do this because we updated the policy with the new parameters - # -- actor - self.policy.act(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[0]) - actions_log_prob_batch = self.policy.get_actions_log_prob(actions_batch) - # -- critic - value_batch = self.policy.evaluate(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[1]) - # -- entropy - # we only keep the entropy of the first augmentation (the original one) - mu_batch = self.policy.action_mean[:original_batch_size] - sigma_batch = self.policy.action_std[:original_batch_size] - entropy_batch = self.policy.entropy[:original_batch_size] - - # KL - if self.desired_kl is not None and self.schedule == "adaptive": - with torch.inference_mode(): - kl = torch.sum( - torch.log(sigma_batch / old_sigma_batch + 1.0e-5) - + (torch.square(old_sigma_batch) + torch.square(old_mu_batch - mu_batch)) - / (2.0 * torch.square(sigma_batch)) - - 0.5, - axis=-1, - ) - kl_mean = torch.mean(kl) - - # Reduce the KL divergence across all GPUs - if self.is_multi_gpu: - torch.distributed.all_reduce(kl_mean, op=torch.distributed.ReduceOp.SUM) - kl_mean /= self.gpu_world_size - - # Update the learning rate - # Perform this adaptation only on the main process - # TODO: Is this needed? If KL-divergence is the "same" across all GPUs, - # then the learning rate should be the same across all GPUs. - if self.gpu_global_rank == 0: - if kl_mean > self.desired_kl * 2.0: - self.learning_rate = max(1e-5, self.learning_rate / 1.5) - elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0: - self.learning_rate = min(1e-2, self.learning_rate * 1.5) - - # Update the learning rate for all GPUs - if self.is_multi_gpu: - lr_tensor = torch.tensor(self.learning_rate, device=self.device) - torch.distributed.broadcast(lr_tensor, src=0) - self.learning_rate = lr_tensor.item() - - # Update the learning rate for all parameter groups - for param_group in self.optimizer.param_groups: - param_group["lr"] = self.learning_rate - - # Surrogate loss - ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) - surrogate = -torch.squeeze(advantages_batch) * ratio - surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( - ratio, 1.0 - self.clip_param, 1.0 + self.clip_param - ) - surrogate_loss = torch.max(surrogate, surrogate_clipped).mean() - - # Value function loss - if self.use_clipped_value_loss: - value_clipped = target_values_batch + (value_batch - target_values_batch).clamp( - -self.clip_param, self.clip_param - ) - value_losses = (value_batch - returns_batch).pow(2) - value_losses_clipped = (value_clipped - returns_batch).pow(2) - value_loss = torch.max(value_losses, value_losses_clipped).mean() - else: - value_loss = (returns_batch - value_batch).pow(2).mean() - - loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy_batch.mean() - - # Symmetry loss - if self.symmetry: - # obtain the symmetric actions - # if we did augmentation before then we don't need to augment again - if not self.symmetry["use_data_augmentation"]: - data_augmentation_func = self.symmetry["data_augmentation_func"] - obs_batch, _ = data_augmentation_func(obs=obs_batch, actions=None, env=self.symmetry["_env"]) - # compute number of augmentations per sample - num_aug = int(obs_batch.shape[0] / original_batch_size) - - # actions predicted by the actor for symmetrically-augmented observations - mean_actions_batch = self.policy.act_inference(obs_batch.detach().clone()) - - # compute the symmetrically augmented actions - # note: we are assuming the first augmentation is the original one. - # We do not use the action_batch from earlier since that action was sampled from the distribution. - # However, the symmetry loss is computed using the mean of the distribution. - action_mean_orig = mean_actions_batch[:original_batch_size] - _, actions_mean_symm_batch = data_augmentation_func( - obs=None, actions=action_mean_orig, env=self.symmetry["_env"] - ) - - # compute the loss (we skip the first augmentation as it is the original one) - mse_loss = torch.nn.MSELoss() - symmetry_loss = mse_loss( - mean_actions_batch[original_batch_size:], actions_mean_symm_batch.detach()[original_batch_size:] - ) - # add the loss to the total loss - if self.symmetry["use_mirror_loss"]: - loss += self.symmetry["mirror_loss_coeff"] * symmetry_loss - else: - symmetry_loss = symmetry_loss.detach() - - # Random Network Distillation loss - # TODO: Move this processing to inside RND module. - if self.rnd: - # extract the rnd_state - # TODO: Check if we still need torch no grad. It is just an affine transformation. - with torch.no_grad(): - rnd_state_batch = self.rnd.get_rnd_state(obs_batch[:original_batch_size]) - rnd_state_batch = self.rnd.state_normalizer(rnd_state_batch) - # predict the embedding and the target - predicted_embedding = self.rnd.predictor(rnd_state_batch) - target_embedding = self.rnd.target(rnd_state_batch).detach() - # compute the loss as the mean squared error - mseloss = torch.nn.MSELoss() - rnd_loss = mseloss(predicted_embedding, target_embedding) - - expert_states = sample_amp_expert - policy_states = sample_amp_policy - - with torch.no_grad(): - expert_states = self.amp_normalizer.normalize_torch(expert_states.to(self.device), self.device) - policy_states = self.amp_normalizer.normalize_torch(policy_states, self.device) - - contact_phase_push = obs_batch['critic'][:, -4] - mask_push = contact_phase_push == 1. - - if mask_push.any(): - policy_d = self.discriminator(policy_states.flatten(1)) - expert_states = expert_states.to(self.device) - expert_d = self.discriminator(expert_states.flatten(1)) - - expert_loss = torch.nn.MSELoss()(expert_d, torch.ones(expert_d.size(), device=self.device)) - policy_loss = torch.nn.MSELoss()(policy_d, -1 * torch.ones(policy_d.size(), device=self.device)) - amp_loss = 0.5 * (expert_loss + policy_loss) - - # grad penalty - grad_pen_loss = self.discriminator.compute_grad_pen(expert_states, lambda_=5) - else: - amp_loss = torch.tensor(0.0, device=self.device) - grad_pen_loss = torch.tensor(0.0, device=self.device) - expert_loss = torch.tensor(0.0, device=self.device) - policy_loss = torch.tensor(0.0, device=self.device) - - loss += (amp_loss + grad_pen_loss) - self.amp_normalizer.update(policy_states.cpu().numpy()) - self.amp_normalizer.update(expert_states.cpu().numpy()) - - # Compute the gradients - # -- For PPO - self.optimizer.zero_grad() - loss.backward() - # -- For RND - if self.rnd: - self.rnd_optimizer.zero_grad() # type: ignore - rnd_loss.backward() - - # Collect gradients from all GPUs - if self.is_multi_gpu: - self.reduce_parameters() - - # Apply the gradients - # -- For PPO - nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) - self.optimizer.step() - # -- For RND - if self.rnd_optimizer: - self.rnd_optimizer.step() - - # Store the losses - mean_value_loss += value_loss.item() - mean_surrogate_loss += surrogate_loss.item() - mean_entropy += entropy_batch.mean().item() - mean_amp_loss += amp_loss.item() - mean_grad_pen_loss += grad_pen_loss.item() - mean_policy_pred += policy_loss.mean().item() - mean_expert_pred += expert_loss.mean().item() - # -- RND loss - if mean_rnd_loss is not None: - mean_rnd_loss += rnd_loss.item() - # -- Symmetry loss - if mean_symmetry_loss is not None: - mean_symmetry_loss += symmetry_loss.item() - - # -- For PPO - num_updates = self.num_learning_epochs * self.num_mini_batches - mean_value_loss /= num_updates - mean_surrogate_loss /= num_updates - mean_entropy /= num_updates - mean_amp_loss /= num_updates - mean_grad_pen_loss /= num_updates - mean_policy_pred /= num_updates - mean_expert_pred /= num_updates - - # -- For RND - if mean_rnd_loss is not None: - mean_rnd_loss /= num_updates - # -- For Symmetry - if mean_symmetry_loss is not None: - mean_symmetry_loss /= num_updates - # -- Clear the storage - self.storage.clear() - - # construct the loss dictionary - loss_dict = { - "value_function": mean_value_loss, - "surrogate": mean_surrogate_loss, - "entropy": mean_entropy, - "amp": mean_amp_loss, - "amp_grad_pen": mean_grad_pen_loss, - "amp_policy_pred": mean_policy_pred, - "amp_expert_pred": mean_expert_pred, - } - if self.rnd: - loss_dict["rnd"] = mean_rnd_loss - if self.symmetry: - loss_dict["symmetry"] = mean_symmetry_loss - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - if self.rnd: - model_params.append(self.rnd.predictor.state_dict()) - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - if self.rnd: - self.rnd.predictor.load_state_dict(model_params[1]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - if self.rnd: - grads += [param.grad.view(-1) for param in self.rnd.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - - # Get all parameters - all_params = self.policy.parameters() - if self.rnd: - all_params = chain(all_params, self.rnd.parameters()) - - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in all_params: - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py deleted file mode 100644 index 3a86e00..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/distillation.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -import torch -import torch.nn as nn - -from rsl_rl.modules import StudentTeacher, StudentTeacherRecurrent -from rsl_rl.storage import RolloutStorage -from rsl_rl.utils import resolve_optimizer - - -class Distillation: - """Distillation algorithm for training a student model to mimic a teacher model.""" - - policy: StudentTeacher | StudentTeacherRecurrent - """The student teacher model.""" - - def __init__( - self, - policy, - num_learning_epochs=1, - gradient_length=15, - learning_rate=1e-3, - max_grad_norm=None, - loss_type="mse", - optimizer="adam", - device="cpu", - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # distillation components - self.policy = policy - self.policy.to(self.device) - self.storage = None # initialized later - - # initialize the optimizer - self.optimizer = resolve_optimizer(optimizer)(self.policy.parameters(), lr=learning_rate) - - # initialize the transition - self.transition = RolloutStorage.Transition() - self.last_hidden_states = None - - # distillation parameters - self.num_learning_epochs = num_learning_epochs - self.gradient_length = gradient_length - self.learning_rate = learning_rate - self.max_grad_norm = max_grad_norm - - # initialize the loss function - loss_fn_dict = { - "mse": nn.functional.mse_loss, - "huber": nn.functional.huber_loss, - } - if loss_type in loss_fn_dict: - self.loss_fn = loss_fn_dict[loss_type] - else: - raise ValueError(f"Unknown loss type: {loss_type}. Supported types are: {list(loss_fn_dict.keys())}") - - self.num_updates = 0 - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs): - # compute the actions - self.transition.actions = self.policy.act(obs).detach() - self.transition.privileged_actions = self.policy.evaluate(obs).detach() - # record the observations - self.transition.observations = obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras): - # update the normalizers - self.policy.update_normalization(obs) - - # record the rewards and dones - self.transition.rewards = rewards - self.transition.dones = dones - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.policy.reset(dones) - - def update(self): - self.num_updates += 1 - mean_behavior_loss = 0 - loss = 0 - cnt = 0 - - for epoch in range(self.num_learning_epochs): - self.policy.reset(hidden_states=self.last_hidden_states) - self.policy.detach_hidden_states() - for obs, _, privileged_actions, dones in self.storage.generator(): - - # inference the student for gradient computation - actions = self.policy.act_inference(obs) - - # behavior cloning loss - behavior_loss = self.loss_fn(actions, privileged_actions) - - # total loss - loss = loss + behavior_loss - mean_behavior_loss += behavior_loss.item() - cnt += 1 - - # gradient step - if cnt % self.gradient_length == 0: - self.optimizer.zero_grad() - loss.backward() - if self.is_multi_gpu: - self.reduce_parameters() - if self.max_grad_norm: - nn.utils.clip_grad_norm_(self.policy.student.parameters(), self.max_grad_norm) - self.optimizer.step() - self.policy.detach_hidden_states() - loss = 0 - - # reset dones - self.policy.reset(dones.view(-1)) - self.policy.detach_hidden_states(dones.view(-1)) - - mean_behavior_loss /= cnt - self.storage.clear() - self.last_hidden_states = self.policy.get_hidden_states() - self.policy.detach_hidden_states() - - # construct the loss dictionary - loss_dict = {"behavior": mean_behavior_loss} - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in self.policy.parameters(): - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py deleted file mode 100644 index 6c21fc5..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/algorithms/ppo.py +++ /dev/null @@ -1,469 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import torch.optim as optim -from itertools import chain - -from rsl_rl.modules import ActorCritic -from rsl_rl.modules.rnd import RandomNetworkDistillation -from rsl_rl.storage import RolloutStorage -from rsl_rl.utils import string_to_callable - - -class PPO: - """Proximal Policy Optimization algorithm (https://arxiv.org/abs/1707.06347).""" - - policy: ActorCritic - """The actor critic module.""" - - def __init__( - self, - policy, - num_learning_epochs=5, - num_mini_batches=4, - clip_param=0.2, - gamma=0.99, - lam=0.95, - value_loss_coef=1.0, - entropy_coef=0.01, - learning_rate=0.001, - max_grad_norm=1.0, - use_clipped_value_loss=True, - schedule="adaptive", - desired_kl=0.01, - device="cpu", - normalize_advantage_per_mini_batch=False, - # RND parameters - rnd_cfg: dict | None = None, - # Symmetry parameters - symmetry_cfg: dict | None = None, - # Distributed training parameters - multi_gpu_cfg: dict | None = None, - ): - # device-related parameters - self.device = device - self.is_multi_gpu = multi_gpu_cfg is not None - # Multi-GPU parameters - if multi_gpu_cfg is not None: - self.gpu_global_rank = multi_gpu_cfg["global_rank"] - self.gpu_world_size = multi_gpu_cfg["world_size"] - else: - self.gpu_global_rank = 0 - self.gpu_world_size = 1 - - # RND components - if rnd_cfg is not None: - # Extract parameters used in ppo - rnd_lr = rnd_cfg.pop("learning_rate", 1e-3) - # Create RND module - self.rnd = RandomNetworkDistillation(device=self.device, **rnd_cfg) - # Create RND optimizer - params = self.rnd.predictor.parameters() - self.rnd_optimizer = optim.Adam(params, lr=rnd_lr) - else: - self.rnd = None - self.rnd_optimizer = None - - # Symmetry components - if symmetry_cfg is not None: - # Check if symmetry is enabled - use_symmetry = symmetry_cfg["use_data_augmentation"] or symmetry_cfg["use_mirror_loss"] - # Print that we are not using symmetry - if not use_symmetry: - print("Symmetry not used for learning. We will use it for logging instead.") - # If function is a string then resolve it to a function - if isinstance(symmetry_cfg["data_augmentation_func"], str): - symmetry_cfg["data_augmentation_func"] = string_to_callable(symmetry_cfg["data_augmentation_func"]) - # Check valid configuration - if symmetry_cfg["use_data_augmentation"] and not callable(symmetry_cfg["data_augmentation_func"]): - raise ValueError( - "Data augmentation enabled but the function is not callable:" - f" {symmetry_cfg['data_augmentation_func']}" - ) - # Store symmetry configuration - self.symmetry = symmetry_cfg - else: - self.symmetry = None - - # PPO components - self.policy = policy - self.policy.to(self.device) - # Create optimizer - self.optimizer = optim.Adam(self.policy.parameters(), lr=learning_rate) - # Create rollout storage - self.storage: RolloutStorage = None # type: ignore - self.transition = RolloutStorage.Transition() - - # PPO parameters - self.clip_param = clip_param - self.num_learning_epochs = num_learning_epochs - self.num_mini_batches = num_mini_batches - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - self.gamma = gamma - self.lam = lam - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.desired_kl = desired_kl - self.schedule = schedule - self.learning_rate = learning_rate - self.normalize_advantage_per_mini_batch = normalize_advantage_per_mini_batch - - def init_storage(self, training_type, num_envs, num_transitions_per_env, obs, actions_shape): - # create rollout storage - self.storage = RolloutStorage( - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - self.device, - ) - - def act(self, obs): - if self.policy.is_recurrent: - self.transition.hidden_states = self.policy.get_hidden_states() - # compute the actions and values - self.transition.actions = self.policy.act(obs).detach() - self.transition.values = self.policy.evaluate(obs).detach() - self.transition.actions_log_prob = self.policy.get_actions_log_prob(self.transition.actions).detach() - self.transition.action_mean = self.policy.action_mean.detach() - self.transition.action_sigma = self.policy.action_std.detach() - # need to record obs before env.step() - self.transition.observations = obs - return self.transition.actions - - def process_env_step(self, obs, rewards, dones, extras): - # update the normalizers - self.policy.update_normalization(obs) - if self.rnd: - self.rnd.update_normalization(obs) - - # Record the rewards and dones - # Note: we clone here because later on we bootstrap the rewards based on timeouts - self.transition.rewards = rewards.clone() - self.transition.dones = dones - - # Compute the intrinsic rewards and add to extrinsic rewards - if self.rnd: - # Compute the intrinsic rewards - self.intrinsic_rewards = self.rnd.get_intrinsic_reward(obs) - # Add intrinsic rewards to extrinsic rewards - self.transition.rewards += self.intrinsic_rewards - - # Bootstrapping on time outs - if "time_outs" in extras: - self.transition.rewards += self.gamma * torch.squeeze( - self.transition.values * extras["time_outs"].unsqueeze(1).to(self.device), 1 - ) - - # record the transition - self.storage.add_transitions(self.transition) - self.transition.clear() - self.policy.reset(dones) - - def compute_returns(self, obs): - # compute value for the last step - last_values = self.policy.evaluate(obs).detach() - self.storage.compute_returns( - last_values, self.gamma, self.lam, normalize_advantage=not self.normalize_advantage_per_mini_batch - ) - - def update(self): # noqa: C901 - mean_value_loss = 0 - mean_surrogate_loss = 0 - mean_entropy = 0 - # -- RND loss - if self.rnd: - mean_rnd_loss = 0 - else: - mean_rnd_loss = None - # -- Symmetry loss - if self.symmetry: - mean_symmetry_loss = 0 - else: - mean_symmetry_loss = None - - # generator for mini batches - if self.policy.is_recurrent: - generator = self.storage.recurrent_mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - else: - generator = self.storage.mini_batch_generator(self.num_mini_batches, self.num_learning_epochs) - - # iterate over batches - for ( - obs_batch, - actions_batch, - target_values_batch, - advantages_batch, - returns_batch, - old_actions_log_prob_batch, - old_mu_batch, - old_sigma_batch, - hid_states_batch, - masks_batch, - ) in generator: - - # number of augmentations per sample - # we start with 1 and increase it if we use symmetry augmentation - num_aug = 1 - # original batch size - # we assume policy group is always there and needs augmentation - original_batch_size = obs_batch.batch_size[0] - - # check if we should normalize advantages per mini batch - if self.normalize_advantage_per_mini_batch: - with torch.no_grad(): - advantages_batch = (advantages_batch - advantages_batch.mean()) / (advantages_batch.std() + 1e-8) - - # Perform symmetric augmentation - if self.symmetry and self.symmetry["use_data_augmentation"]: - # augmentation using symmetry - data_augmentation_func = self.symmetry["data_augmentation_func"] - # returned shape: [batch_size * num_aug, ...] - obs_batch, actions_batch = data_augmentation_func( - obs=obs_batch, - actions=actions_batch, - env=self.symmetry["_env"], - ) - # compute number of augmentations per sample - # we assume policy group is always there and needs augmentation - num_aug = int(obs_batch.batch_size[0] / original_batch_size) - # repeat the rest of the batch - # -- actor - old_actions_log_prob_batch = old_actions_log_prob_batch.repeat(num_aug, 1) - # -- critic - target_values_batch = target_values_batch.repeat(num_aug, 1) - advantages_batch = advantages_batch.repeat(num_aug, 1) - returns_batch = returns_batch.repeat(num_aug, 1) - - # Recompute actions log prob and entropy for current batch of transitions - # Note: we need to do this because we updated the policy with the new parameters - # -- actor - self.policy.act(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[0]) - actions_log_prob_batch = self.policy.get_actions_log_prob(actions_batch) - # -- critic - value_batch = self.policy.evaluate(obs_batch, masks=masks_batch, hidden_states=hid_states_batch[1]) - # -- entropy - # we only keep the entropy of the first augmentation (the original one) - mu_batch = self.policy.action_mean[:original_batch_size] - sigma_batch = self.policy.action_std[:original_batch_size] - entropy_batch = self.policy.entropy[:original_batch_size] - - # KL - if self.desired_kl is not None and self.schedule == "adaptive": - with torch.inference_mode(): - kl = torch.sum( - torch.log(sigma_batch / old_sigma_batch + 1.0e-5) - + (torch.square(old_sigma_batch) + torch.square(old_mu_batch - mu_batch)) - / (2.0 * torch.square(sigma_batch)) - - 0.5, - axis=-1, - ) - kl_mean = torch.mean(kl) - - # Reduce the KL divergence across all GPUs - if self.is_multi_gpu: - torch.distributed.all_reduce(kl_mean, op=torch.distributed.ReduceOp.SUM) - kl_mean /= self.gpu_world_size - - # Update the learning rate - # Perform this adaptation only on the main process - # TODO: Is this needed? If KL-divergence is the "same" across all GPUs, - # then the learning rate should be the same across all GPUs. - if self.gpu_global_rank == 0: - if kl_mean > self.desired_kl * 2.0: - self.learning_rate = max(1e-5, self.learning_rate / 1.5) - elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0: - self.learning_rate = min(1e-2, self.learning_rate * 1.5) - - # Update the learning rate for all GPUs - if self.is_multi_gpu: - lr_tensor = torch.tensor(self.learning_rate, device=self.device) - torch.distributed.broadcast(lr_tensor, src=0) - self.learning_rate = lr_tensor.item() - - # Update the learning rate for all parameter groups - for param_group in self.optimizer.param_groups: - param_group["lr"] = self.learning_rate - - # Surrogate loss - ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) - surrogate = -torch.squeeze(advantages_batch) * ratio - surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( - ratio, 1.0 - self.clip_param, 1.0 + self.clip_param - ) - surrogate_loss = torch.max(surrogate, surrogate_clipped).mean() - - # Value function loss - if self.use_clipped_value_loss: - value_clipped = target_values_batch + (value_batch - target_values_batch).clamp( - -self.clip_param, self.clip_param - ) - value_losses = (value_batch - returns_batch).pow(2) - value_losses_clipped = (value_clipped - returns_batch).pow(2) - value_loss = torch.max(value_losses, value_losses_clipped).mean() - else: - value_loss = (returns_batch - value_batch).pow(2).mean() - - loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy_batch.mean() - - # Symmetry loss - if self.symmetry: - # obtain the symmetric actions - # if we did augmentation before then we don't need to augment again - if not self.symmetry["use_data_augmentation"]: - data_augmentation_func = self.symmetry["data_augmentation_func"] - obs_batch, _ = data_augmentation_func(obs=obs_batch, actions=None, env=self.symmetry["_env"]) - # compute number of augmentations per sample - num_aug = int(obs_batch.shape[0] / original_batch_size) - - # actions predicted by the actor for symmetrically-augmented observations - mean_actions_batch = self.policy.act_inference(obs_batch.detach().clone()) - - # compute the symmetrically augmented actions - # note: we are assuming the first augmentation is the original one. - # We do not use the action_batch from earlier since that action was sampled from the distribution. - # However, the symmetry loss is computed using the mean of the distribution. - action_mean_orig = mean_actions_batch[:original_batch_size] - _, actions_mean_symm_batch = data_augmentation_func( - obs=None, actions=action_mean_orig, env=self.symmetry["_env"] - ) - - # compute the loss (we skip the first augmentation as it is the original one) - mse_loss = torch.nn.MSELoss() - symmetry_loss = mse_loss( - mean_actions_batch[original_batch_size:], actions_mean_symm_batch.detach()[original_batch_size:] - ) - # add the loss to the total loss - if self.symmetry["use_mirror_loss"]: - loss += self.symmetry["mirror_loss_coeff"] * symmetry_loss - else: - symmetry_loss = symmetry_loss.detach() - - # Random Network Distillation loss - # TODO: Move this processing to inside RND module. - if self.rnd: - # extract the rnd_state - # TODO: Check if we still need torch no grad. It is just an affine transformation. - with torch.no_grad(): - rnd_state_batch = self.rnd.get_rnd_state(obs_batch[:original_batch_size]) - rnd_state_batch = self.rnd.state_normalizer(rnd_state_batch) - # predict the embedding and the target - predicted_embedding = self.rnd.predictor(rnd_state_batch) - target_embedding = self.rnd.target(rnd_state_batch).detach() - # compute the loss as the mean squared error - mseloss = torch.nn.MSELoss() - rnd_loss = mseloss(predicted_embedding, target_embedding) - - # Compute the gradients - # -- For PPO - self.optimizer.zero_grad() - loss.backward() - # -- For RND - if self.rnd: - self.rnd_optimizer.zero_grad() # type: ignore - rnd_loss.backward() - - # Collect gradients from all GPUs - if self.is_multi_gpu: - self.reduce_parameters() - - # Apply the gradients - # -- For PPO - nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) - self.optimizer.step() - # -- For RND - if self.rnd_optimizer: - self.rnd_optimizer.step() - - # Store the losses - mean_value_loss += value_loss.item() - mean_surrogate_loss += surrogate_loss.item() - mean_entropy += entropy_batch.mean().item() - # -- RND loss - if mean_rnd_loss is not None: - mean_rnd_loss += rnd_loss.item() - # -- Symmetry loss - if mean_symmetry_loss is not None: - mean_symmetry_loss += symmetry_loss.item() - - # -- For PPO - num_updates = self.num_learning_epochs * self.num_mini_batches - mean_value_loss /= num_updates - mean_surrogate_loss /= num_updates - mean_entropy /= num_updates - # -- For RND - if mean_rnd_loss is not None: - mean_rnd_loss /= num_updates - # -- For Symmetry - if mean_symmetry_loss is not None: - mean_symmetry_loss /= num_updates - # -- Clear the storage - self.storage.clear() - - # construct the loss dictionary - loss_dict = { - "value_function": mean_value_loss, - "surrogate": mean_surrogate_loss, - "entropy": mean_entropy, - } - if self.rnd: - loss_dict["rnd"] = mean_rnd_loss - if self.symmetry: - loss_dict["symmetry"] = mean_symmetry_loss - - return loss_dict - - """ - Helper functions - """ - - def broadcast_parameters(self): - """Broadcast model parameters to all GPUs.""" - # obtain the model parameters on current GPU - model_params = [self.policy.state_dict()] - if self.rnd: - model_params.append(self.rnd.predictor.state_dict()) - # broadcast the model parameters - torch.distributed.broadcast_object_list(model_params, src=0) - # load the model parameters on all GPUs from source GPU - self.policy.load_state_dict(model_params[0]) - if self.rnd: - self.rnd.predictor.load_state_dict(model_params[1]) - - def reduce_parameters(self): - """Collect gradients from all GPUs and average them. - - This function is called after the backward pass to synchronize the gradients across all GPUs. - """ - # Create a tensor to store the gradients - grads = [param.grad.view(-1) for param in self.policy.parameters() if param.grad is not None] - if self.rnd: - grads += [param.grad.view(-1) for param in self.rnd.parameters() if param.grad is not None] - all_grads = torch.cat(grads) - - # Average the gradients across all GPUs - torch.distributed.all_reduce(all_grads, op=torch.distributed.ReduceOp.SUM) - all_grads /= self.gpu_world_size - - # Get all parameters - all_params = self.policy.parameters() - if self.rnd: - all_params = chain(all_params, self.rnd.parameters()) - - # Update the gradients for all parameters with the reduced gradients - offset = 0 - for param in all_params: - if param.grad is not None: - numel = param.numel() - # copy data back from shared buffer - param.grad.data.copy_(all_grads[offset : offset + numel].view_as(param.grad.data)) - # update the offset for the next parameter - offset += numel diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py deleted file mode 100644 index ab7c056..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Submodule defining the environment definitions.""" - -from .vec_env import VecEnv - -__all__ = ["VecEnv"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py deleted file mode 100644 index ae471cf..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -from abc import ABC, abstractmethod -from tensordict import TensorDict - - -class VecEnv(ABC): - """Abstract class for a vectorized environment. - - The vectorized environment is a collection of environments that are synchronized. This means that - the same type of action is applied to all environments and the same type of observation is returned from all - environments. - """ - - num_envs: int - """Number of environments.""" - - num_actions: int - """Number of actions.""" - - max_episode_length: int | torch.Tensor - - max_episode_length_s: float - """Maximum episode length. - - The maximum episode length can be a scalar or a tensor. If it is a scalar, it is the same for all environments. - If it is a tensor, it is the maximum episode length for each environment. This is useful for dynamic episode - lengths. - """ - - episode_length_buf: torch.Tensor - """Buffer for current episode lengths.""" - - device: torch.device | str - """Device to use.""" - - cfg: dict | object - """Configuration object.""" - - reset_env_ids: torch.Tensor | None = None - - contact_phase: torch.Tensor | None = None - """ - Operations. - """ - - @abstractmethod - def get_observations(self) -> TensorDict: - """Return the current observations. - - Returns: - observations (TensorDict): Observations from the environment. - """ - raise NotImplementedError - - @abstractmethod - def get_amp_observations(self) -> TensorDict: - """Return the current AMP observations. - - Returns: - observations (TensorDict): Observations from the environment. - """ - raise NotImplementedError - - @abstractmethod - def step(self, actions: torch.Tensor) -> tuple[TensorDict, torch.Tensor, torch.Tensor, dict]: - """Apply input action to the environment. - - Args: - actions (torch.Tensor): Input actions to apply. Shape: (num_envs, num_actions) - - Returns: - observations (TensorDict): Observations from the environment. - rewards (torch.Tensor): Rewards from the environment. Shape: (num_envs,) - dones (torch.Tensor): Done flags from the environment. Shape: (num_envs,) - extras (dict): Extra information from the environment. - - Observations: - - The observations TensorDict usually contains multiple observation groups. The `obs_groups` - dictionary of the runner configuration specifies which observation groups are used for which - purpose, i.e., it maps the available observation groups to observation sets. The observation sets - (keys of the `obs_groups` dictionary) currently used by rsl_rl are: - - - "policy": Specified observation groups are used as input to the actor/student network. - - "critic": Specified observation groups are used as input to the critic network. - - "teacher": Specified observation groups are used as input to the teacher network. - - "rnd_state": Specified observation groups are used as input to the RND network. - - Incomplete or incorrect configurations are handled in the `resolve_obs_groups()` function in - `rsl_rl/utils/utils.py`. - - Extras: - - The extras dictionary includes metrics such as the episode reward, episode length, etc. The following - dictionary keys are used by rsl_rl: - - - "time_outs" (torch.Tensor): Timeouts for the environments. These correspond to terminations that - happen due to time limits and not due to the environment reaching a terminal state. This is useful - for environments that have a fixed episode length. - - - "log" (dict[str, float | torch.Tensor]): Additional information for logging and debugging purposes. - The key should be a string and start with "/" for namespacing. The value can be a scalar or a - tensor. If it is a tensor, the mean of the tensor is used for logging. - """ - raise NotImplementedError - \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py deleted file mode 100644 index 9afe1a2..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Definitions for neural-network components for RL-agents.""" - -from .actor_critic import ActorCritic -from .actor_critic_recurrent import ActorCriticRecurrent -from .rnd import * -from .student_teacher import StudentTeacher -from .student_teacher_recurrent import StudentTeacherRecurrent -from .symmetry import * -from .discriminator_multi import DiscriminatorMulti -__all__ = [ - "ActorCritic", - "ActorCriticRecurrent", - "StudentTeacher", - "StudentTeacherRecurrent", - "DiscriminatorMulti", -] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py deleted file mode 100644 index 0efc36b..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization - - -class ActorCritic(nn.Module): - is_recurrent = False - - def __init__( - self, - obs, - obs_groups, - num_actions, - actor_obs_normalization=False, - critic_obs_normalization=False, - actor_hidden_dims=[256, 256, 256], - critic_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=1.0, - noise_std_type: str = "scalar", - state_dependent_std=False, - **kwargs, - ): - if kwargs: - print( - "ActorCritic.__init__ got unexpected arguments, which will be ignored: " - + str([key for key in kwargs.keys()]) - ) - super().__init__() - - # get the observation dimensions - self.obs_groups = obs_groups - num_actor_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The ActorCritic module only supports 1D observations." - num_actor_obs += obs[obs_group].shape[-1] - num_critic_obs = 0 - for obs_group in obs_groups["critic"]: - assert len(obs[obs_group].shape) == 2, "The ActorCritic module only supports 1D observations." - num_critic_obs += obs[obs_group].shape[-1] - - self.state_dependent_std = state_dependent_std - # actor - if self.state_dependent_std: - self.actor = MLP(num_actor_obs, [2, num_actions], actor_hidden_dims, activation) - else: - self.actor = MLP(num_actor_obs, num_actions, actor_hidden_dims, activation) - # actor observation normalization - self.actor_obs_normalization = actor_obs_normalization - if actor_obs_normalization: - self.actor_obs_normalizer = EmpiricalNormalization(num_actor_obs) - else: - self.actor_obs_normalizer = torch.nn.Identity() - print(f"Actor MLP: {self.actor}") - - # critic - self.critic = MLP(num_critic_obs, 1, critic_hidden_dims, activation) - # critic observation normalization - self.critic_obs_normalization = critic_obs_normalization - if critic_obs_normalization: - self.critic_obs_normalizer = EmpiricalNormalization(num_critic_obs) - else: - self.critic_obs_normalizer = torch.nn.Identity() - print(f"Critic MLP: {self.critic}") - - # Action noise - self.noise_std_type = noise_std_type - if self.state_dependent_std: - torch.nn.init.zeros_(self.actor[-2].weight[num_actions:]) - if self.noise_std_type == "scalar": - torch.nn.init.constant_(self.actor[-2].bias[num_actions:], init_noise_std) - elif self.noise_std_type == "log": - torch.nn.init.constant_( - self.actor[-2].bias[num_actions:], torch.log(torch.tensor(init_noise_std + 1e-7)) - ) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # Action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None): - pass - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - if self.state_dependent_std: - # compute mean and standard deviation - mean_and_std = self.actor(obs) - if self.noise_std_type == "scalar": - mean, std = torch.unbind(mean_and_std, dim=-2) - elif self.noise_std_type == "log": - mean, log_std = torch.unbind(mean_and_std, dim=-2) - std = torch.exp(log_std) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - # compute mean - mean = self.actor(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs, **kwargs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - self.update_distribution(obs) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - return self.actor(obs) - - def evaluate(self, obs, **kwargs): - obs = self.get_critic_obs(obs) - obs = self.critic_obs_normalizer(obs) - return self.critic(obs) - - def get_actor_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_critic_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["critic"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_actions_log_prob(self, actions): - return self.distribution.log_prob(actions).sum(dim=-1) - - def update_normalization(self, obs): - if self.actor_obs_normalization: - actor_obs = self.get_actor_obs(obs) - self.actor_obs_normalizer.update(actor_obs) - if self.critic_obs_normalization: - critic_obs = self.get_critic_obs(obs) - self.critic_obs_normalizer.update(critic_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the actor-critic model. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters (relevant for, e.g., distillation). - """ - - super().load_state_dict(state_dict, strict=strict) - return True # training resumes diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py deleted file mode 100644 index bba46ca..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import warnings -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization, Memory - - -class ActorCriticRecurrent(nn.Module): - is_recurrent = True - - def __init__( - self, - obs, - obs_groups, - num_actions, - actor_obs_normalization=False, - critic_obs_normalization=False, - actor_hidden_dims=[256, 256, 256], - critic_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=1.0, - noise_std_type: str = "scalar", - state_dependent_std=False, - rnn_type="lstm", - rnn_hidden_dim=256, - rnn_num_layers=1, - **kwargs, - ): - if "rnn_hidden_size" in kwargs: - warnings.warn( - "The argument `rnn_hidden_size` is deprecated and will be removed in a future version. " - "Please use `rnn_hidden_dim` instead.", - DeprecationWarning, - ) - if rnn_hidden_dim == 256: # Only override if the new argument is at its default - rnn_hidden_dim = kwargs.pop("rnn_hidden_size") - if kwargs: - print( - "ActorCriticRecurrent.__init__ got unexpected arguments, which will be ignored: " + str(kwargs.keys()), - ) - super().__init__() - - # get the observation dimensions - self.obs_groups = obs_groups - num_actor_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The ActorCriticRecurrent module only supports 1D observations." - num_actor_obs += obs[obs_group].shape[-1] - num_critic_obs = 0 - for obs_group in obs_groups["critic"]: - assert len(obs[obs_group].shape) == 2, "The ActorCriticRecurrent module only supports 1D observations." - num_critic_obs += obs[obs_group].shape[-1] - - self.state_dependent_std = state_dependent_std - # actor - self.memory_a = Memory(num_actor_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - if self.state_dependent_std: - self.actor = MLP(rnn_hidden_dim, [2, num_actions], actor_hidden_dims, activation) - else: - self.actor = MLP(rnn_hidden_dim, num_actions, actor_hidden_dims, activation) - - # actor observation normalization - self.actor_obs_normalization = actor_obs_normalization - if actor_obs_normalization: - self.actor_obs_normalizer = EmpiricalNormalization(num_actor_obs) - else: - self.actor_obs_normalizer = torch.nn.Identity() - print(f"Actor RNN: {self.memory_a}") - print(f"Actor MLP: {self.actor}") - - # critic - self.memory_c = Memory(num_critic_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - self.critic = MLP(rnn_hidden_dim, 1, critic_hidden_dims, activation) - # critic observation normalization - self.critic_obs_normalization = critic_obs_normalization - if critic_obs_normalization: - self.critic_obs_normalizer = EmpiricalNormalization(num_critic_obs) - else: - self.critic_obs_normalizer = torch.nn.Identity() - print(f"Critic RNN: {self.memory_c}") - print(f"Critic MLP: {self.critic}") - - # Action noise - self.noise_std_type = noise_std_type - if self.state_dependent_std: - torch.nn.init.zeros_(self.actor[-2].weight[num_actions:]) - if self.noise_std_type == "scalar": - torch.nn.init.constant_(self.actor[-2].bias[num_actions:], init_noise_std) - elif self.noise_std_type == "log": - torch.nn.init.constant_( - self.actor[-2].bias[num_actions:], torch.log(torch.tensor(init_noise_std + 1e-7)) - ) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # Action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def reset(self, dones=None): - self.memory_a.reset(dones) - self.memory_c.reset(dones) - - def forward(self): - raise NotImplementedError - - def update_distribution(self, obs): - if self.state_dependent_std: - # compute mean and standard deviation - mean_and_std = self.actor(obs) - if self.noise_std_type == "scalar": - mean, std = torch.unbind(mean_and_std, dim=-2) - elif self.noise_std_type == "log": - mean, log_std = torch.unbind(mean_and_std, dim=-2) - std = torch.exp(log_std) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - # compute mean - mean = self.actor(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs, masks=None, hidden_states=None): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - out_mem = self.memory_a(obs, masks, hidden_states).squeeze(0) - self.update_distribution(out_mem) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - out_mem = self.memory_a(obs).squeeze(0) - return self.actor(out_mem) - - def evaluate(self, obs, masks=None, hidden_states=None): - obs = self.get_critic_obs(obs) - obs = self.critic_obs_normalizer(obs) - out_mem = self.memory_c(obs, masks, hidden_states).squeeze(0) - return self.critic(out_mem) - - def get_actor_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_critic_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["critic"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_actions_log_prob(self, actions): - return self.distribution.log_prob(actions).sum(dim=-1) - - def get_hidden_states(self): - return self.memory_a.hidden_states, self.memory_c.hidden_states - - def update_normalization(self, obs): - if self.actor_obs_normalization: - actor_obs = self.get_actor_obs(obs) - self.actor_obs_normalizer.update(actor_obs) - if self.critic_obs_normalization: - critic_obs = self.get_critic_obs(obs) - self.critic_obs_normalizer.update(critic_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the actor-critic model. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters (relevant for, e.g., distillation). - """ - - super().load_state_dict(state_dict, strict=strict) - return True diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py deleted file mode 100644 index 7899d23..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py +++ /dev/null @@ -1,102 +0,0 @@ -import torch -import torch.nn as nn -from torch import autograd -import torch.nn.utils.spectral_norm as spectral_norm - - -class DiscriminatorMulti(nn.Module): - def __init__( - self, state_dim, amp_reward_coef, hidden_layer_sizes, device, - num_frames=2, task_reward_lerp=0.0, use_lerp=True): - super(DiscriminatorMulti, self).__init__() - - self.device = device - self.state_dim = state_dim - self.use_lerp = use_lerp - self.num_frames = num_frames # 存储帧数参数 - - self.amp_reward_coef = amp_reward_coef - amp_layers = [] - - curr_in_dim = state_dim * num_frames - for hidden_dim in hidden_layer_sizes: - amp_layers.append(spectral_norm(nn.Linear(curr_in_dim, hidden_dim))) - amp_layers.append(nn.ReLU()) - curr_in_dim = hidden_dim - self.trunk = nn.Sequential(*amp_layers).to(device) - self.amp_linear = spectral_norm(nn.Linear(hidden_layer_sizes[-1], 1)).to(device) - - self.trunk.train() - self.amp_linear.train() - - self.task_reward_lerp = task_reward_lerp - - def forward(self, x): - h = self.trunk(x) - d = self.amp_linear(h) - return d - - def compute_grad_pen(self, - expert_states, # 改为接收多帧状态列表 - lambda_=10): - # 将多帧状态沿最后一个维度拼接 - expert_data = expert_states.flatten(1) - expert_data.requires_grad = True - - disc = self.amp_linear(self.trunk(expert_data)) - ones = torch.ones(disc.size(), device=disc.device) - grad = autograd.grad( - outputs=disc, inputs=expert_data, - grad_outputs=ones, create_graph=True, - retain_graph=True, only_inputs=True)[0] - - # Enforce that the grad norm approaches 0. - grad_pen = lambda_ * (grad.norm(2, dim=1) - 0).pow(2).mean() - return grad_pen - - - def get_disc_weights(self): - weights = [] - for m in self.trunk.modules(): - if isinstance(m, nn.Linear): - weights.append(torch.flatten(m.weight)) - - weights.append(torch.flatten(self.amp_linear.weight)) - return weights - - def get_disc_logit_weights(self): - return torch.flatten(self.amp_linear.weight) - - def predict_amp_reward( - self, states, # 改为接收多帧状态列表 - task_reward, normalizer=None): - """ - states: torch.Tensor, shape=(num_envs, num_frames, state_dim) - task_reward: torch.Tensor, shape=(num_envs, 1) - """ - # import ipdb; ipdb.set_trace() - with torch.no_grad(): - self.eval() - if normalizer is not None: - # 对每一帧状态进行归一化 - states = normalizer.normalize_torch(states, self.device) - - # 拼接多帧状态 - state_cat = states.flatten(1) - d = self.amp_linear(self.trunk(state_cat)) - disc_reward = self.amp_reward_coef * torch.clamp(1 - (1/4) * torch.square(d - 1), min=0) - - if self.use_lerp: - if self.task_reward_lerp > 0: - reward = self._lerp_reward(disc_reward, task_reward.unsqueeze(-1)) - self.train() - return reward.squeeze(), d, disc_reward.squeeze() * (1.0 - self.task_reward_lerp) - else: - disc_reward *= 0.02 - reward = task_reward.unsqueeze(-1) + disc_reward - self.train() - return reward.squeeze(), d, disc_reward.squeeze() - - def _lerp_reward(self, disc_r, task_r): - r = (1.0 - self.task_reward_lerp) * disc_r + self.task_reward_lerp * task_r - return r \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py deleted file mode 100644 index 8e65c43..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn - -from rsl_rl.networks import MLP, EmpiricalDiscountedVariationNormalization, EmpiricalNormalization - - -class RandomNetworkDistillation(nn.Module): - """Implementation of Random Network Distillation (RND) [1] - - References: - .. [1] Burda, Yuri, et al. "Exploration by random network distillation." arXiv preprint arXiv:1810.12894 (2018). - """ - - def __init__( - self, - num_states: int, - obs_groups: dict, - num_outputs: int, - predictor_hidden_dims: list[int], - target_hidden_dims: list[int], - activation: str = "elu", - weight: float = 0.0, - state_normalization: bool = False, - reward_normalization: bool = False, - device: str = "cpu", - weight_schedule: dict | None = None, - ): - """Initialize the RND module. - - - If :attr:`state_normalization` is True, then the input state is normalized using an Empirical Normalization layer. - - If :attr:`reward_normalization` is True, then the intrinsic reward is normalized using an Empirical Discounted - Variation Normalization layer. - - .. note:: - If the hidden dimensions are -1 in the predictor and target networks configuration, then the number of states - is used as the hidden dimension. - - Args: - num_states: Number of states/inputs to the predictor and target networks. - num_outputs: Number of outputs (embedding size) of the predictor and target networks. - predictor_hidden_dims: List of hidden dimensions of the predictor network. - target_hidden_dims: List of hidden dimensions of the target network. - activation: Activation function. Defaults to "elu". - weight: Scaling factor of the intrinsic reward. Defaults to 0.0. - state_normalization: Whether to normalize the input state. Defaults to False. - reward_normalization: Whether to normalize the intrinsic reward. Defaults to False. - device: Device to use. Defaults to "cpu". - weight_schedule: The type of schedule to use for the RND weight parameter. - Defaults to None, in which case the weight parameter is constant. - It is a dictionary with the following keys: - - - "mode": The type of schedule to use for the RND weight parameter. - - "constant": Constant weight schedule. - - "step": Step weight schedule. - - "linear": Linear weight schedule. - - For the "step" weight schedule, the following parameters are required: - - - "final_step": The step at which the weight parameter is set to the final value. - - "final_value": The final value of the weight parameter. - - For the "linear" weight schedule, the following parameters are required: - - "initial_step": The step at which the weight parameter is set to the initial value. - - "final_step": The step at which the weight parameter is set to the final value. - - "final_value": The final value of the weight parameter. - """ - # initialize parent class - super().__init__() - - # Store parameters - self.num_states = num_states - self.obs_groups = obs_groups - self.num_outputs = num_outputs - self.initial_weight = weight - self.device = device - self.state_normalization = state_normalization - self.reward_normalization = reward_normalization - - # Normalization of input gates - if state_normalization: - self.state_normalizer = EmpiricalNormalization(shape=[self.num_states], until=1.0e8).to(self.device) - else: - self.state_normalizer = torch.nn.Identity() - # Normalization of intrinsic reward - if reward_normalization: - self.reward_normalizer = EmpiricalDiscountedVariationNormalization(shape=[], until=1.0e8).to(self.device) - else: - self.reward_normalizer = torch.nn.Identity() - - # counter for the number of updates - self.update_counter = 0 - - # resolve weight schedule - if weight_schedule is not None: - self.weight_scheduler_params = weight_schedule - self.weight_scheduler = getattr(self, f"_{weight_schedule['mode']}_weight_schedule") - else: - self.weight_scheduler = None - # Create network architecture - self.predictor = MLP(num_states, num_outputs, predictor_hidden_dims, activation).to(self.device) - self.target = MLP(num_states, num_outputs, target_hidden_dims, activation).to(self.device) - - # make target network not trainable - self.target.eval() - - def get_intrinsic_reward(self, obs) -> torch.Tensor: - # Note: the counter is updated number of env steps per learning iteration - self.update_counter += 1 - # Extract the rnd state from the observation - rnd_state = self.get_rnd_state(obs) - rnd_state = self.state_normalizer(rnd_state) - # Obtain the embedding of the rnd state from the target and predictor networks - target_embedding = self.target(rnd_state).detach() - predictor_embedding = self.predictor(rnd_state).detach() - # Compute the intrinsic reward as the distance between the embeddings - intrinsic_reward = torch.linalg.norm(target_embedding - predictor_embedding, dim=1) - # Normalize intrinsic reward - intrinsic_reward = self.reward_normalizer(intrinsic_reward) - - # Check the weight schedule - if self.weight_scheduler is not None: - self.weight = self.weight_scheduler(step=self.update_counter, **self.weight_scheduler_params) - else: - self.weight = self.initial_weight - # Scale intrinsic reward - intrinsic_reward *= self.weight - - return intrinsic_reward - - def forward(self, *args, **kwargs): - raise RuntimeError("Forward method is not implemented. Use get_intrinsic_reward instead.") - - def train(self, mode: bool = True): - # sets module into training mode - self.predictor.train(mode) - if self.state_normalization: - self.state_normalizer.train(mode) - if self.reward_normalization: - self.reward_normalizer.train(mode) - return self - - def eval(self): - return self.train(False) - - def get_rnd_state(self, obs): - obs_list = [] - for obs_group in self.obs_groups["rnd_state"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def update_normalization(self, obs): - # Normalize the state - if self.state_normalization: - rnd_state = self.get_rnd_state(obs) - self.state_normalizer.update(rnd_state) - - """ - Different weight schedules. - """ - - def _constant_weight_schedule(self, step: int, **kwargs): - return self.initial_weight - - def _step_weight_schedule(self, step: int, final_step: int, final_value: float, **kwargs): - return self.initial_weight if step < final_step else final_value - - def _linear_weight_schedule(self, step: int, initial_step: int, final_step: int, final_value: float, **kwargs): - if step < initial_step: - return self.initial_weight - elif step > final_step: - return final_value - else: - return self.initial_weight + (final_value - self.initial_weight) * (step - initial_step) / ( - final_step - initial_step - ) - - -def resolve_rnd_config(alg_cfg, obs, obs_groups, env): - """Resolve the RND configuration. - - Args: - alg_cfg: The algorithm configuration dictionary. - obs: The observation dictionary. - obs_groups: The observation groups dictionary. - env: The environment. - - Returns: - The resolved algorithm configuration dictionary. - """ - # resolve dimension of rnd gated state - if "rnd_cfg" in alg_cfg and alg_cfg["rnd_cfg"] is not None: - # get dimension of rnd gated state - num_rnd_state = 0 - for obs_group in obs_groups["rnd_state"]: - assert len(obs[obs_group].shape) == 2, "The RND module only supports 1D observations." - num_rnd_state += obs[obs_group].shape[-1] - # add rnd gated state to config - alg_cfg["rnd_cfg"]["num_states"] = num_rnd_state - alg_cfg["rnd_cfg"]["obs_groups"] = obs_groups - # scale down the rnd weight with timestep - alg_cfg["rnd_cfg"]["weight"] *= env.unwrapped.step_dt - return alg_cfg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py deleted file mode 100644 index 6bf1380..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization - - -class StudentTeacher(nn.Module): - is_recurrent = False - - def __init__( - self, - obs, - obs_groups, - num_actions, - student_obs_normalization=False, - teacher_obs_normalization=False, - student_hidden_dims=[256, 256, 256], - teacher_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=0.1, - noise_std_type: str = "scalar", - **kwargs, - ): - if kwargs: - print( - "StudentTeacher.__init__ got unexpected arguments, which will be ignored: " - + str([key for key in kwargs.keys()]) - ) - super().__init__() - - self.loaded_teacher = False # indicates if teacher has been loaded - - # get the observation dimensions - self.obs_groups = obs_groups - num_student_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_student_obs += obs[obs_group].shape[-1] - num_teacher_obs = 0 - for obs_group in obs_groups["teacher"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_teacher_obs += obs[obs_group].shape[-1] - - # student - self.student = MLP(num_student_obs, num_actions, student_hidden_dims, activation) - - # student observation normalization - self.student_obs_normalization = student_obs_normalization - if student_obs_normalization: - self.student_obs_normalizer = EmpiricalNormalization(num_student_obs) - else: - self.student_obs_normalizer = torch.nn.Identity() - - print(f"Student MLP: {self.student}") - - # teacher - self.teacher = MLP(num_teacher_obs, num_actions, teacher_hidden_dims, activation) - self.teacher.eval() - - # teacher observation normalization - self.teacher_obs_normalization = teacher_obs_normalization - if teacher_obs_normalization: - self.teacher_obs_normalizer = EmpiricalNormalization(num_teacher_obs) - else: - self.teacher_obs_normalizer = torch.nn.Identity() - - print(f"Teacher MLP: {self.teacher}") - - # action noise - self.noise_std_type = noise_std_type - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None, hidden_states=None): - pass - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - # compute mean - mean = self.student(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - self.update_distribution(obs) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - return self.student(obs) - - def evaluate(self, obs): - obs = self.get_teacher_obs(obs) - obs = self.teacher_obs_normalizer(obs) - with torch.no_grad(): - return self.teacher(obs) - - def get_student_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_teacher_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["teacher"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_hidden_states(self): - return None - - def detach_hidden_states(self, dones=None): - pass - - def train(self, mode=True): - super().train(mode) - # make sure teacher is in eval mode - self.teacher.eval() - self.teacher_obs_normalizer.eval() - - def update_normalization(self, obs): - if self.student_obs_normalization: - student_obs = self.get_student_obs(obs) - self.student_obs_normalizer.update(student_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the student and teacher networks. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters. - """ - - # check if state_dict contains teacher and student or just teacher parameters - if any("actor" in key for key in state_dict.keys()): # loading parameters from rl training - # rename keys to match teacher and remove critic parameters - teacher_state_dict = {} - teacher_obs_normalizer_state_dict = {} - for key, value in state_dict.items(): - if "actor." in key: - teacher_state_dict[key.replace("actor.", "")] = value - if "actor_obs_normalizer." in key: - teacher_obs_normalizer_state_dict[key.replace("actor_obs_normalizer.", "")] = value - self.teacher.load_state_dict(teacher_state_dict, strict=strict) - self.teacher_obs_normalizer.load_state_dict(teacher_obs_normalizer_state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return False # training does not resume - elif any("student" in key for key in state_dict.keys()): # loading parameters from distillation training - super().load_state_dict(state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return True # training resumes - else: - raise ValueError("state_dict does not contain student or teacher parameters") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py deleted file mode 100644 index 964a2dc..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import warnings -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization, Memory - - -class StudentTeacherRecurrent(nn.Module): - is_recurrent = True - - def __init__( - self, - obs, - obs_groups, - num_actions, - student_obs_normalization=False, - teacher_obs_normalization=False, - student_hidden_dims=[256, 256, 256], - teacher_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=0.1, - noise_std_type: str = "scalar", - rnn_type="lstm", - rnn_hidden_dim=256, - rnn_num_layers=1, - teacher_recurrent=False, - **kwargs, - ): - if "rnn_hidden_size" in kwargs: - warnings.warn( - "The argument `rnn_hidden_size` is deprecated and will be removed in a future version. " - "Please use `rnn_hidden_dim` instead.", - DeprecationWarning, - ) - if rnn_hidden_dim == 256: # Only override if the new argument is at its default - rnn_hidden_dim = kwargs.pop("rnn_hidden_size") - if kwargs: - print( - "StudentTeacherRecurrent.__init__ got unexpected arguments, which will be ignored: " - + str(kwargs.keys()), - ) - super().__init__() - - self.loaded_teacher = False # indicates if teacher has been loaded - self.teacher_recurrent = teacher_recurrent # indicates if teacher is recurrent too - - # get the observation dimensions - self.obs_groups = obs_groups - num_student_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_student_obs += obs[obs_group].shape[-1] - num_teacher_obs = 0 - for obs_group in obs_groups["teacher"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_teacher_obs += obs[obs_group].shape[-1] - - # student - self.memory_s = Memory(num_student_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - self.student = MLP(rnn_hidden_dim, num_actions, student_hidden_dims, activation) - - # student observation normalization - self.student_obs_normalization = student_obs_normalization - if student_obs_normalization: - self.student_obs_normalizer = EmpiricalNormalization(num_student_obs) - else: - self.student_obs_normalizer = torch.nn.Identity() - - print(f"Student RNN: {self.memory_s}") - print(f"Student MLP: {self.student}") - - # teacher - if self.teacher_recurrent: - self.memory_t = Memory( - num_teacher_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim - ) - num_teacher_obs = rnn_hidden_dim - self.teacher = MLP(num_teacher_obs, num_actions, teacher_hidden_dims, activation) - - # teacher observation normalization - self.teacher_obs_normalization = teacher_obs_normalization - if teacher_obs_normalization: - self.teacher_obs_normalizer = EmpiricalNormalization(num_teacher_obs) - else: - self.teacher_obs_normalizer = torch.nn.Identity() - - if self.teacher_recurrent: - print(f"Teacher RNN: {self.memory_t}") - print(f"Teacher MLP: {self.teacher}") - - # action noise - self.noise_std_type = noise_std_type - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None, hidden_states=None): - if hidden_states is None: - hidden_states = (None, None) - self.memory_s.reset(dones, hidden_states[0]) - if self.teacher_recurrent: - self.memory_t.reset(dones, hidden_states[1]) - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - # compute mean - mean = self.student(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - out_mem = self.memory_s(obs).squeeze(0) - self.update_distribution(out_mem) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - out_mem = self.memory_s(obs).squeeze(0) - return self.student(out_mem) - - def evaluate(self, obs): - obs = self.get_teacher_obs(obs) - obs = self.teacher_obs_normalizer(obs) - with torch.no_grad(): - if self.teacher_recurrent: - self.memory_t.eval() - obs = self.memory_t(obs).squeeze(0) - return self.teacher(obs) - - def get_student_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_teacher_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["teacher"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_hidden_states(self): - if self.teacher_recurrent: - return self.memory_s.hidden_states, self.memory_t.hidden_states - else: - return self.memory_s.hidden_states, None - - def detach_hidden_states(self, dones=None): - self.memory_s.detach_hidden_states(dones) - if self.teacher_recurrent: - self.memory_t.detach_hidden_states(dones) - - def train(self, mode=True): - super().train(mode) - # make sure teacher is in eval mode - self.teacher.eval() - self.teacher_obs_normalizer.eval() - - def update_normalization(self, obs): - if self.student_obs_normalization: - student_obs = self.get_student_obs(obs) - self.student_obs_normalizer.update(student_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the student and teacher networks. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters. - """ - - # check if state_dict contains teacher and student or just teacher parameters - if any("actor" in key for key in state_dict.keys()): # loading parameters from rl training - # rename keys to match teacher and remove critic parameters - teacher_state_dict = {} - teacher_obs_normalizer_state_dict = {} - for key, value in state_dict.items(): - if "actor." in key: - teacher_state_dict[key.replace("actor.", "")] = value - if "actor_obs_normalizer." in key: - teacher_obs_normalizer_state_dict[key.replace("actor_obs_normalizer.", "")] = value - self.teacher.load_state_dict(teacher_state_dict, strict=strict) - self.teacher_obs_normalizer.load_state_dict(teacher_obs_normalizer_state_dict, strict=strict) - # also load recurrent memory if teacher is recurrent - if self.teacher_recurrent: - memory_t_state_dict = {} - for key, value in state_dict.items(): - if "memory_a." in key: - memory_t_state_dict[key.replace("memory_a.", "")] = value - self.memory_t.load_state_dict(memory_t_state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return False # training does not resume - elif any("student" in key for key in state_dict.keys()): # loading parameters from distillation training - super().load_state_dict(state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return True # training resumes - else: - raise ValueError("state_dict does not contain student or teacher parameters") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py deleted file mode 100644 index b017515..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - - -def resolve_symmetry_config(alg_cfg, env): - """Resolve the symmetry configuration. - - Args: - alg_cfg: The algorithm configuration dictionary. - env: The environment. - - Returns: - The resolved algorithm configuration dictionary. - """ - - # if using symmetry then pass the environment config object - if "symmetry_cfg" in alg_cfg and alg_cfg["symmetry_cfg"] is not None: - # this is used by the symmetry function for handling different observation terms - alg_cfg["symmetry_cfg"]["_env"] = env - return alg_cfg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py deleted file mode 100644 index c18f487..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Definitions for components of modules.""" - -from .memory import Memory -from .mlp import MLP -from .normalization import EmpiricalDiscountedVariationNormalization, EmpiricalNormalization diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py deleted file mode 100644 index 7577357..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch.nn as nn - -from rsl_rl.utils import unpad_trajectories - - -class Memory(nn.Module): - """Memory module for recurrent networks. - - This module is used to store the hidden states of the policy. - Currently only supports GRU and LSTM. - """ - - def __init__(self, input_size, type="lstm", num_layers=1, hidden_size=256): - super().__init__() - # RNN - rnn_cls = nn.GRU if type.lower() == "gru" else nn.LSTM - self.rnn = rnn_cls(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers) - self.hidden_states = None - - def forward(self, input, masks=None, hidden_states=None): - batch_mode = masks is not None - if batch_mode: - # batch mode: needs saved hidden states - if hidden_states is None: - raise ValueError("Hidden states not passed to memory module during policy update") - out, _ = self.rnn(input, hidden_states) - out = unpad_trajectories(out, masks) - else: - # inference/distillation mode: uses hidden states of last step - out, self.hidden_states = self.rnn(input.unsqueeze(0), self.hidden_states) - return out - - def reset(self, dones=None, hidden_states=None): - if dones is None: # reset all hidden states - if hidden_states is None: - self.hidden_states = None - else: - self.hidden_states = hidden_states - elif self.hidden_states is not None: # reset hidden states of done environments - if hidden_states is None: - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - for hidden_state in self.hidden_states: - hidden_state[..., dones == 1, :] = 0.0 - else: - self.hidden_states[..., dones == 1, :] = 0.0 - else: - NotImplementedError( - "Resetting hidden states of done environments with custom hidden states is not implemented" - ) - - def detach_hidden_states(self, dones=None): - if self.hidden_states is not None: - if dones is None: # detach all hidden states - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - self.hidden_states = tuple(hidden_state.detach() for hidden_state in self.hidden_states) - else: - self.hidden_states = self.hidden_states.detach() - else: # detach hidden states of done environments - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - for hidden_state in self.hidden_states: - hidden_state[..., dones == 1, :] = hidden_state[..., dones == 1, :].detach() - else: - self.hidden_states[..., dones == 1, :] = self.hidden_states[..., dones == 1, :].detach() diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py deleted file mode 100644 index e91574e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from functools import reduce - -from rsl_rl.utils import resolve_nn_activation - - -class MLP(nn.Sequential): - """Multi-layer perceptron. - - The MLP network is a sequence of linear layers and activation functions. The - last layer is a linear layer that outputs the desired dimension unless the - last activation function is specified. - - It provides additional conveniences: - - - If the hidden dimensions have a value of ``-1``, the dimension is inferred - from the input dimension. - - If the output dimension is a tuple, the output is reshaped to the desired - shape. - - """ - - def __init__( - self, - input_dim: int, - output_dim: int | tuple[int] | list[int], - hidden_dims: tuple[int] | list[int], - activation: str = "elu", - last_activation: str | None = None, - ): - """Initialize the MLP. - - Args: - input_dim: Dimension of the input. - output_dim: Dimension of the output. - hidden_dims: Dimensions of the hidden layers. A value of ``-1`` indicates - that the dimension should be inferred from the input dimension. - activation: Activation function. Defaults to "elu". - last_activation: Activation function of the last layer. Defaults to None, - in which case the last layer is linear. - """ - super().__init__() - - # resolve activation functions - activation_mod = resolve_nn_activation(activation) - last_activation_mod = resolve_nn_activation(last_activation) if last_activation is not None else None - # resolve number of hidden dims if they are -1 - hidden_dims_processed = [input_dim if dim == -1 else dim for dim in hidden_dims] - - # create layers sequentially - layers = [] - layers.append(nn.Linear(input_dim, hidden_dims_processed[0])) - layers.append(activation_mod) - - for layer_index in range(len(hidden_dims_processed) - 1): - layers.append(nn.Linear(hidden_dims_processed[layer_index], hidden_dims_processed[layer_index + 1])) - layers.append(activation_mod) - - # add last layer - if isinstance(output_dim, int): - layers.append(nn.Linear(hidden_dims_processed[-1], output_dim)) - else: - # compute the total output dimension - total_out_dim = reduce(lambda x, y: x * y, output_dim) - # add a layer to reshape the output to the desired shape - layers.append(nn.Linear(hidden_dims_processed[-1], total_out_dim)) - layers.append(nn.Unflatten(dim=-1, unflattened_size=output_dim)) - - # add last activation function if specified - if last_activation_mod is not None: - layers.append(last_activation_mod) - - # register the layers - for idx, layer in enumerate(layers): - self.add_module(f"{idx}", layer) - - def init_weights(self, scales: float | tuple[float]): - """Initialize the weights of the MLP. - - Args: - scales: Scale factor for the weights. - """ - - def get_scale(idx) -> float: - """Get the scale factor for the weights of the MLP. - - Args: - idx: Index of the layer. - """ - return scales[idx] if isinstance(scales, (list, tuple)) else scales - - # initialize the weights - for idx, module in enumerate(self): - if isinstance(module, nn.Linear): - nn.init.orthogonal_(module.weight, gain=get_scale(idx)) - nn.init.zeros_(module.bias) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Forward pass of the MLP. - - Args: - x: Input tensor. - """ - for layer in self: - x = layer(x) - return x - - def reset(self, dones=None, hidden_states=None): - pass - - def detach_hidden_states(self, dones=None): - pass diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py deleted file mode 100644 index 5fd9692..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -# Copyright (c) 2020 Preferred Networks, Inc. - -from __future__ import annotations - -import torch -from torch import nn - - -class EmpiricalNormalization(nn.Module): - """Normalize mean and variance of values based on empirical values.""" - - def __init__(self, shape, eps=1e-2, until=None): - """Initialize EmpiricalNormalization module. - - Args: - shape (int or tuple of int): Shape of input values except batch axis. - eps (float): Small value for stability. - until (int or None): If this arg is specified, the module learns input values until the sum of batch sizes - exceeds it. - - Note: The normalization parameters are computed over the whole batch, not for each environment separately. - """ - super().__init__() - self.eps = eps - self.until = until - self.register_buffer("_mean", torch.zeros(shape).unsqueeze(0)) - self.register_buffer("_var", torch.ones(shape).unsqueeze(0)) - self.register_buffer("_std", torch.ones(shape).unsqueeze(0)) - self.register_buffer("count", torch.tensor(0, dtype=torch.long)) - - @property - def mean(self): - return self._mean.squeeze(0).clone() - - @property - def std(self): - return self._std.squeeze(0).clone() - - def forward(self, x): - """Normalize mean and variance of values based on empirical values.""" - - return (x - self._mean) / (self._std + self.eps) - - @torch.jit.unused - def update(self, x): - """Learn input values without computing the output values of them""" - - if not self.training: - return - if self.until is not None and self.count >= self.until: - return - - count_x = x.shape[0] - self.count += count_x - rate = count_x / self.count - var_x = torch.var(x, dim=0, unbiased=False, keepdim=True) - mean_x = torch.mean(x, dim=0, keepdim=True) - delta_mean = mean_x - self._mean - self._mean += rate * delta_mean - self._var += rate * (var_x - self._var + delta_mean * (mean_x - self._mean)) - self._std = torch.sqrt(self._var) - - @torch.jit.unused - def inverse(self, y): - """De-normalize values based on empirical values.""" - - return y * (self._std + self.eps) + self._mean - - -class EmpiricalDiscountedVariationNormalization(nn.Module): - """Reward normalization from Pathak's large scale study on PPO. - - Reward normalization. Since the reward function is non-stationary, it is useful to normalize - the scale of the rewards so that the value function can learn quickly. We did this by dividing - the rewards by a running estimate of the standard deviation of the sum of discounted rewards. - """ - - def __init__(self, shape, eps=1e-2, gamma=0.99, until=None): - super().__init__() - - self.emp_norm = EmpiricalNormalization(shape, eps, until) - self.disc_avg = _DiscountedAverage(gamma) - - def forward(self, rew): - if self.training: - # update discounted rewards - avg = self.disc_avg.update(rew) - # update moments from discounted rewards - self.emp_norm.update(avg) - - # normalize rewards with the empirical std - if self.emp_norm._std > 0: - return rew / self.emp_norm._std - else: - return rew - - -""" -Helper class. -""" - - -class _DiscountedAverage: - r"""Discounted average of rewards. - - The discounted average is defined as: - - .. math:: - - \bar{R}_t = \gamma \bar{R}_{t-1} + r_t - - Args: - gamma (float): Discount factor. - """ - - def __init__(self, gamma): - self.avg = None - self.gamma = gamma - - def update(self, rew: torch.Tensor) -> torch.Tensor: - if self.avg is None: - self.avg = rew - else: - self.avg = self.avg * self.gamma + rew - return self.avg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py deleted file mode 100644 index 61f1682..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of runners for environment-agent interaction.""" - -from .on_policy_runner import OnPolicyRunner # isort:skip -from .distillation_runner import DistillationRunner -from .amp_on_policy_runner import AMPOnPolicyRunner - -__all__ = ["OnPolicyRunner", "DistillationRunner", "AMPOnPolicyRunner"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py deleted file mode 100644 index c0b9b9e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py +++ /dev/null @@ -1,521 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import statistics -import time -import torch -import warnings -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import AMP_PPO -from rsl_rl.env import VecEnv -from rsl_rl.modules import ActorCritic, ActorCriticRecurrent,DiscriminatorMulti, resolve_rnd_config, resolve_symmetry_config -from rsl_rl.utils import resolve_obs_groups, store_code_state, Normalizer, G1_AMPLoader - - -class AMPOnPolicyRunner: - """On-policy runner for training and evaluation of actor-critic methods.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - default_sets = ["critic"] - if "rnd_cfg" in self.alg_cfg and self.alg_cfg["rnd_cfg"] is not None: - default_sets.append("rnd_state") - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets) - - self.amp_data = G1_AMPLoader( - device, - time_between_frames=1/50.0, - preload_transitions=True, - num_preload_transitions=train_cfg["amp_num_preload_transitions"], - motion_files=train_cfg["amp_motion_files"], - num_frames=train_cfg['amp_num_frames'] - ) - - self.amp_observation_dim = self.amp_data.observation_dim if self.cfg["amp_num_obs"] == 0 else self.cfg["amp_num_obs"] - self.amp_num_frames = 0 if self.cfg["amp_num_frames"] == 0 else self.cfg["amp_num_frames"] - self.amp_normalizer = Normalizer(self.amp_observation_dim) - self.discriminator = DiscriminatorMulti( - self.amp_observation_dim, - train_cfg["amp_reward_coef"], - train_cfg["amp_discr_hidden_dims"], - device, - train_cfg["amp_num_frames"], - train_cfg["amp_task_reward_lerp"], - train_cfg['use_lerp'], - ).to(self.device) - - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - amp_obs = self.env.get_amp_observations() - amp_obs = amp_obs.to(self.device) - if self.amp_num_frames != 0: - self.amp_obs_frames = torch.zeros(size=(self.env.num_envs, self.amp_num_frames, self.amp_observation_dim), device=self.device) - self.amp_obs_frames = torch.concat((self.amp_obs_frames[:, 1:], amp_obs.unsqueeze(1)), dim=1) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - step_discrewbuffer = deque(maxlen=100) - - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_single_step_disc_rew = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - # create buffers for logging extrinsic and intrinsic rewards - if self.alg.rnd: - erewbuffer = deque(maxlen=100) - irewbuffer = deque(maxlen=100) - cur_ereward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_ireward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs,amp_obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - - next_amp_obs = self.env.get_amp_observations() - next_amp_obs = next_amp_obs.to(self.device) - next_amp_obs_with_term = torch.clone(next_amp_obs) - - reset_env_ids = self.env.reset_env_ids - terminal_amp_states = self.env.get_amp_observations()[reset_env_ids] - next_amp_obs_with_term[reset_env_ids] = terminal_amp_states - self.amp_obs_frames = torch.concat((self.amp_obs_frames[:, 1:], next_amp_obs_with_term.unsqueeze(1)), dim=1) - - amp_reward = torch.zeros(self.env.num_envs, device=obs.device) - - mask = self.env.contact_phase[:, 0] == 1.0 - if mask.any(): - rewards[mask], logit, disc_reward = self.alg.discriminator.predict_amp_reward( - self.amp_obs_frames[mask], rewards[mask], normalizer=self.alg.amp_normalizer - ) - amp_reward[mask] += disc_reward - - # process the step - self.alg.process_env_step(obs, rewards, dones, extras, next_amp_obs_with_term, self.amp_obs_frames) - self.amp_obs_frames[reset_env_ids] = 0 - - amp_obs = torch.clone(next_amp_obs) - # Extract intrinsic rewards (only for logging) - intrinsic_rewards = self.alg.intrinsic_rewards if self.alg.rnd else None - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - if self.alg.rnd: - cur_ereward_sum += rewards - cur_ireward_sum += intrinsic_rewards # type: ignore - cur_reward_sum += rewards + intrinsic_rewards - else: - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - cur_single_step_disc_rew += amp_reward - # Clear data for completed episodes - # -- common - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - to_extend_disc = (cur_single_step_disc_rew[new_ids] / self.env.max_episode_length_s)[:, 0].cpu().numpy() - step_discrewbuffer.extend(to_extend_disc.tolist()) - cur_single_step_disc_rew[new_ids] = 0 - # -- intrinsic and extrinsic rewards - if self.alg.rnd: - erewbuffer.extend(cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist()) - irewbuffer.extend(cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist()) - cur_ereward_sum[new_ids] = 0 - cur_ireward_sum[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # compute returns - self.alg.compute_returns(obs) - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - def log(self, locs: dict, width: int = 80, pad: int = 35): - # Compute the collection size - collection_size = self.num_steps_per_env * self.env.num_envs * self.gpu_world_size - # Update total time-steps and time - self.tot_timesteps += collection_size - self.tot_time += locs["collection_time"] + locs["learn_time"] - iteration_time = locs["collection_time"] + locs["learn_time"] - - # -- Episode info - ep_string = "" - if locs["ep_infos"]: - for key in locs["ep_infos"][0]: - infotensor = torch.tensor([], device=self.device) - for ep_info in locs["ep_infos"]: - # handle scalar and zero dimensional tensor infos - if key not in ep_info: - continue - if not isinstance(ep_info[key], torch.Tensor): - ep_info[key] = torch.Tensor([ep_info[key]]) - if len(ep_info[key].shape) == 0: - ep_info[key] = ep_info[key].unsqueeze(0) - infotensor = torch.cat((infotensor, ep_info[key].to(self.device))) - value = torch.mean(infotensor) - # log to logger and terminal - if "/" in key: - self.writer.add_scalar(key, value, locs["it"]) - ep_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - else: - self.writer.add_scalar("Episode/" + key, value, locs["it"]) - ep_string += f"""{f'Mean episode {key}:':>{pad}} {value:.4f}\n""" - - mean_std = self.alg.policy.action_std.mean() - fps = int(collection_size / (locs["collection_time"] + locs["learn_time"])) - - # -- Losses - for key, value in locs["loss_dict"].items(): - self.writer.add_scalar(f"Loss/{key}", value, locs["it"]) - self.writer.add_scalar("Loss/learning_rate", self.alg.learning_rate, locs["it"]) - - # -- Policy - self.writer.add_scalar("Policy/mean_noise_std", mean_std.item(), locs["it"]) - - # -- Performance - self.writer.add_scalar("Perf/total_fps", fps, locs["it"]) - self.writer.add_scalar("Perf/collection time", locs["collection_time"], locs["it"]) - self.writer.add_scalar("Perf/learning_time", locs["learn_time"], locs["it"]) - - # -- Training - if len(locs["rewbuffer"]) > 0: - # separate logging for intrinsic and extrinsic rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(locs["erewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(locs["irewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/weight", self.alg.rnd.weight, locs["it"]) - # everything else - self.writer.add_scalar("Train/mean_reward", statistics.mean(locs["rewbuffer"]), locs["it"]) - self.writer.add_scalar("Train/mean_episode_length", statistics.mean(locs["lenbuffer"]), locs["it"]) - self.writer.add_scalar('Train/mean_step_disc_reward', statistics.mean(locs['step_discrewbuffer']), locs['it']) - if self.logger_type != "wandb": # wandb does not support non-integer x-axis logging - self.writer.add_scalar("Train/mean_reward/time", statistics.mean(locs["rewbuffer"]), self.tot_time) - self.writer.add_scalar( - "Train/mean_episode_length/time", statistics.mean(locs["lenbuffer"]), self.tot_time - ) - - str = f" \033[1m Learning iteration {locs['it']}/{locs['tot_iter']} \033[0m " - - if len(locs["rewbuffer"]) > 0: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - f"""{'Step disc reward:':>{pad}} {statistics.mean(locs['step_discrewbuffer']):.2f}\n""" - ) - # -- Losses - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'Mean {key} loss:':>{pad}} {value:.4f}\n""" - # -- Rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - log_string += ( - f"""{'Mean extrinsic reward:':>{pad}} {statistics.mean(locs['erewbuffer']):.2f}\n""" - f"""{'Mean intrinsic reward:':>{pad}} {statistics.mean(locs['irewbuffer']):.2f}\n""" - ) - log_string += f"""{'Mean reward:':>{pad}} {statistics.mean(locs['rewbuffer']):.2f}\n""" - # -- episode info - log_string += f"""{'Mean episode length:':>{pad}} {statistics.mean(locs['lenbuffer']):.2f}\n""" - else: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - - log_string += ep_string - log_string += ( - f"""{'-' * width}\n""" - f"""{'Total timesteps:':>{pad}} {self.tot_timesteps}\n""" - f"""{'Iteration time:':>{pad}} {iteration_time:.2f}s\n""" - f"""{'Time elapsed:':>{pad}} {time.strftime("%H:%M:%S", time.gmtime(self.tot_time))}\n""" - f"""{'ETA:':>{pad}} {time.strftime( - "%H:%M:%S", - time.gmtime( - self.tot_time / (locs['it'] - locs['start_iter'] + 1) - * (locs['start_iter'] + locs['num_learning_iterations'] - locs['it']) - ) - )}\n""" - ) - print(log_string) - - def save(self, path: str, infos=None): - # -- Save model - saved_dict = { - "model_state_dict": self.alg.policy.state_dict(), - "optimizer_state_dict": self.alg.optimizer.state_dict(), - "iter": self.current_learning_iteration, - "infos": infos, - } - # -- Save RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - saved_dict["rnd_state_dict"] = self.alg.rnd.state_dict() - saved_dict["rnd_optimizer_state_dict"] = self.alg.rnd_optimizer.state_dict() - torch.save(saved_dict, path) - - # upload model to external logging service - if self.logger_type in ["neptune", "wandb"] and not self.disable_logs: - self.writer.save_model(path, self.current_learning_iteration) - - def load(self, path: str, load_optimizer: bool = True, map_location: str | None = None): - loaded_dict = torch.load(path, weights_only=False, map_location=map_location) - # -- Load model - resumed_training = self.alg.policy.load_state_dict(loaded_dict["model_state_dict"]) - # -- Load RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.load_state_dict(loaded_dict["rnd_state_dict"]) - # -- load optimizer if used - if load_optimizer and resumed_training: - # -- algorithm optimizer - self.alg.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"]) - # -- RND optimizer if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd_optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"]) - # -- load current learning iteration - if resumed_training: - self.current_learning_iteration = loaded_dict["iter"] - return loaded_dict["infos"] - - def get_inference_policy(self, device=None): - self.eval_mode() # switch to evaluation mode (dropout for example) - if device is not None: - self.alg.policy.to(device) - return self.alg.policy.act_inference - - def train_mode(self): - # -- PPO - self.alg.policy.train() - self.alg.discriminator.train() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.train() - - def eval_mode(self): - # -- PPO - self.alg.policy.eval() - self.alg.discriminator.eval() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.eval() - - def add_git_repo_to_log(self, repo_file_path): - self.git_status_repos.append(repo_file_path) - - """ - Helper functions. - """ - - def _configure_multi_gpu(self): - """Configure multi-gpu training.""" - # check if distributed training is enabled - self.gpu_world_size = int(os.getenv("WORLD_SIZE", "1")) - self.is_distributed = self.gpu_world_size > 1 - - # if not distributed training, set local and global rank to 0 and return - if not self.is_distributed: - self.gpu_local_rank = 0 - self.gpu_global_rank = 0 - self.multi_gpu_cfg = None - return - - # get rank and world size - self.gpu_local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.gpu_global_rank = int(os.getenv("RANK", "0")) - - # make a configuration dictionary - self.multi_gpu_cfg = { - "global_rank": self.gpu_global_rank, # rank of the main process - "local_rank": self.gpu_local_rank, # rank of the current process - "world_size": self.gpu_world_size, # total number of processes - } - - # check if user has device specified for local rank - if self.device != f"cuda:{self.gpu_local_rank}": - raise ValueError( - f"Device '{self.device}' does not match expected device for local rank '{self.gpu_local_rank}'." - ) - # validate multi-gpu configuration - if self.gpu_local_rank >= self.gpu_world_size: - raise ValueError( - f"Local rank '{self.gpu_local_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - if self.gpu_global_rank >= self.gpu_world_size: - raise ValueError( - f"Global rank '{self.gpu_global_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - - # initialize torch distributed - torch.distributed.init_process_group(backend="nccl", rank=self.gpu_global_rank, world_size=self.gpu_world_size) - # set device to the local rank - torch.cuda.set_device(self.gpu_local_rank) - - def _construct_algorithm(self, obs) -> AMP_PPO: - """Construct the actor-critic algorithm.""" - # resolve RND config - self.alg_cfg = resolve_rnd_config(self.alg_cfg, obs, self.cfg["obs_groups"], self.env) - - # resolve symmetry config - self.alg_cfg = resolve_symmetry_config(self.alg_cfg, self.env) - - # resolve deprecated normalization config - if self.cfg.get("empirical_normalization") is not None: - warnings.warn( - "The `empirical_normalization` parameter is deprecated. Please set `actor_obs_normalization` and " - "`critic_obs_normalization` as part of the `policy` configuration instead.", - DeprecationWarning, - ) - if self.policy_cfg.get("actor_obs_normalization") is None: - self.policy_cfg["actor_obs_normalization"] = self.cfg["empirical_normalization"] - if self.policy_cfg.get("critic_obs_normalization") is None: - self.policy_cfg["critic_obs_normalization"] = self.cfg["empirical_normalization"] - - # initialize the actor-critic - actor_critic_class = eval(self.policy_cfg.pop("class_name")) - actor_critic: ActorCritic | ActorCriticRecurrent = actor_critic_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - - alg: AMP_PPO = alg_class(actor_critic, self.discriminator, self.amp_data, self.amp_normalizer, self.amp_num_frames, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg) - - # initialize the storage - alg.init_storage( - "rl", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg - - def _prepare_logging_writer(self): - """Prepares the logging writers.""" - if self.log_dir is not None and self.writer is None and not self.disable_logs: - # Launch either Tensorboard or Neptune & Tensorboard summary writer(s), default: Tensorboard. - self.logger_type = self.cfg.get("logger", "tensorboard") - self.logger_type = self.logger_type.lower() - - if self.logger_type == "neptune": - from rsl_rl.utils.neptune_utils import NeptuneSummaryWriter - - self.writer = NeptuneSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "wandb": - from rsl_rl.utils.wandb_utils import WandbSummaryWriter - - self.writer = WandbSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "tensorboard": - from torch.utils.tensorboard import SummaryWriter - - self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) - else: - raise ValueError("Logger type not found. Please choose 'neptune', 'wandb' or 'tensorboard'.") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py deleted file mode 100644 index 9cc6a8b..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import time -import torch -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import Distillation -from rsl_rl.env import VecEnv -from rsl_rl.modules import StudentTeacher, StudentTeacherRecurrent -from rsl_rl.runners import OnPolicyRunner -from rsl_rl.utils import resolve_obs_groups, store_code_state - - -class DistillationRunner(OnPolicyRunner): - """On-policy runner for training and evaluation of teacher-student training.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets=["teacher"]) - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - # check if teacher is loaded - if not self.alg.policy.loaded_teacher: - raise ValueError("Teacher model parameters not loaded. Please load a teacher model to distill.") - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - # process the step - self.alg.process_env_step(obs, rewards, dones, extras) - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - # Clear data for completed episodes - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - """ - Helper methods. - """ - - def _construct_algorithm(self, obs) -> Distillation: - """Construct the distillation algorithm.""" - # initialize the actor-critic - student_teacher_class = eval(self.policy_cfg.pop("class_name")) - student_teacher: StudentTeacher | StudentTeacherRecurrent = student_teacher_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - alg: Distillation = alg_class( - student_teacher, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg - ) - - # initialize the storage - alg.init_storage( - "distillation", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py deleted file mode 100644 index 36f11f3..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py +++ /dev/null @@ -1,460 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import statistics -import time -import torch -import warnings -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import PPO -from rsl_rl.env import VecEnv -from rsl_rl.modules import ActorCritic, ActorCriticRecurrent, resolve_rnd_config, resolve_symmetry_config -from rsl_rl.utils import resolve_obs_groups, store_code_state - - -class OnPolicyRunner: - """On-policy runner for training and evaluation of actor-critic methods.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - default_sets = ["critic"] - if "rnd_cfg" in self.alg_cfg and self.alg_cfg["rnd_cfg"] is not None: - default_sets.append("rnd_state") - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets) - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # create buffers for logging extrinsic and intrinsic rewards - if self.alg.rnd: - erewbuffer = deque(maxlen=100) - irewbuffer = deque(maxlen=100) - cur_ereward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_ireward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - # process the step - self.alg.process_env_step(obs, rewards, dones, extras) - # Extract intrinsic rewards (only for logging) - intrinsic_rewards = self.alg.intrinsic_rewards if self.alg.rnd else None - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - if self.alg.rnd: - cur_ereward_sum += rewards - cur_ireward_sum += intrinsic_rewards # type: ignore - cur_reward_sum += rewards + intrinsic_rewards - else: - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - # Clear data for completed episodes - # -- common - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - # -- intrinsic and extrinsic rewards - if self.alg.rnd: - erewbuffer.extend(cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist()) - irewbuffer.extend(cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist()) - cur_ereward_sum[new_ids] = 0 - cur_ireward_sum[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # compute returns - self.alg.compute_returns(obs) - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - def log(self, locs: dict, width: int = 80, pad: int = 35): - # Compute the collection size - collection_size = self.num_steps_per_env * self.env.num_envs * self.gpu_world_size - # Update total time-steps and time - self.tot_timesteps += collection_size - self.tot_time += locs["collection_time"] + locs["learn_time"] - iteration_time = locs["collection_time"] + locs["learn_time"] - - # -- Episode info - ep_string = "" - if locs["ep_infos"]: - for key in locs["ep_infos"][0]: - infotensor = torch.tensor([], device=self.device) - for ep_info in locs["ep_infos"]: - # handle scalar and zero dimensional tensor infos - if key not in ep_info: - continue - if not isinstance(ep_info[key], torch.Tensor): - ep_info[key] = torch.Tensor([ep_info[key]]) - if len(ep_info[key].shape) == 0: - ep_info[key] = ep_info[key].unsqueeze(0) - infotensor = torch.cat((infotensor, ep_info[key].to(self.device))) - value = torch.mean(infotensor) - # log to logger and terminal - if "/" in key: - self.writer.add_scalar(key, value, locs["it"]) - ep_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - else: - self.writer.add_scalar("Episode/" + key, value, locs["it"]) - ep_string += f"""{f'Mean episode {key}:':>{pad}} {value:.4f}\n""" - - mean_std = self.alg.policy.action_std.mean() - fps = int(collection_size / (locs["collection_time"] + locs["learn_time"])) - - # -- Losses - for key, value in locs["loss_dict"].items(): - self.writer.add_scalar(f"Loss/{key}", value, locs["it"]) - self.writer.add_scalar("Loss/learning_rate", self.alg.learning_rate, locs["it"]) - - # -- Policy - self.writer.add_scalar("Policy/mean_noise_std", mean_std.item(), locs["it"]) - - # -- Performance - self.writer.add_scalar("Perf/total_fps", fps, locs["it"]) - self.writer.add_scalar("Perf/collection time", locs["collection_time"], locs["it"]) - self.writer.add_scalar("Perf/learning_time", locs["learn_time"], locs["it"]) - - # -- Training - if len(locs["rewbuffer"]) > 0: - # separate logging for intrinsic and extrinsic rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(locs["erewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(locs["irewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/weight", self.alg.rnd.weight, locs["it"]) - # everything else - self.writer.add_scalar("Train/mean_reward", statistics.mean(locs["rewbuffer"]), locs["it"]) - self.writer.add_scalar("Train/mean_episode_length", statistics.mean(locs["lenbuffer"]), locs["it"]) - if self.logger_type != "wandb": # wandb does not support non-integer x-axis logging - self.writer.add_scalar("Train/mean_reward/time", statistics.mean(locs["rewbuffer"]), self.tot_time) - self.writer.add_scalar( - "Train/mean_episode_length/time", statistics.mean(locs["lenbuffer"]), self.tot_time - ) - - str = f" \033[1m Learning iteration {locs['it']}/{locs['tot_iter']} \033[0m " - - if len(locs["rewbuffer"]) > 0: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - # -- Losses - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'Mean {key} loss:':>{pad}} {value:.4f}\n""" - # -- Rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - log_string += ( - f"""{'Mean extrinsic reward:':>{pad}} {statistics.mean(locs['erewbuffer']):.2f}\n""" - f"""{'Mean intrinsic reward:':>{pad}} {statistics.mean(locs['irewbuffer']):.2f}\n""" - ) - log_string += f"""{'Mean reward:':>{pad}} {statistics.mean(locs['rewbuffer']):.2f}\n""" - # -- episode info - log_string += f"""{'Mean episode length:':>{pad}} {statistics.mean(locs['lenbuffer']):.2f}\n""" - else: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - - log_string += ep_string - log_string += ( - f"""{'-' * width}\n""" - f"""{'Total timesteps:':>{pad}} {self.tot_timesteps}\n""" - f"""{'Iteration time:':>{pad}} {iteration_time:.2f}s\n""" - f"""{'Time elapsed:':>{pad}} {time.strftime("%H:%M:%S", time.gmtime(self.tot_time))}\n""" - f"""{'ETA:':>{pad}} {time.strftime( - "%H:%M:%S", - time.gmtime( - self.tot_time / (locs['it'] - locs['start_iter'] + 1) - * (locs['start_iter'] + locs['num_learning_iterations'] - locs['it']) - ) - )}\n""" - ) - print(log_string) - - def save(self, path: str, infos=None): - # -- Save model - saved_dict = { - "model_state_dict": self.alg.policy.state_dict(), - "optimizer_state_dict": self.alg.optimizer.state_dict(), - "iter": self.current_learning_iteration, - "infos": infos, - } - # -- Save RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - saved_dict["rnd_state_dict"] = self.alg.rnd.state_dict() - saved_dict["rnd_optimizer_state_dict"] = self.alg.rnd_optimizer.state_dict() - torch.save(saved_dict, path) - - # upload model to external logging service - if self.logger_type in ["neptune", "wandb"] and not self.disable_logs: - self.writer.save_model(path, self.current_learning_iteration) - - def load(self, path: str, load_optimizer: bool = True, map_location: str | None = None): - loaded_dict = torch.load(path, weights_only=False, map_location=map_location) - # -- Load model - resumed_training = self.alg.policy.load_state_dict(loaded_dict["model_state_dict"]) - # -- Load RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.load_state_dict(loaded_dict["rnd_state_dict"]) - # -- load optimizer if used - if load_optimizer and resumed_training: - # -- algorithm optimizer - self.alg.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"]) - # -- RND optimizer if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd_optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"]) - # -- load current learning iteration - if resumed_training: - self.current_learning_iteration = loaded_dict["iter"] - return loaded_dict["infos"] - - def get_inference_policy(self, device=None): - self.eval_mode() # switch to evaluation mode (dropout for example) - if device is not None: - self.alg.policy.to(device) - return self.alg.policy.act_inference - - def train_mode(self): - # -- PPO - self.alg.policy.train() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.train() - - def eval_mode(self): - # -- PPO - self.alg.policy.eval() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.eval() - - def add_git_repo_to_log(self, repo_file_path): - self.git_status_repos.append(repo_file_path) - - """ - Helper functions. - """ - - def _configure_multi_gpu(self): - """Configure multi-gpu training.""" - # check if distributed training is enabled - self.gpu_world_size = int(os.getenv("WORLD_SIZE", "1")) - self.is_distributed = self.gpu_world_size > 1 - - # if not distributed training, set local and global rank to 0 and return - if not self.is_distributed: - self.gpu_local_rank = 0 - self.gpu_global_rank = 0 - self.multi_gpu_cfg = None - return - - # get rank and world size - self.gpu_local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.gpu_global_rank = int(os.getenv("RANK", "0")) - - # make a configuration dictionary - self.multi_gpu_cfg = { - "global_rank": self.gpu_global_rank, # rank of the main process - "local_rank": self.gpu_local_rank, # rank of the current process - "world_size": self.gpu_world_size, # total number of processes - } - - # check if user has device specified for local rank - if self.device != f"cuda:{self.gpu_local_rank}": - raise ValueError( - f"Device '{self.device}' does not match expected device for local rank '{self.gpu_local_rank}'." - ) - # validate multi-gpu configuration - if self.gpu_local_rank >= self.gpu_world_size: - raise ValueError( - f"Local rank '{self.gpu_local_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - if self.gpu_global_rank >= self.gpu_world_size: - raise ValueError( - f"Global rank '{self.gpu_global_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - - # initialize torch distributed - torch.distributed.init_process_group(backend="nccl", rank=self.gpu_global_rank, world_size=self.gpu_world_size) - # set device to the local rank - torch.cuda.set_device(self.gpu_local_rank) - - def _construct_algorithm(self, obs) -> PPO: - """Construct the actor-critic algorithm.""" - # resolve RND config - self.alg_cfg = resolve_rnd_config(self.alg_cfg, obs, self.cfg["obs_groups"], self.env) - - # resolve symmetry config - self.alg_cfg = resolve_symmetry_config(self.alg_cfg, self.env) - - # resolve deprecated normalization config - if self.cfg.get("empirical_normalization") is not None: - warnings.warn( - "The `empirical_normalization` parameter is deprecated. Please set `actor_obs_normalization` and " - "`critic_obs_normalization` as part of the `policy` configuration instead.", - DeprecationWarning, - ) - if self.policy_cfg.get("actor_obs_normalization") is None: - self.policy_cfg["actor_obs_normalization"] = self.cfg["empirical_normalization"] - if self.policy_cfg.get("critic_obs_normalization") is None: - self.policy_cfg["critic_obs_normalization"] = self.cfg["empirical_normalization"] - - # initialize the actor-critic - actor_critic_class = eval(self.policy_cfg.pop("class_name")) - actor_critic: ActorCritic | ActorCriticRecurrent = actor_critic_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - alg: PPO = alg_class(actor_critic, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg) - - # initialize the storage - alg.init_storage( - "rl", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg - - def _prepare_logging_writer(self): - """Prepares the logging writers.""" - if self.log_dir is not None and self.writer is None and not self.disable_logs: - # Launch either Tensorboard or Neptune & Tensorboard summary writer(s), default: Tensorboard. - self.logger_type = self.cfg.get("logger", "tensorboard") - self.logger_type = self.logger_type.lower() - - if self.logger_type == "neptune": - from rsl_rl.utils.neptune_utils import NeptuneSummaryWriter - - self.writer = NeptuneSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "wandb": - from rsl_rl.utils.wandb_utils import WandbSummaryWriter - - self.writer = WandbSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "tensorboard": - from torch.utils.tensorboard import SummaryWriter - - self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) - else: - raise ValueError("Logger type not found. Please choose 'neptune', 'wandb' or 'tensorboard'.") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py deleted file mode 100644 index 1624330..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of transitions storage for RL-agent.""" - -from .rollout_storage import RolloutStorage -from .replay_buffer_multi import ReplayBufferMulti -__all__ = ["RolloutStorage", "ReplayBufferMulti"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py deleted file mode 100644 index 6462b8e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -import numpy as np - - -class ReplayBufferMulti: - """Fixed-size buffer to store experience tuples.""" - - def __init__(self, obs_dim, buffer_size, num_amp_frames, device): - """Initialize a ReplayBuffer object. - Arguments: - buffer_size (int): maximum size of buffer - """ - self.states = torch.zeros(buffer_size, num_amp_frames, obs_dim).to(device) - self.num_amp_frames = num_amp_frames - self.buffer_size = buffer_size - self.device = device - - self.step = 0 - self.num_samples = 0 - - def insert(self, states): - """Add new states to memory.""" - num_states = states.shape[0] - start_idx = self.step - end_idx = self.step + num_states - if end_idx > self.buffer_size: - self.states[self.step:self.buffer_size] = states[:self.buffer_size - self.step] - self.states[:end_idx - self.buffer_size] = states[self.buffer_size - self.step:] - else: - self.states[start_idx:end_idx] = states - - self.num_samples = min(self.buffer_size, max(end_idx, self.num_samples)) - self.step = (self.step + num_states) % self.buffer_size - - def feed_forward_generator(self, num_mini_batch, mini_batch_size): - for _ in range(num_mini_batch): - sample_idxs = np.random.choice(self.num_samples, size=mini_batch_size) - yield (self.states[sample_idxs].to(self.device)) diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py deleted file mode 100644 index e9309b3..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -from tensordict import TensorDict - -from rsl_rl.utils import split_and_pad_trajectories - - -class RolloutStorage: - class Transition: - def __init__(self): - self.observations = None - self.actions = None - self.privileged_actions = None - self.rewards = None - self.dones = None - self.values = None - self.actions_log_prob = None - self.action_mean = None - self.action_sigma = None - self.hidden_states = None - - def clear(self): - self.__init__() - - def __init__( - self, - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - device="cpu", - ): - # store inputs - self.training_type = training_type - self.device = device - self.num_transitions_per_env = num_transitions_per_env - self.num_envs = num_envs - self.actions_shape = actions_shape - - # Core - self.observations = TensorDict( - {key: torch.zeros(num_transitions_per_env, *value.shape, device=device) for key, value in obs.items()}, - batch_size=[num_transitions_per_env, num_envs], - device=self.device, - ) - self.rewards = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.actions = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.dones = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device).byte() - - # for distillation - if training_type == "distillation": - self.privileged_actions = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - - # for reinforcement learning - if training_type == "rl": - self.values = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.actions_log_prob = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.mu = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.sigma = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.returns = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.advantages = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - - # For RNN networks - self.saved_hidden_states_a = None - self.saved_hidden_states_c = None - - # counter for the number of transitions stored - self.step = 0 - - def add_transitions(self, transition: Transition): - # check if the transition is valid - if self.step >= self.num_transitions_per_env: - raise OverflowError("Rollout buffer overflow! You should call clear() before adding new transitions.") - - # Core - self.observations[self.step].copy_(transition.observations) - self.actions[self.step].copy_(transition.actions) - self.rewards[self.step].copy_(transition.rewards.view(-1, 1)) - self.dones[self.step].copy_(transition.dones.view(-1, 1)) - - # for distillation - if self.training_type == "distillation": - self.privileged_actions[self.step].copy_(transition.privileged_actions) - - # for reinforcement learning - if self.training_type == "rl": - self.values[self.step].copy_(transition.values) - self.actions_log_prob[self.step].copy_(transition.actions_log_prob.view(-1, 1)) - self.mu[self.step].copy_(transition.action_mean) - self.sigma[self.step].copy_(transition.action_sigma) - - # For RNN networks - self._save_hidden_states(transition.hidden_states) - - # increment the counter - self.step += 1 - - def _save_hidden_states(self, hidden_states): - if hidden_states is None or hidden_states == (None, None): - return - # make a tuple out of GRU hidden state sto match the LSTM format - hid_a = hidden_states[0] if isinstance(hidden_states[0], tuple) else (hidden_states[0],) - hid_c = hidden_states[1] if isinstance(hidden_states[1], tuple) else (hidden_states[1],) - # initialize if needed - if self.saved_hidden_states_a is None: - self.saved_hidden_states_a = [ - torch.zeros(self.observations.shape[0], *hid_a[i].shape, device=self.device) for i in range(len(hid_a)) - ] - self.saved_hidden_states_c = [ - torch.zeros(self.observations.shape[0], *hid_c[i].shape, device=self.device) for i in range(len(hid_c)) - ] - # copy the states - for i in range(len(hid_a)): - self.saved_hidden_states_a[i][self.step].copy_(hid_a[i]) - self.saved_hidden_states_c[i][self.step].copy_(hid_c[i]) - - def clear(self): - self.step = 0 - - def compute_returns(self, last_values, gamma, lam, normalize_advantage: bool = True): - advantage = 0 - for step in reversed(range(self.num_transitions_per_env)): - # if we are at the last step, bootstrap the return value - if step == self.num_transitions_per_env - 1: - next_values = last_values - else: - next_values = self.values[step + 1] - # 1 if we are not in a terminal state, 0 otherwise - next_is_not_terminal = 1.0 - self.dones[step].float() - # TD error: r_t + gamma * V(s_{t+1}) - V(s_t) - delta = self.rewards[step] + next_is_not_terminal * gamma * next_values - self.values[step] - # Advantage: A(s_t, a_t) = delta_t + gamma * lambda * A(s_{t+1}, a_{t+1}) - advantage = delta + next_is_not_terminal * gamma * lam * advantage - # Return: R_t = A(s_t, a_t) + V(s_t) - self.returns[step] = advantage + self.values[step] - - # Compute the advantages - self.advantages = self.returns - self.values - # Normalize the advantages if flag is set - # This is to prevent double normalization (i.e. if per minibatch normalization is used) - if normalize_advantage: - self.advantages = (self.advantages - self.advantages.mean()) / (self.advantages.std() + 1e-8) - - # for distillation - def generator(self): - if self.training_type != "distillation": - raise ValueError("This function is only available for distillation training.") - - for i in range(self.num_transitions_per_env): - yield self.observations[i], self.actions[i], self.privileged_actions[i], self.dones[i] - - # for reinforcement learning with feedforward networks - def mini_batch_generator(self, num_mini_batches, num_epochs=8): - if self.training_type != "rl": - raise ValueError("This function is only available for reinforcement learning training.") - batch_size = self.num_envs * self.num_transitions_per_env - mini_batch_size = batch_size // num_mini_batches - indices = torch.randperm(num_mini_batches * mini_batch_size, requires_grad=False, device=self.device) - - # Core - observations = self.observations.flatten(0, 1) - actions = self.actions.flatten(0, 1) - values = self.values.flatten(0, 1) - returns = self.returns.flatten(0, 1) - - # For PPO - old_actions_log_prob = self.actions_log_prob.flatten(0, 1) - advantages = self.advantages.flatten(0, 1) - old_mu = self.mu.flatten(0, 1) - old_sigma = self.sigma.flatten(0, 1) - - for epoch in range(num_epochs): - for i in range(num_mini_batches): - # Select the indices for the mini-batch - start = i * mini_batch_size - end = (i + 1) * mini_batch_size - batch_idx = indices[start:end] - - # Create the mini-batch - # -- Core - obs_batch = observations[batch_idx] - actions_batch = actions[batch_idx] - - # -- For PPO - target_values_batch = values[batch_idx] - returns_batch = returns[batch_idx] - old_actions_log_prob_batch = old_actions_log_prob[batch_idx] - advantages_batch = advantages[batch_idx] - old_mu_batch = old_mu[batch_idx] - old_sigma_batch = old_sigma[batch_idx] - - # yield the mini-batch - yield obs_batch, actions_batch, target_values_batch, advantages_batch, returns_batch, old_actions_log_prob_batch, old_mu_batch, old_sigma_batch, ( - None, - None, - ), None - - # for reinfrocement learning with recurrent networks - def recurrent_mini_batch_generator(self, num_mini_batches, num_epochs=8): - if self.training_type != "rl": - raise ValueError("This function is only available for reinforcement learning training.") - padded_obs_trajectories, trajectory_masks = split_and_pad_trajectories(self.observations, self.dones) - - mini_batch_size = self.num_envs // num_mini_batches - for ep in range(num_epochs): - first_traj = 0 - for i in range(num_mini_batches): - start = i * mini_batch_size - stop = (i + 1) * mini_batch_size - - dones = self.dones.squeeze(-1) - last_was_done = torch.zeros_like(dones, dtype=torch.bool) - last_was_done[1:] = dones[:-1] - last_was_done[0] = True - trajectories_batch_size = torch.sum(last_was_done[:, start:stop]) - last_traj = first_traj + trajectories_batch_size - - masks_batch = trajectory_masks[:, first_traj:last_traj] - obs_batch = padded_obs_trajectories[:, first_traj:last_traj] - actions_batch = self.actions[:, start:stop] - old_mu_batch = self.mu[:, start:stop] - old_sigma_batch = self.sigma[:, start:stop] - returns_batch = self.returns[:, start:stop] - advantages_batch = self.advantages[:, start:stop] - values_batch = self.values[:, start:stop] - old_actions_log_prob_batch = self.actions_log_prob[:, start:stop] - - # reshape to [num_envs, time, num layers, hidden dim] (original shape: [time, num_layers, num_envs, hidden_dim]) - # then take only time steps after dones (flattens num envs and time dimensions), - # take a batch of trajectories and finally reshape back to [num_layers, batch, hidden_dim] - last_was_done = last_was_done.permute(1, 0) - hid_a_batch = [ - saved_hidden_states.permute(2, 0, 1, 3)[last_was_done][first_traj:last_traj] - .transpose(1, 0) - .contiguous() - for saved_hidden_states in self.saved_hidden_states_a - ] - hid_c_batch = [ - saved_hidden_states.permute(2, 0, 1, 3)[last_was_done][first_traj:last_traj] - .transpose(1, 0) - .contiguous() - for saved_hidden_states in self.saved_hidden_states_c - ] - # remove the tuple for GRU - hid_a_batch = hid_a_batch[0] if len(hid_a_batch) == 1 else hid_a_batch - hid_c_batch = hid_c_batch[0] if len(hid_c_batch) == 1 else hid_c_batch - - yield obs_batch, actions_batch, values_batch, advantages_batch, returns_batch, old_actions_log_prob_batch, old_mu_batch, old_sigma_batch, ( - hid_a_batch, - hid_c_batch, - ), masks_batch - - first_traj = last_traj diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py deleted file mode 100644 index f5781f1..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Helper functions.""" - -from .utils import * -from .motion_loader_g1 import G1_AMPLoader - -__all__ = [ - "G1_AMPLoader", -] \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py deleted file mode 100644 index 677c630..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py +++ /dev/null @@ -1,388 +0,0 @@ -import os -from os.path import join as pjoin -import glob -import json -import logging - -import torch -import numpy as np -from pybullet_utils import transformations - -from rsl_rl.utils import motion_util - -_EPS = np.finfo(float).eps * 4.0 -def quaternion_slerp(q0, q1, fraction, spin=0, shortestpath=True): - """Batch quaternion spherical linear interpolation.""" - - out = torch.zeros_like(q0) - - zero_mask = torch.isclose(fraction, torch.zeros_like(fraction)).squeeze() - ones_mask = torch.isclose(fraction, torch.ones_like(fraction)).squeeze() - out[zero_mask] = q0[zero_mask] - out[ones_mask] = q1[ones_mask] - - d = torch.sum(q0 * q1, dim=-1, keepdim=True) - dist_mask = (torch.abs(torch.abs(d) - 1.0) < _EPS).squeeze() - out[dist_mask] = q0[dist_mask] - - if shortestpath: - d_old = torch.clone(d) - d = torch.where(d_old < 0, -d, d) - q1 = torch.where(d_old < 0, -q1, q1) - - angle = torch.acos(d) + spin * torch.pi - angle_mask = (torch.abs(angle) < _EPS).squeeze() - out[angle_mask] = q0[angle_mask] - - final_mask = torch.logical_or(zero_mask, ones_mask) - final_mask = torch.logical_or(final_mask, dist_mask) - final_mask = torch.logical_or(final_mask, angle_mask) - final_mask = torch.logical_not(final_mask) - - isin = 1.0 / angle - q0 *= torch.sin((1.0 - fraction) * angle) * isin - q1 *= torch.sin(fraction * angle) * isin - q0 += q1 - out[final_mask] = q0[final_mask] - return out - - -class G1_AMPLoader: - - def __init__( - self, - device, - time_between_frames, - motion_files, - preload_transitions=False, - num_preload_transitions=1000000, - num_frames=5, - ): - """Expert dataset provides AMP observations from Dog mocap dataset. - - time_between_frames: Amount of time in seconds between transition. - """ - self.device = device - self.time_between_frames = time_between_frames - self.num_frames = num_frames - - # Values to store for each trajectory. - self.trajectories = [] - self.trajectories_full = [] - self.trajectory_names = [] - self.trajectory_idxs = [] - self.trajectory_lens = [] # Traj length in seconds. - self.trajectory_weights = [] - self.trajectory_frame_durations = [] - self.trajectory_num_frames = [] - self.motion_dir = motion_files - # import ipdb; ipdb.set_trace() - for i, motion_file in enumerate(os.listdir(motion_files)): - self.trajectory_names.append(motion_file) - motion_path = pjoin(motion_files, motion_file) - motion_data = np.load(motion_path, allow_pickle=True) - motion_data_processed = np.zeros((motion_data.shape[0],36)) - - for f_i in range(motion_data.shape[0]): - motion_data_processed[f_i, :3] = motion_data[f_i, :3] # base pos - motion_data_processed[f_i, 3:7] = motion_data[f_i, 3:7] # base quat (wxyz) - motion_data_processed[f_i, 7:35] = motion_data[f_i, 7:35] # base vel - ''' - NOTE The order of motion_data_processed is - base pos 0:3, - base quat 3:7, wxyz - dof pos 7:36, (mujoco joint order) - ''' - self.trajectories.append(torch.tensor( - motion_data_processed[:, 7:], - dtype=torch.float32, - device=self.device - )) - - self.trajectories_full.append(torch.tensor( - motion_data_processed, - dtype=torch.float32, - device=self.device - )) - - self.trajectory_idxs.append(i) - self.trajectory_weights.append(1 / len(os.listdir(motion_files))) - frame_duration = 1 / 50 - - self.trajectory_frame_durations.append(frame_duration) - traj_len = (motion_data_processed.shape[0] - 1) * frame_duration # seconds - self.trajectory_lens.append(traj_len) - self.trajectory_num_frames.append(float(motion_data_processed.shape[0])) - print(f"Loaded {traj_len}s. motion from {motion_file}.") - - # Trajectory weights are used to sample some trajectories more than others. - self.trajectory_weights = np.array(self.trajectory_weights) / np.sum(self.trajectory_weights) - self.trajectory_frame_durations = np.array(self.trajectory_frame_durations) - self.trajectory_lens = np.array(self.trajectory_lens) - self.trajectory_num_frames = np.array(self.trajectory_num_frames) - - # Preload transitions. - self.preload_transitions = preload_transitions - if self.preload_transitions: - print(f'Preloading {num_preload_transitions} transitions') - - traj_idxs = self.weighted_traj_idx_sample_batch(num_preload_transitions) - times = self.traj_time_sample_batch(traj_idxs) - self.preloaded_s_prior = self.get_full_frame_at_time_batch(traj_idxs, times - self.time_between_frames) - self.preloaded_s = self.get_full_frame_at_time_batch(traj_idxs, times) - self.preloaded_s_next = self.get_full_frame_at_time_batch(traj_idxs, times + self.time_between_frames) - print(f'Finished preloading') - - # 预加载多帧数据 - self.preloaded_frames = [] - for i in range(self.num_frames): - frame_time = times + (i - (self.num_frames - 2)) * self.time_between_frames - full_frame = self.get_full_frame_at_time_batch(traj_idxs, frame_time) - # 预处理：提前提取并连接需要的列（7:26 和 29:33），避免每次生成时重复切片 - processed_frame = torch.cat([ - full_frame[:, 7:26], - full_frame[:, 29:33] - ], dim=-1) - self.preloaded_frames.append(processed_frame) - print(f'Finished preloading multiple frames') - - self.all_trajectories_full = torch.vstack(self.trajectories_full) - - def weighted_traj_idx_sample(self): - """Get traj idx via weighted sampling.""" - return np.random.choice( - self.trajectory_idxs, p=self.trajectory_weights) - - def weighted_traj_idx_sample_batch(self, size): - """Batch sample traj idxs.""" - return np.random.choice( - self.trajectory_idxs, size=size, p=self.trajectory_weights, - replace=True) - - def traj_time_sample(self, traj_idx): - """Sample random time for traj.""" - subst = self.time_between_frames + self.trajectory_frame_durations[traj_idx] - return max( - 0, (self.trajectory_lens[traj_idx] * np.random.uniform() - subst)) - - def traj_time_sample_batch(self, traj_idxs): - """Sample random time for multiple trajectories.""" - subst = self.time_between_frames + self.trajectory_frame_durations[traj_idxs] - time_samples = self.trajectory_lens[traj_idxs] * np.random.uniform(size=len(traj_idxs)) - subst - return np.maximum(np.zeros_like(time_samples), time_samples) - - def slerp(self, val0, val1, blend): - return (1.0 - blend) * val0 + blend * val1 - - def get_trajectory(self, traj_idx): - """Returns trajectory of AMP observations.""" - return self.trajectories_full[traj_idx] - - def get_frame_at_time(self, traj_idx, time): - """Returns frame for the given trajectory at the specified time.""" - p = float(time) / self.trajectory_lens[traj_idx] - n = self.trajectories[traj_idx].shape[0] - idx_low, idx_high = int(np.floor(p * n)), int(np.ceil(p * n)) - frame_start = self.trajectories[traj_idx][idx_low] - frame_end = self.trajectories[traj_idx][idx_high] - blend = p * n - idx_low - return self.slerp(frame_start, frame_end, blend) - - def get_frame_at_time_batch(self, traj_idxs, times): - """Returns frame for the given trajectory at the specified time.""" - p = times / self.trajectory_lens[traj_idxs] - n = self.trajectory_num_frames[traj_idxs] - idx_low, idx_high = np.floor(p * n).astype(np.int32), np.ceil(p * n).astype(np.int32) - all_frame_starts = torch.zeros(len(traj_idxs), self.observation_dim, device=self.device) - all_frame_ends = torch.zeros(len(traj_idxs), self.observation_dim, device=self.device) - for traj_idx in set(traj_idxs): - trajectory = self.trajectories[traj_idx] - traj_mask = traj_idxs == traj_idx - all_frame_starts[traj_mask] = trajectory[idx_low[traj_mask]] - all_frame_ends[traj_mask] = trajectory[idx_high[traj_mask]] - blend = torch.tensor(p * n - idx_low, device=self.device, dtype=torch.float32).unsqueeze(-1) - return self.slerp(all_frame_starts, all_frame_ends, blend) - - def get_full_frame_at_time(self, traj_idx, time): - """Returns full frame for the given trajectory at the specified time.""" - p = float(time) / self.trajectory_lens[traj_idx] - n = self.trajectories_full[traj_idx].shape[0] - idx_low, idx_high = int(np.floor(p * n)), int(np.ceil(p * n)) - frame_start = self.trajectories_full[traj_idx][idx_low] - frame_end = self.trajectories_full[traj_idx][idx_high] - blend = p * n - idx_low - print(idx_low, idx_high) - return self.blend_frame_pose(frame_start, frame_end, blend) - - def get_full_frame_at_time_batch(self, traj_idxs, times): - p = times / self.trajectory_lens[traj_idxs] - n = self.trajectory_num_frames[traj_idxs] - idx_low, idx_high = np.floor(p * n).astype(np.int32), np.ceil(p * n).astype(np.int32) - all_frame_pos_starts = torch.zeros(len(traj_idxs), 3, device=self.device) - all_frame_pos_ends = torch.zeros(len(traj_idxs), 3, device=self.device) - all_frame_rot_starts = torch.zeros(len(traj_idxs), 4, device=self.device) - all_frame_rot_ends = torch.zeros(len(traj_idxs), 4, device=self.device) - all_frame_amp_starts = torch.zeros(len(traj_idxs), 29, device=self.device) - all_frame_amp_ends = torch.zeros(len(traj_idxs), 29, device=self.device) - for traj_idx in set(traj_idxs): - trajectory = self.trajectories_full[traj_idx] - traj_mask = traj_idxs == traj_idx - all_frame_pos_starts[traj_mask] = G1_AMPLoader.get_root_pos_batch(trajectory[idx_low[traj_mask]]) - all_frame_pos_ends[traj_mask] = G1_AMPLoader.get_root_pos_batch(trajectory[idx_high[traj_mask]]) - all_frame_rot_starts[traj_mask] = G1_AMPLoader.get_root_rot_batch(trajectory[idx_low[traj_mask]]) - all_frame_rot_ends[traj_mask] = G1_AMPLoader.get_root_rot_batch(trajectory[idx_high[traj_mask]]) - all_frame_amp_starts[traj_mask] = trajectory[idx_low[traj_mask]][:, 7:36] # base vel3+ang3, dof vel23+ang23 - all_frame_amp_ends[traj_mask] = trajectory[idx_high[traj_mask]][:, 7:36] # base vel3+ang3, dof vel23+ang23 - blend = torch.tensor(p * n - idx_low, device=self.device, dtype=torch.float32).unsqueeze(-1) - pos_blend = self.slerp(all_frame_pos_starts, all_frame_pos_ends, blend) - rot_blend = quaternion_slerp(all_frame_rot_starts, all_frame_rot_ends, blend) - amp_blend = self.slerp(all_frame_amp_starts, all_frame_amp_ends, blend) - return torch.cat([pos_blend, rot_blend, amp_blend], dim=-1) - - def get_frame(self): - """Returns random frame.""" - traj_idx = self.weighted_traj_idx_sample() - sampled_time = self.traj_time_sample(traj_idx) - return self.get_frame_at_time(traj_idx, sampled_time) - - def get_full_frame(self): - """Returns random full frame.""" - traj_idx = self.weighted_traj_idx_sample() - sampled_time = self.traj_time_sample(traj_idx) - return self.get_full_frame_at_time(traj_idx, sampled_time) - - def get_full_frame_batch(self, num_frames): - if self.preload_transitions: - idxs = np.random.choice( - self.preloaded_s.shape[0], size=num_frames) - return self.preloaded_s[idxs] - else: - traj_idxs = self.weighted_traj_idx_sample_batch(num_frames) - times = self.traj_time_sample_batch(traj_idxs) - return self.get_full_frame_at_time_batch(traj_idxs, times) - - def blend_frame_pose(self, frame0, frame1, blend): - """Linearly interpolate between two frames, including orientation. - - Args: - frame0: First frame to be blended corresponds to (blend = 0). - frame1: Second frame to be blended corresponds to (blend = 1). - blend: Float between [0, 1], specifying the interpolation between - the two frames. - Returns: - An interpolation of the two frames. - """ - root_pos0, root_pos1 = G1_AMPLoader.get_root_pos(frame0), G1_AMPLoader.get_root_pos(frame1) - root_rot0, root_rot1 = G1_AMPLoader.get_root_rot(frame0), G1_AMPLoader.get_root_rot(frame1) - joints0, joints1 = G1_AMPLoader.get_joint_pose(frame0), G1_AMPLoader.get_joint_pose(frame1) - # tar_toe_pos_0, tar_toe_pos_1 = G1_AMPLoader.get_tar_toe_pos_local(frame0), G1_AMPLoader.get_tar_toe_pos_local(frame1) - linear_vel_0, linear_vel_1 = G1_AMPLoader.get_linear_vel(frame0), G1_AMPLoader.get_linear_vel(frame1) - angular_vel_0, angular_vel_1 = G1_AMPLoader.get_angular_vel(frame0), G1_AMPLoader.get_angular_vel(frame1) - joint_vel_0, joint_vel_1 = G1_AMPLoader.get_joint_vel(frame0), G1_AMPLoader.get_joint_vel(frame1) - - blend_root_pos = self.slerp(root_pos0, root_pos1, blend) - blend_root_rot = transformations.quaternion_slerp(root_rot0.cpu().numpy(), root_rot1.cpu().numpy(), blend) - blend_root_rot = torch.tensor(motion_util.standardize_quaternion(blend_root_rot),dtype=torch.float32, device=self.device) - blend_joints = self.slerp(joints0, joints1, blend) - # blend_tar_toe_pos = self.slerp(tar_toe_pos_0, tar_toe_pos_1, blend) - blend_linear_vel = self.slerp(linear_vel_0, linear_vel_1, blend) - blend_angular_vel = self.slerp(angular_vel_0, angular_vel_1, blend) - blend_joints_vel = self.slerp(joint_vel_0, joint_vel_1, blend) - - # return - # torch.cat([ - # blend_root_pos, blend_root_rot, blend_linear_vel, blend_angular_vel, blend_joints, blend_joints_vel]) - return torch.cat([blend_root_pos, blend_root_rot, blend_linear_vel, blend_angular_vel, blend_joints]) - - def feed_forward_generator_23dof_multi(self, num_mini_batch, mini_batch_size): - """Generates a batch of AMP transitions.""" - # import ipdb; ipdb.set_trace() - for _ in range(num_mini_batch): - if self.preload_transitions: - idxs = np.random.choice(self.preloaded_s.shape[0], size=mini_batch_size) - - frames = [] - for i in range(self.num_frames): - # 数据已在预加载时预处理，直接索引即可 - s = self.preloaded_frames[i][idxs] - frames.append(s) - else: - NotImplementedError('preload transition') - yield torch.stack(frames, dim=1) # [batch, num_frames, 16] - - - - - def quaternion_to_euler_array(self, quat): - # Ensure quaternion is in the correct format [x, y, z, w] - x, y, z, w =quat - - # Roll (x-axis rotation) - t0 = +2.0 * (w * x + y * z) - t1 = +1.0 - 2.0 * (x * x + y * y) - roll_x = np.arctan2(t0, t1) - - # Pitch (y-axis rotation) - t2 = +2.0 * (w * y - z * x) - t2 = np.clip(t2, -1.0, 1.0) - pitch_y = np.arcsin(t2) - - # Yaw (z-axis rotation) - t3 = +2.0 * (w * z + x * y) - t4 = +1.0 - 2.0 * (y * y + z * z) - yaw_z = np.arctan2(t3, t4) - - # Returns roll, pitch, yaw in a NumPy array in radians - return np.array([roll_x, pitch_y, yaw_z]) - - def euler_to_quaternion(self, root_rot): - roll, pitch, yaw = root_rot[0], root_rot[1], root_rot[2] - cy = np.cos(yaw * 0.5) - sy = np.sin(yaw * 0.5) - cp = np.cos(pitch * 0.5) - sp = np.sin(pitch * 0.5) - cr = np.cos(roll * 0.5) - sr = np.sin(roll * 0.5) - - qw = cy * cp * cr + sy * sp * sr - qx = cy * cp * sr - sy * sp * cr - qy = sy * cp * sr + cy * sp * cr - qz = sy * cp * cr - cy * sp * sr - - return np.array([qx, qy, qz, qw]) - - @property - def observation_dim(self): - """Size of AMP observations.""" - return self.trajectories[0].shape[1] + 1 - - @property - def num_motions(self): - return len(self.trajectory_names) - @staticmethod - def get_root_pos(pose): - return pose[0:3] - - @staticmethod - def get_root_pos_batch(poses): - return poses[:, 0:3] - - @staticmethod - def get_root_rot(pose): - return pose[3:7] - - @staticmethod - def get_root_rot_batch(poses): - return poses[:, 3:7] - - @staticmethod - def get_joint_pose_batch_12dof(poses): - return poses[:, 13:25] - - @staticmethod - def get_tar_toe_pos_local(pose): - return pose[G1_AMPLoader.TAR_TOE_POS_LOCAL_START_IDX:G1_AMPLoader.TAR_TOE_POS_LOCAL_END_IDX] - - @staticmethod - def get_tar_toe_pos_local_batch(poses): - return poses[:, G1_AMPLoader.TAR_TOE_POS_LOCAL_START_IDX:G1_AMPLoader.TAR_TOE_POS_LOCAL_END_IDX] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py deleted file mode 100644 index 3d49bfa..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility functions for processing motion clips.""" - -import os -import inspect -currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parentdir = os.path.dirname(os.path.dirname(currentdir)) -os.sys.path.insert(0, parentdir) - -import numpy as np - -from rsl_rl.utils import pose3d -# from pybullet_utils import transformations - - -def standardize_quaternion(q): - """Returns a quaternion where q.w >= 0 to remove redundancy due to q = -q. - - Args: - q: A quaternion to be standardized. - - Returns: - A quaternion with q.w >= 0. - - """ - if q[-1] < 0: - q = -q - return q - - -def normalize_rotation_angle(theta): - """Returns a rotation angle normalized between [-pi, pi]. - - Args: - theta: angle of rotation (radians). - - Returns: - An angle of rotation normalized between [-pi, pi]. - - """ - norm_theta = theta - if np.abs(norm_theta) > np.pi: - norm_theta = np.fmod(norm_theta, 2 * np.pi) - if norm_theta >= 0: - norm_theta += -2 * np.pi - else: - norm_theta += 2 * np.pi - - return norm_theta - - -def calc_heading(q): - """Returns the heading of a rotation q, specified as a quaternion. - - The heading represents the rotational component of q along the vertical - axis (z axis). - - Args: - q: A quaternion that the heading is to be computed from. - - Returns: - An angle representing the rotation about the z axis. - - """ - ref_dir = np.array([1, 0, 0]) - rot_dir = pose3d.QuaternionRotatePoint(ref_dir, q) - heading = np.arctan2(rot_dir[1], rot_dir[0]) - return heading - - -# def calc_heading_rot(q): -# """Return a quaternion representing the heading rotation of q along the vertical axis (z axis). - -# Args: -# q: A quaternion that the heading is to be computed from. - -# Returns: -# A quaternion representing the rotation about the z axis. - -# """ -# heading = calc_heading(q) -# q_heading = transformations.quaternion_about_axis(heading, [0, 0, 1]) -# return q_heading diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py deleted file mode 100644 index 3796ec8..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -from dataclasses import asdict -from torch.utils.tensorboard import SummaryWriter - -try: - import neptune -except ModuleNotFoundError: - raise ModuleNotFoundError("neptune-client is required to log to Neptune.") - - -class NeptuneLogger: - def __init__(self, project, token): - self.run = neptune.init_run(project=project, api_token=token) - - def store_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.run["runner_cfg"] = runner_cfg - self.run["policy_cfg"] = policy_cfg - self.run["alg_cfg"] = alg_cfg - self.run["env_cfg"] = asdict(env_cfg) - - -class NeptuneSummaryWriter(SummaryWriter): - """Summary writer for Neptune.""" - - def __init__(self, log_dir: str, flush_secs: int, cfg): - super().__init__(log_dir, flush_secs) - - try: - project = cfg["neptune_project"] - except KeyError: - raise KeyError("Please specify neptune_project in the runner config, e.g. legged_gym.") - - try: - token = os.environ["NEPTUNE_API_TOKEN"] - except KeyError: - raise KeyError( - "Neptune api token not found. Please run or add to ~/.bashrc: export NEPTUNE_API_TOKEN=YOUR_API_TOKEN" - ) - - try: - entity = os.environ["NEPTUNE_USERNAME"] - except KeyError: - raise KeyError( - "Neptune username not found. Please run or add to ~/.bashrc: export NEPTUNE_USERNAME=YOUR_USERNAME" - ) - - neptune_project = entity + "/" + project - - self.neptune_logger = NeptuneLogger(neptune_project, token) - - self.name_map = { - "Train/mean_reward/time": "Train/mean_reward_time", - "Train/mean_episode_length/time": "Train/mean_episode_length_time", - } - - run_name = os.path.split(log_dir)[-1] - - self.neptune_logger.run["log_dir"].log(run_name) - - def _map_path(self, path): - if path in self.name_map: - return self.name_map[path] - else: - return path - - def add_scalar(self, tag, scalar_value, global_step=None, walltime=None, new_style=False): - super().add_scalar( - tag, - scalar_value, - global_step=global_step, - walltime=walltime, - new_style=new_style, - ) - self.neptune_logger.run[self._map_path(tag)].log(scalar_value, step=global_step) - - def stop(self): - self.neptune_logger.run.stop() - - def log_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.neptune_logger.store_config(env_cfg, runner_cfg, alg_cfg, policy_cfg) - - def save_model(self, model_path, iter): - self.neptune_logger.run["model/saved_model_" + str(iter)].upload(model_path) - - def save_file(self, path, iter=None): - name = path.rsplit("/", 1)[-1].split(".")[0] - self.neptune_logger.run["git_diff/" + name].upload(path) diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py deleted file mode 100644 index a4cba1e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py +++ /dev/null @@ -1,283 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for 3D pose conversion.""" -import math -import numpy as np - -# from pybullet_utils import transformations - -VECTOR3_0 = np.zeros(3, dtype=np.float64) -VECTOR3_1 = np.ones(3, dtype=np.float64) -VECTOR3_X = np.array([1, 0, 0], dtype=np.float64) -VECTOR3_Y = np.array([0, 1, 0], dtype=np.float64) -VECTOR3_Z = np.array([0, 0, 1], dtype=np.float64) - -# QUATERNION_IDENTITY is the multiplicative identity 1.0 + 0i + 0j + 0k. -# When interpreted as a rotation, it is the identity rotation. -QUATERNION_IDENTITY = np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float64) - - -def Vector3RandomNormal(sigma, mu=VECTOR3_0): - """Returns a random 3D vector from a normal distribution. - - Each component is selected independently from a normal distribution. - - Args: - sigma: Scale (or stddev) of distribution for all variables. - mu: Mean of distribution for each variable. - - Returns: - A 3D vector in a numpy array. - """ - - random_v3 = np.random.normal(scale=sigma, size=3) + mu - return random_v3 - - -def Vector3RandomUniform(low=VECTOR3_0, high=VECTOR3_1): - """Returns a 3D vector selected uniformly from the input box. - - Args: - low: The min-value corner of the box. - high: The max-value corner of the box. - - Returns: - A 3D vector in a numpy array. - """ - - random_x = np.random.uniform(low=low[0], high=high[0]) - random_y = np.random.uniform(low=low[1], high=high[1]) - random_z = np.random.uniform(low=low[2], high=high[2]) - return np.array([random_x, random_y, random_z]) - - -def Vector3RandomUnit(): - """Returns a random 3D vector with unit length. - - Generates a 3D vector selected uniformly from the unit sphere. - - Returns: - A normalized 3D vector in a numpy array. - """ - longitude = np.random.uniform(low=-math.pi, high=math.pi) - sin_latitude = np.random.uniform(low=-1.0, high=1.0) - cos_latitude = math.sqrt(1.0 - sin_latitude * sin_latitude) - x = math.cos(longitude) * cos_latitude - y = math.sin(longitude) * cos_latitude - z = sin_latitude - return np.array([x, y, z], dtype=np.float64) - - -def QuaternionNormalize(q): - """Normalizes the quaternion to length 1. - - Divides the quaternion by its magnitude. If the magnitude is too - small, returns the quaternion identity value (1.0). - - Args: - q: A quaternion to be normalized. - - Raises: - ValueError: If input quaternion has length near zero. - - Returns: - A quaternion with magnitude 1 in a numpy array [x, y, z, w]. - - """ - q_norm = np.linalg.norm(q) - if np.isclose(q_norm, 0.0): - raise ValueError( - 'Quaternion may not be zero in QuaternionNormalize: |q| = %f, q = %s' % - (q_norm, q)) - return q / q_norm - - -def QuaternionFromAxisAngle(axis, angle): - """Returns a quaternion that generates the given axis-angle rotation. - - Returns the quaternion: sin(angle/2) * axis + cos(angle/2). - - Args: - axis: Axis of rotation, a 3D vector in a numpy array. - angle: The angle of rotation (radians). - - Raises: - ValueError: If input axis is not a normalizable 3D vector. - - Returns: - A unit quaternion in a numpy array. - - """ - if len(axis) != 3: - raise ValueError('Axis vector should have three components: %s' % axis) - axis_norm = np.linalg.norm(axis) - if np.isclose(axis_norm, 0.0): - raise ValueError('Axis vector may not have zero length: |v| = %f, v = %s' % - (axis_norm, axis)) - half_angle = angle * 0.5 - q = np.zeros(4, dtype=np.float64) - q[0:3] = axis - q[0:3] *= math.sin(half_angle) / axis_norm - q[3] = math.cos(half_angle) - return q - - -def QuaternionToAxisAngle(quat, default_axis=VECTOR3_Z, direction_axis=None): - """Calculates axis and angle of rotation performed by a quaternion. - - Calculates the axis and angle of the rotation performed by the quaternion. - The quaternion should have four values and be normalized. - - Args: - quat: Unit quaternion in a numpy array. - default_axis: 3D vector axis used if the rotation is near to zero. Without - this default, small rotations would result in an exception. It is - reasonable to use a default axis for tiny rotations, because zero angle - rotations about any axis are equivalent. - direction_axis: Used to disambiguate rotation directions. If the - direction_axis is specified, the axis of the rotation will be chosen such - that its inner product with the direction_axis is non-negative. - - Raises: - ValueError: If quat is not a normalized quaternion. - - Returns: - axis: Axis of rotation. - angle: Angle in radians. - """ - if len(quat) != 4: - raise ValueError( - 'Quaternion should have four components [x, y, z, w]: %s' % quat) - if not np.isclose(1.0, np.linalg.norm(quat)): - raise ValueError('Quaternion should have unit length: |q| = %f, q = %s' % - (np.linalg.norm(quat), quat)) - axis = quat[:3].copy() - axis_norm = np.linalg.norm(axis) - min_axis_norm = 1e-8 - if axis_norm < min_axis_norm: - axis = default_axis - if len(default_axis) != 3: - raise ValueError('Axis vector should have three components: %s' % axis) - if not np.isclose(np.linalg.norm(axis), 1.0): - raise ValueError('Axis vector should have unit length: |v| = %f, v = %s' % - (np.linalg.norm(axis), axis)) - else: - axis /= axis_norm - sin_half_angle = axis_norm - if direction_axis is not None and np.inner(axis, direction_axis) < 0: - sin_half_angle = -sin_half_angle - axis = -axis - cos_half_angle = quat[3] - half_angle = math.atan2(sin_half_angle, cos_half_angle) - angle = half_angle * 2 - return axis, angle - - -def QuaternionRandomRotation(max_angle=math.pi): - """Creates a random small rotation around a random axis. - - Generates a small rotation with the axis vector selected uniformly - from the unit sphere and an angle selected from a uniform - distribution over [0, max_angle]. - - If the max_angle is not specified, the rotation should be selected - uniformly over all possible rotation angles. - - Args: - max_angle: The maximum angle of rotation (radians). - - Returns: - A unit quaternion in a numpy array. - - """ - - angle = np.random.uniform(low=0, high=max_angle) - axis = Vector3RandomUnit() - return QuaternionFromAxisAngle(axis, angle) - - -# def QuaternionRotatePoint(point, quat): -# """Performs a rotation by quaternion. - -# Rotate the point by the quaternion using quaternion multiplication, -# (q * p * q^-1), without constructing the rotation matrix. - -# Args: -# point: The point to be rotated. -# quat: The rotation represented as a quaternion [x, y, z, w]. - -# Returns: -# A 3D vector in a numpy array. -# """ - -# q_point = np.array([point[0], point[1], point[2], 0.0]) -# quat_inverse = transformations.quaternion_inverse(quat) -# q_point_rotated = transformations.quaternion_multiply( -# transformations.quaternion_multiply(quat, q_point), quat_inverse) -# return q_point_rotated[:3] - - -def IsRotationMatrix(m): - """Returns true if the 3x3 submatrix represents a rotation. - - Args: - m: A transformation matrix. - - Raises: - ValueError: If input is not a matrix of size at least 3x3. - - Returns: - True if the 3x3 submatrix is a rotation (orthogonal). - """ - if len(m.shape) != 2 or m.shape[0] < 3 or m.shape[1] < 3: - raise ValueError('Matrix should be 3x3 or 4x4: %s\n %s' % (m.shape, m)) - rot = m[:3, :3] - eye = np.matmul(rot, np.transpose(rot)) - return np.isclose(eye, np.identity(3), atol=1e-4).all() - -# def ZAxisAlignedRobotPoseTool(robot_pose_tool): -# """Returns the current gripper pose rotated for alignment with the z-axis. - -# Args: -# robot_pose_tool: a pose3d.Pose3d() instance. - -# Returns: -# An instance of pose.Transform representing the current gripper pose -# rotated for alignment with the z-axis. -# """ -# # Align the current pose to the z-axis. -# robot_pose_tool.quaternion = transformations.quaternion_multiply( -# RotationBetween( -# robot_pose_tool.matrix4x4[0:3, 0:3].dot(np.array([0, 0, 1])), -# np.array([0.0, 0.0, -1.0])), robot_pose_tool.quaternion) -# return robot_pose_tool - -# def RotationBetween(a_translation_b, a_translation_c): -# """Computes the rotation from one vector to another. - -# The computed rotation has the property that: - -# a_translation_c = a_rotation_b_to_c * a_translation_b - -# Args: -# a_translation_b: vec3, vector to rotate from -# a_translation_c: vec3, vector to rotate to - -# Returns: -# a_rotation_b_to_c: new Orientation -# """ -# rotation = rotation3.Rotation3.rotation_between( -# a_translation_b, a_translation_c, err_msg='RotationBetween') -# return rotation.quaternion.xyzw diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py deleted file mode 100644 index 3605622..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import git -import importlib -import os -import pathlib -import torch -import warnings -from tensordict import TensorDict -from typing import Callable -import numpy as np -class RunningMeanStd: - def __init__(self, epsilon: float = 1e-4, shape: Tuple[int, ...] = ()): - """ - Calculates the running mean and std of a data stream - https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm - :param epsilon: helps with arithmetic issues - :param shape: the shape of the data stream's output - """ - self.mean = np.zeros(shape, np.float64) - self.var = np.ones(shape, np.float64) - self.count = epsilon - - def update(self, arr: np.ndarray) -> None: - batch_mean = np.mean(arr, axis=0) - batch_var = np.var(arr, axis=0) - batch_count = arr.shape[0] - self.update_from_moments(batch_mean, batch_var, batch_count) - - def update_from_moments(self, batch_mean: np.ndarray, batch_var: np.ndarray, batch_count: int) -> None: - delta = batch_mean - self.mean - tot_count = self.count + batch_count - - new_mean = self.mean + delta * batch_count / tot_count - m_a = self.var * self.count - m_b = batch_var * batch_count - m_2 = m_a + m_b + np.square(delta) * self.count * batch_count / (self.count + batch_count) - new_var = m_2 / (self.count + batch_count) - - new_count = batch_count + self.count - - self.mean = new_mean - self.var = new_var - self.count = new_count - - -class Normalizer(RunningMeanStd): - def __init__(self, input_dim, epsilon=1e-4, clip_obs=10.0): - super().__init__(shape=input_dim) - self.epsilon = epsilon - self.clip_obs = clip_obs - - def normalize(self, input): - return np.clip((input - self.mean) / np.sqrt(self.var + self.epsilon), -self.clip_obs, self.clip_obs) - - def normalize_torch(self, input, device): - mean_torch = torch.tensor(self.mean, device=device, dtype=torch.float32) - std_torch = torch.sqrt(torch.tensor(self.var + self.epsilon, device=device, dtype=torch.float32)) - return torch.clamp((input - mean_torch) / std_torch, -self.clip_obs, self.clip_obs) - - def update_normalizer(self, rollouts, expert_loader): - policy_data_generator = rollouts.feed_forward_generator_amp(None, mini_batch_size=expert_loader.batch_size) - expert_data_generator = expert_loader.dataset.feed_forward_generator_amp(expert_loader.batch_size) - - for expert_batch, policy_batch in zip(expert_data_generator, policy_data_generator): - self.update(torch.vstack(tuple(policy_batch) + tuple(expert_batch)).cpu().numpy()) - - -def resolve_nn_activation(act_name: str) -> torch.nn.Module: - """Resolves the activation function from the name. - - Args: - act_name: The name of the activation function. - - Returns: - The activation function. - - Raises: - ValueError: If the activation function is not found. - """ - act_dict = { - "elu": torch.nn.ELU(), - "selu": torch.nn.SELU(), - "relu": torch.nn.ReLU(), - "crelu": torch.nn.CELU(), - "lrelu": torch.nn.LeakyReLU(), - "tanh": torch.nn.Tanh(), - "sigmoid": torch.nn.Sigmoid(), - "softplus": torch.nn.Softplus(), - "gelu": torch.nn.GELU(), - "swish": torch.nn.SiLU(), - "mish": torch.nn.Mish(), - "identity": torch.nn.Identity(), - } - - act_name = act_name.lower() - if act_name in act_dict: - return act_dict[act_name] - else: - raise ValueError(f"Invalid activation function '{act_name}'. Valid activations are: {list(act_dict.keys())}") - - -def resolve_optimizer(optimizer_name: str) -> torch.optim.Optimizer: - """Resolves the optimizer from the name. - - Args: - optimizer_name: The name of the optimizer. - - Returns: - The optimizer. - - Raises: - ValueError: If the optimizer is not found. - """ - optimizer_dict = { - "adam": torch.optim.Adam, - "adamw": torch.optim.AdamW, - "sgd": torch.optim.SGD, - "rmsprop": torch.optim.RMSprop, - } - - optimizer_name = optimizer_name.lower() - if optimizer_name in optimizer_dict: - return optimizer_dict[optimizer_name] - else: - raise ValueError(f"Invalid optimizer '{optimizer_name}'. Valid optimizers are: {list(optimizer_dict.keys())}") - - -def split_and_pad_trajectories( - tensor: torch.Tensor | TensorDict, dones: torch.Tensor -) -> tuple[torch.Tensor | TensorDict, torch.Tensor]: - """Splits trajectories at done indices. Then concatenates them and pads with zeros up to the length of the longest - trajectory. Returns masks corresponding to valid parts of the trajectories. - - Example: - Input: [[a1, a2, a3, a4 | a5, a6], - [b1, b2 | b3, b4, b5 | b6]] - - Output:[[a1, a2, a3, a4], | [[True, True, True, True], - [a5, a6, 0, 0], | [True, True, False, False], - [b1, b2, 0, 0], | [True, True, False, False], - [b3, b4, b5, 0], | [True, True, True, False], - [b6, 0, 0, 0]] | [True, False, False, False]] - - Assumes that the input has the following order of dimensions: [time, number of envs, additional dimensions] - """ - - dones = dones.clone() - dones[-1] = 1 - # Permute the buffers to have order (num_envs, num_transitions_per_env, ...), for correct reshaping - flat_dones = dones.transpose(1, 0).reshape(-1, 1) - # Get length of trajectory by counting the number of successive not done elements - done_indices = torch.cat((flat_dones.new_tensor([-1], dtype=torch.int64), flat_dones.nonzero()[:, 0])) - trajectory_lengths = done_indices[1:] - done_indices[:-1] - trajectory_lengths_list = trajectory_lengths.tolist() - # Extract the individual trajectories - if isinstance(tensor, TensorDict): - padded_trajectories = {} - for k, v in tensor.items(): - # split the tensor into trajectories - trajectories = torch.split(v.transpose(1, 0).flatten(0, 1), trajectory_lengths_list) - # add at least one full length trajectory - trajectories = trajectories + (torch.zeros(v.shape[0], *v.shape[2:], device=v.device),) - # pad the trajectories to the length of the longest trajectory - padded_trajectories[k] = torch.nn.utils.rnn.pad_sequence(trajectories) - # remove the added tensor - padded_trajectories[k] = padded_trajectories[k][:, :-1] - padded_trajectories = TensorDict( - padded_trajectories, batch_size=[tensor.batch_size[0], len(trajectory_lengths_list)] - ) - else: - # split the tensor into trajectories - trajectories = torch.split(tensor.transpose(1, 0).flatten(0, 1), trajectory_lengths_list) - # add at least one full length trajectory - trajectories = trajectories + (torch.zeros(tensor.shape[0], *tensor.shape[2:], device=tensor.device),) - # pad the trajectories to the length of the longest trajectory - padded_trajectories = torch.nn.utils.rnn.pad_sequence(trajectories) - # remove the added tensor - padded_trajectories = padded_trajectories[:, :-1] - # create masks for the valid parts of the trajectories - trajectory_masks = trajectory_lengths > torch.arange(0, tensor.shape[0], device=tensor.device).unsqueeze(1) - return padded_trajectories, trajectory_masks - - -def unpad_trajectories(trajectories, masks): - """Does the inverse operation of split_and_pad_trajectories()""" - # Need to transpose before and after the masking to have proper reshaping - return ( - trajectories.transpose(1, 0)[masks.transpose(1, 0)] - .view(-1, trajectories.shape[0], trajectories.shape[-1]) - .transpose(1, 0) - ) - - -def store_code_state(logdir, repositories) -> list: - git_log_dir = os.path.join(logdir, "git") - os.makedirs(git_log_dir, exist_ok=True) - file_paths = [] - for repository_file_path in repositories: - try: - repo = git.Repo(repository_file_path, search_parent_directories=True) - t = repo.head.commit.tree - except Exception: - print(f"Could not find git repository in {repository_file_path}. Skipping.") - # skip if not a git repository - continue - # get the name of the repository - repo_name = pathlib.Path(repo.working_dir).name - diff_file_name = os.path.join(git_log_dir, f"{repo_name}.diff") - # check if the diff file already exists - if os.path.isfile(diff_file_name): - continue - # write the diff file - print(f"Storing git diff for '{repo_name}' in: {diff_file_name}") - with open(diff_file_name, "x", encoding="utf-8") as f: - content = f"--- git status ---\n{repo.git.status()} \n\n\n--- git diff ---\n{repo.git.diff(t)}" - f.write(content) - # add the file path to the list of files to be uploaded - file_paths.append(diff_file_name) - return file_paths - - -def string_to_callable(name: str) -> Callable: - """Resolves the module and function names to return the function. - - Args: - name: The function name. The format should be 'module:attribute_name'. - - Raises: - ValueError: When the resolved attribute is not a function. - ValueError: When unable to resolve the attribute. - - Returns: - The function loaded from the module. - """ - try: - mod_name, attr_name = name.split(":") - mod = importlib.import_module(mod_name) - callable_object = getattr(mod, attr_name) - # check if attribute is callable - if callable(callable_object): - return callable_object - else: - raise ValueError(f"The imported object is not callable: '{name}'") - except AttributeError as e: - msg = ( - "We could not interpret the entry as a callable object. The format of input should be" - f" 'module:attribute_name'\nWhile processing input '{name}', received the error:\n {e}." - ) - raise ValueError(msg) - - -def resolve_obs_groups( - obs: TensorDict, obs_groups: dict[str, list[str]], default_sets: list[str] -) -> dict[str, list[str]]: - """Validates the observation configuration and defaults missing observation sets. - - The input is an observation dictionary `obs` containing observation groups and a configuration dictionary - `obs_groups` where the keys are the observation sets and the values are lists of observation groups. - - The configuration dictionary could for example look like: - { - "policy": ["group_1", "group_2"], - "critic": ["group_1", "group_3"] - } - - This means that the 'policy' observation set will contain the observations "group_1" and "group_2" and the - 'critic' observation set will contain the observations "group_1" and "group_3". This function will check that all - the observations in the 'policy' and 'critic' observation sets are present in the observation dictionary from the - environment. - - Additionally, if one of the `default_sets`, e.g. "critic", is not present in the configuration dictionary, - this function will: - - 1. Check if a group with the same name exists in the observations and assign this group to the observation set. - 2. If 1. fails, it will assign the observations from the 'policy' observation set to the default observation set. - - Args: - obs: Observations from the environment in the form of a dictionary. - obs_groups: Observation sets configuration. - default_sets: Reserved observation set names used by the algorithm (besides 'policy'). - If not provided in 'obs_groups', a default behavior gets triggered. - - Returns: - The resolved observation groups. - - Raises: - ValueError: If any observation set is an empty list. - ValueError: If any observation set contains an observation term that is not present in the observations. - """ - # check if policy observation set exists - if "policy" not in obs_groups.keys(): - if "policy" in obs: - obs_groups["policy"] = ["policy"] - warnings.warn( - "The observation configuration dictionary 'obs_groups' must contain the 'policy' key." - " As an observation group with the name 'policy' was found, this is assumed to be the observation set." - " Consider adding the 'policy' key to the 'obs_groups' dictionary for clarity." - " This behavior will be removed in a future version." - ) - else: - raise ValueError( - "The observation configuration dictionary 'obs_groups' must contain the 'policy' key." - f" Found keys: {list(obs_groups.keys())}" - ) - - # check all observation sets for valid observation groups - for set_name, groups in obs_groups.items(): - # check if the list is empty - if len(groups) == 0: - msg = f"The '{set_name}' key in the 'obs_groups' dictionary can not be an empty list." - if set_name in default_sets: - if set_name not in obs: - msg += " Consider removing the key to default to the observations used for the 'policy' set." - else: - msg += ( - f" Consider removing the key to default to the observation '{set_name}' from the environment." - ) - raise ValueError(msg) - # check groups exist inside the observations from the environment - for group in groups: - if group not in obs: - raise ValueError( - f"Observation '{group}' in observation set '{set_name}' not found in the observations from the" - f" environment. Available observations from the environment: {list(obs.keys())}" - ) - - # fill missing observation sets - for default_set_name in default_sets: - if default_set_name not in obs_groups.keys(): - if default_set_name in obs: - obs_groups[default_set_name] = [default_set_name] - warnings.warn( - f"The observation configuration dictionary 'obs_groups' must contain the '{default_set_name}' key." - f" As an observation group with the name '{default_set_name}' was found, this is assumed to be the" - f" observation set. Consider adding the '{default_set_name}' key to the 'obs_groups' dictionary for" - " clarity. This behavior will be removed in a future version." - ) - else: - obs_groups[default_set_name] = obs_groups["policy"].copy() - warnings.warn( - f"The observation configuration dictionary 'obs_groups' must contain the '{default_set_name}' key." - f" As the configuration for '{default_set_name}' is missing, the observations from the 'policy' set" - f" are used. Consider adding the '{default_set_name}' key to the 'obs_groups' dictionary for" - " clarity. This behavior will be removed in a future version." - ) - - # print the final parsed observation sets - print("-" * 80) - print("Resolved observation sets: ") - for set_name, groups in obs_groups.items(): - print("\t", set_name, ": ", groups) - print("-" * 80) - - return obs_groups diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py deleted file mode 100644 index 243e82d..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -from dataclasses import asdict -from torch.utils.tensorboard import SummaryWriter - -try: - import wandb -except ModuleNotFoundError: - raise ModuleNotFoundError("Wandb is required to log to Weights and Biases.") - - -class WandbSummaryWriter(SummaryWriter): - """Summary writer for Weights and Biases.""" - - def __init__(self, log_dir: str, flush_secs: int, cfg): - super().__init__(log_dir, flush_secs) - - # Get the run name - run_name = os.path.split(log_dir)[-1] - - try: - project = cfg["wandb_project"] - except KeyError: - raise KeyError("Please specify wandb_project in the runner config, e.g. legged_gym.") - - try: - entity = os.environ["WANDB_USERNAME"] - except KeyError: - entity = None - - # Initialize wandb - wandb.init(project=project, entity=entity, name=run_name) - - # Add log directory to wandb - wandb.config.update({"log_dir": log_dir}) - - self.name_map = { - "Train/mean_reward/time": "Train/mean_reward_time", - "Train/mean_episode_length/time": "Train/mean_episode_length_time", - } - - def store_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - wandb.config.update({"runner_cfg": runner_cfg}) - wandb.config.update({"policy_cfg": policy_cfg}) - wandb.config.update({"alg_cfg": alg_cfg}) - try: - wandb.config.update({"env_cfg": env_cfg.to_dict()}) - except Exception: - wandb.config.update({"env_cfg": asdict(env_cfg)}) - - def add_scalar(self, tag, scalar_value, global_step=None, walltime=None, new_style=False): - super().add_scalar( - tag, - scalar_value, - global_step=global_step, - walltime=walltime, - new_style=new_style, - ) - wandb.log({self._map_path(tag): scalar_value}, step=global_step) - - def stop(self): - wandb.finish() - - def log_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.store_config(env_cfg, runner_cfg, alg_cfg, policy_cfg) - - def save_model(self, model_path, iter): - wandb.save(model_path, base_path=os.path.dirname(model_path)) - - def save_file(self, path, iter=None): - wandb.save(path, base_path=os.path.dirname(path)) - - """ - Private methods. - """ - - def _map_path(self, path): - if path in self.name_map: - return self.name_map[path] - else: - return path diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py deleted file mode 100644 index ab7c056..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Submodule defining the environment definitions.""" - -from .vec_env import VecEnv - -__all__ = ["VecEnv"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py deleted file mode 100644 index ae471cf..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/env/vec_env.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -from abc import ABC, abstractmethod -from tensordict import TensorDict - - -class VecEnv(ABC): - """Abstract class for a vectorized environment. - - The vectorized environment is a collection of environments that are synchronized. This means that - the same type of action is applied to all environments and the same type of observation is returned from all - environments. - """ - - num_envs: int - """Number of environments.""" - - num_actions: int - """Number of actions.""" - - max_episode_length: int | torch.Tensor - - max_episode_length_s: float - """Maximum episode length. - - The maximum episode length can be a scalar or a tensor. If it is a scalar, it is the same for all environments. - If it is a tensor, it is the maximum episode length for each environment. This is useful for dynamic episode - lengths. - """ - - episode_length_buf: torch.Tensor - """Buffer for current episode lengths.""" - - device: torch.device | str - """Device to use.""" - - cfg: dict | object - """Configuration object.""" - - reset_env_ids: torch.Tensor | None = None - - contact_phase: torch.Tensor | None = None - """ - Operations. - """ - - @abstractmethod - def get_observations(self) -> TensorDict: - """Return the current observations. - - Returns: - observations (TensorDict): Observations from the environment. - """ - raise NotImplementedError - - @abstractmethod - def get_amp_observations(self) -> TensorDict: - """Return the current AMP observations. - - Returns: - observations (TensorDict): Observations from the environment. - """ - raise NotImplementedError - - @abstractmethod - def step(self, actions: torch.Tensor) -> tuple[TensorDict, torch.Tensor, torch.Tensor, dict]: - """Apply input action to the environment. - - Args: - actions (torch.Tensor): Input actions to apply. Shape: (num_envs, num_actions) - - Returns: - observations (TensorDict): Observations from the environment. - rewards (torch.Tensor): Rewards from the environment. Shape: (num_envs,) - dones (torch.Tensor): Done flags from the environment. Shape: (num_envs,) - extras (dict): Extra information from the environment. - - Observations: - - The observations TensorDict usually contains multiple observation groups. The `obs_groups` - dictionary of the runner configuration specifies which observation groups are used for which - purpose, i.e., it maps the available observation groups to observation sets. The observation sets - (keys of the `obs_groups` dictionary) currently used by rsl_rl are: - - - "policy": Specified observation groups are used as input to the actor/student network. - - "critic": Specified observation groups are used as input to the critic network. - - "teacher": Specified observation groups are used as input to the teacher network. - - "rnd_state": Specified observation groups are used as input to the RND network. - - Incomplete or incorrect configurations are handled in the `resolve_obs_groups()` function in - `rsl_rl/utils/utils.py`. - - Extras: - - The extras dictionary includes metrics such as the episode reward, episode length, etc. The following - dictionary keys are used by rsl_rl: - - - "time_outs" (torch.Tensor): Timeouts for the environments. These correspond to terminations that - happen due to time limits and not due to the environment reaching a terminal state. This is useful - for environments that have a fixed episode length. - - - "log" (dict[str, float | torch.Tensor]): Additional information for logging and debugging purposes. - The key should be a string and start with "/" for namespacing. The value can be a scalar or a - tensor. If it is a tensor, the mean of the tensor is used for logging. - """ - raise NotImplementedError - \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py deleted file mode 100644 index 9afe1a2..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Definitions for neural-network components for RL-agents.""" - -from .actor_critic import ActorCritic -from .actor_critic_recurrent import ActorCriticRecurrent -from .rnd import * -from .student_teacher import StudentTeacher -from .student_teacher_recurrent import StudentTeacherRecurrent -from .symmetry import * -from .discriminator_multi import DiscriminatorMulti -__all__ = [ - "ActorCritic", - "ActorCriticRecurrent", - "StudentTeacher", - "StudentTeacherRecurrent", - "DiscriminatorMulti", -] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py deleted file mode 100644 index 0efc36b..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization - - -class ActorCritic(nn.Module): - is_recurrent = False - - def __init__( - self, - obs, - obs_groups, - num_actions, - actor_obs_normalization=False, - critic_obs_normalization=False, - actor_hidden_dims=[256, 256, 256], - critic_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=1.0, - noise_std_type: str = "scalar", - state_dependent_std=False, - **kwargs, - ): - if kwargs: - print( - "ActorCritic.__init__ got unexpected arguments, which will be ignored: " - + str([key for key in kwargs.keys()]) - ) - super().__init__() - - # get the observation dimensions - self.obs_groups = obs_groups - num_actor_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The ActorCritic module only supports 1D observations." - num_actor_obs += obs[obs_group].shape[-1] - num_critic_obs = 0 - for obs_group in obs_groups["critic"]: - assert len(obs[obs_group].shape) == 2, "The ActorCritic module only supports 1D observations." - num_critic_obs += obs[obs_group].shape[-1] - - self.state_dependent_std = state_dependent_std - # actor - if self.state_dependent_std: - self.actor = MLP(num_actor_obs, [2, num_actions], actor_hidden_dims, activation) - else: - self.actor = MLP(num_actor_obs, num_actions, actor_hidden_dims, activation) - # actor observation normalization - self.actor_obs_normalization = actor_obs_normalization - if actor_obs_normalization: - self.actor_obs_normalizer = EmpiricalNormalization(num_actor_obs) - else: - self.actor_obs_normalizer = torch.nn.Identity() - print(f"Actor MLP: {self.actor}") - - # critic - self.critic = MLP(num_critic_obs, 1, critic_hidden_dims, activation) - # critic observation normalization - self.critic_obs_normalization = critic_obs_normalization - if critic_obs_normalization: - self.critic_obs_normalizer = EmpiricalNormalization(num_critic_obs) - else: - self.critic_obs_normalizer = torch.nn.Identity() - print(f"Critic MLP: {self.critic}") - - # Action noise - self.noise_std_type = noise_std_type - if self.state_dependent_std: - torch.nn.init.zeros_(self.actor[-2].weight[num_actions:]) - if self.noise_std_type == "scalar": - torch.nn.init.constant_(self.actor[-2].bias[num_actions:], init_noise_std) - elif self.noise_std_type == "log": - torch.nn.init.constant_( - self.actor[-2].bias[num_actions:], torch.log(torch.tensor(init_noise_std + 1e-7)) - ) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # Action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None): - pass - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - if self.state_dependent_std: - # compute mean and standard deviation - mean_and_std = self.actor(obs) - if self.noise_std_type == "scalar": - mean, std = torch.unbind(mean_and_std, dim=-2) - elif self.noise_std_type == "log": - mean, log_std = torch.unbind(mean_and_std, dim=-2) - std = torch.exp(log_std) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - # compute mean - mean = self.actor(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs, **kwargs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - self.update_distribution(obs) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - return self.actor(obs) - - def evaluate(self, obs, **kwargs): - obs = self.get_critic_obs(obs) - obs = self.critic_obs_normalizer(obs) - return self.critic(obs) - - def get_actor_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_critic_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["critic"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_actions_log_prob(self, actions): - return self.distribution.log_prob(actions).sum(dim=-1) - - def update_normalization(self, obs): - if self.actor_obs_normalization: - actor_obs = self.get_actor_obs(obs) - self.actor_obs_normalizer.update(actor_obs) - if self.critic_obs_normalization: - critic_obs = self.get_critic_obs(obs) - self.critic_obs_normalizer.update(critic_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the actor-critic model. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters (relevant for, e.g., distillation). - """ - - super().load_state_dict(state_dict, strict=strict) - return True # training resumes diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py deleted file mode 100644 index bba46ca..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import warnings -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization, Memory - - -class ActorCriticRecurrent(nn.Module): - is_recurrent = True - - def __init__( - self, - obs, - obs_groups, - num_actions, - actor_obs_normalization=False, - critic_obs_normalization=False, - actor_hidden_dims=[256, 256, 256], - critic_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=1.0, - noise_std_type: str = "scalar", - state_dependent_std=False, - rnn_type="lstm", - rnn_hidden_dim=256, - rnn_num_layers=1, - **kwargs, - ): - if "rnn_hidden_size" in kwargs: - warnings.warn( - "The argument `rnn_hidden_size` is deprecated and will be removed in a future version. " - "Please use `rnn_hidden_dim` instead.", - DeprecationWarning, - ) - if rnn_hidden_dim == 256: # Only override if the new argument is at its default - rnn_hidden_dim = kwargs.pop("rnn_hidden_size") - if kwargs: - print( - "ActorCriticRecurrent.__init__ got unexpected arguments, which will be ignored: " + str(kwargs.keys()), - ) - super().__init__() - - # get the observation dimensions - self.obs_groups = obs_groups - num_actor_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The ActorCriticRecurrent module only supports 1D observations." - num_actor_obs += obs[obs_group].shape[-1] - num_critic_obs = 0 - for obs_group in obs_groups["critic"]: - assert len(obs[obs_group].shape) == 2, "The ActorCriticRecurrent module only supports 1D observations." - num_critic_obs += obs[obs_group].shape[-1] - - self.state_dependent_std = state_dependent_std - # actor - self.memory_a = Memory(num_actor_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - if self.state_dependent_std: - self.actor = MLP(rnn_hidden_dim, [2, num_actions], actor_hidden_dims, activation) - else: - self.actor = MLP(rnn_hidden_dim, num_actions, actor_hidden_dims, activation) - - # actor observation normalization - self.actor_obs_normalization = actor_obs_normalization - if actor_obs_normalization: - self.actor_obs_normalizer = EmpiricalNormalization(num_actor_obs) - else: - self.actor_obs_normalizer = torch.nn.Identity() - print(f"Actor RNN: {self.memory_a}") - print(f"Actor MLP: {self.actor}") - - # critic - self.memory_c = Memory(num_critic_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - self.critic = MLP(rnn_hidden_dim, 1, critic_hidden_dims, activation) - # critic observation normalization - self.critic_obs_normalization = critic_obs_normalization - if critic_obs_normalization: - self.critic_obs_normalizer = EmpiricalNormalization(num_critic_obs) - else: - self.critic_obs_normalizer = torch.nn.Identity() - print(f"Critic RNN: {self.memory_c}") - print(f"Critic MLP: {self.critic}") - - # Action noise - self.noise_std_type = noise_std_type - if self.state_dependent_std: - torch.nn.init.zeros_(self.actor[-2].weight[num_actions:]) - if self.noise_std_type == "scalar": - torch.nn.init.constant_(self.actor[-2].bias[num_actions:], init_noise_std) - elif self.noise_std_type == "log": - torch.nn.init.constant_( - self.actor[-2].bias[num_actions:], torch.log(torch.tensor(init_noise_std + 1e-7)) - ) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # Action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def reset(self, dones=None): - self.memory_a.reset(dones) - self.memory_c.reset(dones) - - def forward(self): - raise NotImplementedError - - def update_distribution(self, obs): - if self.state_dependent_std: - # compute mean and standard deviation - mean_and_std = self.actor(obs) - if self.noise_std_type == "scalar": - mean, std = torch.unbind(mean_and_std, dim=-2) - elif self.noise_std_type == "log": - mean, log_std = torch.unbind(mean_and_std, dim=-2) - std = torch.exp(log_std) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - # compute mean - mean = self.actor(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs, masks=None, hidden_states=None): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - out_mem = self.memory_a(obs, masks, hidden_states).squeeze(0) - self.update_distribution(out_mem) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - out_mem = self.memory_a(obs).squeeze(0) - return self.actor(out_mem) - - def evaluate(self, obs, masks=None, hidden_states=None): - obs = self.get_critic_obs(obs) - obs = self.critic_obs_normalizer(obs) - out_mem = self.memory_c(obs, masks, hidden_states).squeeze(0) - return self.critic(out_mem) - - def get_actor_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_critic_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["critic"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_actions_log_prob(self, actions): - return self.distribution.log_prob(actions).sum(dim=-1) - - def get_hidden_states(self): - return self.memory_a.hidden_states, self.memory_c.hidden_states - - def update_normalization(self, obs): - if self.actor_obs_normalization: - actor_obs = self.get_actor_obs(obs) - self.actor_obs_normalizer.update(actor_obs) - if self.critic_obs_normalization: - critic_obs = self.get_critic_obs(obs) - self.critic_obs_normalizer.update(critic_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the actor-critic model. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters (relevant for, e.g., distillation). - """ - - super().load_state_dict(state_dict, strict=strict) - return True diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py deleted file mode 100644 index 7899d23..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py +++ /dev/null @@ -1,102 +0,0 @@ -import torch -import torch.nn as nn -from torch import autograd -import torch.nn.utils.spectral_norm as spectral_norm - - -class DiscriminatorMulti(nn.Module): - def __init__( - self, state_dim, amp_reward_coef, hidden_layer_sizes, device, - num_frames=2, task_reward_lerp=0.0, use_lerp=True): - super(DiscriminatorMulti, self).__init__() - - self.device = device - self.state_dim = state_dim - self.use_lerp = use_lerp - self.num_frames = num_frames # 存储帧数参数 - - self.amp_reward_coef = amp_reward_coef - amp_layers = [] - - curr_in_dim = state_dim * num_frames - for hidden_dim in hidden_layer_sizes: - amp_layers.append(spectral_norm(nn.Linear(curr_in_dim, hidden_dim))) - amp_layers.append(nn.ReLU()) - curr_in_dim = hidden_dim - self.trunk = nn.Sequential(*amp_layers).to(device) - self.amp_linear = spectral_norm(nn.Linear(hidden_layer_sizes[-1], 1)).to(device) - - self.trunk.train() - self.amp_linear.train() - - self.task_reward_lerp = task_reward_lerp - - def forward(self, x): - h = self.trunk(x) - d = self.amp_linear(h) - return d - - def compute_grad_pen(self, - expert_states, # 改为接收多帧状态列表 - lambda_=10): - # 将多帧状态沿最后一个维度拼接 - expert_data = expert_states.flatten(1) - expert_data.requires_grad = True - - disc = self.amp_linear(self.trunk(expert_data)) - ones = torch.ones(disc.size(), device=disc.device) - grad = autograd.grad( - outputs=disc, inputs=expert_data, - grad_outputs=ones, create_graph=True, - retain_graph=True, only_inputs=True)[0] - - # Enforce that the grad norm approaches 0. - grad_pen = lambda_ * (grad.norm(2, dim=1) - 0).pow(2).mean() - return grad_pen - - - def get_disc_weights(self): - weights = [] - for m in self.trunk.modules(): - if isinstance(m, nn.Linear): - weights.append(torch.flatten(m.weight)) - - weights.append(torch.flatten(self.amp_linear.weight)) - return weights - - def get_disc_logit_weights(self): - return torch.flatten(self.amp_linear.weight) - - def predict_amp_reward( - self, states, # 改为接收多帧状态列表 - task_reward, normalizer=None): - """ - states: torch.Tensor, shape=(num_envs, num_frames, state_dim) - task_reward: torch.Tensor, shape=(num_envs, 1) - """ - # import ipdb; ipdb.set_trace() - with torch.no_grad(): - self.eval() - if normalizer is not None: - # 对每一帧状态进行归一化 - states = normalizer.normalize_torch(states, self.device) - - # 拼接多帧状态 - state_cat = states.flatten(1) - d = self.amp_linear(self.trunk(state_cat)) - disc_reward = self.amp_reward_coef * torch.clamp(1 - (1/4) * torch.square(d - 1), min=0) - - if self.use_lerp: - if self.task_reward_lerp > 0: - reward = self._lerp_reward(disc_reward, task_reward.unsqueeze(-1)) - self.train() - return reward.squeeze(), d, disc_reward.squeeze() * (1.0 - self.task_reward_lerp) - else: - disc_reward *= 0.02 - reward = task_reward.unsqueeze(-1) + disc_reward - self.train() - return reward.squeeze(), d, disc_reward.squeeze() - - def _lerp_reward(self, disc_r, task_r): - r = (1.0 - self.task_reward_lerp) * disc_r + self.task_reward_lerp * task_r - return r \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py deleted file mode 100644 index 8e65c43..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/rnd.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn - -from rsl_rl.networks import MLP, EmpiricalDiscountedVariationNormalization, EmpiricalNormalization - - -class RandomNetworkDistillation(nn.Module): - """Implementation of Random Network Distillation (RND) [1] - - References: - .. [1] Burda, Yuri, et al. "Exploration by random network distillation." arXiv preprint arXiv:1810.12894 (2018). - """ - - def __init__( - self, - num_states: int, - obs_groups: dict, - num_outputs: int, - predictor_hidden_dims: list[int], - target_hidden_dims: list[int], - activation: str = "elu", - weight: float = 0.0, - state_normalization: bool = False, - reward_normalization: bool = False, - device: str = "cpu", - weight_schedule: dict | None = None, - ): - """Initialize the RND module. - - - If :attr:`state_normalization` is True, then the input state is normalized using an Empirical Normalization layer. - - If :attr:`reward_normalization` is True, then the intrinsic reward is normalized using an Empirical Discounted - Variation Normalization layer. - - .. note:: - If the hidden dimensions are -1 in the predictor and target networks configuration, then the number of states - is used as the hidden dimension. - - Args: - num_states: Number of states/inputs to the predictor and target networks. - num_outputs: Number of outputs (embedding size) of the predictor and target networks. - predictor_hidden_dims: List of hidden dimensions of the predictor network. - target_hidden_dims: List of hidden dimensions of the target network. - activation: Activation function. Defaults to "elu". - weight: Scaling factor of the intrinsic reward. Defaults to 0.0. - state_normalization: Whether to normalize the input state. Defaults to False. - reward_normalization: Whether to normalize the intrinsic reward. Defaults to False. - device: Device to use. Defaults to "cpu". - weight_schedule: The type of schedule to use for the RND weight parameter. - Defaults to None, in which case the weight parameter is constant. - It is a dictionary with the following keys: - - - "mode": The type of schedule to use for the RND weight parameter. - - "constant": Constant weight schedule. - - "step": Step weight schedule. - - "linear": Linear weight schedule. - - For the "step" weight schedule, the following parameters are required: - - - "final_step": The step at which the weight parameter is set to the final value. - - "final_value": The final value of the weight parameter. - - For the "linear" weight schedule, the following parameters are required: - - "initial_step": The step at which the weight parameter is set to the initial value. - - "final_step": The step at which the weight parameter is set to the final value. - - "final_value": The final value of the weight parameter. - """ - # initialize parent class - super().__init__() - - # Store parameters - self.num_states = num_states - self.obs_groups = obs_groups - self.num_outputs = num_outputs - self.initial_weight = weight - self.device = device - self.state_normalization = state_normalization - self.reward_normalization = reward_normalization - - # Normalization of input gates - if state_normalization: - self.state_normalizer = EmpiricalNormalization(shape=[self.num_states], until=1.0e8).to(self.device) - else: - self.state_normalizer = torch.nn.Identity() - # Normalization of intrinsic reward - if reward_normalization: - self.reward_normalizer = EmpiricalDiscountedVariationNormalization(shape=[], until=1.0e8).to(self.device) - else: - self.reward_normalizer = torch.nn.Identity() - - # counter for the number of updates - self.update_counter = 0 - - # resolve weight schedule - if weight_schedule is not None: - self.weight_scheduler_params = weight_schedule - self.weight_scheduler = getattr(self, f"_{weight_schedule['mode']}_weight_schedule") - else: - self.weight_scheduler = None - # Create network architecture - self.predictor = MLP(num_states, num_outputs, predictor_hidden_dims, activation).to(self.device) - self.target = MLP(num_states, num_outputs, target_hidden_dims, activation).to(self.device) - - # make target network not trainable - self.target.eval() - - def get_intrinsic_reward(self, obs) -> torch.Tensor: - # Note: the counter is updated number of env steps per learning iteration - self.update_counter += 1 - # Extract the rnd state from the observation - rnd_state = self.get_rnd_state(obs) - rnd_state = self.state_normalizer(rnd_state) - # Obtain the embedding of the rnd state from the target and predictor networks - target_embedding = self.target(rnd_state).detach() - predictor_embedding = self.predictor(rnd_state).detach() - # Compute the intrinsic reward as the distance between the embeddings - intrinsic_reward = torch.linalg.norm(target_embedding - predictor_embedding, dim=1) - # Normalize intrinsic reward - intrinsic_reward = self.reward_normalizer(intrinsic_reward) - - # Check the weight schedule - if self.weight_scheduler is not None: - self.weight = self.weight_scheduler(step=self.update_counter, **self.weight_scheduler_params) - else: - self.weight = self.initial_weight - # Scale intrinsic reward - intrinsic_reward *= self.weight - - return intrinsic_reward - - def forward(self, *args, **kwargs): - raise RuntimeError("Forward method is not implemented. Use get_intrinsic_reward instead.") - - def train(self, mode: bool = True): - # sets module into training mode - self.predictor.train(mode) - if self.state_normalization: - self.state_normalizer.train(mode) - if self.reward_normalization: - self.reward_normalizer.train(mode) - return self - - def eval(self): - return self.train(False) - - def get_rnd_state(self, obs): - obs_list = [] - for obs_group in self.obs_groups["rnd_state"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def update_normalization(self, obs): - # Normalize the state - if self.state_normalization: - rnd_state = self.get_rnd_state(obs) - self.state_normalizer.update(rnd_state) - - """ - Different weight schedules. - """ - - def _constant_weight_schedule(self, step: int, **kwargs): - return self.initial_weight - - def _step_weight_schedule(self, step: int, final_step: int, final_value: float, **kwargs): - return self.initial_weight if step < final_step else final_value - - def _linear_weight_schedule(self, step: int, initial_step: int, final_step: int, final_value: float, **kwargs): - if step < initial_step: - return self.initial_weight - elif step > final_step: - return final_value - else: - return self.initial_weight + (final_value - self.initial_weight) * (step - initial_step) / ( - final_step - initial_step - ) - - -def resolve_rnd_config(alg_cfg, obs, obs_groups, env): - """Resolve the RND configuration. - - Args: - alg_cfg: The algorithm configuration dictionary. - obs: The observation dictionary. - obs_groups: The observation groups dictionary. - env: The environment. - - Returns: - The resolved algorithm configuration dictionary. - """ - # resolve dimension of rnd gated state - if "rnd_cfg" in alg_cfg and alg_cfg["rnd_cfg"] is not None: - # get dimension of rnd gated state - num_rnd_state = 0 - for obs_group in obs_groups["rnd_state"]: - assert len(obs[obs_group].shape) == 2, "The RND module only supports 1D observations." - num_rnd_state += obs[obs_group].shape[-1] - # add rnd gated state to config - alg_cfg["rnd_cfg"]["num_states"] = num_rnd_state - alg_cfg["rnd_cfg"]["obs_groups"] = obs_groups - # scale down the rnd weight with timestep - alg_cfg["rnd_cfg"]["weight"] *= env.unwrapped.step_dt - return alg_cfg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py deleted file mode 100644 index 6bf1380..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization - - -class StudentTeacher(nn.Module): - is_recurrent = False - - def __init__( - self, - obs, - obs_groups, - num_actions, - student_obs_normalization=False, - teacher_obs_normalization=False, - student_hidden_dims=[256, 256, 256], - teacher_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=0.1, - noise_std_type: str = "scalar", - **kwargs, - ): - if kwargs: - print( - "StudentTeacher.__init__ got unexpected arguments, which will be ignored: " - + str([key for key in kwargs.keys()]) - ) - super().__init__() - - self.loaded_teacher = False # indicates if teacher has been loaded - - # get the observation dimensions - self.obs_groups = obs_groups - num_student_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_student_obs += obs[obs_group].shape[-1] - num_teacher_obs = 0 - for obs_group in obs_groups["teacher"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_teacher_obs += obs[obs_group].shape[-1] - - # student - self.student = MLP(num_student_obs, num_actions, student_hidden_dims, activation) - - # student observation normalization - self.student_obs_normalization = student_obs_normalization - if student_obs_normalization: - self.student_obs_normalizer = EmpiricalNormalization(num_student_obs) - else: - self.student_obs_normalizer = torch.nn.Identity() - - print(f"Student MLP: {self.student}") - - # teacher - self.teacher = MLP(num_teacher_obs, num_actions, teacher_hidden_dims, activation) - self.teacher.eval() - - # teacher observation normalization - self.teacher_obs_normalization = teacher_obs_normalization - if teacher_obs_normalization: - self.teacher_obs_normalizer = EmpiricalNormalization(num_teacher_obs) - else: - self.teacher_obs_normalizer = torch.nn.Identity() - - print(f"Teacher MLP: {self.teacher}") - - # action noise - self.noise_std_type = noise_std_type - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None, hidden_states=None): - pass - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - # compute mean - mean = self.student(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - self.update_distribution(obs) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - return self.student(obs) - - def evaluate(self, obs): - obs = self.get_teacher_obs(obs) - obs = self.teacher_obs_normalizer(obs) - with torch.no_grad(): - return self.teacher(obs) - - def get_student_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_teacher_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["teacher"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_hidden_states(self): - return None - - def detach_hidden_states(self, dones=None): - pass - - def train(self, mode=True): - super().train(mode) - # make sure teacher is in eval mode - self.teacher.eval() - self.teacher_obs_normalizer.eval() - - def update_normalization(self, obs): - if self.student_obs_normalization: - student_obs = self.get_student_obs(obs) - self.student_obs_normalizer.update(student_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the student and teacher networks. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters. - """ - - # check if state_dict contains teacher and student or just teacher parameters - if any("actor" in key for key in state_dict.keys()): # loading parameters from rl training - # rename keys to match teacher and remove critic parameters - teacher_state_dict = {} - teacher_obs_normalizer_state_dict = {} - for key, value in state_dict.items(): - if "actor." in key: - teacher_state_dict[key.replace("actor.", "")] = value - if "actor_obs_normalizer." in key: - teacher_obs_normalizer_state_dict[key.replace("actor_obs_normalizer.", "")] = value - self.teacher.load_state_dict(teacher_state_dict, strict=strict) - self.teacher_obs_normalizer.load_state_dict(teacher_obs_normalizer_state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return False # training does not resume - elif any("student" in key for key in state_dict.keys()): # loading parameters from distillation training - super().load_state_dict(state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return True # training resumes - else: - raise ValueError("state_dict does not contain student or teacher parameters") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py deleted file mode 100644 index 964a2dc..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import warnings -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization, Memory - - -class StudentTeacherRecurrent(nn.Module): - is_recurrent = True - - def __init__( - self, - obs, - obs_groups, - num_actions, - student_obs_normalization=False, - teacher_obs_normalization=False, - student_hidden_dims=[256, 256, 256], - teacher_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=0.1, - noise_std_type: str = "scalar", - rnn_type="lstm", - rnn_hidden_dim=256, - rnn_num_layers=1, - teacher_recurrent=False, - **kwargs, - ): - if "rnn_hidden_size" in kwargs: - warnings.warn( - "The argument `rnn_hidden_size` is deprecated and will be removed in a future version. " - "Please use `rnn_hidden_dim` instead.", - DeprecationWarning, - ) - if rnn_hidden_dim == 256: # Only override if the new argument is at its default - rnn_hidden_dim = kwargs.pop("rnn_hidden_size") - if kwargs: - print( - "StudentTeacherRecurrent.__init__ got unexpected arguments, which will be ignored: " - + str(kwargs.keys()), - ) - super().__init__() - - self.loaded_teacher = False # indicates if teacher has been loaded - self.teacher_recurrent = teacher_recurrent # indicates if teacher is recurrent too - - # get the observation dimensions - self.obs_groups = obs_groups - num_student_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_student_obs += obs[obs_group].shape[-1] - num_teacher_obs = 0 - for obs_group in obs_groups["teacher"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_teacher_obs += obs[obs_group].shape[-1] - - # student - self.memory_s = Memory(num_student_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - self.student = MLP(rnn_hidden_dim, num_actions, student_hidden_dims, activation) - - # student observation normalization - self.student_obs_normalization = student_obs_normalization - if student_obs_normalization: - self.student_obs_normalizer = EmpiricalNormalization(num_student_obs) - else: - self.student_obs_normalizer = torch.nn.Identity() - - print(f"Student RNN: {self.memory_s}") - print(f"Student MLP: {self.student}") - - # teacher - if self.teacher_recurrent: - self.memory_t = Memory( - num_teacher_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim - ) - num_teacher_obs = rnn_hidden_dim - self.teacher = MLP(num_teacher_obs, num_actions, teacher_hidden_dims, activation) - - # teacher observation normalization - self.teacher_obs_normalization = teacher_obs_normalization - if teacher_obs_normalization: - self.teacher_obs_normalizer = EmpiricalNormalization(num_teacher_obs) - else: - self.teacher_obs_normalizer = torch.nn.Identity() - - if self.teacher_recurrent: - print(f"Teacher RNN: {self.memory_t}") - print(f"Teacher MLP: {self.teacher}") - - # action noise - self.noise_std_type = noise_std_type - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None, hidden_states=None): - if hidden_states is None: - hidden_states = (None, None) - self.memory_s.reset(dones, hidden_states[0]) - if self.teacher_recurrent: - self.memory_t.reset(dones, hidden_states[1]) - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - # compute mean - mean = self.student(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - out_mem = self.memory_s(obs).squeeze(0) - self.update_distribution(out_mem) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - out_mem = self.memory_s(obs).squeeze(0) - return self.student(out_mem) - - def evaluate(self, obs): - obs = self.get_teacher_obs(obs) - obs = self.teacher_obs_normalizer(obs) - with torch.no_grad(): - if self.teacher_recurrent: - self.memory_t.eval() - obs = self.memory_t(obs).squeeze(0) - return self.teacher(obs) - - def get_student_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_teacher_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["teacher"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_hidden_states(self): - if self.teacher_recurrent: - return self.memory_s.hidden_states, self.memory_t.hidden_states - else: - return self.memory_s.hidden_states, None - - def detach_hidden_states(self, dones=None): - self.memory_s.detach_hidden_states(dones) - if self.teacher_recurrent: - self.memory_t.detach_hidden_states(dones) - - def train(self, mode=True): - super().train(mode) - # make sure teacher is in eval mode - self.teacher.eval() - self.teacher_obs_normalizer.eval() - - def update_normalization(self, obs): - if self.student_obs_normalization: - student_obs = self.get_student_obs(obs) - self.student_obs_normalizer.update(student_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the student and teacher networks. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters. - """ - - # check if state_dict contains teacher and student or just teacher parameters - if any("actor" in key for key in state_dict.keys()): # loading parameters from rl training - # rename keys to match teacher and remove critic parameters - teacher_state_dict = {} - teacher_obs_normalizer_state_dict = {} - for key, value in state_dict.items(): - if "actor." in key: - teacher_state_dict[key.replace("actor.", "")] = value - if "actor_obs_normalizer." in key: - teacher_obs_normalizer_state_dict[key.replace("actor_obs_normalizer.", "")] = value - self.teacher.load_state_dict(teacher_state_dict, strict=strict) - self.teacher_obs_normalizer.load_state_dict(teacher_obs_normalizer_state_dict, strict=strict) - # also load recurrent memory if teacher is recurrent - if self.teacher_recurrent: - memory_t_state_dict = {} - for key, value in state_dict.items(): - if "memory_a." in key: - memory_t_state_dict[key.replace("memory_a.", "")] = value - self.memory_t.load_state_dict(memory_t_state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return False # training does not resume - elif any("student" in key for key in state_dict.keys()): # loading parameters from distillation training - super().load_state_dict(state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return True # training resumes - else: - raise ValueError("state_dict does not contain student or teacher parameters") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py deleted file mode 100644 index b017515..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/modules/symmetry.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - - -def resolve_symmetry_config(alg_cfg, env): - """Resolve the symmetry configuration. - - Args: - alg_cfg: The algorithm configuration dictionary. - env: The environment. - - Returns: - The resolved algorithm configuration dictionary. - """ - - # if using symmetry then pass the environment config object - if "symmetry_cfg" in alg_cfg and alg_cfg["symmetry_cfg"] is not None: - # this is used by the symmetry function for handling different observation terms - alg_cfg["symmetry_cfg"]["_env"] = env - return alg_cfg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py deleted file mode 100644 index c18f487..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Definitions for components of modules.""" - -from .memory import Memory -from .mlp import MLP -from .normalization import EmpiricalDiscountedVariationNormalization, EmpiricalNormalization diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py deleted file mode 100644 index 7577357..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/memory.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch.nn as nn - -from rsl_rl.utils import unpad_trajectories - - -class Memory(nn.Module): - """Memory module for recurrent networks. - - This module is used to store the hidden states of the policy. - Currently only supports GRU and LSTM. - """ - - def __init__(self, input_size, type="lstm", num_layers=1, hidden_size=256): - super().__init__() - # RNN - rnn_cls = nn.GRU if type.lower() == "gru" else nn.LSTM - self.rnn = rnn_cls(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers) - self.hidden_states = None - - def forward(self, input, masks=None, hidden_states=None): - batch_mode = masks is not None - if batch_mode: - # batch mode: needs saved hidden states - if hidden_states is None: - raise ValueError("Hidden states not passed to memory module during policy update") - out, _ = self.rnn(input, hidden_states) - out = unpad_trajectories(out, masks) - else: - # inference/distillation mode: uses hidden states of last step - out, self.hidden_states = self.rnn(input.unsqueeze(0), self.hidden_states) - return out - - def reset(self, dones=None, hidden_states=None): - if dones is None: # reset all hidden states - if hidden_states is None: - self.hidden_states = None - else: - self.hidden_states = hidden_states - elif self.hidden_states is not None: # reset hidden states of done environments - if hidden_states is None: - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - for hidden_state in self.hidden_states: - hidden_state[..., dones == 1, :] = 0.0 - else: - self.hidden_states[..., dones == 1, :] = 0.0 - else: - NotImplementedError( - "Resetting hidden states of done environments with custom hidden states is not implemented" - ) - - def detach_hidden_states(self, dones=None): - if self.hidden_states is not None: - if dones is None: # detach all hidden states - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - self.hidden_states = tuple(hidden_state.detach() for hidden_state in self.hidden_states) - else: - self.hidden_states = self.hidden_states.detach() - else: # detach hidden states of done environments - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - for hidden_state in self.hidden_states: - hidden_state[..., dones == 1, :] = hidden_state[..., dones == 1, :].detach() - else: - self.hidden_states[..., dones == 1, :] = self.hidden_states[..., dones == 1, :].detach() diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py deleted file mode 100644 index e91574e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/mlp.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from functools import reduce - -from rsl_rl.utils import resolve_nn_activation - - -class MLP(nn.Sequential): - """Multi-layer perceptron. - - The MLP network is a sequence of linear layers and activation functions. The - last layer is a linear layer that outputs the desired dimension unless the - last activation function is specified. - - It provides additional conveniences: - - - If the hidden dimensions have a value of ``-1``, the dimension is inferred - from the input dimension. - - If the output dimension is a tuple, the output is reshaped to the desired - shape. - - """ - - def __init__( - self, - input_dim: int, - output_dim: int | tuple[int] | list[int], - hidden_dims: tuple[int] | list[int], - activation: str = "elu", - last_activation: str | None = None, - ): - """Initialize the MLP. - - Args: - input_dim: Dimension of the input. - output_dim: Dimension of the output. - hidden_dims: Dimensions of the hidden layers. A value of ``-1`` indicates - that the dimension should be inferred from the input dimension. - activation: Activation function. Defaults to "elu". - last_activation: Activation function of the last layer. Defaults to None, - in which case the last layer is linear. - """ - super().__init__() - - # resolve activation functions - activation_mod = resolve_nn_activation(activation) - last_activation_mod = resolve_nn_activation(last_activation) if last_activation is not None else None - # resolve number of hidden dims if they are -1 - hidden_dims_processed = [input_dim if dim == -1 else dim for dim in hidden_dims] - - # create layers sequentially - layers = [] - layers.append(nn.Linear(input_dim, hidden_dims_processed[0])) - layers.append(activation_mod) - - for layer_index in range(len(hidden_dims_processed) - 1): - layers.append(nn.Linear(hidden_dims_processed[layer_index], hidden_dims_processed[layer_index + 1])) - layers.append(activation_mod) - - # add last layer - if isinstance(output_dim, int): - layers.append(nn.Linear(hidden_dims_processed[-1], output_dim)) - else: - # compute the total output dimension - total_out_dim = reduce(lambda x, y: x * y, output_dim) - # add a layer to reshape the output to the desired shape - layers.append(nn.Linear(hidden_dims_processed[-1], total_out_dim)) - layers.append(nn.Unflatten(dim=-1, unflattened_size=output_dim)) - - # add last activation function if specified - if last_activation_mod is not None: - layers.append(last_activation_mod) - - # register the layers - for idx, layer in enumerate(layers): - self.add_module(f"{idx}", layer) - - def init_weights(self, scales: float | tuple[float]): - """Initialize the weights of the MLP. - - Args: - scales: Scale factor for the weights. - """ - - def get_scale(idx) -> float: - """Get the scale factor for the weights of the MLP. - - Args: - idx: Index of the layer. - """ - return scales[idx] if isinstance(scales, (list, tuple)) else scales - - # initialize the weights - for idx, module in enumerate(self): - if isinstance(module, nn.Linear): - nn.init.orthogonal_(module.weight, gain=get_scale(idx)) - nn.init.zeros_(module.bias) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Forward pass of the MLP. - - Args: - x: Input tensor. - """ - for layer in self: - x = layer(x) - return x - - def reset(self, dones=None, hidden_states=None): - pass - - def detach_hidden_states(self, dones=None): - pass diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py deleted file mode 100644 index 5fd9692..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/networks/normalization.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -# Copyright (c) 2020 Preferred Networks, Inc. - -from __future__ import annotations - -import torch -from torch import nn - - -class EmpiricalNormalization(nn.Module): - """Normalize mean and variance of values based on empirical values.""" - - def __init__(self, shape, eps=1e-2, until=None): - """Initialize EmpiricalNormalization module. - - Args: - shape (int or tuple of int): Shape of input values except batch axis. - eps (float): Small value for stability. - until (int or None): If this arg is specified, the module learns input values until the sum of batch sizes - exceeds it. - - Note: The normalization parameters are computed over the whole batch, not for each environment separately. - """ - super().__init__() - self.eps = eps - self.until = until - self.register_buffer("_mean", torch.zeros(shape).unsqueeze(0)) - self.register_buffer("_var", torch.ones(shape).unsqueeze(0)) - self.register_buffer("_std", torch.ones(shape).unsqueeze(0)) - self.register_buffer("count", torch.tensor(0, dtype=torch.long)) - - @property - def mean(self): - return self._mean.squeeze(0).clone() - - @property - def std(self): - return self._std.squeeze(0).clone() - - def forward(self, x): - """Normalize mean and variance of values based on empirical values.""" - - return (x - self._mean) / (self._std + self.eps) - - @torch.jit.unused - def update(self, x): - """Learn input values without computing the output values of them""" - - if not self.training: - return - if self.until is not None and self.count >= self.until: - return - - count_x = x.shape[0] - self.count += count_x - rate = count_x / self.count - var_x = torch.var(x, dim=0, unbiased=False, keepdim=True) - mean_x = torch.mean(x, dim=0, keepdim=True) - delta_mean = mean_x - self._mean - self._mean += rate * delta_mean - self._var += rate * (var_x - self._var + delta_mean * (mean_x - self._mean)) - self._std = torch.sqrt(self._var) - - @torch.jit.unused - def inverse(self, y): - """De-normalize values based on empirical values.""" - - return y * (self._std + self.eps) + self._mean - - -class EmpiricalDiscountedVariationNormalization(nn.Module): - """Reward normalization from Pathak's large scale study on PPO. - - Reward normalization. Since the reward function is non-stationary, it is useful to normalize - the scale of the rewards so that the value function can learn quickly. We did this by dividing - the rewards by a running estimate of the standard deviation of the sum of discounted rewards. - """ - - def __init__(self, shape, eps=1e-2, gamma=0.99, until=None): - super().__init__() - - self.emp_norm = EmpiricalNormalization(shape, eps, until) - self.disc_avg = _DiscountedAverage(gamma) - - def forward(self, rew): - if self.training: - # update discounted rewards - avg = self.disc_avg.update(rew) - # update moments from discounted rewards - self.emp_norm.update(avg) - - # normalize rewards with the empirical std - if self.emp_norm._std > 0: - return rew / self.emp_norm._std - else: - return rew - - -""" -Helper class. -""" - - -class _DiscountedAverage: - r"""Discounted average of rewards. - - The discounted average is defined as: - - .. math:: - - \bar{R}_t = \gamma \bar{R}_{t-1} + r_t - - Args: - gamma (float): Discount factor. - """ - - def __init__(self, gamma): - self.avg = None - self.gamma = gamma - - def update(self, rew: torch.Tensor) -> torch.Tensor: - if self.avg is None: - self.avg = rew - else: - self.avg = self.avg * self.gamma + rew - return self.avg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py deleted file mode 100644 index 61f1682..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of runners for environment-agent interaction.""" - -from .on_policy_runner import OnPolicyRunner # isort:skip -from .distillation_runner import DistillationRunner -from .amp_on_policy_runner import AMPOnPolicyRunner - -__all__ = ["OnPolicyRunner", "DistillationRunner", "AMPOnPolicyRunner"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py deleted file mode 100644 index c0b9b9e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py +++ /dev/null @@ -1,521 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import statistics -import time -import torch -import warnings -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import AMP_PPO -from rsl_rl.env import VecEnv -from rsl_rl.modules import ActorCritic, ActorCriticRecurrent,DiscriminatorMulti, resolve_rnd_config, resolve_symmetry_config -from rsl_rl.utils import resolve_obs_groups, store_code_state, Normalizer, G1_AMPLoader - - -class AMPOnPolicyRunner: - """On-policy runner for training and evaluation of actor-critic methods.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - default_sets = ["critic"] - if "rnd_cfg" in self.alg_cfg and self.alg_cfg["rnd_cfg"] is not None: - default_sets.append("rnd_state") - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets) - - self.amp_data = G1_AMPLoader( - device, - time_between_frames=1/50.0, - preload_transitions=True, - num_preload_transitions=train_cfg["amp_num_preload_transitions"], - motion_files=train_cfg["amp_motion_files"], - num_frames=train_cfg['amp_num_frames'] - ) - - self.amp_observation_dim = self.amp_data.observation_dim if self.cfg["amp_num_obs"] == 0 else self.cfg["amp_num_obs"] - self.amp_num_frames = 0 if self.cfg["amp_num_frames"] == 0 else self.cfg["amp_num_frames"] - self.amp_normalizer = Normalizer(self.amp_observation_dim) - self.discriminator = DiscriminatorMulti( - self.amp_observation_dim, - train_cfg["amp_reward_coef"], - train_cfg["amp_discr_hidden_dims"], - device, - train_cfg["amp_num_frames"], - train_cfg["amp_task_reward_lerp"], - train_cfg['use_lerp'], - ).to(self.device) - - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - amp_obs = self.env.get_amp_observations() - amp_obs = amp_obs.to(self.device) - if self.amp_num_frames != 0: - self.amp_obs_frames = torch.zeros(size=(self.env.num_envs, self.amp_num_frames, self.amp_observation_dim), device=self.device) - self.amp_obs_frames = torch.concat((self.amp_obs_frames[:, 1:], amp_obs.unsqueeze(1)), dim=1) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - step_discrewbuffer = deque(maxlen=100) - - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_single_step_disc_rew = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - # create buffers for logging extrinsic and intrinsic rewards - if self.alg.rnd: - erewbuffer = deque(maxlen=100) - irewbuffer = deque(maxlen=100) - cur_ereward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_ireward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs,amp_obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - - next_amp_obs = self.env.get_amp_observations() - next_amp_obs = next_amp_obs.to(self.device) - next_amp_obs_with_term = torch.clone(next_amp_obs) - - reset_env_ids = self.env.reset_env_ids - terminal_amp_states = self.env.get_amp_observations()[reset_env_ids] - next_amp_obs_with_term[reset_env_ids] = terminal_amp_states - self.amp_obs_frames = torch.concat((self.amp_obs_frames[:, 1:], next_amp_obs_with_term.unsqueeze(1)), dim=1) - - amp_reward = torch.zeros(self.env.num_envs, device=obs.device) - - mask = self.env.contact_phase[:, 0] == 1.0 - if mask.any(): - rewards[mask], logit, disc_reward = self.alg.discriminator.predict_amp_reward( - self.amp_obs_frames[mask], rewards[mask], normalizer=self.alg.amp_normalizer - ) - amp_reward[mask] += disc_reward - - # process the step - self.alg.process_env_step(obs, rewards, dones, extras, next_amp_obs_with_term, self.amp_obs_frames) - self.amp_obs_frames[reset_env_ids] = 0 - - amp_obs = torch.clone(next_amp_obs) - # Extract intrinsic rewards (only for logging) - intrinsic_rewards = self.alg.intrinsic_rewards if self.alg.rnd else None - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - if self.alg.rnd: - cur_ereward_sum += rewards - cur_ireward_sum += intrinsic_rewards # type: ignore - cur_reward_sum += rewards + intrinsic_rewards - else: - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - cur_single_step_disc_rew += amp_reward - # Clear data for completed episodes - # -- common - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - to_extend_disc = (cur_single_step_disc_rew[new_ids] / self.env.max_episode_length_s)[:, 0].cpu().numpy() - step_discrewbuffer.extend(to_extend_disc.tolist()) - cur_single_step_disc_rew[new_ids] = 0 - # -- intrinsic and extrinsic rewards - if self.alg.rnd: - erewbuffer.extend(cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist()) - irewbuffer.extend(cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist()) - cur_ereward_sum[new_ids] = 0 - cur_ireward_sum[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # compute returns - self.alg.compute_returns(obs) - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - def log(self, locs: dict, width: int = 80, pad: int = 35): - # Compute the collection size - collection_size = self.num_steps_per_env * self.env.num_envs * self.gpu_world_size - # Update total time-steps and time - self.tot_timesteps += collection_size - self.tot_time += locs["collection_time"] + locs["learn_time"] - iteration_time = locs["collection_time"] + locs["learn_time"] - - # -- Episode info - ep_string = "" - if locs["ep_infos"]: - for key in locs["ep_infos"][0]: - infotensor = torch.tensor([], device=self.device) - for ep_info in locs["ep_infos"]: - # handle scalar and zero dimensional tensor infos - if key not in ep_info: - continue - if not isinstance(ep_info[key], torch.Tensor): - ep_info[key] = torch.Tensor([ep_info[key]]) - if len(ep_info[key].shape) == 0: - ep_info[key] = ep_info[key].unsqueeze(0) - infotensor = torch.cat((infotensor, ep_info[key].to(self.device))) - value = torch.mean(infotensor) - # log to logger and terminal - if "/" in key: - self.writer.add_scalar(key, value, locs["it"]) - ep_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - else: - self.writer.add_scalar("Episode/" + key, value, locs["it"]) - ep_string += f"""{f'Mean episode {key}:':>{pad}} {value:.4f}\n""" - - mean_std = self.alg.policy.action_std.mean() - fps = int(collection_size / (locs["collection_time"] + locs["learn_time"])) - - # -- Losses - for key, value in locs["loss_dict"].items(): - self.writer.add_scalar(f"Loss/{key}", value, locs["it"]) - self.writer.add_scalar("Loss/learning_rate", self.alg.learning_rate, locs["it"]) - - # -- Policy - self.writer.add_scalar("Policy/mean_noise_std", mean_std.item(), locs["it"]) - - # -- Performance - self.writer.add_scalar("Perf/total_fps", fps, locs["it"]) - self.writer.add_scalar("Perf/collection time", locs["collection_time"], locs["it"]) - self.writer.add_scalar("Perf/learning_time", locs["learn_time"], locs["it"]) - - # -- Training - if len(locs["rewbuffer"]) > 0: - # separate logging for intrinsic and extrinsic rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(locs["erewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(locs["irewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/weight", self.alg.rnd.weight, locs["it"]) - # everything else - self.writer.add_scalar("Train/mean_reward", statistics.mean(locs["rewbuffer"]), locs["it"]) - self.writer.add_scalar("Train/mean_episode_length", statistics.mean(locs["lenbuffer"]), locs["it"]) - self.writer.add_scalar('Train/mean_step_disc_reward', statistics.mean(locs['step_discrewbuffer']), locs['it']) - if self.logger_type != "wandb": # wandb does not support non-integer x-axis logging - self.writer.add_scalar("Train/mean_reward/time", statistics.mean(locs["rewbuffer"]), self.tot_time) - self.writer.add_scalar( - "Train/mean_episode_length/time", statistics.mean(locs["lenbuffer"]), self.tot_time - ) - - str = f" \033[1m Learning iteration {locs['it']}/{locs['tot_iter']} \033[0m " - - if len(locs["rewbuffer"]) > 0: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - f"""{'Step disc reward:':>{pad}} {statistics.mean(locs['step_discrewbuffer']):.2f}\n""" - ) - # -- Losses - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'Mean {key} loss:':>{pad}} {value:.4f}\n""" - # -- Rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - log_string += ( - f"""{'Mean extrinsic reward:':>{pad}} {statistics.mean(locs['erewbuffer']):.2f}\n""" - f"""{'Mean intrinsic reward:':>{pad}} {statistics.mean(locs['irewbuffer']):.2f}\n""" - ) - log_string += f"""{'Mean reward:':>{pad}} {statistics.mean(locs['rewbuffer']):.2f}\n""" - # -- episode info - log_string += f"""{'Mean episode length:':>{pad}} {statistics.mean(locs['lenbuffer']):.2f}\n""" - else: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - - log_string += ep_string - log_string += ( - f"""{'-' * width}\n""" - f"""{'Total timesteps:':>{pad}} {self.tot_timesteps}\n""" - f"""{'Iteration time:':>{pad}} {iteration_time:.2f}s\n""" - f"""{'Time elapsed:':>{pad}} {time.strftime("%H:%M:%S", time.gmtime(self.tot_time))}\n""" - f"""{'ETA:':>{pad}} {time.strftime( - "%H:%M:%S", - time.gmtime( - self.tot_time / (locs['it'] - locs['start_iter'] + 1) - * (locs['start_iter'] + locs['num_learning_iterations'] - locs['it']) - ) - )}\n""" - ) - print(log_string) - - def save(self, path: str, infos=None): - # -- Save model - saved_dict = { - "model_state_dict": self.alg.policy.state_dict(), - "optimizer_state_dict": self.alg.optimizer.state_dict(), - "iter": self.current_learning_iteration, - "infos": infos, - } - # -- Save RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - saved_dict["rnd_state_dict"] = self.alg.rnd.state_dict() - saved_dict["rnd_optimizer_state_dict"] = self.alg.rnd_optimizer.state_dict() - torch.save(saved_dict, path) - - # upload model to external logging service - if self.logger_type in ["neptune", "wandb"] and not self.disable_logs: - self.writer.save_model(path, self.current_learning_iteration) - - def load(self, path: str, load_optimizer: bool = True, map_location: str | None = None): - loaded_dict = torch.load(path, weights_only=False, map_location=map_location) - # -- Load model - resumed_training = self.alg.policy.load_state_dict(loaded_dict["model_state_dict"]) - # -- Load RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.load_state_dict(loaded_dict["rnd_state_dict"]) - # -- load optimizer if used - if load_optimizer and resumed_training: - # -- algorithm optimizer - self.alg.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"]) - # -- RND optimizer if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd_optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"]) - # -- load current learning iteration - if resumed_training: - self.current_learning_iteration = loaded_dict["iter"] - return loaded_dict["infos"] - - def get_inference_policy(self, device=None): - self.eval_mode() # switch to evaluation mode (dropout for example) - if device is not None: - self.alg.policy.to(device) - return self.alg.policy.act_inference - - def train_mode(self): - # -- PPO - self.alg.policy.train() - self.alg.discriminator.train() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.train() - - def eval_mode(self): - # -- PPO - self.alg.policy.eval() - self.alg.discriminator.eval() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.eval() - - def add_git_repo_to_log(self, repo_file_path): - self.git_status_repos.append(repo_file_path) - - """ - Helper functions. - """ - - def _configure_multi_gpu(self): - """Configure multi-gpu training.""" - # check if distributed training is enabled - self.gpu_world_size = int(os.getenv("WORLD_SIZE", "1")) - self.is_distributed = self.gpu_world_size > 1 - - # if not distributed training, set local and global rank to 0 and return - if not self.is_distributed: - self.gpu_local_rank = 0 - self.gpu_global_rank = 0 - self.multi_gpu_cfg = None - return - - # get rank and world size - self.gpu_local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.gpu_global_rank = int(os.getenv("RANK", "0")) - - # make a configuration dictionary - self.multi_gpu_cfg = { - "global_rank": self.gpu_global_rank, # rank of the main process - "local_rank": self.gpu_local_rank, # rank of the current process - "world_size": self.gpu_world_size, # total number of processes - } - - # check if user has device specified for local rank - if self.device != f"cuda:{self.gpu_local_rank}": - raise ValueError( - f"Device '{self.device}' does not match expected device for local rank '{self.gpu_local_rank}'." - ) - # validate multi-gpu configuration - if self.gpu_local_rank >= self.gpu_world_size: - raise ValueError( - f"Local rank '{self.gpu_local_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - if self.gpu_global_rank >= self.gpu_world_size: - raise ValueError( - f"Global rank '{self.gpu_global_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - - # initialize torch distributed - torch.distributed.init_process_group(backend="nccl", rank=self.gpu_global_rank, world_size=self.gpu_world_size) - # set device to the local rank - torch.cuda.set_device(self.gpu_local_rank) - - def _construct_algorithm(self, obs) -> AMP_PPO: - """Construct the actor-critic algorithm.""" - # resolve RND config - self.alg_cfg = resolve_rnd_config(self.alg_cfg, obs, self.cfg["obs_groups"], self.env) - - # resolve symmetry config - self.alg_cfg = resolve_symmetry_config(self.alg_cfg, self.env) - - # resolve deprecated normalization config - if self.cfg.get("empirical_normalization") is not None: - warnings.warn( - "The `empirical_normalization` parameter is deprecated. Please set `actor_obs_normalization` and " - "`critic_obs_normalization` as part of the `policy` configuration instead.", - DeprecationWarning, - ) - if self.policy_cfg.get("actor_obs_normalization") is None: - self.policy_cfg["actor_obs_normalization"] = self.cfg["empirical_normalization"] - if self.policy_cfg.get("critic_obs_normalization") is None: - self.policy_cfg["critic_obs_normalization"] = self.cfg["empirical_normalization"] - - # initialize the actor-critic - actor_critic_class = eval(self.policy_cfg.pop("class_name")) - actor_critic: ActorCritic | ActorCriticRecurrent = actor_critic_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - - alg: AMP_PPO = alg_class(actor_critic, self.discriminator, self.amp_data, self.amp_normalizer, self.amp_num_frames, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg) - - # initialize the storage - alg.init_storage( - "rl", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg - - def _prepare_logging_writer(self): - """Prepares the logging writers.""" - if self.log_dir is not None and self.writer is None and not self.disable_logs: - # Launch either Tensorboard or Neptune & Tensorboard summary writer(s), default: Tensorboard. - self.logger_type = self.cfg.get("logger", "tensorboard") - self.logger_type = self.logger_type.lower() - - if self.logger_type == "neptune": - from rsl_rl.utils.neptune_utils import NeptuneSummaryWriter - - self.writer = NeptuneSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "wandb": - from rsl_rl.utils.wandb_utils import WandbSummaryWriter - - self.writer = WandbSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "tensorboard": - from torch.utils.tensorboard import SummaryWriter - - self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) - else: - raise ValueError("Logger type not found. Please choose 'neptune', 'wandb' or 'tensorboard'.") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py deleted file mode 100644 index 9cc6a8b..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import time -import torch -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import Distillation -from rsl_rl.env import VecEnv -from rsl_rl.modules import StudentTeacher, StudentTeacherRecurrent -from rsl_rl.runners import OnPolicyRunner -from rsl_rl.utils import resolve_obs_groups, store_code_state - - -class DistillationRunner(OnPolicyRunner): - """On-policy runner for training and evaluation of teacher-student training.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets=["teacher"]) - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - # check if teacher is loaded - if not self.alg.policy.loaded_teacher: - raise ValueError("Teacher model parameters not loaded. Please load a teacher model to distill.") - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - # process the step - self.alg.process_env_step(obs, rewards, dones, extras) - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - # Clear data for completed episodes - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - """ - Helper methods. - """ - - def _construct_algorithm(self, obs) -> Distillation: - """Construct the distillation algorithm.""" - # initialize the actor-critic - student_teacher_class = eval(self.policy_cfg.pop("class_name")) - student_teacher: StudentTeacher | StudentTeacherRecurrent = student_teacher_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - alg: Distillation = alg_class( - student_teacher, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg - ) - - # initialize the storage - alg.init_storage( - "distillation", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py deleted file mode 100644 index 36f11f3..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py +++ /dev/null @@ -1,460 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import statistics -import time -import torch -import warnings -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import PPO -from rsl_rl.env import VecEnv -from rsl_rl.modules import ActorCritic, ActorCriticRecurrent, resolve_rnd_config, resolve_symmetry_config -from rsl_rl.utils import resolve_obs_groups, store_code_state - - -class OnPolicyRunner: - """On-policy runner for training and evaluation of actor-critic methods.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - default_sets = ["critic"] - if "rnd_cfg" in self.alg_cfg and self.alg_cfg["rnd_cfg"] is not None: - default_sets.append("rnd_state") - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets) - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # create buffers for logging extrinsic and intrinsic rewards - if self.alg.rnd: - erewbuffer = deque(maxlen=100) - irewbuffer = deque(maxlen=100) - cur_ereward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_ireward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - # process the step - self.alg.process_env_step(obs, rewards, dones, extras) - # Extract intrinsic rewards (only for logging) - intrinsic_rewards = self.alg.intrinsic_rewards if self.alg.rnd else None - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - if self.alg.rnd: - cur_ereward_sum += rewards - cur_ireward_sum += intrinsic_rewards # type: ignore - cur_reward_sum += rewards + intrinsic_rewards - else: - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - # Clear data for completed episodes - # -- common - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - # -- intrinsic and extrinsic rewards - if self.alg.rnd: - erewbuffer.extend(cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist()) - irewbuffer.extend(cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist()) - cur_ereward_sum[new_ids] = 0 - cur_ireward_sum[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # compute returns - self.alg.compute_returns(obs) - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - def log(self, locs: dict, width: int = 80, pad: int = 35): - # Compute the collection size - collection_size = self.num_steps_per_env * self.env.num_envs * self.gpu_world_size - # Update total time-steps and time - self.tot_timesteps += collection_size - self.tot_time += locs["collection_time"] + locs["learn_time"] - iteration_time = locs["collection_time"] + locs["learn_time"] - - # -- Episode info - ep_string = "" - if locs["ep_infos"]: - for key in locs["ep_infos"][0]: - infotensor = torch.tensor([], device=self.device) - for ep_info in locs["ep_infos"]: - # handle scalar and zero dimensional tensor infos - if key not in ep_info: - continue - if not isinstance(ep_info[key], torch.Tensor): - ep_info[key] = torch.Tensor([ep_info[key]]) - if len(ep_info[key].shape) == 0: - ep_info[key] = ep_info[key].unsqueeze(0) - infotensor = torch.cat((infotensor, ep_info[key].to(self.device))) - value = torch.mean(infotensor) - # log to logger and terminal - if "/" in key: - self.writer.add_scalar(key, value, locs["it"]) - ep_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - else: - self.writer.add_scalar("Episode/" + key, value, locs["it"]) - ep_string += f"""{f'Mean episode {key}:':>{pad}} {value:.4f}\n""" - - mean_std = self.alg.policy.action_std.mean() - fps = int(collection_size / (locs["collection_time"] + locs["learn_time"])) - - # -- Losses - for key, value in locs["loss_dict"].items(): - self.writer.add_scalar(f"Loss/{key}", value, locs["it"]) - self.writer.add_scalar("Loss/learning_rate", self.alg.learning_rate, locs["it"]) - - # -- Policy - self.writer.add_scalar("Policy/mean_noise_std", mean_std.item(), locs["it"]) - - # -- Performance - self.writer.add_scalar("Perf/total_fps", fps, locs["it"]) - self.writer.add_scalar("Perf/collection time", locs["collection_time"], locs["it"]) - self.writer.add_scalar("Perf/learning_time", locs["learn_time"], locs["it"]) - - # -- Training - if len(locs["rewbuffer"]) > 0: - # separate logging for intrinsic and extrinsic rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(locs["erewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(locs["irewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/weight", self.alg.rnd.weight, locs["it"]) - # everything else - self.writer.add_scalar("Train/mean_reward", statistics.mean(locs["rewbuffer"]), locs["it"]) - self.writer.add_scalar("Train/mean_episode_length", statistics.mean(locs["lenbuffer"]), locs["it"]) - if self.logger_type != "wandb": # wandb does not support non-integer x-axis logging - self.writer.add_scalar("Train/mean_reward/time", statistics.mean(locs["rewbuffer"]), self.tot_time) - self.writer.add_scalar( - "Train/mean_episode_length/time", statistics.mean(locs["lenbuffer"]), self.tot_time - ) - - str = f" \033[1m Learning iteration {locs['it']}/{locs['tot_iter']} \033[0m " - - if len(locs["rewbuffer"]) > 0: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - # -- Losses - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'Mean {key} loss:':>{pad}} {value:.4f}\n""" - # -- Rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - log_string += ( - f"""{'Mean extrinsic reward:':>{pad}} {statistics.mean(locs['erewbuffer']):.2f}\n""" - f"""{'Mean intrinsic reward:':>{pad}} {statistics.mean(locs['irewbuffer']):.2f}\n""" - ) - log_string += f"""{'Mean reward:':>{pad}} {statistics.mean(locs['rewbuffer']):.2f}\n""" - # -- episode info - log_string += f"""{'Mean episode length:':>{pad}} {statistics.mean(locs['lenbuffer']):.2f}\n""" - else: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - - log_string += ep_string - log_string += ( - f"""{'-' * width}\n""" - f"""{'Total timesteps:':>{pad}} {self.tot_timesteps}\n""" - f"""{'Iteration time:':>{pad}} {iteration_time:.2f}s\n""" - f"""{'Time elapsed:':>{pad}} {time.strftime("%H:%M:%S", time.gmtime(self.tot_time))}\n""" - f"""{'ETA:':>{pad}} {time.strftime( - "%H:%M:%S", - time.gmtime( - self.tot_time / (locs['it'] - locs['start_iter'] + 1) - * (locs['start_iter'] + locs['num_learning_iterations'] - locs['it']) - ) - )}\n""" - ) - print(log_string) - - def save(self, path: str, infos=None): - # -- Save model - saved_dict = { - "model_state_dict": self.alg.policy.state_dict(), - "optimizer_state_dict": self.alg.optimizer.state_dict(), - "iter": self.current_learning_iteration, - "infos": infos, - } - # -- Save RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - saved_dict["rnd_state_dict"] = self.alg.rnd.state_dict() - saved_dict["rnd_optimizer_state_dict"] = self.alg.rnd_optimizer.state_dict() - torch.save(saved_dict, path) - - # upload model to external logging service - if self.logger_type in ["neptune", "wandb"] and not self.disable_logs: - self.writer.save_model(path, self.current_learning_iteration) - - def load(self, path: str, load_optimizer: bool = True, map_location: str | None = None): - loaded_dict = torch.load(path, weights_only=False, map_location=map_location) - # -- Load model - resumed_training = self.alg.policy.load_state_dict(loaded_dict["model_state_dict"]) - # -- Load RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.load_state_dict(loaded_dict["rnd_state_dict"]) - # -- load optimizer if used - if load_optimizer and resumed_training: - # -- algorithm optimizer - self.alg.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"]) - # -- RND optimizer if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd_optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"]) - # -- load current learning iteration - if resumed_training: - self.current_learning_iteration = loaded_dict["iter"] - return loaded_dict["infos"] - - def get_inference_policy(self, device=None): - self.eval_mode() # switch to evaluation mode (dropout for example) - if device is not None: - self.alg.policy.to(device) - return self.alg.policy.act_inference - - def train_mode(self): - # -- PPO - self.alg.policy.train() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.train() - - def eval_mode(self): - # -- PPO - self.alg.policy.eval() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.eval() - - def add_git_repo_to_log(self, repo_file_path): - self.git_status_repos.append(repo_file_path) - - """ - Helper functions. - """ - - def _configure_multi_gpu(self): - """Configure multi-gpu training.""" - # check if distributed training is enabled - self.gpu_world_size = int(os.getenv("WORLD_SIZE", "1")) - self.is_distributed = self.gpu_world_size > 1 - - # if not distributed training, set local and global rank to 0 and return - if not self.is_distributed: - self.gpu_local_rank = 0 - self.gpu_global_rank = 0 - self.multi_gpu_cfg = None - return - - # get rank and world size - self.gpu_local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.gpu_global_rank = int(os.getenv("RANK", "0")) - - # make a configuration dictionary - self.multi_gpu_cfg = { - "global_rank": self.gpu_global_rank, # rank of the main process - "local_rank": self.gpu_local_rank, # rank of the current process - "world_size": self.gpu_world_size, # total number of processes - } - - # check if user has device specified for local rank - if self.device != f"cuda:{self.gpu_local_rank}": - raise ValueError( - f"Device '{self.device}' does not match expected device for local rank '{self.gpu_local_rank}'." - ) - # validate multi-gpu configuration - if self.gpu_local_rank >= self.gpu_world_size: - raise ValueError( - f"Local rank '{self.gpu_local_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - if self.gpu_global_rank >= self.gpu_world_size: - raise ValueError( - f"Global rank '{self.gpu_global_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - - # initialize torch distributed - torch.distributed.init_process_group(backend="nccl", rank=self.gpu_global_rank, world_size=self.gpu_world_size) - # set device to the local rank - torch.cuda.set_device(self.gpu_local_rank) - - def _construct_algorithm(self, obs) -> PPO: - """Construct the actor-critic algorithm.""" - # resolve RND config - self.alg_cfg = resolve_rnd_config(self.alg_cfg, obs, self.cfg["obs_groups"], self.env) - - # resolve symmetry config - self.alg_cfg = resolve_symmetry_config(self.alg_cfg, self.env) - - # resolve deprecated normalization config - if self.cfg.get("empirical_normalization") is not None: - warnings.warn( - "The `empirical_normalization` parameter is deprecated. Please set `actor_obs_normalization` and " - "`critic_obs_normalization` as part of the `policy` configuration instead.", - DeprecationWarning, - ) - if self.policy_cfg.get("actor_obs_normalization") is None: - self.policy_cfg["actor_obs_normalization"] = self.cfg["empirical_normalization"] - if self.policy_cfg.get("critic_obs_normalization") is None: - self.policy_cfg["critic_obs_normalization"] = self.cfg["empirical_normalization"] - - # initialize the actor-critic - actor_critic_class = eval(self.policy_cfg.pop("class_name")) - actor_critic: ActorCritic | ActorCriticRecurrent = actor_critic_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - alg: PPO = alg_class(actor_critic, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg) - - # initialize the storage - alg.init_storage( - "rl", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg - - def _prepare_logging_writer(self): - """Prepares the logging writers.""" - if self.log_dir is not None and self.writer is None and not self.disable_logs: - # Launch either Tensorboard or Neptune & Tensorboard summary writer(s), default: Tensorboard. - self.logger_type = self.cfg.get("logger", "tensorboard") - self.logger_type = self.logger_type.lower() - - if self.logger_type == "neptune": - from rsl_rl.utils.neptune_utils import NeptuneSummaryWriter - - self.writer = NeptuneSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "wandb": - from rsl_rl.utils.wandb_utils import WandbSummaryWriter - - self.writer = WandbSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "tensorboard": - from torch.utils.tensorboard import SummaryWriter - - self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) - else: - raise ValueError("Logger type not found. Please choose 'neptune', 'wandb' or 'tensorboard'.") diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py deleted file mode 100644 index 1624330..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of transitions storage for RL-agent.""" - -from .rollout_storage import RolloutStorage -from .replay_buffer_multi import ReplayBufferMulti -__all__ = ["RolloutStorage", "ReplayBufferMulti"] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py deleted file mode 100644 index 6462b8e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -import numpy as np - - -class ReplayBufferMulti: - """Fixed-size buffer to store experience tuples.""" - - def __init__(self, obs_dim, buffer_size, num_amp_frames, device): - """Initialize a ReplayBuffer object. - Arguments: - buffer_size (int): maximum size of buffer - """ - self.states = torch.zeros(buffer_size, num_amp_frames, obs_dim).to(device) - self.num_amp_frames = num_amp_frames - self.buffer_size = buffer_size - self.device = device - - self.step = 0 - self.num_samples = 0 - - def insert(self, states): - """Add new states to memory.""" - num_states = states.shape[0] - start_idx = self.step - end_idx = self.step + num_states - if end_idx > self.buffer_size: - self.states[self.step:self.buffer_size] = states[:self.buffer_size - self.step] - self.states[:end_idx - self.buffer_size] = states[self.buffer_size - self.step:] - else: - self.states[start_idx:end_idx] = states - - self.num_samples = min(self.buffer_size, max(end_idx, self.num_samples)) - self.step = (self.step + num_states) % self.buffer_size - - def feed_forward_generator(self, num_mini_batch, mini_batch_size): - for _ in range(num_mini_batch): - sample_idxs = np.random.choice(self.num_samples, size=mini_batch_size) - yield (self.states[sample_idxs].to(self.device)) diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py deleted file mode 100644 index e9309b3..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -from tensordict import TensorDict - -from rsl_rl.utils import split_and_pad_trajectories - - -class RolloutStorage: - class Transition: - def __init__(self): - self.observations = None - self.actions = None - self.privileged_actions = None - self.rewards = None - self.dones = None - self.values = None - self.actions_log_prob = None - self.action_mean = None - self.action_sigma = None - self.hidden_states = None - - def clear(self): - self.__init__() - - def __init__( - self, - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - device="cpu", - ): - # store inputs - self.training_type = training_type - self.device = device - self.num_transitions_per_env = num_transitions_per_env - self.num_envs = num_envs - self.actions_shape = actions_shape - - # Core - self.observations = TensorDict( - {key: torch.zeros(num_transitions_per_env, *value.shape, device=device) for key, value in obs.items()}, - batch_size=[num_transitions_per_env, num_envs], - device=self.device, - ) - self.rewards = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.actions = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.dones = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device).byte() - - # for distillation - if training_type == "distillation": - self.privileged_actions = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - - # for reinforcement learning - if training_type == "rl": - self.values = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.actions_log_prob = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.mu = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.sigma = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.returns = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.advantages = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - - # For RNN networks - self.saved_hidden_states_a = None - self.saved_hidden_states_c = None - - # counter for the number of transitions stored - self.step = 0 - - def add_transitions(self, transition: Transition): - # check if the transition is valid - if self.step >= self.num_transitions_per_env: - raise OverflowError("Rollout buffer overflow! You should call clear() before adding new transitions.") - - # Core - self.observations[self.step].copy_(transition.observations) - self.actions[self.step].copy_(transition.actions) - self.rewards[self.step].copy_(transition.rewards.view(-1, 1)) - self.dones[self.step].copy_(transition.dones.view(-1, 1)) - - # for distillation - if self.training_type == "distillation": - self.privileged_actions[self.step].copy_(transition.privileged_actions) - - # for reinforcement learning - if self.training_type == "rl": - self.values[self.step].copy_(transition.values) - self.actions_log_prob[self.step].copy_(transition.actions_log_prob.view(-1, 1)) - self.mu[self.step].copy_(transition.action_mean) - self.sigma[self.step].copy_(transition.action_sigma) - - # For RNN networks - self._save_hidden_states(transition.hidden_states) - - # increment the counter - self.step += 1 - - def _save_hidden_states(self, hidden_states): - if hidden_states is None or hidden_states == (None, None): - return - # make a tuple out of GRU hidden state sto match the LSTM format - hid_a = hidden_states[0] if isinstance(hidden_states[0], tuple) else (hidden_states[0],) - hid_c = hidden_states[1] if isinstance(hidden_states[1], tuple) else (hidden_states[1],) - # initialize if needed - if self.saved_hidden_states_a is None: - self.saved_hidden_states_a = [ - torch.zeros(self.observations.shape[0], *hid_a[i].shape, device=self.device) for i in range(len(hid_a)) - ] - self.saved_hidden_states_c = [ - torch.zeros(self.observations.shape[0], *hid_c[i].shape, device=self.device) for i in range(len(hid_c)) - ] - # copy the states - for i in range(len(hid_a)): - self.saved_hidden_states_a[i][self.step].copy_(hid_a[i]) - self.saved_hidden_states_c[i][self.step].copy_(hid_c[i]) - - def clear(self): - self.step = 0 - - def compute_returns(self, last_values, gamma, lam, normalize_advantage: bool = True): - advantage = 0 - for step in reversed(range(self.num_transitions_per_env)): - # if we are at the last step, bootstrap the return value - if step == self.num_transitions_per_env - 1: - next_values = last_values - else: - next_values = self.values[step + 1] - # 1 if we are not in a terminal state, 0 otherwise - next_is_not_terminal = 1.0 - self.dones[step].float() - # TD error: r_t + gamma * V(s_{t+1}) - V(s_t) - delta = self.rewards[step] + next_is_not_terminal * gamma * next_values - self.values[step] - # Advantage: A(s_t, a_t) = delta_t + gamma * lambda * A(s_{t+1}, a_{t+1}) - advantage = delta + next_is_not_terminal * gamma * lam * advantage - # Return: R_t = A(s_t, a_t) + V(s_t) - self.returns[step] = advantage + self.values[step] - - # Compute the advantages - self.advantages = self.returns - self.values - # Normalize the advantages if flag is set - # This is to prevent double normalization (i.e. if per minibatch normalization is used) - if normalize_advantage: - self.advantages = (self.advantages - self.advantages.mean()) / (self.advantages.std() + 1e-8) - - # for distillation - def generator(self): - if self.training_type != "distillation": - raise ValueError("This function is only available for distillation training.") - - for i in range(self.num_transitions_per_env): - yield self.observations[i], self.actions[i], self.privileged_actions[i], self.dones[i] - - # for reinforcement learning with feedforward networks - def mini_batch_generator(self, num_mini_batches, num_epochs=8): - if self.training_type != "rl": - raise ValueError("This function is only available for reinforcement learning training.") - batch_size = self.num_envs * self.num_transitions_per_env - mini_batch_size = batch_size // num_mini_batches - indices = torch.randperm(num_mini_batches * mini_batch_size, requires_grad=False, device=self.device) - - # Core - observations = self.observations.flatten(0, 1) - actions = self.actions.flatten(0, 1) - values = self.values.flatten(0, 1) - returns = self.returns.flatten(0, 1) - - # For PPO - old_actions_log_prob = self.actions_log_prob.flatten(0, 1) - advantages = self.advantages.flatten(0, 1) - old_mu = self.mu.flatten(0, 1) - old_sigma = self.sigma.flatten(0, 1) - - for epoch in range(num_epochs): - for i in range(num_mini_batches): - # Select the indices for the mini-batch - start = i * mini_batch_size - end = (i + 1) * mini_batch_size - batch_idx = indices[start:end] - - # Create the mini-batch - # -- Core - obs_batch = observations[batch_idx] - actions_batch = actions[batch_idx] - - # -- For PPO - target_values_batch = values[batch_idx] - returns_batch = returns[batch_idx] - old_actions_log_prob_batch = old_actions_log_prob[batch_idx] - advantages_batch = advantages[batch_idx] - old_mu_batch = old_mu[batch_idx] - old_sigma_batch = old_sigma[batch_idx] - - # yield the mini-batch - yield obs_batch, actions_batch, target_values_batch, advantages_batch, returns_batch, old_actions_log_prob_batch, old_mu_batch, old_sigma_batch, ( - None, - None, - ), None - - # for reinfrocement learning with recurrent networks - def recurrent_mini_batch_generator(self, num_mini_batches, num_epochs=8): - if self.training_type != "rl": - raise ValueError("This function is only available for reinforcement learning training.") - padded_obs_trajectories, trajectory_masks = split_and_pad_trajectories(self.observations, self.dones) - - mini_batch_size = self.num_envs // num_mini_batches - for ep in range(num_epochs): - first_traj = 0 - for i in range(num_mini_batches): - start = i * mini_batch_size - stop = (i + 1) * mini_batch_size - - dones = self.dones.squeeze(-1) - last_was_done = torch.zeros_like(dones, dtype=torch.bool) - last_was_done[1:] = dones[:-1] - last_was_done[0] = True - trajectories_batch_size = torch.sum(last_was_done[:, start:stop]) - last_traj = first_traj + trajectories_batch_size - - masks_batch = trajectory_masks[:, first_traj:last_traj] - obs_batch = padded_obs_trajectories[:, first_traj:last_traj] - actions_batch = self.actions[:, start:stop] - old_mu_batch = self.mu[:, start:stop] - old_sigma_batch = self.sigma[:, start:stop] - returns_batch = self.returns[:, start:stop] - advantages_batch = self.advantages[:, start:stop] - values_batch = self.values[:, start:stop] - old_actions_log_prob_batch = self.actions_log_prob[:, start:stop] - - # reshape to [num_envs, time, num layers, hidden dim] (original shape: [time, num_layers, num_envs, hidden_dim]) - # then take only time steps after dones (flattens num envs and time dimensions), - # take a batch of trajectories and finally reshape back to [num_layers, batch, hidden_dim] - last_was_done = last_was_done.permute(1, 0) - hid_a_batch = [ - saved_hidden_states.permute(2, 0, 1, 3)[last_was_done][first_traj:last_traj] - .transpose(1, 0) - .contiguous() - for saved_hidden_states in self.saved_hidden_states_a - ] - hid_c_batch = [ - saved_hidden_states.permute(2, 0, 1, 3)[last_was_done][first_traj:last_traj] - .transpose(1, 0) - .contiguous() - for saved_hidden_states in self.saved_hidden_states_c - ] - # remove the tuple for GRU - hid_a_batch = hid_a_batch[0] if len(hid_a_batch) == 1 else hid_a_batch - hid_c_batch = hid_c_batch[0] if len(hid_c_batch) == 1 else hid_c_batch - - yield obs_batch, actions_batch, values_batch, advantages_batch, returns_batch, old_actions_log_prob_batch, old_mu_batch, old_sigma_batch, ( - hid_a_batch, - hid_c_batch, - ), masks_batch - - first_traj = last_traj diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py deleted file mode 100644 index f5781f1..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Helper functions.""" - -from .utils import * -from .motion_loader_g1 import G1_AMPLoader - -__all__ = [ - "G1_AMPLoader", -] \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py deleted file mode 100644 index 677c630..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py +++ /dev/null @@ -1,388 +0,0 @@ -import os -from os.path import join as pjoin -import glob -import json -import logging - -import torch -import numpy as np -from pybullet_utils import transformations - -from rsl_rl.utils import motion_util - -_EPS = np.finfo(float).eps * 4.0 -def quaternion_slerp(q0, q1, fraction, spin=0, shortestpath=True): - """Batch quaternion spherical linear interpolation.""" - - out = torch.zeros_like(q0) - - zero_mask = torch.isclose(fraction, torch.zeros_like(fraction)).squeeze() - ones_mask = torch.isclose(fraction, torch.ones_like(fraction)).squeeze() - out[zero_mask] = q0[zero_mask] - out[ones_mask] = q1[ones_mask] - - d = torch.sum(q0 * q1, dim=-1, keepdim=True) - dist_mask = (torch.abs(torch.abs(d) - 1.0) < _EPS).squeeze() - out[dist_mask] = q0[dist_mask] - - if shortestpath: - d_old = torch.clone(d) - d = torch.where(d_old < 0, -d, d) - q1 = torch.where(d_old < 0, -q1, q1) - - angle = torch.acos(d) + spin * torch.pi - angle_mask = (torch.abs(angle) < _EPS).squeeze() - out[angle_mask] = q0[angle_mask] - - final_mask = torch.logical_or(zero_mask, ones_mask) - final_mask = torch.logical_or(final_mask, dist_mask) - final_mask = torch.logical_or(final_mask, angle_mask) - final_mask = torch.logical_not(final_mask) - - isin = 1.0 / angle - q0 *= torch.sin((1.0 - fraction) * angle) * isin - q1 *= torch.sin(fraction * angle) * isin - q0 += q1 - out[final_mask] = q0[final_mask] - return out - - -class G1_AMPLoader: - - def __init__( - self, - device, - time_between_frames, - motion_files, - preload_transitions=False, - num_preload_transitions=1000000, - num_frames=5, - ): - """Expert dataset provides AMP observations from Dog mocap dataset. - - time_between_frames: Amount of time in seconds between transition. - """ - self.device = device - self.time_between_frames = time_between_frames - self.num_frames = num_frames - - # Values to store for each trajectory. - self.trajectories = [] - self.trajectories_full = [] - self.trajectory_names = [] - self.trajectory_idxs = [] - self.trajectory_lens = [] # Traj length in seconds. - self.trajectory_weights = [] - self.trajectory_frame_durations = [] - self.trajectory_num_frames = [] - self.motion_dir = motion_files - # import ipdb; ipdb.set_trace() - for i, motion_file in enumerate(os.listdir(motion_files)): - self.trajectory_names.append(motion_file) - motion_path = pjoin(motion_files, motion_file) - motion_data = np.load(motion_path, allow_pickle=True) - motion_data_processed = np.zeros((motion_data.shape[0],36)) - - for f_i in range(motion_data.shape[0]): - motion_data_processed[f_i, :3] = motion_data[f_i, :3] # base pos - motion_data_processed[f_i, 3:7] = motion_data[f_i, 3:7] # base quat (wxyz) - motion_data_processed[f_i, 7:35] = motion_data[f_i, 7:35] # base vel - ''' - NOTE The order of motion_data_processed is - base pos 0:3, - base quat 3:7, wxyz - dof pos 7:36, (mujoco joint order) - ''' - self.trajectories.append(torch.tensor( - motion_data_processed[:, 7:], - dtype=torch.float32, - device=self.device - )) - - self.trajectories_full.append(torch.tensor( - motion_data_processed, - dtype=torch.float32, - device=self.device - )) - - self.trajectory_idxs.append(i) - self.trajectory_weights.append(1 / len(os.listdir(motion_files))) - frame_duration = 1 / 50 - - self.trajectory_frame_durations.append(frame_duration) - traj_len = (motion_data_processed.shape[0] - 1) * frame_duration # seconds - self.trajectory_lens.append(traj_len) - self.trajectory_num_frames.append(float(motion_data_processed.shape[0])) - print(f"Loaded {traj_len}s. motion from {motion_file}.") - - # Trajectory weights are used to sample some trajectories more than others. - self.trajectory_weights = np.array(self.trajectory_weights) / np.sum(self.trajectory_weights) - self.trajectory_frame_durations = np.array(self.trajectory_frame_durations) - self.trajectory_lens = np.array(self.trajectory_lens) - self.trajectory_num_frames = np.array(self.trajectory_num_frames) - - # Preload transitions. - self.preload_transitions = preload_transitions - if self.preload_transitions: - print(f'Preloading {num_preload_transitions} transitions') - - traj_idxs = self.weighted_traj_idx_sample_batch(num_preload_transitions) - times = self.traj_time_sample_batch(traj_idxs) - self.preloaded_s_prior = self.get_full_frame_at_time_batch(traj_idxs, times - self.time_between_frames) - self.preloaded_s = self.get_full_frame_at_time_batch(traj_idxs, times) - self.preloaded_s_next = self.get_full_frame_at_time_batch(traj_idxs, times + self.time_between_frames) - print(f'Finished preloading') - - # 预加载多帧数据 - self.preloaded_frames = [] - for i in range(self.num_frames): - frame_time = times + (i - (self.num_frames - 2)) * self.time_between_frames - full_frame = self.get_full_frame_at_time_batch(traj_idxs, frame_time) - # 预处理：提前提取并连接需要的列（7:26 和 29:33），避免每次生成时重复切片 - processed_frame = torch.cat([ - full_frame[:, 7:26], - full_frame[:, 29:33] - ], dim=-1) - self.preloaded_frames.append(processed_frame) - print(f'Finished preloading multiple frames') - - self.all_trajectories_full = torch.vstack(self.trajectories_full) - - def weighted_traj_idx_sample(self): - """Get traj idx via weighted sampling.""" - return np.random.choice( - self.trajectory_idxs, p=self.trajectory_weights) - - def weighted_traj_idx_sample_batch(self, size): - """Batch sample traj idxs.""" - return np.random.choice( - self.trajectory_idxs, size=size, p=self.trajectory_weights, - replace=True) - - def traj_time_sample(self, traj_idx): - """Sample random time for traj.""" - subst = self.time_between_frames + self.trajectory_frame_durations[traj_idx] - return max( - 0, (self.trajectory_lens[traj_idx] * np.random.uniform() - subst)) - - def traj_time_sample_batch(self, traj_idxs): - """Sample random time for multiple trajectories.""" - subst = self.time_between_frames + self.trajectory_frame_durations[traj_idxs] - time_samples = self.trajectory_lens[traj_idxs] * np.random.uniform(size=len(traj_idxs)) - subst - return np.maximum(np.zeros_like(time_samples), time_samples) - - def slerp(self, val0, val1, blend): - return (1.0 - blend) * val0 + blend * val1 - - def get_trajectory(self, traj_idx): - """Returns trajectory of AMP observations.""" - return self.trajectories_full[traj_idx] - - def get_frame_at_time(self, traj_idx, time): - """Returns frame for the given trajectory at the specified time.""" - p = float(time) / self.trajectory_lens[traj_idx] - n = self.trajectories[traj_idx].shape[0] - idx_low, idx_high = int(np.floor(p * n)), int(np.ceil(p * n)) - frame_start = self.trajectories[traj_idx][idx_low] - frame_end = self.trajectories[traj_idx][idx_high] - blend = p * n - idx_low - return self.slerp(frame_start, frame_end, blend) - - def get_frame_at_time_batch(self, traj_idxs, times): - """Returns frame for the given trajectory at the specified time.""" - p = times / self.trajectory_lens[traj_idxs] - n = self.trajectory_num_frames[traj_idxs] - idx_low, idx_high = np.floor(p * n).astype(np.int32), np.ceil(p * n).astype(np.int32) - all_frame_starts = torch.zeros(len(traj_idxs), self.observation_dim, device=self.device) - all_frame_ends = torch.zeros(len(traj_idxs), self.observation_dim, device=self.device) - for traj_idx in set(traj_idxs): - trajectory = self.trajectories[traj_idx] - traj_mask = traj_idxs == traj_idx - all_frame_starts[traj_mask] = trajectory[idx_low[traj_mask]] - all_frame_ends[traj_mask] = trajectory[idx_high[traj_mask]] - blend = torch.tensor(p * n - idx_low, device=self.device, dtype=torch.float32).unsqueeze(-1) - return self.slerp(all_frame_starts, all_frame_ends, blend) - - def get_full_frame_at_time(self, traj_idx, time): - """Returns full frame for the given trajectory at the specified time.""" - p = float(time) / self.trajectory_lens[traj_idx] - n = self.trajectories_full[traj_idx].shape[0] - idx_low, idx_high = int(np.floor(p * n)), int(np.ceil(p * n)) - frame_start = self.trajectories_full[traj_idx][idx_low] - frame_end = self.trajectories_full[traj_idx][idx_high] - blend = p * n - idx_low - print(idx_low, idx_high) - return self.blend_frame_pose(frame_start, frame_end, blend) - - def get_full_frame_at_time_batch(self, traj_idxs, times): - p = times / self.trajectory_lens[traj_idxs] - n = self.trajectory_num_frames[traj_idxs] - idx_low, idx_high = np.floor(p * n).astype(np.int32), np.ceil(p * n).astype(np.int32) - all_frame_pos_starts = torch.zeros(len(traj_idxs), 3, device=self.device) - all_frame_pos_ends = torch.zeros(len(traj_idxs), 3, device=self.device) - all_frame_rot_starts = torch.zeros(len(traj_idxs), 4, device=self.device) - all_frame_rot_ends = torch.zeros(len(traj_idxs), 4, device=self.device) - all_frame_amp_starts = torch.zeros(len(traj_idxs), 29, device=self.device) - all_frame_amp_ends = torch.zeros(len(traj_idxs), 29, device=self.device) - for traj_idx in set(traj_idxs): - trajectory = self.trajectories_full[traj_idx] - traj_mask = traj_idxs == traj_idx - all_frame_pos_starts[traj_mask] = G1_AMPLoader.get_root_pos_batch(trajectory[idx_low[traj_mask]]) - all_frame_pos_ends[traj_mask] = G1_AMPLoader.get_root_pos_batch(trajectory[idx_high[traj_mask]]) - all_frame_rot_starts[traj_mask] = G1_AMPLoader.get_root_rot_batch(trajectory[idx_low[traj_mask]]) - all_frame_rot_ends[traj_mask] = G1_AMPLoader.get_root_rot_batch(trajectory[idx_high[traj_mask]]) - all_frame_amp_starts[traj_mask] = trajectory[idx_low[traj_mask]][:, 7:36] # base vel3+ang3, dof vel23+ang23 - all_frame_amp_ends[traj_mask] = trajectory[idx_high[traj_mask]][:, 7:36] # base vel3+ang3, dof vel23+ang23 - blend = torch.tensor(p * n - idx_low, device=self.device, dtype=torch.float32).unsqueeze(-1) - pos_blend = self.slerp(all_frame_pos_starts, all_frame_pos_ends, blend) - rot_blend = quaternion_slerp(all_frame_rot_starts, all_frame_rot_ends, blend) - amp_blend = self.slerp(all_frame_amp_starts, all_frame_amp_ends, blend) - return torch.cat([pos_blend, rot_blend, amp_blend], dim=-1) - - def get_frame(self): - """Returns random frame.""" - traj_idx = self.weighted_traj_idx_sample() - sampled_time = self.traj_time_sample(traj_idx) - return self.get_frame_at_time(traj_idx, sampled_time) - - def get_full_frame(self): - """Returns random full frame.""" - traj_idx = self.weighted_traj_idx_sample() - sampled_time = self.traj_time_sample(traj_idx) - return self.get_full_frame_at_time(traj_idx, sampled_time) - - def get_full_frame_batch(self, num_frames): - if self.preload_transitions: - idxs = np.random.choice( - self.preloaded_s.shape[0], size=num_frames) - return self.preloaded_s[idxs] - else: - traj_idxs = self.weighted_traj_idx_sample_batch(num_frames) - times = self.traj_time_sample_batch(traj_idxs) - return self.get_full_frame_at_time_batch(traj_idxs, times) - - def blend_frame_pose(self, frame0, frame1, blend): - """Linearly interpolate between two frames, including orientation. - - Args: - frame0: First frame to be blended corresponds to (blend = 0). - frame1: Second frame to be blended corresponds to (blend = 1). - blend: Float between [0, 1], specifying the interpolation between - the two frames. - Returns: - An interpolation of the two frames. - """ - root_pos0, root_pos1 = G1_AMPLoader.get_root_pos(frame0), G1_AMPLoader.get_root_pos(frame1) - root_rot0, root_rot1 = G1_AMPLoader.get_root_rot(frame0), G1_AMPLoader.get_root_rot(frame1) - joints0, joints1 = G1_AMPLoader.get_joint_pose(frame0), G1_AMPLoader.get_joint_pose(frame1) - # tar_toe_pos_0, tar_toe_pos_1 = G1_AMPLoader.get_tar_toe_pos_local(frame0), G1_AMPLoader.get_tar_toe_pos_local(frame1) - linear_vel_0, linear_vel_1 = G1_AMPLoader.get_linear_vel(frame0), G1_AMPLoader.get_linear_vel(frame1) - angular_vel_0, angular_vel_1 = G1_AMPLoader.get_angular_vel(frame0), G1_AMPLoader.get_angular_vel(frame1) - joint_vel_0, joint_vel_1 = G1_AMPLoader.get_joint_vel(frame0), G1_AMPLoader.get_joint_vel(frame1) - - blend_root_pos = self.slerp(root_pos0, root_pos1, blend) - blend_root_rot = transformations.quaternion_slerp(root_rot0.cpu().numpy(), root_rot1.cpu().numpy(), blend) - blend_root_rot = torch.tensor(motion_util.standardize_quaternion(blend_root_rot),dtype=torch.float32, device=self.device) - blend_joints = self.slerp(joints0, joints1, blend) - # blend_tar_toe_pos = self.slerp(tar_toe_pos_0, tar_toe_pos_1, blend) - blend_linear_vel = self.slerp(linear_vel_0, linear_vel_1, blend) - blend_angular_vel = self.slerp(angular_vel_0, angular_vel_1, blend) - blend_joints_vel = self.slerp(joint_vel_0, joint_vel_1, blend) - - # return - # torch.cat([ - # blend_root_pos, blend_root_rot, blend_linear_vel, blend_angular_vel, blend_joints, blend_joints_vel]) - return torch.cat([blend_root_pos, blend_root_rot, blend_linear_vel, blend_angular_vel, blend_joints]) - - def feed_forward_generator_23dof_multi(self, num_mini_batch, mini_batch_size): - """Generates a batch of AMP transitions.""" - # import ipdb; ipdb.set_trace() - for _ in range(num_mini_batch): - if self.preload_transitions: - idxs = np.random.choice(self.preloaded_s.shape[0], size=mini_batch_size) - - frames = [] - for i in range(self.num_frames): - # 数据已在预加载时预处理，直接索引即可 - s = self.preloaded_frames[i][idxs] - frames.append(s) - else: - NotImplementedError('preload transition') - yield torch.stack(frames, dim=1) # [batch, num_frames, 16] - - - - - def quaternion_to_euler_array(self, quat): - # Ensure quaternion is in the correct format [x, y, z, w] - x, y, z, w =quat - - # Roll (x-axis rotation) - t0 = +2.0 * (w * x + y * z) - t1 = +1.0 - 2.0 * (x * x + y * y) - roll_x = np.arctan2(t0, t1) - - # Pitch (y-axis rotation) - t2 = +2.0 * (w * y - z * x) - t2 = np.clip(t2, -1.0, 1.0) - pitch_y = np.arcsin(t2) - - # Yaw (z-axis rotation) - t3 = +2.0 * (w * z + x * y) - t4 = +1.0 - 2.0 * (y * y + z * z) - yaw_z = np.arctan2(t3, t4) - - # Returns roll, pitch, yaw in a NumPy array in radians - return np.array([roll_x, pitch_y, yaw_z]) - - def euler_to_quaternion(self, root_rot): - roll, pitch, yaw = root_rot[0], root_rot[1], root_rot[2] - cy = np.cos(yaw * 0.5) - sy = np.sin(yaw * 0.5) - cp = np.cos(pitch * 0.5) - sp = np.sin(pitch * 0.5) - cr = np.cos(roll * 0.5) - sr = np.sin(roll * 0.5) - - qw = cy * cp * cr + sy * sp * sr - qx = cy * cp * sr - sy * sp * cr - qy = sy * cp * sr + cy * sp * cr - qz = sy * cp * cr - cy * sp * sr - - return np.array([qx, qy, qz, qw]) - - @property - def observation_dim(self): - """Size of AMP observations.""" - return self.trajectories[0].shape[1] + 1 - - @property - def num_motions(self): - return len(self.trajectory_names) - @staticmethod - def get_root_pos(pose): - return pose[0:3] - - @staticmethod - def get_root_pos_batch(poses): - return poses[:, 0:3] - - @staticmethod - def get_root_rot(pose): - return pose[3:7] - - @staticmethod - def get_root_rot_batch(poses): - return poses[:, 3:7] - - @staticmethod - def get_joint_pose_batch_12dof(poses): - return poses[:, 13:25] - - @staticmethod - def get_tar_toe_pos_local(pose): - return pose[G1_AMPLoader.TAR_TOE_POS_LOCAL_START_IDX:G1_AMPLoader.TAR_TOE_POS_LOCAL_END_IDX] - - @staticmethod - def get_tar_toe_pos_local_batch(poses): - return poses[:, G1_AMPLoader.TAR_TOE_POS_LOCAL_START_IDX:G1_AMPLoader.TAR_TOE_POS_LOCAL_END_IDX] diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py deleted file mode 100644 index 3d49bfa..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/motion_util.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility functions for processing motion clips.""" - -import os -import inspect -currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parentdir = os.path.dirname(os.path.dirname(currentdir)) -os.sys.path.insert(0, parentdir) - -import numpy as np - -from rsl_rl.utils import pose3d -# from pybullet_utils import transformations - - -def standardize_quaternion(q): - """Returns a quaternion where q.w >= 0 to remove redundancy due to q = -q. - - Args: - q: A quaternion to be standardized. - - Returns: - A quaternion with q.w >= 0. - - """ - if q[-1] < 0: - q = -q - return q - - -def normalize_rotation_angle(theta): - """Returns a rotation angle normalized between [-pi, pi]. - - Args: - theta: angle of rotation (radians). - - Returns: - An angle of rotation normalized between [-pi, pi]. - - """ - norm_theta = theta - if np.abs(norm_theta) > np.pi: - norm_theta = np.fmod(norm_theta, 2 * np.pi) - if norm_theta >= 0: - norm_theta += -2 * np.pi - else: - norm_theta += 2 * np.pi - - return norm_theta - - -def calc_heading(q): - """Returns the heading of a rotation q, specified as a quaternion. - - The heading represents the rotational component of q along the vertical - axis (z axis). - - Args: - q: A quaternion that the heading is to be computed from. - - Returns: - An angle representing the rotation about the z axis. - - """ - ref_dir = np.array([1, 0, 0]) - rot_dir = pose3d.QuaternionRotatePoint(ref_dir, q) - heading = np.arctan2(rot_dir[1], rot_dir[0]) - return heading - - -# def calc_heading_rot(q): -# """Return a quaternion representing the heading rotation of q along the vertical axis (z axis). - -# Args: -# q: A quaternion that the heading is to be computed from. - -# Returns: -# A quaternion representing the rotation about the z axis. - -# """ -# heading = calc_heading(q) -# q_heading = transformations.quaternion_about_axis(heading, [0, 0, 1]) -# return q_heading diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py deleted file mode 100644 index 3796ec8..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -from dataclasses import asdict -from torch.utils.tensorboard import SummaryWriter - -try: - import neptune -except ModuleNotFoundError: - raise ModuleNotFoundError("neptune-client is required to log to Neptune.") - - -class NeptuneLogger: - def __init__(self, project, token): - self.run = neptune.init_run(project=project, api_token=token) - - def store_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.run["runner_cfg"] = runner_cfg - self.run["policy_cfg"] = policy_cfg - self.run["alg_cfg"] = alg_cfg - self.run["env_cfg"] = asdict(env_cfg) - - -class NeptuneSummaryWriter(SummaryWriter): - """Summary writer for Neptune.""" - - def __init__(self, log_dir: str, flush_secs: int, cfg): - super().__init__(log_dir, flush_secs) - - try: - project = cfg["neptune_project"] - except KeyError: - raise KeyError("Please specify neptune_project in the runner config, e.g. legged_gym.") - - try: - token = os.environ["NEPTUNE_API_TOKEN"] - except KeyError: - raise KeyError( - "Neptune api token not found. Please run or add to ~/.bashrc: export NEPTUNE_API_TOKEN=YOUR_API_TOKEN" - ) - - try: - entity = os.environ["NEPTUNE_USERNAME"] - except KeyError: - raise KeyError( - "Neptune username not found. Please run or add to ~/.bashrc: export NEPTUNE_USERNAME=YOUR_USERNAME" - ) - - neptune_project = entity + "/" + project - - self.neptune_logger = NeptuneLogger(neptune_project, token) - - self.name_map = { - "Train/mean_reward/time": "Train/mean_reward_time", - "Train/mean_episode_length/time": "Train/mean_episode_length_time", - } - - run_name = os.path.split(log_dir)[-1] - - self.neptune_logger.run["log_dir"].log(run_name) - - def _map_path(self, path): - if path in self.name_map: - return self.name_map[path] - else: - return path - - def add_scalar(self, tag, scalar_value, global_step=None, walltime=None, new_style=False): - super().add_scalar( - tag, - scalar_value, - global_step=global_step, - walltime=walltime, - new_style=new_style, - ) - self.neptune_logger.run[self._map_path(tag)].log(scalar_value, step=global_step) - - def stop(self): - self.neptune_logger.run.stop() - - def log_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.neptune_logger.store_config(env_cfg, runner_cfg, alg_cfg, policy_cfg) - - def save_model(self, model_path, iter): - self.neptune_logger.run["model/saved_model_" + str(iter)].upload(model_path) - - def save_file(self, path, iter=None): - name = path.rsplit("/", 1)[-1].split(".")[0] - self.neptune_logger.run["git_diff/" + name].upload(path) diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py deleted file mode 100644 index a4cba1e..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/pose3d.py +++ /dev/null @@ -1,283 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for 3D pose conversion.""" -import math -import numpy as np - -# from pybullet_utils import transformations - -VECTOR3_0 = np.zeros(3, dtype=np.float64) -VECTOR3_1 = np.ones(3, dtype=np.float64) -VECTOR3_X = np.array([1, 0, 0], dtype=np.float64) -VECTOR3_Y = np.array([0, 1, 0], dtype=np.float64) -VECTOR3_Z = np.array([0, 0, 1], dtype=np.float64) - -# QUATERNION_IDENTITY is the multiplicative identity 1.0 + 0i + 0j + 0k. -# When interpreted as a rotation, it is the identity rotation. -QUATERNION_IDENTITY = np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float64) - - -def Vector3RandomNormal(sigma, mu=VECTOR3_0): - """Returns a random 3D vector from a normal distribution. - - Each component is selected independently from a normal distribution. - - Args: - sigma: Scale (or stddev) of distribution for all variables. - mu: Mean of distribution for each variable. - - Returns: - A 3D vector in a numpy array. - """ - - random_v3 = np.random.normal(scale=sigma, size=3) + mu - return random_v3 - - -def Vector3RandomUniform(low=VECTOR3_0, high=VECTOR3_1): - """Returns a 3D vector selected uniformly from the input box. - - Args: - low: The min-value corner of the box. - high: The max-value corner of the box. - - Returns: - A 3D vector in a numpy array. - """ - - random_x = np.random.uniform(low=low[0], high=high[0]) - random_y = np.random.uniform(low=low[1], high=high[1]) - random_z = np.random.uniform(low=low[2], high=high[2]) - return np.array([random_x, random_y, random_z]) - - -def Vector3RandomUnit(): - """Returns a random 3D vector with unit length. - - Generates a 3D vector selected uniformly from the unit sphere. - - Returns: - A normalized 3D vector in a numpy array. - """ - longitude = np.random.uniform(low=-math.pi, high=math.pi) - sin_latitude = np.random.uniform(low=-1.0, high=1.0) - cos_latitude = math.sqrt(1.0 - sin_latitude * sin_latitude) - x = math.cos(longitude) * cos_latitude - y = math.sin(longitude) * cos_latitude - z = sin_latitude - return np.array([x, y, z], dtype=np.float64) - - -def QuaternionNormalize(q): - """Normalizes the quaternion to length 1. - - Divides the quaternion by its magnitude. If the magnitude is too - small, returns the quaternion identity value (1.0). - - Args: - q: A quaternion to be normalized. - - Raises: - ValueError: If input quaternion has length near zero. - - Returns: - A quaternion with magnitude 1 in a numpy array [x, y, z, w]. - - """ - q_norm = np.linalg.norm(q) - if np.isclose(q_norm, 0.0): - raise ValueError( - 'Quaternion may not be zero in QuaternionNormalize: |q| = %f, q = %s' % - (q_norm, q)) - return q / q_norm - - -def QuaternionFromAxisAngle(axis, angle): - """Returns a quaternion that generates the given axis-angle rotation. - - Returns the quaternion: sin(angle/2) * axis + cos(angle/2). - - Args: - axis: Axis of rotation, a 3D vector in a numpy array. - angle: The angle of rotation (radians). - - Raises: - ValueError: If input axis is not a normalizable 3D vector. - - Returns: - A unit quaternion in a numpy array. - - """ - if len(axis) != 3: - raise ValueError('Axis vector should have three components: %s' % axis) - axis_norm = np.linalg.norm(axis) - if np.isclose(axis_norm, 0.0): - raise ValueError('Axis vector may not have zero length: |v| = %f, v = %s' % - (axis_norm, axis)) - half_angle = angle * 0.5 - q = np.zeros(4, dtype=np.float64) - q[0:3] = axis - q[0:3] *= math.sin(half_angle) / axis_norm - q[3] = math.cos(half_angle) - return q - - -def QuaternionToAxisAngle(quat, default_axis=VECTOR3_Z, direction_axis=None): - """Calculates axis and angle of rotation performed by a quaternion. - - Calculates the axis and angle of the rotation performed by the quaternion. - The quaternion should have four values and be normalized. - - Args: - quat: Unit quaternion in a numpy array. - default_axis: 3D vector axis used if the rotation is near to zero. Without - this default, small rotations would result in an exception. It is - reasonable to use a default axis for tiny rotations, because zero angle - rotations about any axis are equivalent. - direction_axis: Used to disambiguate rotation directions. If the - direction_axis is specified, the axis of the rotation will be chosen such - that its inner product with the direction_axis is non-negative. - - Raises: - ValueError: If quat is not a normalized quaternion. - - Returns: - axis: Axis of rotation. - angle: Angle in radians. - """ - if len(quat) != 4: - raise ValueError( - 'Quaternion should have four components [x, y, z, w]: %s' % quat) - if not np.isclose(1.0, np.linalg.norm(quat)): - raise ValueError('Quaternion should have unit length: |q| = %f, q = %s' % - (np.linalg.norm(quat), quat)) - axis = quat[:3].copy() - axis_norm = np.linalg.norm(axis) - min_axis_norm = 1e-8 - if axis_norm < min_axis_norm: - axis = default_axis - if len(default_axis) != 3: - raise ValueError('Axis vector should have three components: %s' % axis) - if not np.isclose(np.linalg.norm(axis), 1.0): - raise ValueError('Axis vector should have unit length: |v| = %f, v = %s' % - (np.linalg.norm(axis), axis)) - else: - axis /= axis_norm - sin_half_angle = axis_norm - if direction_axis is not None and np.inner(axis, direction_axis) < 0: - sin_half_angle = -sin_half_angle - axis = -axis - cos_half_angle = quat[3] - half_angle = math.atan2(sin_half_angle, cos_half_angle) - angle = half_angle * 2 - return axis, angle - - -def QuaternionRandomRotation(max_angle=math.pi): - """Creates a random small rotation around a random axis. - - Generates a small rotation with the axis vector selected uniformly - from the unit sphere and an angle selected from a uniform - distribution over [0, max_angle]. - - If the max_angle is not specified, the rotation should be selected - uniformly over all possible rotation angles. - - Args: - max_angle: The maximum angle of rotation (radians). - - Returns: - A unit quaternion in a numpy array. - - """ - - angle = np.random.uniform(low=0, high=max_angle) - axis = Vector3RandomUnit() - return QuaternionFromAxisAngle(axis, angle) - - -# def QuaternionRotatePoint(point, quat): -# """Performs a rotation by quaternion. - -# Rotate the point by the quaternion using quaternion multiplication, -# (q * p * q^-1), without constructing the rotation matrix. - -# Args: -# point: The point to be rotated. -# quat: The rotation represented as a quaternion [x, y, z, w]. - -# Returns: -# A 3D vector in a numpy array. -# """ - -# q_point = np.array([point[0], point[1], point[2], 0.0]) -# quat_inverse = transformations.quaternion_inverse(quat) -# q_point_rotated = transformations.quaternion_multiply( -# transformations.quaternion_multiply(quat, q_point), quat_inverse) -# return q_point_rotated[:3] - - -def IsRotationMatrix(m): - """Returns true if the 3x3 submatrix represents a rotation. - - Args: - m: A transformation matrix. - - Raises: - ValueError: If input is not a matrix of size at least 3x3. - - Returns: - True if the 3x3 submatrix is a rotation (orthogonal). - """ - if len(m.shape) != 2 or m.shape[0] < 3 or m.shape[1] < 3: - raise ValueError('Matrix should be 3x3 or 4x4: %s\n %s' % (m.shape, m)) - rot = m[:3, :3] - eye = np.matmul(rot, np.transpose(rot)) - return np.isclose(eye, np.identity(3), atol=1e-4).all() - -# def ZAxisAlignedRobotPoseTool(robot_pose_tool): -# """Returns the current gripper pose rotated for alignment with the z-axis. - -# Args: -# robot_pose_tool: a pose3d.Pose3d() instance. - -# Returns: -# An instance of pose.Transform representing the current gripper pose -# rotated for alignment with the z-axis. -# """ -# # Align the current pose to the z-axis. -# robot_pose_tool.quaternion = transformations.quaternion_multiply( -# RotationBetween( -# robot_pose_tool.matrix4x4[0:3, 0:3].dot(np.array([0, 0, 1])), -# np.array([0.0, 0.0, -1.0])), robot_pose_tool.quaternion) -# return robot_pose_tool - -# def RotationBetween(a_translation_b, a_translation_c): -# """Computes the rotation from one vector to another. - -# The computed rotation has the property that: - -# a_translation_c = a_rotation_b_to_c * a_translation_b - -# Args: -# a_translation_b: vec3, vector to rotate from -# a_translation_c: vec3, vector to rotate to - -# Returns: -# a_rotation_b_to_c: new Orientation -# """ -# rotation = rotation3.Rotation3.rotation_between( -# a_translation_b, a_translation_c, err_msg='RotationBetween') -# return rotation.quaternion.xyzw diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py deleted file mode 100644 index 3605622..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/utils.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import git -import importlib -import os -import pathlib -import torch -import warnings -from tensordict import TensorDict -from typing import Callable -import numpy as np -class RunningMeanStd: - def __init__(self, epsilon: float = 1e-4, shape: Tuple[int, ...] = ()): - """ - Calculates the running mean and std of a data stream - https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm - :param epsilon: helps with arithmetic issues - :param shape: the shape of the data stream's output - """ - self.mean = np.zeros(shape, np.float64) - self.var = np.ones(shape, np.float64) - self.count = epsilon - - def update(self, arr: np.ndarray) -> None: - batch_mean = np.mean(arr, axis=0) - batch_var = np.var(arr, axis=0) - batch_count = arr.shape[0] - self.update_from_moments(batch_mean, batch_var, batch_count) - - def update_from_moments(self, batch_mean: np.ndarray, batch_var: np.ndarray, batch_count: int) -> None: - delta = batch_mean - self.mean - tot_count = self.count + batch_count - - new_mean = self.mean + delta * batch_count / tot_count - m_a = self.var * self.count - m_b = batch_var * batch_count - m_2 = m_a + m_b + np.square(delta) * self.count * batch_count / (self.count + batch_count) - new_var = m_2 / (self.count + batch_count) - - new_count = batch_count + self.count - - self.mean = new_mean - self.var = new_var - self.count = new_count - - -class Normalizer(RunningMeanStd): - def __init__(self, input_dim, epsilon=1e-4, clip_obs=10.0): - super().__init__(shape=input_dim) - self.epsilon = epsilon - self.clip_obs = clip_obs - - def normalize(self, input): - return np.clip((input - self.mean) / np.sqrt(self.var + self.epsilon), -self.clip_obs, self.clip_obs) - - def normalize_torch(self, input, device): - mean_torch = torch.tensor(self.mean, device=device, dtype=torch.float32) - std_torch = torch.sqrt(torch.tensor(self.var + self.epsilon, device=device, dtype=torch.float32)) - return torch.clamp((input - mean_torch) / std_torch, -self.clip_obs, self.clip_obs) - - def update_normalizer(self, rollouts, expert_loader): - policy_data_generator = rollouts.feed_forward_generator_amp(None, mini_batch_size=expert_loader.batch_size) - expert_data_generator = expert_loader.dataset.feed_forward_generator_amp(expert_loader.batch_size) - - for expert_batch, policy_batch in zip(expert_data_generator, policy_data_generator): - self.update(torch.vstack(tuple(policy_batch) + tuple(expert_batch)).cpu().numpy()) - - -def resolve_nn_activation(act_name: str) -> torch.nn.Module: - """Resolves the activation function from the name. - - Args: - act_name: The name of the activation function. - - Returns: - The activation function. - - Raises: - ValueError: If the activation function is not found. - """ - act_dict = { - "elu": torch.nn.ELU(), - "selu": torch.nn.SELU(), - "relu": torch.nn.ReLU(), - "crelu": torch.nn.CELU(), - "lrelu": torch.nn.LeakyReLU(), - "tanh": torch.nn.Tanh(), - "sigmoid": torch.nn.Sigmoid(), - "softplus": torch.nn.Softplus(), - "gelu": torch.nn.GELU(), - "swish": torch.nn.SiLU(), - "mish": torch.nn.Mish(), - "identity": torch.nn.Identity(), - } - - act_name = act_name.lower() - if act_name in act_dict: - return act_dict[act_name] - else: - raise ValueError(f"Invalid activation function '{act_name}'. Valid activations are: {list(act_dict.keys())}") - - -def resolve_optimizer(optimizer_name: str) -> torch.optim.Optimizer: - """Resolves the optimizer from the name. - - Args: - optimizer_name: The name of the optimizer. - - Returns: - The optimizer. - - Raises: - ValueError: If the optimizer is not found. - """ - optimizer_dict = { - "adam": torch.optim.Adam, - "adamw": torch.optim.AdamW, - "sgd": torch.optim.SGD, - "rmsprop": torch.optim.RMSprop, - } - - optimizer_name = optimizer_name.lower() - if optimizer_name in optimizer_dict: - return optimizer_dict[optimizer_name] - else: - raise ValueError(f"Invalid optimizer '{optimizer_name}'. Valid optimizers are: {list(optimizer_dict.keys())}") - - -def split_and_pad_trajectories( - tensor: torch.Tensor | TensorDict, dones: torch.Tensor -) -> tuple[torch.Tensor | TensorDict, torch.Tensor]: - """Splits trajectories at done indices. Then concatenates them and pads with zeros up to the length of the longest - trajectory. Returns masks corresponding to valid parts of the trajectories. - - Example: - Input: [[a1, a2, a3, a4 | a5, a6], - [b1, b2 | b3, b4, b5 | b6]] - - Output:[[a1, a2, a3, a4], | [[True, True, True, True], - [a5, a6, 0, 0], | [True, True, False, False], - [b1, b2, 0, 0], | [True, True, False, False], - [b3, b4, b5, 0], | [True, True, True, False], - [b6, 0, 0, 0]] | [True, False, False, False]] - - Assumes that the input has the following order of dimensions: [time, number of envs, additional dimensions] - """ - - dones = dones.clone() - dones[-1] = 1 - # Permute the buffers to have order (num_envs, num_transitions_per_env, ...), for correct reshaping - flat_dones = dones.transpose(1, 0).reshape(-1, 1) - # Get length of trajectory by counting the number of successive not done elements - done_indices = torch.cat((flat_dones.new_tensor([-1], dtype=torch.int64), flat_dones.nonzero()[:, 0])) - trajectory_lengths = done_indices[1:] - done_indices[:-1] - trajectory_lengths_list = trajectory_lengths.tolist() - # Extract the individual trajectories - if isinstance(tensor, TensorDict): - padded_trajectories = {} - for k, v in tensor.items(): - # split the tensor into trajectories - trajectories = torch.split(v.transpose(1, 0).flatten(0, 1), trajectory_lengths_list) - # add at least one full length trajectory - trajectories = trajectories + (torch.zeros(v.shape[0], *v.shape[2:], device=v.device),) - # pad the trajectories to the length of the longest trajectory - padded_trajectories[k] = torch.nn.utils.rnn.pad_sequence(trajectories) - # remove the added tensor - padded_trajectories[k] = padded_trajectories[k][:, :-1] - padded_trajectories = TensorDict( - padded_trajectories, batch_size=[tensor.batch_size[0], len(trajectory_lengths_list)] - ) - else: - # split the tensor into trajectories - trajectories = torch.split(tensor.transpose(1, 0).flatten(0, 1), trajectory_lengths_list) - # add at least one full length trajectory - trajectories = trajectories + (torch.zeros(tensor.shape[0], *tensor.shape[2:], device=tensor.device),) - # pad the trajectories to the length of the longest trajectory - padded_trajectories = torch.nn.utils.rnn.pad_sequence(trajectories) - # remove the added tensor - padded_trajectories = padded_trajectories[:, :-1] - # create masks for the valid parts of the trajectories - trajectory_masks = trajectory_lengths > torch.arange(0, tensor.shape[0], device=tensor.device).unsqueeze(1) - return padded_trajectories, trajectory_masks - - -def unpad_trajectories(trajectories, masks): - """Does the inverse operation of split_and_pad_trajectories()""" - # Need to transpose before and after the masking to have proper reshaping - return ( - trajectories.transpose(1, 0)[masks.transpose(1, 0)] - .view(-1, trajectories.shape[0], trajectories.shape[-1]) - .transpose(1, 0) - ) - - -def store_code_state(logdir, repositories) -> list: - git_log_dir = os.path.join(logdir, "git") - os.makedirs(git_log_dir, exist_ok=True) - file_paths = [] - for repository_file_path in repositories: - try: - repo = git.Repo(repository_file_path, search_parent_directories=True) - t = repo.head.commit.tree - except Exception: - print(f"Could not find git repository in {repository_file_path}. Skipping.") - # skip if not a git repository - continue - # get the name of the repository - repo_name = pathlib.Path(repo.working_dir).name - diff_file_name = os.path.join(git_log_dir, f"{repo_name}.diff") - # check if the diff file already exists - if os.path.isfile(diff_file_name): - continue - # write the diff file - print(f"Storing git diff for '{repo_name}' in: {diff_file_name}") - with open(diff_file_name, "x", encoding="utf-8") as f: - content = f"--- git status ---\n{repo.git.status()} \n\n\n--- git diff ---\n{repo.git.diff(t)}" - f.write(content) - # add the file path to the list of files to be uploaded - file_paths.append(diff_file_name) - return file_paths - - -def string_to_callable(name: str) -> Callable: - """Resolves the module and function names to return the function. - - Args: - name: The function name. The format should be 'module:attribute_name'. - - Raises: - ValueError: When the resolved attribute is not a function. - ValueError: When unable to resolve the attribute. - - Returns: - The function loaded from the module. - """ - try: - mod_name, attr_name = name.split(":") - mod = importlib.import_module(mod_name) - callable_object = getattr(mod, attr_name) - # check if attribute is callable - if callable(callable_object): - return callable_object - else: - raise ValueError(f"The imported object is not callable: '{name}'") - except AttributeError as e: - msg = ( - "We could not interpret the entry as a callable object. The format of input should be" - f" 'module:attribute_name'\nWhile processing input '{name}', received the error:\n {e}." - ) - raise ValueError(msg) - - -def resolve_obs_groups( - obs: TensorDict, obs_groups: dict[str, list[str]], default_sets: list[str] -) -> dict[str, list[str]]: - """Validates the observation configuration and defaults missing observation sets. - - The input is an observation dictionary `obs` containing observation groups and a configuration dictionary - `obs_groups` where the keys are the observation sets and the values are lists of observation groups. - - The configuration dictionary could for example look like: - { - "policy": ["group_1", "group_2"], - "critic": ["group_1", "group_3"] - } - - This means that the 'policy' observation set will contain the observations "group_1" and "group_2" and the - 'critic' observation set will contain the observations "group_1" and "group_3". This function will check that all - the observations in the 'policy' and 'critic' observation sets are present in the observation dictionary from the - environment. - - Additionally, if one of the `default_sets`, e.g. "critic", is not present in the configuration dictionary, - this function will: - - 1. Check if a group with the same name exists in the observations and assign this group to the observation set. - 2. If 1. fails, it will assign the observations from the 'policy' observation set to the default observation set. - - Args: - obs: Observations from the environment in the form of a dictionary. - obs_groups: Observation sets configuration. - default_sets: Reserved observation set names used by the algorithm (besides 'policy'). - If not provided in 'obs_groups', a default behavior gets triggered. - - Returns: - The resolved observation groups. - - Raises: - ValueError: If any observation set is an empty list. - ValueError: If any observation set contains an observation term that is not present in the observations. - """ - # check if policy observation set exists - if "policy" not in obs_groups.keys(): - if "policy" in obs: - obs_groups["policy"] = ["policy"] - warnings.warn( - "The observation configuration dictionary 'obs_groups' must contain the 'policy' key." - " As an observation group with the name 'policy' was found, this is assumed to be the observation set." - " Consider adding the 'policy' key to the 'obs_groups' dictionary for clarity." - " This behavior will be removed in a future version." - ) - else: - raise ValueError( - "The observation configuration dictionary 'obs_groups' must contain the 'policy' key." - f" Found keys: {list(obs_groups.keys())}" - ) - - # check all observation sets for valid observation groups - for set_name, groups in obs_groups.items(): - # check if the list is empty - if len(groups) == 0: - msg = f"The '{set_name}' key in the 'obs_groups' dictionary can not be an empty list." - if set_name in default_sets: - if set_name not in obs: - msg += " Consider removing the key to default to the observations used for the 'policy' set." - else: - msg += ( - f" Consider removing the key to default to the observation '{set_name}' from the environment." - ) - raise ValueError(msg) - # check groups exist inside the observations from the environment - for group in groups: - if group not in obs: - raise ValueError( - f"Observation '{group}' in observation set '{set_name}' not found in the observations from the" - f" environment. Available observations from the environment: {list(obs.keys())}" - ) - - # fill missing observation sets - for default_set_name in default_sets: - if default_set_name not in obs_groups.keys(): - if default_set_name in obs: - obs_groups[default_set_name] = [default_set_name] - warnings.warn( - f"The observation configuration dictionary 'obs_groups' must contain the '{default_set_name}' key." - f" As an observation group with the name '{default_set_name}' was found, this is assumed to be the" - f" observation set. Consider adding the '{default_set_name}' key to the 'obs_groups' dictionary for" - " clarity. This behavior will be removed in a future version." - ) - else: - obs_groups[default_set_name] = obs_groups["policy"].copy() - warnings.warn( - f"The observation configuration dictionary 'obs_groups' must contain the '{default_set_name}' key." - f" As the configuration for '{default_set_name}' is missing, the observations from the 'policy' set" - f" are used. Consider adding the '{default_set_name}' key to the 'obs_groups' dictionary for" - " clarity. This behavior will be removed in a future version." - ) - - # print the final parsed observation sets - print("-" * 80) - print("Resolved observation sets: ") - for set_name, groups in obs_groups.items(): - print("\t", set_name, ": ", groups) - print("-" * 80) - - return obs_groups diff --git a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py b/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py deleted file mode 100644 index 243e82d..0000000 --- a/rsl_rl/build/lib/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -from dataclasses import asdict -from torch.utils.tensorboard import SummaryWriter - -try: - import wandb -except ModuleNotFoundError: - raise ModuleNotFoundError("Wandb is required to log to Weights and Biases.") - - -class WandbSummaryWriter(SummaryWriter): - """Summary writer for Weights and Biases.""" - - def __init__(self, log_dir: str, flush_secs: int, cfg): - super().__init__(log_dir, flush_secs) - - # Get the run name - run_name = os.path.split(log_dir)[-1] - - try: - project = cfg["wandb_project"] - except KeyError: - raise KeyError("Please specify wandb_project in the runner config, e.g. legged_gym.") - - try: - entity = os.environ["WANDB_USERNAME"] - except KeyError: - entity = None - - # Initialize wandb - wandb.init(project=project, entity=entity, name=run_name) - - # Add log directory to wandb - wandb.config.update({"log_dir": log_dir}) - - self.name_map = { - "Train/mean_reward/time": "Train/mean_reward_time", - "Train/mean_episode_length/time": "Train/mean_episode_length_time", - } - - def store_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - wandb.config.update({"runner_cfg": runner_cfg}) - wandb.config.update({"policy_cfg": policy_cfg}) - wandb.config.update({"alg_cfg": alg_cfg}) - try: - wandb.config.update({"env_cfg": env_cfg.to_dict()}) - except Exception: - wandb.config.update({"env_cfg": asdict(env_cfg)}) - - def add_scalar(self, tag, scalar_value, global_step=None, walltime=None, new_style=False): - super().add_scalar( - tag, - scalar_value, - global_step=global_step, - walltime=walltime, - new_style=new_style, - ) - wandb.log({self._map_path(tag): scalar_value}, step=global_step) - - def stop(self): - wandb.finish() - - def log_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.store_config(env_cfg, runner_cfg, alg_cfg, policy_cfg) - - def save_model(self, model_path, iter): - wandb.save(model_path, base_path=os.path.dirname(model_path)) - - def save_file(self, path, iter=None): - wandb.save(path, base_path=os.path.dirname(path)) - - """ - Private methods. - """ - - def _map_path(self, path): - if path in self.name_map: - return self.name_map[path] - else: - return path diff --git a/rsl_rl/build/lib/rsl_rl/env/__init__.py b/rsl_rl/build/lib/rsl_rl/env/__init__.py deleted file mode 100644 index ab7c056..0000000 --- a/rsl_rl/build/lib/rsl_rl/env/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Submodule defining the environment definitions.""" - -from .vec_env import VecEnv - -__all__ = ["VecEnv"] diff --git a/rsl_rl/build/lib/rsl_rl/env/vec_env.py b/rsl_rl/build/lib/rsl_rl/env/vec_env.py deleted file mode 100644 index ae471cf..0000000 --- a/rsl_rl/build/lib/rsl_rl/env/vec_env.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -from abc import ABC, abstractmethod -from tensordict import TensorDict - - -class VecEnv(ABC): - """Abstract class for a vectorized environment. - - The vectorized environment is a collection of environments that are synchronized. This means that - the same type of action is applied to all environments and the same type of observation is returned from all - environments. - """ - - num_envs: int - """Number of environments.""" - - num_actions: int - """Number of actions.""" - - max_episode_length: int | torch.Tensor - - max_episode_length_s: float - """Maximum episode length. - - The maximum episode length can be a scalar or a tensor. If it is a scalar, it is the same for all environments. - If it is a tensor, it is the maximum episode length for each environment. This is useful for dynamic episode - lengths. - """ - - episode_length_buf: torch.Tensor - """Buffer for current episode lengths.""" - - device: torch.device | str - """Device to use.""" - - cfg: dict | object - """Configuration object.""" - - reset_env_ids: torch.Tensor | None = None - - contact_phase: torch.Tensor | None = None - """ - Operations. - """ - - @abstractmethod - def get_observations(self) -> TensorDict: - """Return the current observations. - - Returns: - observations (TensorDict): Observations from the environment. - """ - raise NotImplementedError - - @abstractmethod - def get_amp_observations(self) -> TensorDict: - """Return the current AMP observations. - - Returns: - observations (TensorDict): Observations from the environment. - """ - raise NotImplementedError - - @abstractmethod - def step(self, actions: torch.Tensor) -> tuple[TensorDict, torch.Tensor, torch.Tensor, dict]: - """Apply input action to the environment. - - Args: - actions (torch.Tensor): Input actions to apply. Shape: (num_envs, num_actions) - - Returns: - observations (TensorDict): Observations from the environment. - rewards (torch.Tensor): Rewards from the environment. Shape: (num_envs,) - dones (torch.Tensor): Done flags from the environment. Shape: (num_envs,) - extras (dict): Extra information from the environment. - - Observations: - - The observations TensorDict usually contains multiple observation groups. The `obs_groups` - dictionary of the runner configuration specifies which observation groups are used for which - purpose, i.e., it maps the available observation groups to observation sets. The observation sets - (keys of the `obs_groups` dictionary) currently used by rsl_rl are: - - - "policy": Specified observation groups are used as input to the actor/student network. - - "critic": Specified observation groups are used as input to the critic network. - - "teacher": Specified observation groups are used as input to the teacher network. - - "rnd_state": Specified observation groups are used as input to the RND network. - - Incomplete or incorrect configurations are handled in the `resolve_obs_groups()` function in - `rsl_rl/utils/utils.py`. - - Extras: - - The extras dictionary includes metrics such as the episode reward, episode length, etc. The following - dictionary keys are used by rsl_rl: - - - "time_outs" (torch.Tensor): Timeouts for the environments. These correspond to terminations that - happen due to time limits and not due to the environment reaching a terminal state. This is useful - for environments that have a fixed episode length. - - - "log" (dict[str, float | torch.Tensor]): Additional information for logging and debugging purposes. - The key should be a string and start with "/" for namespacing. The value can be a scalar or a - tensor. If it is a tensor, the mean of the tensor is used for logging. - """ - raise NotImplementedError - \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/modules/__init__.py b/rsl_rl/build/lib/rsl_rl/modules/__init__.py deleted file mode 100644 index 9afe1a2..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Definitions for neural-network components for RL-agents.""" - -from .actor_critic import ActorCritic -from .actor_critic_recurrent import ActorCriticRecurrent -from .rnd import * -from .student_teacher import StudentTeacher -from .student_teacher_recurrent import StudentTeacherRecurrent -from .symmetry import * -from .discriminator_multi import DiscriminatorMulti -__all__ = [ - "ActorCritic", - "ActorCriticRecurrent", - "StudentTeacher", - "StudentTeacherRecurrent", - "DiscriminatorMulti", -] diff --git a/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py b/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py deleted file mode 100644 index 0efc36b..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/actor_critic.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization - - -class ActorCritic(nn.Module): - is_recurrent = False - - def __init__( - self, - obs, - obs_groups, - num_actions, - actor_obs_normalization=False, - critic_obs_normalization=False, - actor_hidden_dims=[256, 256, 256], - critic_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=1.0, - noise_std_type: str = "scalar", - state_dependent_std=False, - **kwargs, - ): - if kwargs: - print( - "ActorCritic.__init__ got unexpected arguments, which will be ignored: " - + str([key for key in kwargs.keys()]) - ) - super().__init__() - - # get the observation dimensions - self.obs_groups = obs_groups - num_actor_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The ActorCritic module only supports 1D observations." - num_actor_obs += obs[obs_group].shape[-1] - num_critic_obs = 0 - for obs_group in obs_groups["critic"]: - assert len(obs[obs_group].shape) == 2, "The ActorCritic module only supports 1D observations." - num_critic_obs += obs[obs_group].shape[-1] - - self.state_dependent_std = state_dependent_std - # actor - if self.state_dependent_std: - self.actor = MLP(num_actor_obs, [2, num_actions], actor_hidden_dims, activation) - else: - self.actor = MLP(num_actor_obs, num_actions, actor_hidden_dims, activation) - # actor observation normalization - self.actor_obs_normalization = actor_obs_normalization - if actor_obs_normalization: - self.actor_obs_normalizer = EmpiricalNormalization(num_actor_obs) - else: - self.actor_obs_normalizer = torch.nn.Identity() - print(f"Actor MLP: {self.actor}") - - # critic - self.critic = MLP(num_critic_obs, 1, critic_hidden_dims, activation) - # critic observation normalization - self.critic_obs_normalization = critic_obs_normalization - if critic_obs_normalization: - self.critic_obs_normalizer = EmpiricalNormalization(num_critic_obs) - else: - self.critic_obs_normalizer = torch.nn.Identity() - print(f"Critic MLP: {self.critic}") - - # Action noise - self.noise_std_type = noise_std_type - if self.state_dependent_std: - torch.nn.init.zeros_(self.actor[-2].weight[num_actions:]) - if self.noise_std_type == "scalar": - torch.nn.init.constant_(self.actor[-2].bias[num_actions:], init_noise_std) - elif self.noise_std_type == "log": - torch.nn.init.constant_( - self.actor[-2].bias[num_actions:], torch.log(torch.tensor(init_noise_std + 1e-7)) - ) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # Action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None): - pass - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - if self.state_dependent_std: - # compute mean and standard deviation - mean_and_std = self.actor(obs) - if self.noise_std_type == "scalar": - mean, std = torch.unbind(mean_and_std, dim=-2) - elif self.noise_std_type == "log": - mean, log_std = torch.unbind(mean_and_std, dim=-2) - std = torch.exp(log_std) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - # compute mean - mean = self.actor(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs, **kwargs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - self.update_distribution(obs) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - return self.actor(obs) - - def evaluate(self, obs, **kwargs): - obs = self.get_critic_obs(obs) - obs = self.critic_obs_normalizer(obs) - return self.critic(obs) - - def get_actor_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_critic_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["critic"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_actions_log_prob(self, actions): - return self.distribution.log_prob(actions).sum(dim=-1) - - def update_normalization(self, obs): - if self.actor_obs_normalization: - actor_obs = self.get_actor_obs(obs) - self.actor_obs_normalizer.update(actor_obs) - if self.critic_obs_normalization: - critic_obs = self.get_critic_obs(obs) - self.critic_obs_normalizer.update(critic_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the actor-critic model. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters (relevant for, e.g., distillation). - """ - - super().load_state_dict(state_dict, strict=strict) - return True # training resumes diff --git a/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py b/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py deleted file mode 100644 index bba46ca..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/actor_critic_recurrent.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import warnings -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization, Memory - - -class ActorCriticRecurrent(nn.Module): - is_recurrent = True - - def __init__( - self, - obs, - obs_groups, - num_actions, - actor_obs_normalization=False, - critic_obs_normalization=False, - actor_hidden_dims=[256, 256, 256], - critic_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=1.0, - noise_std_type: str = "scalar", - state_dependent_std=False, - rnn_type="lstm", - rnn_hidden_dim=256, - rnn_num_layers=1, - **kwargs, - ): - if "rnn_hidden_size" in kwargs: - warnings.warn( - "The argument `rnn_hidden_size` is deprecated and will be removed in a future version. " - "Please use `rnn_hidden_dim` instead.", - DeprecationWarning, - ) - if rnn_hidden_dim == 256: # Only override if the new argument is at its default - rnn_hidden_dim = kwargs.pop("rnn_hidden_size") - if kwargs: - print( - "ActorCriticRecurrent.__init__ got unexpected arguments, which will be ignored: " + str(kwargs.keys()), - ) - super().__init__() - - # get the observation dimensions - self.obs_groups = obs_groups - num_actor_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The ActorCriticRecurrent module only supports 1D observations." - num_actor_obs += obs[obs_group].shape[-1] - num_critic_obs = 0 - for obs_group in obs_groups["critic"]: - assert len(obs[obs_group].shape) == 2, "The ActorCriticRecurrent module only supports 1D observations." - num_critic_obs += obs[obs_group].shape[-1] - - self.state_dependent_std = state_dependent_std - # actor - self.memory_a = Memory(num_actor_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - if self.state_dependent_std: - self.actor = MLP(rnn_hidden_dim, [2, num_actions], actor_hidden_dims, activation) - else: - self.actor = MLP(rnn_hidden_dim, num_actions, actor_hidden_dims, activation) - - # actor observation normalization - self.actor_obs_normalization = actor_obs_normalization - if actor_obs_normalization: - self.actor_obs_normalizer = EmpiricalNormalization(num_actor_obs) - else: - self.actor_obs_normalizer = torch.nn.Identity() - print(f"Actor RNN: {self.memory_a}") - print(f"Actor MLP: {self.actor}") - - # critic - self.memory_c = Memory(num_critic_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - self.critic = MLP(rnn_hidden_dim, 1, critic_hidden_dims, activation) - # critic observation normalization - self.critic_obs_normalization = critic_obs_normalization - if critic_obs_normalization: - self.critic_obs_normalizer = EmpiricalNormalization(num_critic_obs) - else: - self.critic_obs_normalizer = torch.nn.Identity() - print(f"Critic RNN: {self.memory_c}") - print(f"Critic MLP: {self.critic}") - - # Action noise - self.noise_std_type = noise_std_type - if self.state_dependent_std: - torch.nn.init.zeros_(self.actor[-2].weight[num_actions:]) - if self.noise_std_type == "scalar": - torch.nn.init.constant_(self.actor[-2].bias[num_actions:], init_noise_std) - elif self.noise_std_type == "log": - torch.nn.init.constant_( - self.actor[-2].bias[num_actions:], torch.log(torch.tensor(init_noise_std + 1e-7)) - ) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # Action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def reset(self, dones=None): - self.memory_a.reset(dones) - self.memory_c.reset(dones) - - def forward(self): - raise NotImplementedError - - def update_distribution(self, obs): - if self.state_dependent_std: - # compute mean and standard deviation - mean_and_std = self.actor(obs) - if self.noise_std_type == "scalar": - mean, std = torch.unbind(mean_and_std, dim=-2) - elif self.noise_std_type == "log": - mean, log_std = torch.unbind(mean_and_std, dim=-2) - std = torch.exp(log_std) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - else: - # compute mean - mean = self.actor(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs, masks=None, hidden_states=None): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - out_mem = self.memory_a(obs, masks, hidden_states).squeeze(0) - self.update_distribution(out_mem) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_actor_obs(obs) - obs = self.actor_obs_normalizer(obs) - out_mem = self.memory_a(obs).squeeze(0) - return self.actor(out_mem) - - def evaluate(self, obs, masks=None, hidden_states=None): - obs = self.get_critic_obs(obs) - obs = self.critic_obs_normalizer(obs) - out_mem = self.memory_c(obs, masks, hidden_states).squeeze(0) - return self.critic(out_mem) - - def get_actor_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_critic_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["critic"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_actions_log_prob(self, actions): - return self.distribution.log_prob(actions).sum(dim=-1) - - def get_hidden_states(self): - return self.memory_a.hidden_states, self.memory_c.hidden_states - - def update_normalization(self, obs): - if self.actor_obs_normalization: - actor_obs = self.get_actor_obs(obs) - self.actor_obs_normalizer.update(actor_obs) - if self.critic_obs_normalization: - critic_obs = self.get_critic_obs(obs) - self.critic_obs_normalizer.update(critic_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the actor-critic model. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters (relevant for, e.g., distillation). - """ - - super().load_state_dict(state_dict, strict=strict) - return True diff --git a/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py b/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py deleted file mode 100644 index 7899d23..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/discriminator_multi.py +++ /dev/null @@ -1,102 +0,0 @@ -import torch -import torch.nn as nn -from torch import autograd -import torch.nn.utils.spectral_norm as spectral_norm - - -class DiscriminatorMulti(nn.Module): - def __init__( - self, state_dim, amp_reward_coef, hidden_layer_sizes, device, - num_frames=2, task_reward_lerp=0.0, use_lerp=True): - super(DiscriminatorMulti, self).__init__() - - self.device = device - self.state_dim = state_dim - self.use_lerp = use_lerp - self.num_frames = num_frames # 存储帧数参数 - - self.amp_reward_coef = amp_reward_coef - amp_layers = [] - - curr_in_dim = state_dim * num_frames - for hidden_dim in hidden_layer_sizes: - amp_layers.append(spectral_norm(nn.Linear(curr_in_dim, hidden_dim))) - amp_layers.append(nn.ReLU()) - curr_in_dim = hidden_dim - self.trunk = nn.Sequential(*amp_layers).to(device) - self.amp_linear = spectral_norm(nn.Linear(hidden_layer_sizes[-1], 1)).to(device) - - self.trunk.train() - self.amp_linear.train() - - self.task_reward_lerp = task_reward_lerp - - def forward(self, x): - h = self.trunk(x) - d = self.amp_linear(h) - return d - - def compute_grad_pen(self, - expert_states, # 改为接收多帧状态列表 - lambda_=10): - # 将多帧状态沿最后一个维度拼接 - expert_data = expert_states.flatten(1) - expert_data.requires_grad = True - - disc = self.amp_linear(self.trunk(expert_data)) - ones = torch.ones(disc.size(), device=disc.device) - grad = autograd.grad( - outputs=disc, inputs=expert_data, - grad_outputs=ones, create_graph=True, - retain_graph=True, only_inputs=True)[0] - - # Enforce that the grad norm approaches 0. - grad_pen = lambda_ * (grad.norm(2, dim=1) - 0).pow(2).mean() - return grad_pen - - - def get_disc_weights(self): - weights = [] - for m in self.trunk.modules(): - if isinstance(m, nn.Linear): - weights.append(torch.flatten(m.weight)) - - weights.append(torch.flatten(self.amp_linear.weight)) - return weights - - def get_disc_logit_weights(self): - return torch.flatten(self.amp_linear.weight) - - def predict_amp_reward( - self, states, # 改为接收多帧状态列表 - task_reward, normalizer=None): - """ - states: torch.Tensor, shape=(num_envs, num_frames, state_dim) - task_reward: torch.Tensor, shape=(num_envs, 1) - """ - # import ipdb; ipdb.set_trace() - with torch.no_grad(): - self.eval() - if normalizer is not None: - # 对每一帧状态进行归一化 - states = normalizer.normalize_torch(states, self.device) - - # 拼接多帧状态 - state_cat = states.flatten(1) - d = self.amp_linear(self.trunk(state_cat)) - disc_reward = self.amp_reward_coef * torch.clamp(1 - (1/4) * torch.square(d - 1), min=0) - - if self.use_lerp: - if self.task_reward_lerp > 0: - reward = self._lerp_reward(disc_reward, task_reward.unsqueeze(-1)) - self.train() - return reward.squeeze(), d, disc_reward.squeeze() * (1.0 - self.task_reward_lerp) - else: - disc_reward *= 0.02 - reward = task_reward.unsqueeze(-1) + disc_reward - self.train() - return reward.squeeze(), d, disc_reward.squeeze() - - def _lerp_reward(self, disc_r, task_r): - r = (1.0 - self.task_reward_lerp) * disc_r + self.task_reward_lerp * task_r - return r \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/modules/rnd.py b/rsl_rl/build/lib/rsl_rl/modules/rnd.py deleted file mode 100644 index 8e65c43..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/rnd.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn - -from rsl_rl.networks import MLP, EmpiricalDiscountedVariationNormalization, EmpiricalNormalization - - -class RandomNetworkDistillation(nn.Module): - """Implementation of Random Network Distillation (RND) [1] - - References: - .. [1] Burda, Yuri, et al. "Exploration by random network distillation." arXiv preprint arXiv:1810.12894 (2018). - """ - - def __init__( - self, - num_states: int, - obs_groups: dict, - num_outputs: int, - predictor_hidden_dims: list[int], - target_hidden_dims: list[int], - activation: str = "elu", - weight: float = 0.0, - state_normalization: bool = False, - reward_normalization: bool = False, - device: str = "cpu", - weight_schedule: dict | None = None, - ): - """Initialize the RND module. - - - If :attr:`state_normalization` is True, then the input state is normalized using an Empirical Normalization layer. - - If :attr:`reward_normalization` is True, then the intrinsic reward is normalized using an Empirical Discounted - Variation Normalization layer. - - .. note:: - If the hidden dimensions are -1 in the predictor and target networks configuration, then the number of states - is used as the hidden dimension. - - Args: - num_states: Number of states/inputs to the predictor and target networks. - num_outputs: Number of outputs (embedding size) of the predictor and target networks. - predictor_hidden_dims: List of hidden dimensions of the predictor network. - target_hidden_dims: List of hidden dimensions of the target network. - activation: Activation function. Defaults to "elu". - weight: Scaling factor of the intrinsic reward. Defaults to 0.0. - state_normalization: Whether to normalize the input state. Defaults to False. - reward_normalization: Whether to normalize the intrinsic reward. Defaults to False. - device: Device to use. Defaults to "cpu". - weight_schedule: The type of schedule to use for the RND weight parameter. - Defaults to None, in which case the weight parameter is constant. - It is a dictionary with the following keys: - - - "mode": The type of schedule to use for the RND weight parameter. - - "constant": Constant weight schedule. - - "step": Step weight schedule. - - "linear": Linear weight schedule. - - For the "step" weight schedule, the following parameters are required: - - - "final_step": The step at which the weight parameter is set to the final value. - - "final_value": The final value of the weight parameter. - - For the "linear" weight schedule, the following parameters are required: - - "initial_step": The step at which the weight parameter is set to the initial value. - - "final_step": The step at which the weight parameter is set to the final value. - - "final_value": The final value of the weight parameter. - """ - # initialize parent class - super().__init__() - - # Store parameters - self.num_states = num_states - self.obs_groups = obs_groups - self.num_outputs = num_outputs - self.initial_weight = weight - self.device = device - self.state_normalization = state_normalization - self.reward_normalization = reward_normalization - - # Normalization of input gates - if state_normalization: - self.state_normalizer = EmpiricalNormalization(shape=[self.num_states], until=1.0e8).to(self.device) - else: - self.state_normalizer = torch.nn.Identity() - # Normalization of intrinsic reward - if reward_normalization: - self.reward_normalizer = EmpiricalDiscountedVariationNormalization(shape=[], until=1.0e8).to(self.device) - else: - self.reward_normalizer = torch.nn.Identity() - - # counter for the number of updates - self.update_counter = 0 - - # resolve weight schedule - if weight_schedule is not None: - self.weight_scheduler_params = weight_schedule - self.weight_scheduler = getattr(self, f"_{weight_schedule['mode']}_weight_schedule") - else: - self.weight_scheduler = None - # Create network architecture - self.predictor = MLP(num_states, num_outputs, predictor_hidden_dims, activation).to(self.device) - self.target = MLP(num_states, num_outputs, target_hidden_dims, activation).to(self.device) - - # make target network not trainable - self.target.eval() - - def get_intrinsic_reward(self, obs) -> torch.Tensor: - # Note: the counter is updated number of env steps per learning iteration - self.update_counter += 1 - # Extract the rnd state from the observation - rnd_state = self.get_rnd_state(obs) - rnd_state = self.state_normalizer(rnd_state) - # Obtain the embedding of the rnd state from the target and predictor networks - target_embedding = self.target(rnd_state).detach() - predictor_embedding = self.predictor(rnd_state).detach() - # Compute the intrinsic reward as the distance between the embeddings - intrinsic_reward = torch.linalg.norm(target_embedding - predictor_embedding, dim=1) - # Normalize intrinsic reward - intrinsic_reward = self.reward_normalizer(intrinsic_reward) - - # Check the weight schedule - if self.weight_scheduler is not None: - self.weight = self.weight_scheduler(step=self.update_counter, **self.weight_scheduler_params) - else: - self.weight = self.initial_weight - # Scale intrinsic reward - intrinsic_reward *= self.weight - - return intrinsic_reward - - def forward(self, *args, **kwargs): - raise RuntimeError("Forward method is not implemented. Use get_intrinsic_reward instead.") - - def train(self, mode: bool = True): - # sets module into training mode - self.predictor.train(mode) - if self.state_normalization: - self.state_normalizer.train(mode) - if self.reward_normalization: - self.reward_normalizer.train(mode) - return self - - def eval(self): - return self.train(False) - - def get_rnd_state(self, obs): - obs_list = [] - for obs_group in self.obs_groups["rnd_state"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def update_normalization(self, obs): - # Normalize the state - if self.state_normalization: - rnd_state = self.get_rnd_state(obs) - self.state_normalizer.update(rnd_state) - - """ - Different weight schedules. - """ - - def _constant_weight_schedule(self, step: int, **kwargs): - return self.initial_weight - - def _step_weight_schedule(self, step: int, final_step: int, final_value: float, **kwargs): - return self.initial_weight if step < final_step else final_value - - def _linear_weight_schedule(self, step: int, initial_step: int, final_step: int, final_value: float, **kwargs): - if step < initial_step: - return self.initial_weight - elif step > final_step: - return final_value - else: - return self.initial_weight + (final_value - self.initial_weight) * (step - initial_step) / ( - final_step - initial_step - ) - - -def resolve_rnd_config(alg_cfg, obs, obs_groups, env): - """Resolve the RND configuration. - - Args: - alg_cfg: The algorithm configuration dictionary. - obs: The observation dictionary. - obs_groups: The observation groups dictionary. - env: The environment. - - Returns: - The resolved algorithm configuration dictionary. - """ - # resolve dimension of rnd gated state - if "rnd_cfg" in alg_cfg and alg_cfg["rnd_cfg"] is not None: - # get dimension of rnd gated state - num_rnd_state = 0 - for obs_group in obs_groups["rnd_state"]: - assert len(obs[obs_group].shape) == 2, "The RND module only supports 1D observations." - num_rnd_state += obs[obs_group].shape[-1] - # add rnd gated state to config - alg_cfg["rnd_cfg"]["num_states"] = num_rnd_state - alg_cfg["rnd_cfg"]["obs_groups"] = obs_groups - # scale down the rnd weight with timestep - alg_cfg["rnd_cfg"]["weight"] *= env.unwrapped.step_dt - return alg_cfg diff --git a/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py b/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py deleted file mode 100644 index 6bf1380..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/student_teacher.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization - - -class StudentTeacher(nn.Module): - is_recurrent = False - - def __init__( - self, - obs, - obs_groups, - num_actions, - student_obs_normalization=False, - teacher_obs_normalization=False, - student_hidden_dims=[256, 256, 256], - teacher_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=0.1, - noise_std_type: str = "scalar", - **kwargs, - ): - if kwargs: - print( - "StudentTeacher.__init__ got unexpected arguments, which will be ignored: " - + str([key for key in kwargs.keys()]) - ) - super().__init__() - - self.loaded_teacher = False # indicates if teacher has been loaded - - # get the observation dimensions - self.obs_groups = obs_groups - num_student_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_student_obs += obs[obs_group].shape[-1] - num_teacher_obs = 0 - for obs_group in obs_groups["teacher"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_teacher_obs += obs[obs_group].shape[-1] - - # student - self.student = MLP(num_student_obs, num_actions, student_hidden_dims, activation) - - # student observation normalization - self.student_obs_normalization = student_obs_normalization - if student_obs_normalization: - self.student_obs_normalizer = EmpiricalNormalization(num_student_obs) - else: - self.student_obs_normalizer = torch.nn.Identity() - - print(f"Student MLP: {self.student}") - - # teacher - self.teacher = MLP(num_teacher_obs, num_actions, teacher_hidden_dims, activation) - self.teacher.eval() - - # teacher observation normalization - self.teacher_obs_normalization = teacher_obs_normalization - if teacher_obs_normalization: - self.teacher_obs_normalizer = EmpiricalNormalization(num_teacher_obs) - else: - self.teacher_obs_normalizer = torch.nn.Identity() - - print(f"Teacher MLP: {self.teacher}") - - # action noise - self.noise_std_type = noise_std_type - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None, hidden_states=None): - pass - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - # compute mean - mean = self.student(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - self.update_distribution(obs) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - return self.student(obs) - - def evaluate(self, obs): - obs = self.get_teacher_obs(obs) - obs = self.teacher_obs_normalizer(obs) - with torch.no_grad(): - return self.teacher(obs) - - def get_student_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_teacher_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["teacher"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_hidden_states(self): - return None - - def detach_hidden_states(self, dones=None): - pass - - def train(self, mode=True): - super().train(mode) - # make sure teacher is in eval mode - self.teacher.eval() - self.teacher_obs_normalizer.eval() - - def update_normalization(self, obs): - if self.student_obs_normalization: - student_obs = self.get_student_obs(obs) - self.student_obs_normalizer.update(student_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the student and teacher networks. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters. - """ - - # check if state_dict contains teacher and student or just teacher parameters - if any("actor" in key for key in state_dict.keys()): # loading parameters from rl training - # rename keys to match teacher and remove critic parameters - teacher_state_dict = {} - teacher_obs_normalizer_state_dict = {} - for key, value in state_dict.items(): - if "actor." in key: - teacher_state_dict[key.replace("actor.", "")] = value - if "actor_obs_normalizer." in key: - teacher_obs_normalizer_state_dict[key.replace("actor_obs_normalizer.", "")] = value - self.teacher.load_state_dict(teacher_state_dict, strict=strict) - self.teacher_obs_normalizer.load_state_dict(teacher_obs_normalizer_state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return False # training does not resume - elif any("student" in key for key in state_dict.keys()): # loading parameters from distillation training - super().load_state_dict(state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return True # training resumes - else: - raise ValueError("state_dict does not contain student or teacher parameters") diff --git a/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py b/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py deleted file mode 100644 index 964a2dc..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/student_teacher_recurrent.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -import warnings -from torch.distributions import Normal - -from rsl_rl.networks import MLP, EmpiricalNormalization, Memory - - -class StudentTeacherRecurrent(nn.Module): - is_recurrent = True - - def __init__( - self, - obs, - obs_groups, - num_actions, - student_obs_normalization=False, - teacher_obs_normalization=False, - student_hidden_dims=[256, 256, 256], - teacher_hidden_dims=[256, 256, 256], - activation="elu", - init_noise_std=0.1, - noise_std_type: str = "scalar", - rnn_type="lstm", - rnn_hidden_dim=256, - rnn_num_layers=1, - teacher_recurrent=False, - **kwargs, - ): - if "rnn_hidden_size" in kwargs: - warnings.warn( - "The argument `rnn_hidden_size` is deprecated and will be removed in a future version. " - "Please use `rnn_hidden_dim` instead.", - DeprecationWarning, - ) - if rnn_hidden_dim == 256: # Only override if the new argument is at its default - rnn_hidden_dim = kwargs.pop("rnn_hidden_size") - if kwargs: - print( - "StudentTeacherRecurrent.__init__ got unexpected arguments, which will be ignored: " - + str(kwargs.keys()), - ) - super().__init__() - - self.loaded_teacher = False # indicates if teacher has been loaded - self.teacher_recurrent = teacher_recurrent # indicates if teacher is recurrent too - - # get the observation dimensions - self.obs_groups = obs_groups - num_student_obs = 0 - for obs_group in obs_groups["policy"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_student_obs += obs[obs_group].shape[-1] - num_teacher_obs = 0 - for obs_group in obs_groups["teacher"]: - assert len(obs[obs_group].shape) == 2, "The StudentTeacher module only supports 1D observations." - num_teacher_obs += obs[obs_group].shape[-1] - - # student - self.memory_s = Memory(num_student_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim) - self.student = MLP(rnn_hidden_dim, num_actions, student_hidden_dims, activation) - - # student observation normalization - self.student_obs_normalization = student_obs_normalization - if student_obs_normalization: - self.student_obs_normalizer = EmpiricalNormalization(num_student_obs) - else: - self.student_obs_normalizer = torch.nn.Identity() - - print(f"Student RNN: {self.memory_s}") - print(f"Student MLP: {self.student}") - - # teacher - if self.teacher_recurrent: - self.memory_t = Memory( - num_teacher_obs, type=rnn_type, num_layers=rnn_num_layers, hidden_size=rnn_hidden_dim - ) - num_teacher_obs = rnn_hidden_dim - self.teacher = MLP(num_teacher_obs, num_actions, teacher_hidden_dims, activation) - - # teacher observation normalization - self.teacher_obs_normalization = teacher_obs_normalization - if teacher_obs_normalization: - self.teacher_obs_normalizer = EmpiricalNormalization(num_teacher_obs) - else: - self.teacher_obs_normalizer = torch.nn.Identity() - - if self.teacher_recurrent: - print(f"Teacher RNN: {self.memory_t}") - print(f"Teacher MLP: {self.teacher}") - - # action noise - self.noise_std_type = noise_std_type - if self.noise_std_type == "scalar": - self.std = nn.Parameter(init_noise_std * torch.ones(num_actions)) - elif self.noise_std_type == "log": - self.log_std = nn.Parameter(torch.log(init_noise_std * torch.ones(num_actions))) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - - # action distribution (populated in update_distribution) - self.distribution = None - # disable args validation for speedup - Normal.set_default_validate_args(False) - - def reset(self, dones=None, hidden_states=None): - if hidden_states is None: - hidden_states = (None, None) - self.memory_s.reset(dones, hidden_states[0]) - if self.teacher_recurrent: - self.memory_t.reset(dones, hidden_states[1]) - - def forward(self): - raise NotImplementedError - - @property - def action_mean(self): - return self.distribution.mean - - @property - def action_std(self): - return self.distribution.stddev - - @property - def entropy(self): - return self.distribution.entropy().sum(dim=-1) - - def update_distribution(self, obs): - # compute mean - mean = self.student(obs) - # compute standard deviation - if self.noise_std_type == "scalar": - std = self.std.expand_as(mean) - elif self.noise_std_type == "log": - std = torch.exp(self.log_std).expand_as(mean) - else: - raise ValueError(f"Unknown standard deviation type: {self.noise_std_type}. Should be 'scalar' or 'log'") - # create distribution - self.distribution = Normal(mean, std) - - def act(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - out_mem = self.memory_s(obs).squeeze(0) - self.update_distribution(out_mem) - return self.distribution.sample() - - def act_inference(self, obs): - obs = self.get_student_obs(obs) - obs = self.student_obs_normalizer(obs) - out_mem = self.memory_s(obs).squeeze(0) - return self.student(out_mem) - - def evaluate(self, obs): - obs = self.get_teacher_obs(obs) - obs = self.teacher_obs_normalizer(obs) - with torch.no_grad(): - if self.teacher_recurrent: - self.memory_t.eval() - obs = self.memory_t(obs).squeeze(0) - return self.teacher(obs) - - def get_student_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["policy"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_teacher_obs(self, obs): - obs_list = [] - for obs_group in self.obs_groups["teacher"]: - obs_list.append(obs[obs_group]) - return torch.cat(obs_list, dim=-1) - - def get_hidden_states(self): - if self.teacher_recurrent: - return self.memory_s.hidden_states, self.memory_t.hidden_states - else: - return self.memory_s.hidden_states, None - - def detach_hidden_states(self, dones=None): - self.memory_s.detach_hidden_states(dones) - if self.teacher_recurrent: - self.memory_t.detach_hidden_states(dones) - - def train(self, mode=True): - super().train(mode) - # make sure teacher is in eval mode - self.teacher.eval() - self.teacher_obs_normalizer.eval() - - def update_normalization(self, obs): - if self.student_obs_normalization: - student_obs = self.get_student_obs(obs) - self.student_obs_normalizer.update(student_obs) - - def load_state_dict(self, state_dict, strict=True): - """Load the parameters of the student and teacher networks. - - Args: - state_dict (dict): State dictionary of the model. - strict (bool): Whether to strictly enforce that the keys in state_dict match the keys returned by this - module's state_dict() function. - - Returns: - bool: Whether this training resumes a previous training. This flag is used by the `load()` function of - `OnPolicyRunner` to determine how to load further parameters. - """ - - # check if state_dict contains teacher and student or just teacher parameters - if any("actor" in key for key in state_dict.keys()): # loading parameters from rl training - # rename keys to match teacher and remove critic parameters - teacher_state_dict = {} - teacher_obs_normalizer_state_dict = {} - for key, value in state_dict.items(): - if "actor." in key: - teacher_state_dict[key.replace("actor.", "")] = value - if "actor_obs_normalizer." in key: - teacher_obs_normalizer_state_dict[key.replace("actor_obs_normalizer.", "")] = value - self.teacher.load_state_dict(teacher_state_dict, strict=strict) - self.teacher_obs_normalizer.load_state_dict(teacher_obs_normalizer_state_dict, strict=strict) - # also load recurrent memory if teacher is recurrent - if self.teacher_recurrent: - memory_t_state_dict = {} - for key, value in state_dict.items(): - if "memory_a." in key: - memory_t_state_dict[key.replace("memory_a.", "")] = value - self.memory_t.load_state_dict(memory_t_state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return False # training does not resume - elif any("student" in key for key in state_dict.keys()): # loading parameters from distillation training - super().load_state_dict(state_dict, strict=strict) - # set flag for successfully loading the parameters - self.loaded_teacher = True - self.teacher.eval() - self.teacher_obs_normalizer.eval() - return True # training resumes - else: - raise ValueError("state_dict does not contain student or teacher parameters") diff --git a/rsl_rl/build/lib/rsl_rl/modules/symmetry.py b/rsl_rl/build/lib/rsl_rl/modules/symmetry.py deleted file mode 100644 index b017515..0000000 --- a/rsl_rl/build/lib/rsl_rl/modules/symmetry.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - - -def resolve_symmetry_config(alg_cfg, env): - """Resolve the symmetry configuration. - - Args: - alg_cfg: The algorithm configuration dictionary. - env: The environment. - - Returns: - The resolved algorithm configuration dictionary. - """ - - # if using symmetry then pass the environment config object - if "symmetry_cfg" in alg_cfg and alg_cfg["symmetry_cfg"] is not None: - # this is used by the symmetry function for handling different observation terms - alg_cfg["symmetry_cfg"]["_env"] = env - return alg_cfg diff --git a/rsl_rl/build/lib/rsl_rl/networks/__init__.py b/rsl_rl/build/lib/rsl_rl/networks/__init__.py deleted file mode 100644 index c18f487..0000000 --- a/rsl_rl/build/lib/rsl_rl/networks/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Definitions for components of modules.""" - -from .memory import Memory -from .mlp import MLP -from .normalization import EmpiricalDiscountedVariationNormalization, EmpiricalNormalization diff --git a/rsl_rl/build/lib/rsl_rl/networks/memory.py b/rsl_rl/build/lib/rsl_rl/networks/memory.py deleted file mode 100644 index 7577357..0000000 --- a/rsl_rl/build/lib/rsl_rl/networks/memory.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch.nn as nn - -from rsl_rl.utils import unpad_trajectories - - -class Memory(nn.Module): - """Memory module for recurrent networks. - - This module is used to store the hidden states of the policy. - Currently only supports GRU and LSTM. - """ - - def __init__(self, input_size, type="lstm", num_layers=1, hidden_size=256): - super().__init__() - # RNN - rnn_cls = nn.GRU if type.lower() == "gru" else nn.LSTM - self.rnn = rnn_cls(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers) - self.hidden_states = None - - def forward(self, input, masks=None, hidden_states=None): - batch_mode = masks is not None - if batch_mode: - # batch mode: needs saved hidden states - if hidden_states is None: - raise ValueError("Hidden states not passed to memory module during policy update") - out, _ = self.rnn(input, hidden_states) - out = unpad_trajectories(out, masks) - else: - # inference/distillation mode: uses hidden states of last step - out, self.hidden_states = self.rnn(input.unsqueeze(0), self.hidden_states) - return out - - def reset(self, dones=None, hidden_states=None): - if dones is None: # reset all hidden states - if hidden_states is None: - self.hidden_states = None - else: - self.hidden_states = hidden_states - elif self.hidden_states is not None: # reset hidden states of done environments - if hidden_states is None: - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - for hidden_state in self.hidden_states: - hidden_state[..., dones == 1, :] = 0.0 - else: - self.hidden_states[..., dones == 1, :] = 0.0 - else: - NotImplementedError( - "Resetting hidden states of done environments with custom hidden states is not implemented" - ) - - def detach_hidden_states(self, dones=None): - if self.hidden_states is not None: - if dones is None: # detach all hidden states - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - self.hidden_states = tuple(hidden_state.detach() for hidden_state in self.hidden_states) - else: - self.hidden_states = self.hidden_states.detach() - else: # detach hidden states of done environments - if isinstance(self.hidden_states, tuple): # tuple in case of LSTM - for hidden_state in self.hidden_states: - hidden_state[..., dones == 1, :] = hidden_state[..., dones == 1, :].detach() - else: - self.hidden_states[..., dones == 1, :] = self.hidden_states[..., dones == 1, :].detach() diff --git a/rsl_rl/build/lib/rsl_rl/networks/mlp.py b/rsl_rl/build/lib/rsl_rl/networks/mlp.py deleted file mode 100644 index e91574e..0000000 --- a/rsl_rl/build/lib/rsl_rl/networks/mlp.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -import torch.nn as nn -from functools import reduce - -from rsl_rl.utils import resolve_nn_activation - - -class MLP(nn.Sequential): - """Multi-layer perceptron. - - The MLP network is a sequence of linear layers and activation functions. The - last layer is a linear layer that outputs the desired dimension unless the - last activation function is specified. - - It provides additional conveniences: - - - If the hidden dimensions have a value of ``-1``, the dimension is inferred - from the input dimension. - - If the output dimension is a tuple, the output is reshaped to the desired - shape. - - """ - - def __init__( - self, - input_dim: int, - output_dim: int | tuple[int] | list[int], - hidden_dims: tuple[int] | list[int], - activation: str = "elu", - last_activation: str | None = None, - ): - """Initialize the MLP. - - Args: - input_dim: Dimension of the input. - output_dim: Dimension of the output. - hidden_dims: Dimensions of the hidden layers. A value of ``-1`` indicates - that the dimension should be inferred from the input dimension. - activation: Activation function. Defaults to "elu". - last_activation: Activation function of the last layer. Defaults to None, - in which case the last layer is linear. - """ - super().__init__() - - # resolve activation functions - activation_mod = resolve_nn_activation(activation) - last_activation_mod = resolve_nn_activation(last_activation) if last_activation is not None else None - # resolve number of hidden dims if they are -1 - hidden_dims_processed = [input_dim if dim == -1 else dim for dim in hidden_dims] - - # create layers sequentially - layers = [] - layers.append(nn.Linear(input_dim, hidden_dims_processed[0])) - layers.append(activation_mod) - - for layer_index in range(len(hidden_dims_processed) - 1): - layers.append(nn.Linear(hidden_dims_processed[layer_index], hidden_dims_processed[layer_index + 1])) - layers.append(activation_mod) - - # add last layer - if isinstance(output_dim, int): - layers.append(nn.Linear(hidden_dims_processed[-1], output_dim)) - else: - # compute the total output dimension - total_out_dim = reduce(lambda x, y: x * y, output_dim) - # add a layer to reshape the output to the desired shape - layers.append(nn.Linear(hidden_dims_processed[-1], total_out_dim)) - layers.append(nn.Unflatten(dim=-1, unflattened_size=output_dim)) - - # add last activation function if specified - if last_activation_mod is not None: - layers.append(last_activation_mod) - - # register the layers - for idx, layer in enumerate(layers): - self.add_module(f"{idx}", layer) - - def init_weights(self, scales: float | tuple[float]): - """Initialize the weights of the MLP. - - Args: - scales: Scale factor for the weights. - """ - - def get_scale(idx) -> float: - """Get the scale factor for the weights of the MLP. - - Args: - idx: Index of the layer. - """ - return scales[idx] if isinstance(scales, (list, tuple)) else scales - - # initialize the weights - for idx, module in enumerate(self): - if isinstance(module, nn.Linear): - nn.init.orthogonal_(module.weight, gain=get_scale(idx)) - nn.init.zeros_(module.bias) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Forward pass of the MLP. - - Args: - x: Input tensor. - """ - for layer in self: - x = layer(x) - return x - - def reset(self, dones=None, hidden_states=None): - pass - - def detach_hidden_states(self, dones=None): - pass diff --git a/rsl_rl/build/lib/rsl_rl/networks/normalization.py b/rsl_rl/build/lib/rsl_rl/networks/normalization.py deleted file mode 100644 index 5fd9692..0000000 --- a/rsl_rl/build/lib/rsl_rl/networks/normalization.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -# Copyright (c) 2020 Preferred Networks, Inc. - -from __future__ import annotations - -import torch -from torch import nn - - -class EmpiricalNormalization(nn.Module): - """Normalize mean and variance of values based on empirical values.""" - - def __init__(self, shape, eps=1e-2, until=None): - """Initialize EmpiricalNormalization module. - - Args: - shape (int or tuple of int): Shape of input values except batch axis. - eps (float): Small value for stability. - until (int or None): If this arg is specified, the module learns input values until the sum of batch sizes - exceeds it. - - Note: The normalization parameters are computed over the whole batch, not for each environment separately. - """ - super().__init__() - self.eps = eps - self.until = until - self.register_buffer("_mean", torch.zeros(shape).unsqueeze(0)) - self.register_buffer("_var", torch.ones(shape).unsqueeze(0)) - self.register_buffer("_std", torch.ones(shape).unsqueeze(0)) - self.register_buffer("count", torch.tensor(0, dtype=torch.long)) - - @property - def mean(self): - return self._mean.squeeze(0).clone() - - @property - def std(self): - return self._std.squeeze(0).clone() - - def forward(self, x): - """Normalize mean and variance of values based on empirical values.""" - - return (x - self._mean) / (self._std + self.eps) - - @torch.jit.unused - def update(self, x): - """Learn input values without computing the output values of them""" - - if not self.training: - return - if self.until is not None and self.count >= self.until: - return - - count_x = x.shape[0] - self.count += count_x - rate = count_x / self.count - var_x = torch.var(x, dim=0, unbiased=False, keepdim=True) - mean_x = torch.mean(x, dim=0, keepdim=True) - delta_mean = mean_x - self._mean - self._mean += rate * delta_mean - self._var += rate * (var_x - self._var + delta_mean * (mean_x - self._mean)) - self._std = torch.sqrt(self._var) - - @torch.jit.unused - def inverse(self, y): - """De-normalize values based on empirical values.""" - - return y * (self._std + self.eps) + self._mean - - -class EmpiricalDiscountedVariationNormalization(nn.Module): - """Reward normalization from Pathak's large scale study on PPO. - - Reward normalization. Since the reward function is non-stationary, it is useful to normalize - the scale of the rewards so that the value function can learn quickly. We did this by dividing - the rewards by a running estimate of the standard deviation of the sum of discounted rewards. - """ - - def __init__(self, shape, eps=1e-2, gamma=0.99, until=None): - super().__init__() - - self.emp_norm = EmpiricalNormalization(shape, eps, until) - self.disc_avg = _DiscountedAverage(gamma) - - def forward(self, rew): - if self.training: - # update discounted rewards - avg = self.disc_avg.update(rew) - # update moments from discounted rewards - self.emp_norm.update(avg) - - # normalize rewards with the empirical std - if self.emp_norm._std > 0: - return rew / self.emp_norm._std - else: - return rew - - -""" -Helper class. -""" - - -class _DiscountedAverage: - r"""Discounted average of rewards. - - The discounted average is defined as: - - .. math:: - - \bar{R}_t = \gamma \bar{R}_{t-1} + r_t - - Args: - gamma (float): Discount factor. - """ - - def __init__(self, gamma): - self.avg = None - self.gamma = gamma - - def update(self, rew: torch.Tensor) -> torch.Tensor: - if self.avg is None: - self.avg = rew - else: - self.avg = self.avg * self.gamma + rew - return self.avg diff --git a/rsl_rl/build/lib/rsl_rl/runners/__init__.py b/rsl_rl/build/lib/rsl_rl/runners/__init__.py deleted file mode 100644 index 61f1682..0000000 --- a/rsl_rl/build/lib/rsl_rl/runners/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of runners for environment-agent interaction.""" - -from .on_policy_runner import OnPolicyRunner # isort:skip -from .distillation_runner import DistillationRunner -from .amp_on_policy_runner import AMPOnPolicyRunner - -__all__ = ["OnPolicyRunner", "DistillationRunner", "AMPOnPolicyRunner"] diff --git a/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py b/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py deleted file mode 100644 index c0b9b9e..0000000 --- a/rsl_rl/build/lib/rsl_rl/runners/amp_on_policy_runner.py +++ /dev/null @@ -1,521 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import statistics -import time -import torch -import warnings -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import AMP_PPO -from rsl_rl.env import VecEnv -from rsl_rl.modules import ActorCritic, ActorCriticRecurrent,DiscriminatorMulti, resolve_rnd_config, resolve_symmetry_config -from rsl_rl.utils import resolve_obs_groups, store_code_state, Normalizer, G1_AMPLoader - - -class AMPOnPolicyRunner: - """On-policy runner for training and evaluation of actor-critic methods.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - default_sets = ["critic"] - if "rnd_cfg" in self.alg_cfg and self.alg_cfg["rnd_cfg"] is not None: - default_sets.append("rnd_state") - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets) - - self.amp_data = G1_AMPLoader( - device, - time_between_frames=1/50.0, - preload_transitions=True, - num_preload_transitions=train_cfg["amp_num_preload_transitions"], - motion_files=train_cfg["amp_motion_files"], - num_frames=train_cfg['amp_num_frames'] - ) - - self.amp_observation_dim = self.amp_data.observation_dim if self.cfg["amp_num_obs"] == 0 else self.cfg["amp_num_obs"] - self.amp_num_frames = 0 if self.cfg["amp_num_frames"] == 0 else self.cfg["amp_num_frames"] - self.amp_normalizer = Normalizer(self.amp_observation_dim) - self.discriminator = DiscriminatorMulti( - self.amp_observation_dim, - train_cfg["amp_reward_coef"], - train_cfg["amp_discr_hidden_dims"], - device, - train_cfg["amp_num_frames"], - train_cfg["amp_task_reward_lerp"], - train_cfg['use_lerp'], - ).to(self.device) - - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - amp_obs = self.env.get_amp_observations() - amp_obs = amp_obs.to(self.device) - if self.amp_num_frames != 0: - self.amp_obs_frames = torch.zeros(size=(self.env.num_envs, self.amp_num_frames, self.amp_observation_dim), device=self.device) - self.amp_obs_frames = torch.concat((self.amp_obs_frames[:, 1:], amp_obs.unsqueeze(1)), dim=1) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - step_discrewbuffer = deque(maxlen=100) - - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_single_step_disc_rew = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - # create buffers for logging extrinsic and intrinsic rewards - if self.alg.rnd: - erewbuffer = deque(maxlen=100) - irewbuffer = deque(maxlen=100) - cur_ereward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_ireward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs,amp_obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - - next_amp_obs = self.env.get_amp_observations() - next_amp_obs = next_amp_obs.to(self.device) - next_amp_obs_with_term = torch.clone(next_amp_obs) - - reset_env_ids = self.env.reset_env_ids - terminal_amp_states = self.env.get_amp_observations()[reset_env_ids] - next_amp_obs_with_term[reset_env_ids] = terminal_amp_states - self.amp_obs_frames = torch.concat((self.amp_obs_frames[:, 1:], next_amp_obs_with_term.unsqueeze(1)), dim=1) - - amp_reward = torch.zeros(self.env.num_envs, device=obs.device) - - mask = self.env.contact_phase[:, 0] == 1.0 - if mask.any(): - rewards[mask], logit, disc_reward = self.alg.discriminator.predict_amp_reward( - self.amp_obs_frames[mask], rewards[mask], normalizer=self.alg.amp_normalizer - ) - amp_reward[mask] += disc_reward - - # process the step - self.alg.process_env_step(obs, rewards, dones, extras, next_amp_obs_with_term, self.amp_obs_frames) - self.amp_obs_frames[reset_env_ids] = 0 - - amp_obs = torch.clone(next_amp_obs) - # Extract intrinsic rewards (only for logging) - intrinsic_rewards = self.alg.intrinsic_rewards if self.alg.rnd else None - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - if self.alg.rnd: - cur_ereward_sum += rewards - cur_ireward_sum += intrinsic_rewards # type: ignore - cur_reward_sum += rewards + intrinsic_rewards - else: - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - cur_single_step_disc_rew += amp_reward - # Clear data for completed episodes - # -- common - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - to_extend_disc = (cur_single_step_disc_rew[new_ids] / self.env.max_episode_length_s)[:, 0].cpu().numpy() - step_discrewbuffer.extend(to_extend_disc.tolist()) - cur_single_step_disc_rew[new_ids] = 0 - # -- intrinsic and extrinsic rewards - if self.alg.rnd: - erewbuffer.extend(cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist()) - irewbuffer.extend(cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist()) - cur_ereward_sum[new_ids] = 0 - cur_ireward_sum[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # compute returns - self.alg.compute_returns(obs) - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - def log(self, locs: dict, width: int = 80, pad: int = 35): - # Compute the collection size - collection_size = self.num_steps_per_env * self.env.num_envs * self.gpu_world_size - # Update total time-steps and time - self.tot_timesteps += collection_size - self.tot_time += locs["collection_time"] + locs["learn_time"] - iteration_time = locs["collection_time"] + locs["learn_time"] - - # -- Episode info - ep_string = "" - if locs["ep_infos"]: - for key in locs["ep_infos"][0]: - infotensor = torch.tensor([], device=self.device) - for ep_info in locs["ep_infos"]: - # handle scalar and zero dimensional tensor infos - if key not in ep_info: - continue - if not isinstance(ep_info[key], torch.Tensor): - ep_info[key] = torch.Tensor([ep_info[key]]) - if len(ep_info[key].shape) == 0: - ep_info[key] = ep_info[key].unsqueeze(0) - infotensor = torch.cat((infotensor, ep_info[key].to(self.device))) - value = torch.mean(infotensor) - # log to logger and terminal - if "/" in key: - self.writer.add_scalar(key, value, locs["it"]) - ep_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - else: - self.writer.add_scalar("Episode/" + key, value, locs["it"]) - ep_string += f"""{f'Mean episode {key}:':>{pad}} {value:.4f}\n""" - - mean_std = self.alg.policy.action_std.mean() - fps = int(collection_size / (locs["collection_time"] + locs["learn_time"])) - - # -- Losses - for key, value in locs["loss_dict"].items(): - self.writer.add_scalar(f"Loss/{key}", value, locs["it"]) - self.writer.add_scalar("Loss/learning_rate", self.alg.learning_rate, locs["it"]) - - # -- Policy - self.writer.add_scalar("Policy/mean_noise_std", mean_std.item(), locs["it"]) - - # -- Performance - self.writer.add_scalar("Perf/total_fps", fps, locs["it"]) - self.writer.add_scalar("Perf/collection time", locs["collection_time"], locs["it"]) - self.writer.add_scalar("Perf/learning_time", locs["learn_time"], locs["it"]) - - # -- Training - if len(locs["rewbuffer"]) > 0: - # separate logging for intrinsic and extrinsic rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(locs["erewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(locs["irewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/weight", self.alg.rnd.weight, locs["it"]) - # everything else - self.writer.add_scalar("Train/mean_reward", statistics.mean(locs["rewbuffer"]), locs["it"]) - self.writer.add_scalar("Train/mean_episode_length", statistics.mean(locs["lenbuffer"]), locs["it"]) - self.writer.add_scalar('Train/mean_step_disc_reward', statistics.mean(locs['step_discrewbuffer']), locs['it']) - if self.logger_type != "wandb": # wandb does not support non-integer x-axis logging - self.writer.add_scalar("Train/mean_reward/time", statistics.mean(locs["rewbuffer"]), self.tot_time) - self.writer.add_scalar( - "Train/mean_episode_length/time", statistics.mean(locs["lenbuffer"]), self.tot_time - ) - - str = f" \033[1m Learning iteration {locs['it']}/{locs['tot_iter']} \033[0m " - - if len(locs["rewbuffer"]) > 0: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - f"""{'Step disc reward:':>{pad}} {statistics.mean(locs['step_discrewbuffer']):.2f}\n""" - ) - # -- Losses - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'Mean {key} loss:':>{pad}} {value:.4f}\n""" - # -- Rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - log_string += ( - f"""{'Mean extrinsic reward:':>{pad}} {statistics.mean(locs['erewbuffer']):.2f}\n""" - f"""{'Mean intrinsic reward:':>{pad}} {statistics.mean(locs['irewbuffer']):.2f}\n""" - ) - log_string += f"""{'Mean reward:':>{pad}} {statistics.mean(locs['rewbuffer']):.2f}\n""" - # -- episode info - log_string += f"""{'Mean episode length:':>{pad}} {statistics.mean(locs['lenbuffer']):.2f}\n""" - else: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - - log_string += ep_string - log_string += ( - f"""{'-' * width}\n""" - f"""{'Total timesteps:':>{pad}} {self.tot_timesteps}\n""" - f"""{'Iteration time:':>{pad}} {iteration_time:.2f}s\n""" - f"""{'Time elapsed:':>{pad}} {time.strftime("%H:%M:%S", time.gmtime(self.tot_time))}\n""" - f"""{'ETA:':>{pad}} {time.strftime( - "%H:%M:%S", - time.gmtime( - self.tot_time / (locs['it'] - locs['start_iter'] + 1) - * (locs['start_iter'] + locs['num_learning_iterations'] - locs['it']) - ) - )}\n""" - ) - print(log_string) - - def save(self, path: str, infos=None): - # -- Save model - saved_dict = { - "model_state_dict": self.alg.policy.state_dict(), - "optimizer_state_dict": self.alg.optimizer.state_dict(), - "iter": self.current_learning_iteration, - "infos": infos, - } - # -- Save RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - saved_dict["rnd_state_dict"] = self.alg.rnd.state_dict() - saved_dict["rnd_optimizer_state_dict"] = self.alg.rnd_optimizer.state_dict() - torch.save(saved_dict, path) - - # upload model to external logging service - if self.logger_type in ["neptune", "wandb"] and not self.disable_logs: - self.writer.save_model(path, self.current_learning_iteration) - - def load(self, path: str, load_optimizer: bool = True, map_location: str | None = None): - loaded_dict = torch.load(path, weights_only=False, map_location=map_location) - # -- Load model - resumed_training = self.alg.policy.load_state_dict(loaded_dict["model_state_dict"]) - # -- Load RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.load_state_dict(loaded_dict["rnd_state_dict"]) - # -- load optimizer if used - if load_optimizer and resumed_training: - # -- algorithm optimizer - self.alg.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"]) - # -- RND optimizer if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd_optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"]) - # -- load current learning iteration - if resumed_training: - self.current_learning_iteration = loaded_dict["iter"] - return loaded_dict["infos"] - - def get_inference_policy(self, device=None): - self.eval_mode() # switch to evaluation mode (dropout for example) - if device is not None: - self.alg.policy.to(device) - return self.alg.policy.act_inference - - def train_mode(self): - # -- PPO - self.alg.policy.train() - self.alg.discriminator.train() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.train() - - def eval_mode(self): - # -- PPO - self.alg.policy.eval() - self.alg.discriminator.eval() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.eval() - - def add_git_repo_to_log(self, repo_file_path): - self.git_status_repos.append(repo_file_path) - - """ - Helper functions. - """ - - def _configure_multi_gpu(self): - """Configure multi-gpu training.""" - # check if distributed training is enabled - self.gpu_world_size = int(os.getenv("WORLD_SIZE", "1")) - self.is_distributed = self.gpu_world_size > 1 - - # if not distributed training, set local and global rank to 0 and return - if not self.is_distributed: - self.gpu_local_rank = 0 - self.gpu_global_rank = 0 - self.multi_gpu_cfg = None - return - - # get rank and world size - self.gpu_local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.gpu_global_rank = int(os.getenv("RANK", "0")) - - # make a configuration dictionary - self.multi_gpu_cfg = { - "global_rank": self.gpu_global_rank, # rank of the main process - "local_rank": self.gpu_local_rank, # rank of the current process - "world_size": self.gpu_world_size, # total number of processes - } - - # check if user has device specified for local rank - if self.device != f"cuda:{self.gpu_local_rank}": - raise ValueError( - f"Device '{self.device}' does not match expected device for local rank '{self.gpu_local_rank}'." - ) - # validate multi-gpu configuration - if self.gpu_local_rank >= self.gpu_world_size: - raise ValueError( - f"Local rank '{self.gpu_local_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - if self.gpu_global_rank >= self.gpu_world_size: - raise ValueError( - f"Global rank '{self.gpu_global_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - - # initialize torch distributed - torch.distributed.init_process_group(backend="nccl", rank=self.gpu_global_rank, world_size=self.gpu_world_size) - # set device to the local rank - torch.cuda.set_device(self.gpu_local_rank) - - def _construct_algorithm(self, obs) -> AMP_PPO: - """Construct the actor-critic algorithm.""" - # resolve RND config - self.alg_cfg = resolve_rnd_config(self.alg_cfg, obs, self.cfg["obs_groups"], self.env) - - # resolve symmetry config - self.alg_cfg = resolve_symmetry_config(self.alg_cfg, self.env) - - # resolve deprecated normalization config - if self.cfg.get("empirical_normalization") is not None: - warnings.warn( - "The `empirical_normalization` parameter is deprecated. Please set `actor_obs_normalization` and " - "`critic_obs_normalization` as part of the `policy` configuration instead.", - DeprecationWarning, - ) - if self.policy_cfg.get("actor_obs_normalization") is None: - self.policy_cfg["actor_obs_normalization"] = self.cfg["empirical_normalization"] - if self.policy_cfg.get("critic_obs_normalization") is None: - self.policy_cfg["critic_obs_normalization"] = self.cfg["empirical_normalization"] - - # initialize the actor-critic - actor_critic_class = eval(self.policy_cfg.pop("class_name")) - actor_critic: ActorCritic | ActorCriticRecurrent = actor_critic_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - - alg: AMP_PPO = alg_class(actor_critic, self.discriminator, self.amp_data, self.amp_normalizer, self.amp_num_frames, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg) - - # initialize the storage - alg.init_storage( - "rl", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg - - def _prepare_logging_writer(self): - """Prepares the logging writers.""" - if self.log_dir is not None and self.writer is None and not self.disable_logs: - # Launch either Tensorboard or Neptune & Tensorboard summary writer(s), default: Tensorboard. - self.logger_type = self.cfg.get("logger", "tensorboard") - self.logger_type = self.logger_type.lower() - - if self.logger_type == "neptune": - from rsl_rl.utils.neptune_utils import NeptuneSummaryWriter - - self.writer = NeptuneSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "wandb": - from rsl_rl.utils.wandb_utils import WandbSummaryWriter - - self.writer = WandbSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "tensorboard": - from torch.utils.tensorboard import SummaryWriter - - self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) - else: - raise ValueError("Logger type not found. Please choose 'neptune', 'wandb' or 'tensorboard'.") diff --git a/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py b/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py deleted file mode 100644 index 9cc6a8b..0000000 --- a/rsl_rl/build/lib/rsl_rl/runners/distillation_runner.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import time -import torch -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import Distillation -from rsl_rl.env import VecEnv -from rsl_rl.modules import StudentTeacher, StudentTeacherRecurrent -from rsl_rl.runners import OnPolicyRunner -from rsl_rl.utils import resolve_obs_groups, store_code_state - - -class DistillationRunner(OnPolicyRunner): - """On-policy runner for training and evaluation of teacher-student training.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets=["teacher"]) - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - # check if teacher is loaded - if not self.alg.policy.loaded_teacher: - raise ValueError("Teacher model parameters not loaded. Please load a teacher model to distill.") - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - # process the step - self.alg.process_env_step(obs, rewards, dones, extras) - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - # Clear data for completed episodes - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - """ - Helper methods. - """ - - def _construct_algorithm(self, obs) -> Distillation: - """Construct the distillation algorithm.""" - # initialize the actor-critic - student_teacher_class = eval(self.policy_cfg.pop("class_name")) - student_teacher: StudentTeacher | StudentTeacherRecurrent = student_teacher_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - alg: Distillation = alg_class( - student_teacher, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg - ) - - # initialize the storage - alg.init_storage( - "distillation", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg diff --git a/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py b/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py deleted file mode 100644 index 36f11f3..0000000 --- a/rsl_rl/build/lib/rsl_rl/runners/on_policy_runner.py +++ /dev/null @@ -1,460 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -import statistics -import time -import torch -import warnings -from collections import deque - -import rsl_rl -from rsl_rl.algorithms import PPO -from rsl_rl.env import VecEnv -from rsl_rl.modules import ActorCritic, ActorCriticRecurrent, resolve_rnd_config, resolve_symmetry_config -from rsl_rl.utils import resolve_obs_groups, store_code_state - - -class OnPolicyRunner: - """On-policy runner for training and evaluation of actor-critic methods.""" - - def __init__(self, env: VecEnv, train_cfg: dict, log_dir: str | None = None, device="cpu"): - self.cfg = train_cfg - self.alg_cfg = train_cfg["algorithm"] - self.policy_cfg = train_cfg["policy"] - self.device = device - self.env = env - - # check if multi-gpu is enabled - self._configure_multi_gpu() - - # store training configuration - self.num_steps_per_env = self.cfg["num_steps_per_env"] - self.save_interval = self.cfg["save_interval"] - - # query observations from environment for algorithm construction - obs = self.env.get_observations() - default_sets = ["critic"] - if "rnd_cfg" in self.alg_cfg and self.alg_cfg["rnd_cfg"] is not None: - default_sets.append("rnd_state") - self.cfg["obs_groups"] = resolve_obs_groups(obs, self.cfg["obs_groups"], default_sets) - - # create the algorithm - self.alg = self._construct_algorithm(obs) - - # Decide whether to disable logging - # We only log from the process with rank 0 (main process) - self.disable_logs = self.is_distributed and self.gpu_global_rank != 0 - - # Logging - self.log_dir = log_dir - self.writer = None - self.tot_timesteps = 0 - self.tot_time = 0 - self.current_learning_iteration = 0 - self.git_status_repos = [rsl_rl.__file__] - - def learn(self, num_learning_iterations: int, init_at_random_ep_len: bool = False): # noqa: C901 - # initialize writer - self._prepare_logging_writer() - - # randomize initial episode lengths (for exploration) - if init_at_random_ep_len: - self.env.episode_length_buf = torch.randint_like( - self.env.episode_length_buf, high=int(self.env.max_episode_length) - ) - - # start learning - obs = self.env.get_observations().to(self.device) - self.train_mode() # switch to train mode (for dropout for example) - - # Book keeping - ep_infos = [] - rewbuffer = deque(maxlen=100) - lenbuffer = deque(maxlen=100) - cur_reward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_episode_length = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # create buffers for logging extrinsic and intrinsic rewards - if self.alg.rnd: - erewbuffer = deque(maxlen=100) - irewbuffer = deque(maxlen=100) - cur_ereward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - cur_ireward_sum = torch.zeros(self.env.num_envs, dtype=torch.float, device=self.device) - - # Ensure all parameters are in-synced - if self.is_distributed: - print(f"Synchronizing parameters for rank {self.gpu_global_rank}...") - self.alg.broadcast_parameters() - - # Start training - start_iter = self.current_learning_iteration - tot_iter = start_iter + num_learning_iterations - for it in range(start_iter, tot_iter): - start = time.time() - # Rollout - with torch.inference_mode(): - for _ in range(self.num_steps_per_env): - # Sample actions - actions = self.alg.act(obs) - # Step the environment - obs, rewards, dones, extras = self.env.step(actions.to(self.env.device)) - # Move to device - obs, rewards, dones = (obs.to(self.device), rewards.to(self.device), dones.to(self.device)) - # process the step - self.alg.process_env_step(obs, rewards, dones, extras) - # Extract intrinsic rewards (only for logging) - intrinsic_rewards = self.alg.intrinsic_rewards if self.alg.rnd else None - # book keeping - if self.log_dir is not None: - if "episode" in extras: - ep_infos.append(extras["episode"]) - elif "log" in extras: - ep_infos.append(extras["log"]) - # Update rewards - if self.alg.rnd: - cur_ereward_sum += rewards - cur_ireward_sum += intrinsic_rewards # type: ignore - cur_reward_sum += rewards + intrinsic_rewards - else: - cur_reward_sum += rewards - # Update episode length - cur_episode_length += 1 - # Clear data for completed episodes - # -- common - new_ids = (dones > 0).nonzero(as_tuple=False) - rewbuffer.extend(cur_reward_sum[new_ids][:, 0].cpu().numpy().tolist()) - lenbuffer.extend(cur_episode_length[new_ids][:, 0].cpu().numpy().tolist()) - cur_reward_sum[new_ids] = 0 - cur_episode_length[new_ids] = 0 - # -- intrinsic and extrinsic rewards - if self.alg.rnd: - erewbuffer.extend(cur_ereward_sum[new_ids][:, 0].cpu().numpy().tolist()) - irewbuffer.extend(cur_ireward_sum[new_ids][:, 0].cpu().numpy().tolist()) - cur_ereward_sum[new_ids] = 0 - cur_ireward_sum[new_ids] = 0 - - stop = time.time() - collection_time = stop - start - start = stop - - # compute returns - self.alg.compute_returns(obs) - - # update policy - loss_dict = self.alg.update() - - stop = time.time() - learn_time = stop - start - self.current_learning_iteration = it - # log info - if self.log_dir is not None and not self.disable_logs: - # Log information - self.log(locals()) - # Save model - if it % self.save_interval == 0: - self.save(os.path.join(self.log_dir, f"model_{it}.pt")) - - # Clear episode infos - ep_infos.clear() - # Save code state - if it == start_iter and not self.disable_logs: - # obtain all the diff files - git_file_paths = store_code_state(self.log_dir, self.git_status_repos) - # if possible store them to wandb - if self.logger_type in ["wandb", "neptune"] and git_file_paths: - for path in git_file_paths: - self.writer.save_file(path) - - # Save the final model after training - if self.log_dir is not None and not self.disable_logs: - self.save(os.path.join(self.log_dir, f"model_{self.current_learning_iteration}.pt")) - - def log(self, locs: dict, width: int = 80, pad: int = 35): - # Compute the collection size - collection_size = self.num_steps_per_env * self.env.num_envs * self.gpu_world_size - # Update total time-steps and time - self.tot_timesteps += collection_size - self.tot_time += locs["collection_time"] + locs["learn_time"] - iteration_time = locs["collection_time"] + locs["learn_time"] - - # -- Episode info - ep_string = "" - if locs["ep_infos"]: - for key in locs["ep_infos"][0]: - infotensor = torch.tensor([], device=self.device) - for ep_info in locs["ep_infos"]: - # handle scalar and zero dimensional tensor infos - if key not in ep_info: - continue - if not isinstance(ep_info[key], torch.Tensor): - ep_info[key] = torch.Tensor([ep_info[key]]) - if len(ep_info[key].shape) == 0: - ep_info[key] = ep_info[key].unsqueeze(0) - infotensor = torch.cat((infotensor, ep_info[key].to(self.device))) - value = torch.mean(infotensor) - # log to logger and terminal - if "/" in key: - self.writer.add_scalar(key, value, locs["it"]) - ep_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - else: - self.writer.add_scalar("Episode/" + key, value, locs["it"]) - ep_string += f"""{f'Mean episode {key}:':>{pad}} {value:.4f}\n""" - - mean_std = self.alg.policy.action_std.mean() - fps = int(collection_size / (locs["collection_time"] + locs["learn_time"])) - - # -- Losses - for key, value in locs["loss_dict"].items(): - self.writer.add_scalar(f"Loss/{key}", value, locs["it"]) - self.writer.add_scalar("Loss/learning_rate", self.alg.learning_rate, locs["it"]) - - # -- Policy - self.writer.add_scalar("Policy/mean_noise_std", mean_std.item(), locs["it"]) - - # -- Performance - self.writer.add_scalar("Perf/total_fps", fps, locs["it"]) - self.writer.add_scalar("Perf/collection time", locs["collection_time"], locs["it"]) - self.writer.add_scalar("Perf/learning_time", locs["learn_time"], locs["it"]) - - # -- Training - if len(locs["rewbuffer"]) > 0: - # separate logging for intrinsic and extrinsic rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.writer.add_scalar("Rnd/mean_extrinsic_reward", statistics.mean(locs["erewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/mean_intrinsic_reward", statistics.mean(locs["irewbuffer"]), locs["it"]) - self.writer.add_scalar("Rnd/weight", self.alg.rnd.weight, locs["it"]) - # everything else - self.writer.add_scalar("Train/mean_reward", statistics.mean(locs["rewbuffer"]), locs["it"]) - self.writer.add_scalar("Train/mean_episode_length", statistics.mean(locs["lenbuffer"]), locs["it"]) - if self.logger_type != "wandb": # wandb does not support non-integer x-axis logging - self.writer.add_scalar("Train/mean_reward/time", statistics.mean(locs["rewbuffer"]), self.tot_time) - self.writer.add_scalar( - "Train/mean_episode_length/time", statistics.mean(locs["lenbuffer"]), self.tot_time - ) - - str = f" \033[1m Learning iteration {locs['it']}/{locs['tot_iter']} \033[0m " - - if len(locs["rewbuffer"]) > 0: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - # -- Losses - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'Mean {key} loss:':>{pad}} {value:.4f}\n""" - # -- Rewards - if hasattr(self.alg, "rnd") and self.alg.rnd: - log_string += ( - f"""{'Mean extrinsic reward:':>{pad}} {statistics.mean(locs['erewbuffer']):.2f}\n""" - f"""{'Mean intrinsic reward:':>{pad}} {statistics.mean(locs['irewbuffer']):.2f}\n""" - ) - log_string += f"""{'Mean reward:':>{pad}} {statistics.mean(locs['rewbuffer']):.2f}\n""" - # -- episode info - log_string += f"""{'Mean episode length:':>{pad}} {statistics.mean(locs['lenbuffer']):.2f}\n""" - else: - log_string = ( - f"""{'#' * width}\n""" - f"""{str.center(width, ' ')}\n\n""" - f"""{'Computation:':>{pad}} {fps:.0f} steps/s (collection: {locs[ - 'collection_time']:.3f}s, learning {locs['learn_time']:.3f}s)\n""" - f"""{'Mean action noise std:':>{pad}} {mean_std.item():.2f}\n""" - ) - for key, value in locs["loss_dict"].items(): - log_string += f"""{f'{key}:':>{pad}} {value:.4f}\n""" - - log_string += ep_string - log_string += ( - f"""{'-' * width}\n""" - f"""{'Total timesteps:':>{pad}} {self.tot_timesteps}\n""" - f"""{'Iteration time:':>{pad}} {iteration_time:.2f}s\n""" - f"""{'Time elapsed:':>{pad}} {time.strftime("%H:%M:%S", time.gmtime(self.tot_time))}\n""" - f"""{'ETA:':>{pad}} {time.strftime( - "%H:%M:%S", - time.gmtime( - self.tot_time / (locs['it'] - locs['start_iter'] + 1) - * (locs['start_iter'] + locs['num_learning_iterations'] - locs['it']) - ) - )}\n""" - ) - print(log_string) - - def save(self, path: str, infos=None): - # -- Save model - saved_dict = { - "model_state_dict": self.alg.policy.state_dict(), - "optimizer_state_dict": self.alg.optimizer.state_dict(), - "iter": self.current_learning_iteration, - "infos": infos, - } - # -- Save RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - saved_dict["rnd_state_dict"] = self.alg.rnd.state_dict() - saved_dict["rnd_optimizer_state_dict"] = self.alg.rnd_optimizer.state_dict() - torch.save(saved_dict, path) - - # upload model to external logging service - if self.logger_type in ["neptune", "wandb"] and not self.disable_logs: - self.writer.save_model(path, self.current_learning_iteration) - - def load(self, path: str, load_optimizer: bool = True, map_location: str | None = None): - loaded_dict = torch.load(path, weights_only=False, map_location=map_location) - # -- Load model - resumed_training = self.alg.policy.load_state_dict(loaded_dict["model_state_dict"]) - # -- Load RND model if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.load_state_dict(loaded_dict["rnd_state_dict"]) - # -- load optimizer if used - if load_optimizer and resumed_training: - # -- algorithm optimizer - self.alg.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"]) - # -- RND optimizer if used - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd_optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"]) - # -- load current learning iteration - if resumed_training: - self.current_learning_iteration = loaded_dict["iter"] - return loaded_dict["infos"] - - def get_inference_policy(self, device=None): - self.eval_mode() # switch to evaluation mode (dropout for example) - if device is not None: - self.alg.policy.to(device) - return self.alg.policy.act_inference - - def train_mode(self): - # -- PPO - self.alg.policy.train() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.train() - - def eval_mode(self): - # -- PPO - self.alg.policy.eval() - # -- RND - if hasattr(self.alg, "rnd") and self.alg.rnd: - self.alg.rnd.eval() - - def add_git_repo_to_log(self, repo_file_path): - self.git_status_repos.append(repo_file_path) - - """ - Helper functions. - """ - - def _configure_multi_gpu(self): - """Configure multi-gpu training.""" - # check if distributed training is enabled - self.gpu_world_size = int(os.getenv("WORLD_SIZE", "1")) - self.is_distributed = self.gpu_world_size > 1 - - # if not distributed training, set local and global rank to 0 and return - if not self.is_distributed: - self.gpu_local_rank = 0 - self.gpu_global_rank = 0 - self.multi_gpu_cfg = None - return - - # get rank and world size - self.gpu_local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.gpu_global_rank = int(os.getenv("RANK", "0")) - - # make a configuration dictionary - self.multi_gpu_cfg = { - "global_rank": self.gpu_global_rank, # rank of the main process - "local_rank": self.gpu_local_rank, # rank of the current process - "world_size": self.gpu_world_size, # total number of processes - } - - # check if user has device specified for local rank - if self.device != f"cuda:{self.gpu_local_rank}": - raise ValueError( - f"Device '{self.device}' does not match expected device for local rank '{self.gpu_local_rank}'." - ) - # validate multi-gpu configuration - if self.gpu_local_rank >= self.gpu_world_size: - raise ValueError( - f"Local rank '{self.gpu_local_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - if self.gpu_global_rank >= self.gpu_world_size: - raise ValueError( - f"Global rank '{self.gpu_global_rank}' is greater than or equal to world size '{self.gpu_world_size}'." - ) - - # initialize torch distributed - torch.distributed.init_process_group(backend="nccl", rank=self.gpu_global_rank, world_size=self.gpu_world_size) - # set device to the local rank - torch.cuda.set_device(self.gpu_local_rank) - - def _construct_algorithm(self, obs) -> PPO: - """Construct the actor-critic algorithm.""" - # resolve RND config - self.alg_cfg = resolve_rnd_config(self.alg_cfg, obs, self.cfg["obs_groups"], self.env) - - # resolve symmetry config - self.alg_cfg = resolve_symmetry_config(self.alg_cfg, self.env) - - # resolve deprecated normalization config - if self.cfg.get("empirical_normalization") is not None: - warnings.warn( - "The `empirical_normalization` parameter is deprecated. Please set `actor_obs_normalization` and " - "`critic_obs_normalization` as part of the `policy` configuration instead.", - DeprecationWarning, - ) - if self.policy_cfg.get("actor_obs_normalization") is None: - self.policy_cfg["actor_obs_normalization"] = self.cfg["empirical_normalization"] - if self.policy_cfg.get("critic_obs_normalization") is None: - self.policy_cfg["critic_obs_normalization"] = self.cfg["empirical_normalization"] - - # initialize the actor-critic - actor_critic_class = eval(self.policy_cfg.pop("class_name")) - actor_critic: ActorCritic | ActorCriticRecurrent = actor_critic_class( - obs, self.cfg["obs_groups"], self.env.num_actions, **self.policy_cfg - ).to(self.device) - - # initialize the algorithm - alg_class = eval(self.alg_cfg.pop("class_name")) - alg: PPO = alg_class(actor_critic, device=self.device, **self.alg_cfg, multi_gpu_cfg=self.multi_gpu_cfg) - - # initialize the storage - alg.init_storage( - "rl", - self.env.num_envs, - self.num_steps_per_env, - obs, - [self.env.num_actions], - ) - - return alg - - def _prepare_logging_writer(self): - """Prepares the logging writers.""" - if self.log_dir is not None and self.writer is None and not self.disable_logs: - # Launch either Tensorboard or Neptune & Tensorboard summary writer(s), default: Tensorboard. - self.logger_type = self.cfg.get("logger", "tensorboard") - self.logger_type = self.logger_type.lower() - - if self.logger_type == "neptune": - from rsl_rl.utils.neptune_utils import NeptuneSummaryWriter - - self.writer = NeptuneSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "wandb": - from rsl_rl.utils.wandb_utils import WandbSummaryWriter - - self.writer = WandbSummaryWriter(log_dir=self.log_dir, flush_secs=10, cfg=self.cfg) - self.writer.log_config(self.env.cfg, self.cfg, self.alg_cfg, self.policy_cfg) - elif self.logger_type == "tensorboard": - from torch.utils.tensorboard import SummaryWriter - - self.writer = SummaryWriter(log_dir=self.log_dir, flush_secs=10) - else: - raise ValueError("Logger type not found. Please choose 'neptune', 'wandb' or 'tensorboard'.") diff --git a/rsl_rl/build/lib/rsl_rl/storage/__init__.py b/rsl_rl/build/lib/rsl_rl/storage/__init__.py deleted file mode 100644 index 1624330..0000000 --- a/rsl_rl/build/lib/rsl_rl/storage/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Implementation of transitions storage for RL-agent.""" - -from .rollout_storage import RolloutStorage -from .replay_buffer_multi import ReplayBufferMulti -__all__ = ["RolloutStorage", "ReplayBufferMulti"] diff --git a/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py b/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py deleted file mode 100644 index 6462b8e..0000000 --- a/rsl_rl/build/lib/rsl_rl/storage/replay_buffer_multi.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -import numpy as np - - -class ReplayBufferMulti: - """Fixed-size buffer to store experience tuples.""" - - def __init__(self, obs_dim, buffer_size, num_amp_frames, device): - """Initialize a ReplayBuffer object. - Arguments: - buffer_size (int): maximum size of buffer - """ - self.states = torch.zeros(buffer_size, num_amp_frames, obs_dim).to(device) - self.num_amp_frames = num_amp_frames - self.buffer_size = buffer_size - self.device = device - - self.step = 0 - self.num_samples = 0 - - def insert(self, states): - """Add new states to memory.""" - num_states = states.shape[0] - start_idx = self.step - end_idx = self.step + num_states - if end_idx > self.buffer_size: - self.states[self.step:self.buffer_size] = states[:self.buffer_size - self.step] - self.states[:end_idx - self.buffer_size] = states[self.buffer_size - self.step:] - else: - self.states[start_idx:end_idx] = states - - self.num_samples = min(self.buffer_size, max(end_idx, self.num_samples)) - self.step = (self.step + num_states) % self.buffer_size - - def feed_forward_generator(self, num_mini_batch, mini_batch_size): - for _ in range(num_mini_batch): - sample_idxs = np.random.choice(self.num_samples, size=mini_batch_size) - yield (self.states[sample_idxs].to(self.device)) diff --git a/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py b/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py deleted file mode 100644 index e9309b3..0000000 --- a/rsl_rl/build/lib/rsl_rl/storage/rollout_storage.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch -from tensordict import TensorDict - -from rsl_rl.utils import split_and_pad_trajectories - - -class RolloutStorage: - class Transition: - def __init__(self): - self.observations = None - self.actions = None - self.privileged_actions = None - self.rewards = None - self.dones = None - self.values = None - self.actions_log_prob = None - self.action_mean = None - self.action_sigma = None - self.hidden_states = None - - def clear(self): - self.__init__() - - def __init__( - self, - training_type, - num_envs, - num_transitions_per_env, - obs, - actions_shape, - device="cpu", - ): - # store inputs - self.training_type = training_type - self.device = device - self.num_transitions_per_env = num_transitions_per_env - self.num_envs = num_envs - self.actions_shape = actions_shape - - # Core - self.observations = TensorDict( - {key: torch.zeros(num_transitions_per_env, *value.shape, device=device) for key, value in obs.items()}, - batch_size=[num_transitions_per_env, num_envs], - device=self.device, - ) - self.rewards = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.actions = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.dones = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device).byte() - - # for distillation - if training_type == "distillation": - self.privileged_actions = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - - # for reinforcement learning - if training_type == "rl": - self.values = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.actions_log_prob = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.mu = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.sigma = torch.zeros(num_transitions_per_env, num_envs, *actions_shape, device=self.device) - self.returns = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - self.advantages = torch.zeros(num_transitions_per_env, num_envs, 1, device=self.device) - - # For RNN networks - self.saved_hidden_states_a = None - self.saved_hidden_states_c = None - - # counter for the number of transitions stored - self.step = 0 - - def add_transitions(self, transition: Transition): - # check if the transition is valid - if self.step >= self.num_transitions_per_env: - raise OverflowError("Rollout buffer overflow! You should call clear() before adding new transitions.") - - # Core - self.observations[self.step].copy_(transition.observations) - self.actions[self.step].copy_(transition.actions) - self.rewards[self.step].copy_(transition.rewards.view(-1, 1)) - self.dones[self.step].copy_(transition.dones.view(-1, 1)) - - # for distillation - if self.training_type == "distillation": - self.privileged_actions[self.step].copy_(transition.privileged_actions) - - # for reinforcement learning - if self.training_type == "rl": - self.values[self.step].copy_(transition.values) - self.actions_log_prob[self.step].copy_(transition.actions_log_prob.view(-1, 1)) - self.mu[self.step].copy_(transition.action_mean) - self.sigma[self.step].copy_(transition.action_sigma) - - # For RNN networks - self._save_hidden_states(transition.hidden_states) - - # increment the counter - self.step += 1 - - def _save_hidden_states(self, hidden_states): - if hidden_states is None or hidden_states == (None, None): - return - # make a tuple out of GRU hidden state sto match the LSTM format - hid_a = hidden_states[0] if isinstance(hidden_states[0], tuple) else (hidden_states[0],) - hid_c = hidden_states[1] if isinstance(hidden_states[1], tuple) else (hidden_states[1],) - # initialize if needed - if self.saved_hidden_states_a is None: - self.saved_hidden_states_a = [ - torch.zeros(self.observations.shape[0], *hid_a[i].shape, device=self.device) for i in range(len(hid_a)) - ] - self.saved_hidden_states_c = [ - torch.zeros(self.observations.shape[0], *hid_c[i].shape, device=self.device) for i in range(len(hid_c)) - ] - # copy the states - for i in range(len(hid_a)): - self.saved_hidden_states_a[i][self.step].copy_(hid_a[i]) - self.saved_hidden_states_c[i][self.step].copy_(hid_c[i]) - - def clear(self): - self.step = 0 - - def compute_returns(self, last_values, gamma, lam, normalize_advantage: bool = True): - advantage = 0 - for step in reversed(range(self.num_transitions_per_env)): - # if we are at the last step, bootstrap the return value - if step == self.num_transitions_per_env - 1: - next_values = last_values - else: - next_values = self.values[step + 1] - # 1 if we are not in a terminal state, 0 otherwise - next_is_not_terminal = 1.0 - self.dones[step].float() - # TD error: r_t + gamma * V(s_{t+1}) - V(s_t) - delta = self.rewards[step] + next_is_not_terminal * gamma * next_values - self.values[step] - # Advantage: A(s_t, a_t) = delta_t + gamma * lambda * A(s_{t+1}, a_{t+1}) - advantage = delta + next_is_not_terminal * gamma * lam * advantage - # Return: R_t = A(s_t, a_t) + V(s_t) - self.returns[step] = advantage + self.values[step] - - # Compute the advantages - self.advantages = self.returns - self.values - # Normalize the advantages if flag is set - # This is to prevent double normalization (i.e. if per minibatch normalization is used) - if normalize_advantage: - self.advantages = (self.advantages - self.advantages.mean()) / (self.advantages.std() + 1e-8) - - # for distillation - def generator(self): - if self.training_type != "distillation": - raise ValueError("This function is only available for distillation training.") - - for i in range(self.num_transitions_per_env): - yield self.observations[i], self.actions[i], self.privileged_actions[i], self.dones[i] - - # for reinforcement learning with feedforward networks - def mini_batch_generator(self, num_mini_batches, num_epochs=8): - if self.training_type != "rl": - raise ValueError("This function is only available for reinforcement learning training.") - batch_size = self.num_envs * self.num_transitions_per_env - mini_batch_size = batch_size // num_mini_batches - indices = torch.randperm(num_mini_batches * mini_batch_size, requires_grad=False, device=self.device) - - # Core - observations = self.observations.flatten(0, 1) - actions = self.actions.flatten(0, 1) - values = self.values.flatten(0, 1) - returns = self.returns.flatten(0, 1) - - # For PPO - old_actions_log_prob = self.actions_log_prob.flatten(0, 1) - advantages = self.advantages.flatten(0, 1) - old_mu = self.mu.flatten(0, 1) - old_sigma = self.sigma.flatten(0, 1) - - for epoch in range(num_epochs): - for i in range(num_mini_batches): - # Select the indices for the mini-batch - start = i * mini_batch_size - end = (i + 1) * mini_batch_size - batch_idx = indices[start:end] - - # Create the mini-batch - # -- Core - obs_batch = observations[batch_idx] - actions_batch = actions[batch_idx] - - # -- For PPO - target_values_batch = values[batch_idx] - returns_batch = returns[batch_idx] - old_actions_log_prob_batch = old_actions_log_prob[batch_idx] - advantages_batch = advantages[batch_idx] - old_mu_batch = old_mu[batch_idx] - old_sigma_batch = old_sigma[batch_idx] - - # yield the mini-batch - yield obs_batch, actions_batch, target_values_batch, advantages_batch, returns_batch, old_actions_log_prob_batch, old_mu_batch, old_sigma_batch, ( - None, - None, - ), None - - # for reinfrocement learning with recurrent networks - def recurrent_mini_batch_generator(self, num_mini_batches, num_epochs=8): - if self.training_type != "rl": - raise ValueError("This function is only available for reinforcement learning training.") - padded_obs_trajectories, trajectory_masks = split_and_pad_trajectories(self.observations, self.dones) - - mini_batch_size = self.num_envs // num_mini_batches - for ep in range(num_epochs): - first_traj = 0 - for i in range(num_mini_batches): - start = i * mini_batch_size - stop = (i + 1) * mini_batch_size - - dones = self.dones.squeeze(-1) - last_was_done = torch.zeros_like(dones, dtype=torch.bool) - last_was_done[1:] = dones[:-1] - last_was_done[0] = True - trajectories_batch_size = torch.sum(last_was_done[:, start:stop]) - last_traj = first_traj + trajectories_batch_size - - masks_batch = trajectory_masks[:, first_traj:last_traj] - obs_batch = padded_obs_trajectories[:, first_traj:last_traj] - actions_batch = self.actions[:, start:stop] - old_mu_batch = self.mu[:, start:stop] - old_sigma_batch = self.sigma[:, start:stop] - returns_batch = self.returns[:, start:stop] - advantages_batch = self.advantages[:, start:stop] - values_batch = self.values[:, start:stop] - old_actions_log_prob_batch = self.actions_log_prob[:, start:stop] - - # reshape to [num_envs, time, num layers, hidden dim] (original shape: [time, num_layers, num_envs, hidden_dim]) - # then take only time steps after dones (flattens num envs and time dimensions), - # take a batch of trajectories and finally reshape back to [num_layers, batch, hidden_dim] - last_was_done = last_was_done.permute(1, 0) - hid_a_batch = [ - saved_hidden_states.permute(2, 0, 1, 3)[last_was_done][first_traj:last_traj] - .transpose(1, 0) - .contiguous() - for saved_hidden_states in self.saved_hidden_states_a - ] - hid_c_batch = [ - saved_hidden_states.permute(2, 0, 1, 3)[last_was_done][first_traj:last_traj] - .transpose(1, 0) - .contiguous() - for saved_hidden_states in self.saved_hidden_states_c - ] - # remove the tuple for GRU - hid_a_batch = hid_a_batch[0] if len(hid_a_batch) == 1 else hid_a_batch - hid_c_batch = hid_c_batch[0] if len(hid_c_batch) == 1 else hid_c_batch - - yield obs_batch, actions_batch, values_batch, advantages_batch, returns_batch, old_actions_log_prob_batch, old_mu_batch, old_sigma_batch, ( - hid_a_batch, - hid_c_batch, - ), masks_batch - - first_traj = last_traj diff --git a/rsl_rl/build/lib/rsl_rl/utils/__init__.py b/rsl_rl/build/lib/rsl_rl/utils/__init__.py deleted file mode 100644 index f5781f1..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Helper functions.""" - -from .utils import * -from .motion_loader_g1 import G1_AMPLoader - -__all__ = [ - "G1_AMPLoader", -] \ No newline at end of file diff --git a/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py b/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py deleted file mode 100644 index 677c630..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/motion_loader_g1.py +++ /dev/null @@ -1,388 +0,0 @@ -import os -from os.path import join as pjoin -import glob -import json -import logging - -import torch -import numpy as np -from pybullet_utils import transformations - -from rsl_rl.utils import motion_util - -_EPS = np.finfo(float).eps * 4.0 -def quaternion_slerp(q0, q1, fraction, spin=0, shortestpath=True): - """Batch quaternion spherical linear interpolation.""" - - out = torch.zeros_like(q0) - - zero_mask = torch.isclose(fraction, torch.zeros_like(fraction)).squeeze() - ones_mask = torch.isclose(fraction, torch.ones_like(fraction)).squeeze() - out[zero_mask] = q0[zero_mask] - out[ones_mask] = q1[ones_mask] - - d = torch.sum(q0 * q1, dim=-1, keepdim=True) - dist_mask = (torch.abs(torch.abs(d) - 1.0) < _EPS).squeeze() - out[dist_mask] = q0[dist_mask] - - if shortestpath: - d_old = torch.clone(d) - d = torch.where(d_old < 0, -d, d) - q1 = torch.where(d_old < 0, -q1, q1) - - angle = torch.acos(d) + spin * torch.pi - angle_mask = (torch.abs(angle) < _EPS).squeeze() - out[angle_mask] = q0[angle_mask] - - final_mask = torch.logical_or(zero_mask, ones_mask) - final_mask = torch.logical_or(final_mask, dist_mask) - final_mask = torch.logical_or(final_mask, angle_mask) - final_mask = torch.logical_not(final_mask) - - isin = 1.0 / angle - q0 *= torch.sin((1.0 - fraction) * angle) * isin - q1 *= torch.sin(fraction * angle) * isin - q0 += q1 - out[final_mask] = q0[final_mask] - return out - - -class G1_AMPLoader: - - def __init__( - self, - device, - time_between_frames, - motion_files, - preload_transitions=False, - num_preload_transitions=1000000, - num_frames=5, - ): - """Expert dataset provides AMP observations from Dog mocap dataset. - - time_between_frames: Amount of time in seconds between transition. - """ - self.device = device - self.time_between_frames = time_between_frames - self.num_frames = num_frames - - # Values to store for each trajectory. - self.trajectories = [] - self.trajectories_full = [] - self.trajectory_names = [] - self.trajectory_idxs = [] - self.trajectory_lens = [] # Traj length in seconds. - self.trajectory_weights = [] - self.trajectory_frame_durations = [] - self.trajectory_num_frames = [] - self.motion_dir = motion_files - # import ipdb; ipdb.set_trace() - for i, motion_file in enumerate(os.listdir(motion_files)): - self.trajectory_names.append(motion_file) - motion_path = pjoin(motion_files, motion_file) - motion_data = np.load(motion_path, allow_pickle=True) - motion_data_processed = np.zeros((motion_data.shape[0],36)) - - for f_i in range(motion_data.shape[0]): - motion_data_processed[f_i, :3] = motion_data[f_i, :3] # base pos - motion_data_processed[f_i, 3:7] = motion_data[f_i, 3:7] # base quat (wxyz) - motion_data_processed[f_i, 7:35] = motion_data[f_i, 7:35] # base vel - ''' - NOTE The order of motion_data_processed is - base pos 0:3, - base quat 3:7, wxyz - dof pos 7:36, (mujoco joint order) - ''' - self.trajectories.append(torch.tensor( - motion_data_processed[:, 7:], - dtype=torch.float32, - device=self.device - )) - - self.trajectories_full.append(torch.tensor( - motion_data_processed, - dtype=torch.float32, - device=self.device - )) - - self.trajectory_idxs.append(i) - self.trajectory_weights.append(1 / len(os.listdir(motion_files))) - frame_duration = 1 / 50 - - self.trajectory_frame_durations.append(frame_duration) - traj_len = (motion_data_processed.shape[0] - 1) * frame_duration # seconds - self.trajectory_lens.append(traj_len) - self.trajectory_num_frames.append(float(motion_data_processed.shape[0])) - print(f"Loaded {traj_len}s. motion from {motion_file}.") - - # Trajectory weights are used to sample some trajectories more than others. - self.trajectory_weights = np.array(self.trajectory_weights) / np.sum(self.trajectory_weights) - self.trajectory_frame_durations = np.array(self.trajectory_frame_durations) - self.trajectory_lens = np.array(self.trajectory_lens) - self.trajectory_num_frames = np.array(self.trajectory_num_frames) - - # Preload transitions. - self.preload_transitions = preload_transitions - if self.preload_transitions: - print(f'Preloading {num_preload_transitions} transitions') - - traj_idxs = self.weighted_traj_idx_sample_batch(num_preload_transitions) - times = self.traj_time_sample_batch(traj_idxs) - self.preloaded_s_prior = self.get_full_frame_at_time_batch(traj_idxs, times - self.time_between_frames) - self.preloaded_s = self.get_full_frame_at_time_batch(traj_idxs, times) - self.preloaded_s_next = self.get_full_frame_at_time_batch(traj_idxs, times + self.time_between_frames) - print(f'Finished preloading') - - # 预加载多帧数据 - self.preloaded_frames = [] - for i in range(self.num_frames): - frame_time = times + (i - (self.num_frames - 2)) * self.time_between_frames - full_frame = self.get_full_frame_at_time_batch(traj_idxs, frame_time) - # 预处理：提前提取并连接需要的列（7:26 和 29:33），避免每次生成时重复切片 - processed_frame = torch.cat([ - full_frame[:, 7:26], - full_frame[:, 29:33] - ], dim=-1) - self.preloaded_frames.append(processed_frame) - print(f'Finished preloading multiple frames') - - self.all_trajectories_full = torch.vstack(self.trajectories_full) - - def weighted_traj_idx_sample(self): - """Get traj idx via weighted sampling.""" - return np.random.choice( - self.trajectory_idxs, p=self.trajectory_weights) - - def weighted_traj_idx_sample_batch(self, size): - """Batch sample traj idxs.""" - return np.random.choice( - self.trajectory_idxs, size=size, p=self.trajectory_weights, - replace=True) - - def traj_time_sample(self, traj_idx): - """Sample random time for traj.""" - subst = self.time_between_frames + self.trajectory_frame_durations[traj_idx] - return max( - 0, (self.trajectory_lens[traj_idx] * np.random.uniform() - subst)) - - def traj_time_sample_batch(self, traj_idxs): - """Sample random time for multiple trajectories.""" - subst = self.time_between_frames + self.trajectory_frame_durations[traj_idxs] - time_samples = self.trajectory_lens[traj_idxs] * np.random.uniform(size=len(traj_idxs)) - subst - return np.maximum(np.zeros_like(time_samples), time_samples) - - def slerp(self, val0, val1, blend): - return (1.0 - blend) * val0 + blend * val1 - - def get_trajectory(self, traj_idx): - """Returns trajectory of AMP observations.""" - return self.trajectories_full[traj_idx] - - def get_frame_at_time(self, traj_idx, time): - """Returns frame for the given trajectory at the specified time.""" - p = float(time) / self.trajectory_lens[traj_idx] - n = self.trajectories[traj_idx].shape[0] - idx_low, idx_high = int(np.floor(p * n)), int(np.ceil(p * n)) - frame_start = self.trajectories[traj_idx][idx_low] - frame_end = self.trajectories[traj_idx][idx_high] - blend = p * n - idx_low - return self.slerp(frame_start, frame_end, blend) - - def get_frame_at_time_batch(self, traj_idxs, times): - """Returns frame for the given trajectory at the specified time.""" - p = times / self.trajectory_lens[traj_idxs] - n = self.trajectory_num_frames[traj_idxs] - idx_low, idx_high = np.floor(p * n).astype(np.int32), np.ceil(p * n).astype(np.int32) - all_frame_starts = torch.zeros(len(traj_idxs), self.observation_dim, device=self.device) - all_frame_ends = torch.zeros(len(traj_idxs), self.observation_dim, device=self.device) - for traj_idx in set(traj_idxs): - trajectory = self.trajectories[traj_idx] - traj_mask = traj_idxs == traj_idx - all_frame_starts[traj_mask] = trajectory[idx_low[traj_mask]] - all_frame_ends[traj_mask] = trajectory[idx_high[traj_mask]] - blend = torch.tensor(p * n - idx_low, device=self.device, dtype=torch.float32).unsqueeze(-1) - return self.slerp(all_frame_starts, all_frame_ends, blend) - - def get_full_frame_at_time(self, traj_idx, time): - """Returns full frame for the given trajectory at the specified time.""" - p = float(time) / self.trajectory_lens[traj_idx] - n = self.trajectories_full[traj_idx].shape[0] - idx_low, idx_high = int(np.floor(p * n)), int(np.ceil(p * n)) - frame_start = self.trajectories_full[traj_idx][idx_low] - frame_end = self.trajectories_full[traj_idx][idx_high] - blend = p * n - idx_low - print(idx_low, idx_high) - return self.blend_frame_pose(frame_start, frame_end, blend) - - def get_full_frame_at_time_batch(self, traj_idxs, times): - p = times / self.trajectory_lens[traj_idxs] - n = self.trajectory_num_frames[traj_idxs] - idx_low, idx_high = np.floor(p * n).astype(np.int32), np.ceil(p * n).astype(np.int32) - all_frame_pos_starts = torch.zeros(len(traj_idxs), 3, device=self.device) - all_frame_pos_ends = torch.zeros(len(traj_idxs), 3, device=self.device) - all_frame_rot_starts = torch.zeros(len(traj_idxs), 4, device=self.device) - all_frame_rot_ends = torch.zeros(len(traj_idxs), 4, device=self.device) - all_frame_amp_starts = torch.zeros(len(traj_idxs), 29, device=self.device) - all_frame_amp_ends = torch.zeros(len(traj_idxs), 29, device=self.device) - for traj_idx in set(traj_idxs): - trajectory = self.trajectories_full[traj_idx] - traj_mask = traj_idxs == traj_idx - all_frame_pos_starts[traj_mask] = G1_AMPLoader.get_root_pos_batch(trajectory[idx_low[traj_mask]]) - all_frame_pos_ends[traj_mask] = G1_AMPLoader.get_root_pos_batch(trajectory[idx_high[traj_mask]]) - all_frame_rot_starts[traj_mask] = G1_AMPLoader.get_root_rot_batch(trajectory[idx_low[traj_mask]]) - all_frame_rot_ends[traj_mask] = G1_AMPLoader.get_root_rot_batch(trajectory[idx_high[traj_mask]]) - all_frame_amp_starts[traj_mask] = trajectory[idx_low[traj_mask]][:, 7:36] # base vel3+ang3, dof vel23+ang23 - all_frame_amp_ends[traj_mask] = trajectory[idx_high[traj_mask]][:, 7:36] # base vel3+ang3, dof vel23+ang23 - blend = torch.tensor(p * n - idx_low, device=self.device, dtype=torch.float32).unsqueeze(-1) - pos_blend = self.slerp(all_frame_pos_starts, all_frame_pos_ends, blend) - rot_blend = quaternion_slerp(all_frame_rot_starts, all_frame_rot_ends, blend) - amp_blend = self.slerp(all_frame_amp_starts, all_frame_amp_ends, blend) - return torch.cat([pos_blend, rot_blend, amp_blend], dim=-1) - - def get_frame(self): - """Returns random frame.""" - traj_idx = self.weighted_traj_idx_sample() - sampled_time = self.traj_time_sample(traj_idx) - return self.get_frame_at_time(traj_idx, sampled_time) - - def get_full_frame(self): - """Returns random full frame.""" - traj_idx = self.weighted_traj_idx_sample() - sampled_time = self.traj_time_sample(traj_idx) - return self.get_full_frame_at_time(traj_idx, sampled_time) - - def get_full_frame_batch(self, num_frames): - if self.preload_transitions: - idxs = np.random.choice( - self.preloaded_s.shape[0], size=num_frames) - return self.preloaded_s[idxs] - else: - traj_idxs = self.weighted_traj_idx_sample_batch(num_frames) - times = self.traj_time_sample_batch(traj_idxs) - return self.get_full_frame_at_time_batch(traj_idxs, times) - - def blend_frame_pose(self, frame0, frame1, blend): - """Linearly interpolate between two frames, including orientation. - - Args: - frame0: First frame to be blended corresponds to (blend = 0). - frame1: Second frame to be blended corresponds to (blend = 1). - blend: Float between [0, 1], specifying the interpolation between - the two frames. - Returns: - An interpolation of the two frames. - """ - root_pos0, root_pos1 = G1_AMPLoader.get_root_pos(frame0), G1_AMPLoader.get_root_pos(frame1) - root_rot0, root_rot1 = G1_AMPLoader.get_root_rot(frame0), G1_AMPLoader.get_root_rot(frame1) - joints0, joints1 = G1_AMPLoader.get_joint_pose(frame0), G1_AMPLoader.get_joint_pose(frame1) - # tar_toe_pos_0, tar_toe_pos_1 = G1_AMPLoader.get_tar_toe_pos_local(frame0), G1_AMPLoader.get_tar_toe_pos_local(frame1) - linear_vel_0, linear_vel_1 = G1_AMPLoader.get_linear_vel(frame0), G1_AMPLoader.get_linear_vel(frame1) - angular_vel_0, angular_vel_1 = G1_AMPLoader.get_angular_vel(frame0), G1_AMPLoader.get_angular_vel(frame1) - joint_vel_0, joint_vel_1 = G1_AMPLoader.get_joint_vel(frame0), G1_AMPLoader.get_joint_vel(frame1) - - blend_root_pos = self.slerp(root_pos0, root_pos1, blend) - blend_root_rot = transformations.quaternion_slerp(root_rot0.cpu().numpy(), root_rot1.cpu().numpy(), blend) - blend_root_rot = torch.tensor(motion_util.standardize_quaternion(blend_root_rot),dtype=torch.float32, device=self.device) - blend_joints = self.slerp(joints0, joints1, blend) - # blend_tar_toe_pos = self.slerp(tar_toe_pos_0, tar_toe_pos_1, blend) - blend_linear_vel = self.slerp(linear_vel_0, linear_vel_1, blend) - blend_angular_vel = self.slerp(angular_vel_0, angular_vel_1, blend) - blend_joints_vel = self.slerp(joint_vel_0, joint_vel_1, blend) - - # return - # torch.cat([ - # blend_root_pos, blend_root_rot, blend_linear_vel, blend_angular_vel, blend_joints, blend_joints_vel]) - return torch.cat([blend_root_pos, blend_root_rot, blend_linear_vel, blend_angular_vel, blend_joints]) - - def feed_forward_generator_23dof_multi(self, num_mini_batch, mini_batch_size): - """Generates a batch of AMP transitions.""" - # import ipdb; ipdb.set_trace() - for _ in range(num_mini_batch): - if self.preload_transitions: - idxs = np.random.choice(self.preloaded_s.shape[0], size=mini_batch_size) - - frames = [] - for i in range(self.num_frames): - # 数据已在预加载时预处理，直接索引即可 - s = self.preloaded_frames[i][idxs] - frames.append(s) - else: - NotImplementedError('preload transition') - yield torch.stack(frames, dim=1) # [batch, num_frames, 16] - - - - - def quaternion_to_euler_array(self, quat): - # Ensure quaternion is in the correct format [x, y, z, w] - x, y, z, w =quat - - # Roll (x-axis rotation) - t0 = +2.0 * (w * x + y * z) - t1 = +1.0 - 2.0 * (x * x + y * y) - roll_x = np.arctan2(t0, t1) - - # Pitch (y-axis rotation) - t2 = +2.0 * (w * y - z * x) - t2 = np.clip(t2, -1.0, 1.0) - pitch_y = np.arcsin(t2) - - # Yaw (z-axis rotation) - t3 = +2.0 * (w * z + x * y) - t4 = +1.0 - 2.0 * (y * y + z * z) - yaw_z = np.arctan2(t3, t4) - - # Returns roll, pitch, yaw in a NumPy array in radians - return np.array([roll_x, pitch_y, yaw_z]) - - def euler_to_quaternion(self, root_rot): - roll, pitch, yaw = root_rot[0], root_rot[1], root_rot[2] - cy = np.cos(yaw * 0.5) - sy = np.sin(yaw * 0.5) - cp = np.cos(pitch * 0.5) - sp = np.sin(pitch * 0.5) - cr = np.cos(roll * 0.5) - sr = np.sin(roll * 0.5) - - qw = cy * cp * cr + sy * sp * sr - qx = cy * cp * sr - sy * sp * cr - qy = sy * cp * sr + cy * sp * cr - qz = sy * cp * cr - cy * sp * sr - - return np.array([qx, qy, qz, qw]) - - @property - def observation_dim(self): - """Size of AMP observations.""" - return self.trajectories[0].shape[1] + 1 - - @property - def num_motions(self): - return len(self.trajectory_names) - @staticmethod - def get_root_pos(pose): - return pose[0:3] - - @staticmethod - def get_root_pos_batch(poses): - return poses[:, 0:3] - - @staticmethod - def get_root_rot(pose): - return pose[3:7] - - @staticmethod - def get_root_rot_batch(poses): - return poses[:, 3:7] - - @staticmethod - def get_joint_pose_batch_12dof(poses): - return poses[:, 13:25] - - @staticmethod - def get_tar_toe_pos_local(pose): - return pose[G1_AMPLoader.TAR_TOE_POS_LOCAL_START_IDX:G1_AMPLoader.TAR_TOE_POS_LOCAL_END_IDX] - - @staticmethod - def get_tar_toe_pos_local_batch(poses): - return poses[:, G1_AMPLoader.TAR_TOE_POS_LOCAL_START_IDX:G1_AMPLoader.TAR_TOE_POS_LOCAL_END_IDX] diff --git a/rsl_rl/build/lib/rsl_rl/utils/motion_util.py b/rsl_rl/build/lib/rsl_rl/utils/motion_util.py deleted file mode 100644 index 3d49bfa..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/motion_util.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility functions for processing motion clips.""" - -import os -import inspect -currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parentdir = os.path.dirname(os.path.dirname(currentdir)) -os.sys.path.insert(0, parentdir) - -import numpy as np - -from rsl_rl.utils import pose3d -# from pybullet_utils import transformations - - -def standardize_quaternion(q): - """Returns a quaternion where q.w >= 0 to remove redundancy due to q = -q. - - Args: - q: A quaternion to be standardized. - - Returns: - A quaternion with q.w >= 0. - - """ - if q[-1] < 0: - q = -q - return q - - -def normalize_rotation_angle(theta): - """Returns a rotation angle normalized between [-pi, pi]. - - Args: - theta: angle of rotation (radians). - - Returns: - An angle of rotation normalized between [-pi, pi]. - - """ - norm_theta = theta - if np.abs(norm_theta) > np.pi: - norm_theta = np.fmod(norm_theta, 2 * np.pi) - if norm_theta >= 0: - norm_theta += -2 * np.pi - else: - norm_theta += 2 * np.pi - - return norm_theta - - -def calc_heading(q): - """Returns the heading of a rotation q, specified as a quaternion. - - The heading represents the rotational component of q along the vertical - axis (z axis). - - Args: - q: A quaternion that the heading is to be computed from. - - Returns: - An angle representing the rotation about the z axis. - - """ - ref_dir = np.array([1, 0, 0]) - rot_dir = pose3d.QuaternionRotatePoint(ref_dir, q) - heading = np.arctan2(rot_dir[1], rot_dir[0]) - return heading - - -# def calc_heading_rot(q): -# """Return a quaternion representing the heading rotation of q along the vertical axis (z axis). - -# Args: -# q: A quaternion that the heading is to be computed from. - -# Returns: -# A quaternion representing the rotation about the z axis. - -# """ -# heading = calc_heading(q) -# q_heading = transformations.quaternion_about_axis(heading, [0, 0, 1]) -# return q_heading diff --git a/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py b/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py deleted file mode 100644 index 3796ec8..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/neptune_utils.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -from dataclasses import asdict -from torch.utils.tensorboard import SummaryWriter - -try: - import neptune -except ModuleNotFoundError: - raise ModuleNotFoundError("neptune-client is required to log to Neptune.") - - -class NeptuneLogger: - def __init__(self, project, token): - self.run = neptune.init_run(project=project, api_token=token) - - def store_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.run["runner_cfg"] = runner_cfg - self.run["policy_cfg"] = policy_cfg - self.run["alg_cfg"] = alg_cfg - self.run["env_cfg"] = asdict(env_cfg) - - -class NeptuneSummaryWriter(SummaryWriter): - """Summary writer for Neptune.""" - - def __init__(self, log_dir: str, flush_secs: int, cfg): - super().__init__(log_dir, flush_secs) - - try: - project = cfg["neptune_project"] - except KeyError: - raise KeyError("Please specify neptune_project in the runner config, e.g. legged_gym.") - - try: - token = os.environ["NEPTUNE_API_TOKEN"] - except KeyError: - raise KeyError( - "Neptune api token not found. Please run or add to ~/.bashrc: export NEPTUNE_API_TOKEN=YOUR_API_TOKEN" - ) - - try: - entity = os.environ["NEPTUNE_USERNAME"] - except KeyError: - raise KeyError( - "Neptune username not found. Please run or add to ~/.bashrc: export NEPTUNE_USERNAME=YOUR_USERNAME" - ) - - neptune_project = entity + "/" + project - - self.neptune_logger = NeptuneLogger(neptune_project, token) - - self.name_map = { - "Train/mean_reward/time": "Train/mean_reward_time", - "Train/mean_episode_length/time": "Train/mean_episode_length_time", - } - - run_name = os.path.split(log_dir)[-1] - - self.neptune_logger.run["log_dir"].log(run_name) - - def _map_path(self, path): - if path in self.name_map: - return self.name_map[path] - else: - return path - - def add_scalar(self, tag, scalar_value, global_step=None, walltime=None, new_style=False): - super().add_scalar( - tag, - scalar_value, - global_step=global_step, - walltime=walltime, - new_style=new_style, - ) - self.neptune_logger.run[self._map_path(tag)].log(scalar_value, step=global_step) - - def stop(self): - self.neptune_logger.run.stop() - - def log_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.neptune_logger.store_config(env_cfg, runner_cfg, alg_cfg, policy_cfg) - - def save_model(self, model_path, iter): - self.neptune_logger.run["model/saved_model_" + str(iter)].upload(model_path) - - def save_file(self, path, iter=None): - name = path.rsplit("/", 1)[-1].split(".")[0] - self.neptune_logger.run["git_diff/" + name].upload(path) diff --git a/rsl_rl/build/lib/rsl_rl/utils/pose3d.py b/rsl_rl/build/lib/rsl_rl/utils/pose3d.py deleted file mode 100644 index a4cba1e..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/pose3d.py +++ /dev/null @@ -1,283 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for 3D pose conversion.""" -import math -import numpy as np - -# from pybullet_utils import transformations - -VECTOR3_0 = np.zeros(3, dtype=np.float64) -VECTOR3_1 = np.ones(3, dtype=np.float64) -VECTOR3_X = np.array([1, 0, 0], dtype=np.float64) -VECTOR3_Y = np.array([0, 1, 0], dtype=np.float64) -VECTOR3_Z = np.array([0, 0, 1], dtype=np.float64) - -# QUATERNION_IDENTITY is the multiplicative identity 1.0 + 0i + 0j + 0k. -# When interpreted as a rotation, it is the identity rotation. -QUATERNION_IDENTITY = np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float64) - - -def Vector3RandomNormal(sigma, mu=VECTOR3_0): - """Returns a random 3D vector from a normal distribution. - - Each component is selected independently from a normal distribution. - - Args: - sigma: Scale (or stddev) of distribution for all variables. - mu: Mean of distribution for each variable. - - Returns: - A 3D vector in a numpy array. - """ - - random_v3 = np.random.normal(scale=sigma, size=3) + mu - return random_v3 - - -def Vector3RandomUniform(low=VECTOR3_0, high=VECTOR3_1): - """Returns a 3D vector selected uniformly from the input box. - - Args: - low: The min-value corner of the box. - high: The max-value corner of the box. - - Returns: - A 3D vector in a numpy array. - """ - - random_x = np.random.uniform(low=low[0], high=high[0]) - random_y = np.random.uniform(low=low[1], high=high[1]) - random_z = np.random.uniform(low=low[2], high=high[2]) - return np.array([random_x, random_y, random_z]) - - -def Vector3RandomUnit(): - """Returns a random 3D vector with unit length. - - Generates a 3D vector selected uniformly from the unit sphere. - - Returns: - A normalized 3D vector in a numpy array. - """ - longitude = np.random.uniform(low=-math.pi, high=math.pi) - sin_latitude = np.random.uniform(low=-1.0, high=1.0) - cos_latitude = math.sqrt(1.0 - sin_latitude * sin_latitude) - x = math.cos(longitude) * cos_latitude - y = math.sin(longitude) * cos_latitude - z = sin_latitude - return np.array([x, y, z], dtype=np.float64) - - -def QuaternionNormalize(q): - """Normalizes the quaternion to length 1. - - Divides the quaternion by its magnitude. If the magnitude is too - small, returns the quaternion identity value (1.0). - - Args: - q: A quaternion to be normalized. - - Raises: - ValueError: If input quaternion has length near zero. - - Returns: - A quaternion with magnitude 1 in a numpy array [x, y, z, w]. - - """ - q_norm = np.linalg.norm(q) - if np.isclose(q_norm, 0.0): - raise ValueError( - 'Quaternion may not be zero in QuaternionNormalize: |q| = %f, q = %s' % - (q_norm, q)) - return q / q_norm - - -def QuaternionFromAxisAngle(axis, angle): - """Returns a quaternion that generates the given axis-angle rotation. - - Returns the quaternion: sin(angle/2) * axis + cos(angle/2). - - Args: - axis: Axis of rotation, a 3D vector in a numpy array. - angle: The angle of rotation (radians). - - Raises: - ValueError: If input axis is not a normalizable 3D vector. - - Returns: - A unit quaternion in a numpy array. - - """ - if len(axis) != 3: - raise ValueError('Axis vector should have three components: %s' % axis) - axis_norm = np.linalg.norm(axis) - if np.isclose(axis_norm, 0.0): - raise ValueError('Axis vector may not have zero length: |v| = %f, v = %s' % - (axis_norm, axis)) - half_angle = angle * 0.5 - q = np.zeros(4, dtype=np.float64) - q[0:3] = axis - q[0:3] *= math.sin(half_angle) / axis_norm - q[3] = math.cos(half_angle) - return q - - -def QuaternionToAxisAngle(quat, default_axis=VECTOR3_Z, direction_axis=None): - """Calculates axis and angle of rotation performed by a quaternion. - - Calculates the axis and angle of the rotation performed by the quaternion. - The quaternion should have four values and be normalized. - - Args: - quat: Unit quaternion in a numpy array. - default_axis: 3D vector axis used if the rotation is near to zero. Without - this default, small rotations would result in an exception. It is - reasonable to use a default axis for tiny rotations, because zero angle - rotations about any axis are equivalent. - direction_axis: Used to disambiguate rotation directions. If the - direction_axis is specified, the axis of the rotation will be chosen such - that its inner product with the direction_axis is non-negative. - - Raises: - ValueError: If quat is not a normalized quaternion. - - Returns: - axis: Axis of rotation. - angle: Angle in radians. - """ - if len(quat) != 4: - raise ValueError( - 'Quaternion should have four components [x, y, z, w]: %s' % quat) - if not np.isclose(1.0, np.linalg.norm(quat)): - raise ValueError('Quaternion should have unit length: |q| = %f, q = %s' % - (np.linalg.norm(quat), quat)) - axis = quat[:3].copy() - axis_norm = np.linalg.norm(axis) - min_axis_norm = 1e-8 - if axis_norm < min_axis_norm: - axis = default_axis - if len(default_axis) != 3: - raise ValueError('Axis vector should have three components: %s' % axis) - if not np.isclose(np.linalg.norm(axis), 1.0): - raise ValueError('Axis vector should have unit length: |v| = %f, v = %s' % - (np.linalg.norm(axis), axis)) - else: - axis /= axis_norm - sin_half_angle = axis_norm - if direction_axis is not None and np.inner(axis, direction_axis) < 0: - sin_half_angle = -sin_half_angle - axis = -axis - cos_half_angle = quat[3] - half_angle = math.atan2(sin_half_angle, cos_half_angle) - angle = half_angle * 2 - return axis, angle - - -def QuaternionRandomRotation(max_angle=math.pi): - """Creates a random small rotation around a random axis. - - Generates a small rotation with the axis vector selected uniformly - from the unit sphere and an angle selected from a uniform - distribution over [0, max_angle]. - - If the max_angle is not specified, the rotation should be selected - uniformly over all possible rotation angles. - - Args: - max_angle: The maximum angle of rotation (radians). - - Returns: - A unit quaternion in a numpy array. - - """ - - angle = np.random.uniform(low=0, high=max_angle) - axis = Vector3RandomUnit() - return QuaternionFromAxisAngle(axis, angle) - - -# def QuaternionRotatePoint(point, quat): -# """Performs a rotation by quaternion. - -# Rotate the point by the quaternion using quaternion multiplication, -# (q * p * q^-1), without constructing the rotation matrix. - -# Args: -# point: The point to be rotated. -# quat: The rotation represented as a quaternion [x, y, z, w]. - -# Returns: -# A 3D vector in a numpy array. -# """ - -# q_point = np.array([point[0], point[1], point[2], 0.0]) -# quat_inverse = transformations.quaternion_inverse(quat) -# q_point_rotated = transformations.quaternion_multiply( -# transformations.quaternion_multiply(quat, q_point), quat_inverse) -# return q_point_rotated[:3] - - -def IsRotationMatrix(m): - """Returns true if the 3x3 submatrix represents a rotation. - - Args: - m: A transformation matrix. - - Raises: - ValueError: If input is not a matrix of size at least 3x3. - - Returns: - True if the 3x3 submatrix is a rotation (orthogonal). - """ - if len(m.shape) != 2 or m.shape[0] < 3 or m.shape[1] < 3: - raise ValueError('Matrix should be 3x3 or 4x4: %s\n %s' % (m.shape, m)) - rot = m[:3, :3] - eye = np.matmul(rot, np.transpose(rot)) - return np.isclose(eye, np.identity(3), atol=1e-4).all() - -# def ZAxisAlignedRobotPoseTool(robot_pose_tool): -# """Returns the current gripper pose rotated for alignment with the z-axis. - -# Args: -# robot_pose_tool: a pose3d.Pose3d() instance. - -# Returns: -# An instance of pose.Transform representing the current gripper pose -# rotated for alignment with the z-axis. -# """ -# # Align the current pose to the z-axis. -# robot_pose_tool.quaternion = transformations.quaternion_multiply( -# RotationBetween( -# robot_pose_tool.matrix4x4[0:3, 0:3].dot(np.array([0, 0, 1])), -# np.array([0.0, 0.0, -1.0])), robot_pose_tool.quaternion) -# return robot_pose_tool - -# def RotationBetween(a_translation_b, a_translation_c): -# """Computes the rotation from one vector to another. - -# The computed rotation has the property that: - -# a_translation_c = a_rotation_b_to_c * a_translation_b - -# Args: -# a_translation_b: vec3, vector to rotate from -# a_translation_c: vec3, vector to rotate to - -# Returns: -# a_rotation_b_to_c: new Orientation -# """ -# rotation = rotation3.Rotation3.rotation_between( -# a_translation_b, a_translation_c, err_msg='RotationBetween') -# return rotation.quaternion.xyzw diff --git a/rsl_rl/build/lib/rsl_rl/utils/utils.py b/rsl_rl/build/lib/rsl_rl/utils/utils.py deleted file mode 100644 index 3605622..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/utils.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import git -import importlib -import os -import pathlib -import torch -import warnings -from tensordict import TensorDict -from typing import Callable -import numpy as np -class RunningMeanStd: - def __init__(self, epsilon: float = 1e-4, shape: Tuple[int, ...] = ()): - """ - Calculates the running mean and std of a data stream - https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm - :param epsilon: helps with arithmetic issues - :param shape: the shape of the data stream's output - """ - self.mean = np.zeros(shape, np.float64) - self.var = np.ones(shape, np.float64) - self.count = epsilon - - def update(self, arr: np.ndarray) -> None: - batch_mean = np.mean(arr, axis=0) - batch_var = np.var(arr, axis=0) - batch_count = arr.shape[0] - self.update_from_moments(batch_mean, batch_var, batch_count) - - def update_from_moments(self, batch_mean: np.ndarray, batch_var: np.ndarray, batch_count: int) -> None: - delta = batch_mean - self.mean - tot_count = self.count + batch_count - - new_mean = self.mean + delta * batch_count / tot_count - m_a = self.var * self.count - m_b = batch_var * batch_count - m_2 = m_a + m_b + np.square(delta) * self.count * batch_count / (self.count + batch_count) - new_var = m_2 / (self.count + batch_count) - - new_count = batch_count + self.count - - self.mean = new_mean - self.var = new_var - self.count = new_count - - -class Normalizer(RunningMeanStd): - def __init__(self, input_dim, epsilon=1e-4, clip_obs=10.0): - super().__init__(shape=input_dim) - self.epsilon = epsilon - self.clip_obs = clip_obs - - def normalize(self, input): - return np.clip((input - self.mean) / np.sqrt(self.var + self.epsilon), -self.clip_obs, self.clip_obs) - - def normalize_torch(self, input, device): - mean_torch = torch.tensor(self.mean, device=device, dtype=torch.float32) - std_torch = torch.sqrt(torch.tensor(self.var + self.epsilon, device=device, dtype=torch.float32)) - return torch.clamp((input - mean_torch) / std_torch, -self.clip_obs, self.clip_obs) - - def update_normalizer(self, rollouts, expert_loader): - policy_data_generator = rollouts.feed_forward_generator_amp(None, mini_batch_size=expert_loader.batch_size) - expert_data_generator = expert_loader.dataset.feed_forward_generator_amp(expert_loader.batch_size) - - for expert_batch, policy_batch in zip(expert_data_generator, policy_data_generator): - self.update(torch.vstack(tuple(policy_batch) + tuple(expert_batch)).cpu().numpy()) - - -def resolve_nn_activation(act_name: str) -> torch.nn.Module: - """Resolves the activation function from the name. - - Args: - act_name: The name of the activation function. - - Returns: - The activation function. - - Raises: - ValueError: If the activation function is not found. - """ - act_dict = { - "elu": torch.nn.ELU(), - "selu": torch.nn.SELU(), - "relu": torch.nn.ReLU(), - "crelu": torch.nn.CELU(), - "lrelu": torch.nn.LeakyReLU(), - "tanh": torch.nn.Tanh(), - "sigmoid": torch.nn.Sigmoid(), - "softplus": torch.nn.Softplus(), - "gelu": torch.nn.GELU(), - "swish": torch.nn.SiLU(), - "mish": torch.nn.Mish(), - "identity": torch.nn.Identity(), - } - - act_name = act_name.lower() - if act_name in act_dict: - return act_dict[act_name] - else: - raise ValueError(f"Invalid activation function '{act_name}'. Valid activations are: {list(act_dict.keys())}") - - -def resolve_optimizer(optimizer_name: str) -> torch.optim.Optimizer: - """Resolves the optimizer from the name. - - Args: - optimizer_name: The name of the optimizer. - - Returns: - The optimizer. - - Raises: - ValueError: If the optimizer is not found. - """ - optimizer_dict = { - "adam": torch.optim.Adam, - "adamw": torch.optim.AdamW, - "sgd": torch.optim.SGD, - "rmsprop": torch.optim.RMSprop, - } - - optimizer_name = optimizer_name.lower() - if optimizer_name in optimizer_dict: - return optimizer_dict[optimizer_name] - else: - raise ValueError(f"Invalid optimizer '{optimizer_name}'. Valid optimizers are: {list(optimizer_dict.keys())}") - - -def split_and_pad_trajectories( - tensor: torch.Tensor | TensorDict, dones: torch.Tensor -) -> tuple[torch.Tensor | TensorDict, torch.Tensor]: - """Splits trajectories at done indices. Then concatenates them and pads with zeros up to the length of the longest - trajectory. Returns masks corresponding to valid parts of the trajectories. - - Example: - Input: [[a1, a2, a3, a4 | a5, a6], - [b1, b2 | b3, b4, b5 | b6]] - - Output:[[a1, a2, a3, a4], | [[True, True, True, True], - [a5, a6, 0, 0], | [True, True, False, False], - [b1, b2, 0, 0], | [True, True, False, False], - [b3, b4, b5, 0], | [True, True, True, False], - [b6, 0, 0, 0]] | [True, False, False, False]] - - Assumes that the input has the following order of dimensions: [time, number of envs, additional dimensions] - """ - - dones = dones.clone() - dones[-1] = 1 - # Permute the buffers to have order (num_envs, num_transitions_per_env, ...), for correct reshaping - flat_dones = dones.transpose(1, 0).reshape(-1, 1) - # Get length of trajectory by counting the number of successive not done elements - done_indices = torch.cat((flat_dones.new_tensor([-1], dtype=torch.int64), flat_dones.nonzero()[:, 0])) - trajectory_lengths = done_indices[1:] - done_indices[:-1] - trajectory_lengths_list = trajectory_lengths.tolist() - # Extract the individual trajectories - if isinstance(tensor, TensorDict): - padded_trajectories = {} - for k, v in tensor.items(): - # split the tensor into trajectories - trajectories = torch.split(v.transpose(1, 0).flatten(0, 1), trajectory_lengths_list) - # add at least one full length trajectory - trajectories = trajectories + (torch.zeros(v.shape[0], *v.shape[2:], device=v.device),) - # pad the trajectories to the length of the longest trajectory - padded_trajectories[k] = torch.nn.utils.rnn.pad_sequence(trajectories) - # remove the added tensor - padded_trajectories[k] = padded_trajectories[k][:, :-1] - padded_trajectories = TensorDict( - padded_trajectories, batch_size=[tensor.batch_size[0], len(trajectory_lengths_list)] - ) - else: - # split the tensor into trajectories - trajectories = torch.split(tensor.transpose(1, 0).flatten(0, 1), trajectory_lengths_list) - # add at least one full length trajectory - trajectories = trajectories + (torch.zeros(tensor.shape[0], *tensor.shape[2:], device=tensor.device),) - # pad the trajectories to the length of the longest trajectory - padded_trajectories = torch.nn.utils.rnn.pad_sequence(trajectories) - # remove the added tensor - padded_trajectories = padded_trajectories[:, :-1] - # create masks for the valid parts of the trajectories - trajectory_masks = trajectory_lengths > torch.arange(0, tensor.shape[0], device=tensor.device).unsqueeze(1) - return padded_trajectories, trajectory_masks - - -def unpad_trajectories(trajectories, masks): - """Does the inverse operation of split_and_pad_trajectories()""" - # Need to transpose before and after the masking to have proper reshaping - return ( - trajectories.transpose(1, 0)[masks.transpose(1, 0)] - .view(-1, trajectories.shape[0], trajectories.shape[-1]) - .transpose(1, 0) - ) - - -def store_code_state(logdir, repositories) -> list: - git_log_dir = os.path.join(logdir, "git") - os.makedirs(git_log_dir, exist_ok=True) - file_paths = [] - for repository_file_path in repositories: - try: - repo = git.Repo(repository_file_path, search_parent_directories=True) - t = repo.head.commit.tree - except Exception: - print(f"Could not find git repository in {repository_file_path}. Skipping.") - # skip if not a git repository - continue - # get the name of the repository - repo_name = pathlib.Path(repo.working_dir).name - diff_file_name = os.path.join(git_log_dir, f"{repo_name}.diff") - # check if the diff file already exists - if os.path.isfile(diff_file_name): - continue - # write the diff file - print(f"Storing git diff for '{repo_name}' in: {diff_file_name}") - with open(diff_file_name, "x", encoding="utf-8") as f: - content = f"--- git status ---\n{repo.git.status()} \n\n\n--- git diff ---\n{repo.git.diff(t)}" - f.write(content) - # add the file path to the list of files to be uploaded - file_paths.append(diff_file_name) - return file_paths - - -def string_to_callable(name: str) -> Callable: - """Resolves the module and function names to return the function. - - Args: - name: The function name. The format should be 'module:attribute_name'. - - Raises: - ValueError: When the resolved attribute is not a function. - ValueError: When unable to resolve the attribute. - - Returns: - The function loaded from the module. - """ - try: - mod_name, attr_name = name.split(":") - mod = importlib.import_module(mod_name) - callable_object = getattr(mod, attr_name) - # check if attribute is callable - if callable(callable_object): - return callable_object - else: - raise ValueError(f"The imported object is not callable: '{name}'") - except AttributeError as e: - msg = ( - "We could not interpret the entry as a callable object. The format of input should be" - f" 'module:attribute_name'\nWhile processing input '{name}', received the error:\n {e}." - ) - raise ValueError(msg) - - -def resolve_obs_groups( - obs: TensorDict, obs_groups: dict[str, list[str]], default_sets: list[str] -) -> dict[str, list[str]]: - """Validates the observation configuration and defaults missing observation sets. - - The input is an observation dictionary `obs` containing observation groups and a configuration dictionary - `obs_groups` where the keys are the observation sets and the values are lists of observation groups. - - The configuration dictionary could for example look like: - { - "policy": ["group_1", "group_2"], - "critic": ["group_1", "group_3"] - } - - This means that the 'policy' observation set will contain the observations "group_1" and "group_2" and the - 'critic' observation set will contain the observations "group_1" and "group_3". This function will check that all - the observations in the 'policy' and 'critic' observation sets are present in the observation dictionary from the - environment. - - Additionally, if one of the `default_sets`, e.g. "critic", is not present in the configuration dictionary, - this function will: - - 1. Check if a group with the same name exists in the observations and assign this group to the observation set. - 2. If 1. fails, it will assign the observations from the 'policy' observation set to the default observation set. - - Args: - obs: Observations from the environment in the form of a dictionary. - obs_groups: Observation sets configuration. - default_sets: Reserved observation set names used by the algorithm (besides 'policy'). - If not provided in 'obs_groups', a default behavior gets triggered. - - Returns: - The resolved observation groups. - - Raises: - ValueError: If any observation set is an empty list. - ValueError: If any observation set contains an observation term that is not present in the observations. - """ - # check if policy observation set exists - if "policy" not in obs_groups.keys(): - if "policy" in obs: - obs_groups["policy"] = ["policy"] - warnings.warn( - "The observation configuration dictionary 'obs_groups' must contain the 'policy' key." - " As an observation group with the name 'policy' was found, this is assumed to be the observation set." - " Consider adding the 'policy' key to the 'obs_groups' dictionary for clarity." - " This behavior will be removed in a future version." - ) - else: - raise ValueError( - "The observation configuration dictionary 'obs_groups' must contain the 'policy' key." - f" Found keys: {list(obs_groups.keys())}" - ) - - # check all observation sets for valid observation groups - for set_name, groups in obs_groups.items(): - # check if the list is empty - if len(groups) == 0: - msg = f"The '{set_name}' key in the 'obs_groups' dictionary can not be an empty list." - if set_name in default_sets: - if set_name not in obs: - msg += " Consider removing the key to default to the observations used for the 'policy' set." - else: - msg += ( - f" Consider removing the key to default to the observation '{set_name}' from the environment." - ) - raise ValueError(msg) - # check groups exist inside the observations from the environment - for group in groups: - if group not in obs: - raise ValueError( - f"Observation '{group}' in observation set '{set_name}' not found in the observations from the" - f" environment. Available observations from the environment: {list(obs.keys())}" - ) - - # fill missing observation sets - for default_set_name in default_sets: - if default_set_name not in obs_groups.keys(): - if default_set_name in obs: - obs_groups[default_set_name] = [default_set_name] - warnings.warn( - f"The observation configuration dictionary 'obs_groups' must contain the '{default_set_name}' key." - f" As an observation group with the name '{default_set_name}' was found, this is assumed to be the" - f" observation set. Consider adding the '{default_set_name}' key to the 'obs_groups' dictionary for" - " clarity. This behavior will be removed in a future version." - ) - else: - obs_groups[default_set_name] = obs_groups["policy"].copy() - warnings.warn( - f"The observation configuration dictionary 'obs_groups' must contain the '{default_set_name}' key." - f" As the configuration for '{default_set_name}' is missing, the observations from the 'policy' set" - f" are used. Consider adding the '{default_set_name}' key to the 'obs_groups' dictionary for" - " clarity. This behavior will be removed in a future version." - ) - - # print the final parsed observation sets - print("-" * 80) - print("Resolved observation sets: ") - for set_name, groups in obs_groups.items(): - print("\t", set_name, ": ", groups) - print("-" * 80) - - return obs_groups diff --git a/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py b/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py deleted file mode 100644 index 243e82d..0000000 --- a/rsl_rl/build/lib/rsl_rl/utils/wandb_utils.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2021-2025, ETH Zurich and NVIDIA CORPORATION -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import os -from dataclasses import asdict -from torch.utils.tensorboard import SummaryWriter - -try: - import wandb -except ModuleNotFoundError: - raise ModuleNotFoundError("Wandb is required to log to Weights and Biases.") - - -class WandbSummaryWriter(SummaryWriter): - """Summary writer for Weights and Biases.""" - - def __init__(self, log_dir: str, flush_secs: int, cfg): - super().__init__(log_dir, flush_secs) - - # Get the run name - run_name = os.path.split(log_dir)[-1] - - try: - project = cfg["wandb_project"] - except KeyError: - raise KeyError("Please specify wandb_project in the runner config, e.g. legged_gym.") - - try: - entity = os.environ["WANDB_USERNAME"] - except KeyError: - entity = None - - # Initialize wandb - wandb.init(project=project, entity=entity, name=run_name) - - # Add log directory to wandb - wandb.config.update({"log_dir": log_dir}) - - self.name_map = { - "Train/mean_reward/time": "Train/mean_reward_time", - "Train/mean_episode_length/time": "Train/mean_episode_length_time", - } - - def store_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - wandb.config.update({"runner_cfg": runner_cfg}) - wandb.config.update({"policy_cfg": policy_cfg}) - wandb.config.update({"alg_cfg": alg_cfg}) - try: - wandb.config.update({"env_cfg": env_cfg.to_dict()}) - except Exception: - wandb.config.update({"env_cfg": asdict(env_cfg)}) - - def add_scalar(self, tag, scalar_value, global_step=None, walltime=None, new_style=False): - super().add_scalar( - tag, - scalar_value, - global_step=global_step, - walltime=walltime, - new_style=new_style, - ) - wandb.log({self._map_path(tag): scalar_value}, step=global_step) - - def stop(self): - wandb.finish() - - def log_config(self, env_cfg, runner_cfg, alg_cfg, policy_cfg): - self.store_config(env_cfg, runner_cfg, alg_cfg, policy_cfg) - - def save_model(self, model_path, iter): - wandb.save(model_path, base_path=os.path.dirname(model_path)) - - def save_file(self, path, iter=None): - wandb.save(path, base_path=os.path.dirname(path)) - - """ - Private methods. - """ - - def _map_path(self, path): - if path in self.name_map: - return self.name_map[path] - else: - return path diff --git a/rsl_rl/env/__pycache__/__init__.cpython-312.pyc b/rsl_rl/env/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 4ed9b78504f2308b8e6400356f38f04a67e4b243..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 288 zcmX@j%ge<81k+?*WxfE?k3k$5V1hC}O92_v8B!Rc7*ZHhm~t3%nWC5&8B&ITbzgcEae;^S8`dMenx3-VqSh`N_=s4Vo7RJeqvDy(02Wz;+*)R9DSe>`tk8V2bILf>lIYq r;;?}@)2;~QZICO9MS#QyW=2NFk30;FW=|QUA8-qIXf?7IaR4O%P}@=F diff --git a/rsl_rl/env/__pycache__/vec_env.cpython-312.pyc b/rsl_rl/env/__pycache__/vec_env.cpython-312.pyc deleted file mode 100644 index 1ded7acd6a0b33dbf264ff6ca3e61df3e88bde62..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4395 zcmcgvO>Y~=8QvwA6!m2}lIl3I8;|L>5CfSy6h#3c2%y$R-~v_)+wFltuo~_T$*uRR zGqaK@^boWO(Efq?So9d=59B}e5j6%eO--vU|2obSxNgaHLi8${y~$DoiFk@PBhm+606-#EwAtV zeYS;J$rS(CLF*1 z4tD0DD`=;r8=#v+`*WaLg_2$by;Rc6ps$qlO0VHx#r!pz`zyU>FzB2B{ps_BIBFr1daW30~}P5Mg6{o>l`yc9Z< zF&GSF5)q3+V8u|?Emrb+5bL}ZXOW9dl`j%y)KG`3*kiwi4$>QSjlblp5r5{2G*F2z z+)%_rJ<1z{FyT61zr5wDZ3#g z+>$DEWw-?)Tkv>-r<;!Rt3yCl)%5Un6EEG_jUPeK8Ktuit*@_GEA7u)x6cT${{qoJ zz5vmkhrgIY>i>hPyavtXbqoeT`6kGGoowb&It|E;3Lvwa?-HAvG<+9>SHp1r`?t=; zKWiZyA+*OVh|>&t!UQ~`{E{bJIMt3E&bU|^pKPNV2k@^46`n{lNF5}q%N~z-Dz@3i zd6aLmN|bm0gFpZ8e404XPl^LLr@Ey6qjJ4SS+jUtHllNZkYUT1k34Xh?LDlrLCA;5 z{FDq|g|m2u>Iv4AjR5q0n&JvkNg|VSU&wiaY&G(?C(ESx6QqD@B(h1$V42F0y2mIt z$YlXaN0fy+KuV~dU_+T?siG1i*dAE9W?^p%AJQDPh08G&8p1$kaV#Y99t6Wo8aq@f zykHOj#)DDdjb0$9sBM{|(t;SS z1ROH0+L!X|6Kv5=5x|9dlm6GloDx7AsH-%gUK7?i+N6)#5B7rt`rs4en>3+xe5v)j_A+EWS z^PvG)3=8dHr7!xKnUy)P?6J=^#y|NEu8^rwl7&9&ldceb&H;u?%?<)RV(qPVaX&HM z2h`x<6XPQLJOs!dkIT~@54ogB;j?$5ix*y(Js40Hrbh)Jrbpjq;gs|kInSJCoxZuq z?4@Os#8os;otk_Dtxi5y_8^LSAC&Z>k#dw4>EUA8IML)i?5af!=pG;0rhP1WCOk}= z=8Yno*zP#;HvZ%a4RGnxNemASxEGf z>;49nwA`6Gld!CHgq47?we(KV$gV z`m)w&v`*0^um1kwv2*+6we=%s{n)vAa{cWi=k1d>ZyhvHwtX?I`4bZA=AkIe8IQ{VO!jivARS1)c2Z8Z@pt+ vephy}M(wYnt30jSw*AeLW#9S2TKcgo-AE)YW{ zVwfcotE5bDJf|{My|UMwd$RwzRr_+FvNCJbfy~vg9A0;ft~)Y0R+Dmq(E_pjBQB?_ z;e7Od|A?OmExa(=%}$J+XKOCLx7D}zGlE*b6TBp3(C&FO4qS;~2w8+2;N^THWP~tN5fUaeW+LE*P>m(r=}N@jBx^>6)M^$sT^o-0HeItI-2G4O zSmW-HyOmPh+f~?R-MmD_z)iH6i%LnzCv*iC^yi?6SL>Fv;>BsLs5XKTcSb#bYp7*H zFAD1@8OzKWNoMdmDzuS8xviex?}XUlW9%d_(uwX#WyztO>;&PX>zFE z8O9Mqx5mOir8cRpZg(YZy0u&^5UFty7wE$+7HAgOVgvMtV=AR~-~|G#(IV}SV<+e~ zA8pUM!{N)6y-6R7-j6%?dCs}#o_o%@=gvR4T=fj3?{)u6)V_&fet{J;*^L5wF9wAL zMr1@b&Lr4zmc_CqZW*`GuQg#Cx6w9h+@5fZJ80P!uS;;_T*5i-WEl(dEF;=4GNMDW z#ac}SrSJPzqdeX)V+quKij@GXH-u$5sfN{PQdY*G?ifpwL^vL>=+{Duna?-Kn2Zos}xU78Z_NGpgR=cN|;3W0+sl|u?-Bv-utUMhWReLz+`x`O9d!K z5?Qsd2@Lea3Y%~C;@4dAD09gbu}m@%%WF1}hSyvzK+Vp0HdRa$$a=*-kr ztQ(}vNEtieXfD+%*3)Q0gE2*D|IPIj;gqeX9@ewPq@nb!Tu%l@g^H`Lz2lcz5_|}^ zx0T?VrPz>JDt&8nSZtstELa+6mNHh6=`1_dFdt4@brN|zf8?33LF0~USXAnZ{3!Jn zXxv$9F-r!G8Rz>JqlINIv+~m$Q0k(T%D+xZRWQ0nial~6pG4GH53vild6!Cr8lRgh$@Lfm`tV;lB_C&!nvtvWJ)*} zjmL#Ek`SGglSC3n1cC=dcT6Ya(a6lFSV`|bH6;mU(1k=&OvNQ3DaU67B{e;rB&s6p z7!{Id6p6fH_+`OQ{}Yn#h)~BQZFngHxMw56r)dAvyy3DGn*IX@6SOV5HJ+TDJoLej zzJE8L-v4FqH|4W(@|-LvYFHM-L=;5njVMJSs58^ji16j6pm1_3nTm_Jjse5L84!|0 z7yw8E&jcRP?b9SGtGZQDiEcY9%_zDZ$d-uCnS9dgf%ecONv5V1pg@vg#8h}%(pvzp zDuqO8T9QRzTu4zxf?kPk52N3ux6@AurvnGU3oUq?1YH}w_5NN!Yln%EDcvH=I(HbM zM%5XDMqKYO*mXZRy{$$MfO5;ps3IX%A!LVc!+qBsX^AA2knT7qMJK0J-5CK-85pRB zblaI|SRrVMbcZU*3M`@iTj6+0I!FkVohQO1oRCxiZA;1$FatpAb-+o)q4Q!Cm>4~i zLf@O9FQNMsNd^8)gi~=f^aiY7M23N&DS>9)rbzJ#-3r_xT{yTwP9;LX6ZG7P04<~s zo0|+aO+`iEMo5e%6ur5y-txA3oN36H?xq4>Ts;{;VCZ7#j@;C1$9%Z@A`%kGNh zAaB{R&ifWmF8h{V&u`kHZQ5~ti?(UE#_ygxcE`?i1_5wq*R1<1LhM|OEC-h4{N|n7 z=AG9e8`k*YxiJGG$2_uU*0#n8pEP*#4O?$EY`x{{UVLrk`RfhY!F}2N$FtABlJmVf zJNnL%+qC1k?A{|E9m)A#oE?4p$R`8)aq=TSYuIwjzyHShod4MD=*N8DZU4{*+$xv% z@74T!bN(l0N7wkiTb%21!{X?bBTGjX_hsGwH7@YEgK6(v46gKMhYoz$ne!e6t~Gq( ze-eTA{S0Uc9Qe?k3miwF{kH?b4?0#m@_`X8Fp>-GL!kW#lwUk`<&~vZ7LR4!+t#>8 zflEC@2%&48t|j~^|!8Pvb_NC z;JT+J@7bbxwk%KG^aO7;wk>e;+=6S~m2Vu-8V6QvAJnhbU*Eav&ikLx{7>9??#9uq z|L{lq|FLm=y{RqV^oZ8<$cp`D)6jiW*IPRlj?5ofB=ckW)&Z?`V5MP&&$jOVi_~Aw z{Q1mJ+J4&o-o#J?{ZZH_VixKEM=}8 z{NV8F;p_5GQa?@qIGugrwcN9>Ys0VS21B`?(=dy_?G-jg56s@wi_G@j=G=MCuW|mB zwvV~(|97ybxYsZn=B2TrW8UmD-e` z{|;At24gqXDOSxT7;us?D7BDXP}6M*DJ+xcpjb{_11LIB=^^Y)x0h2`)`1umblNX4 zRBQyzBji1Onx`+z_5JTr>;St3KEr)9Q)2=&LwDL+gJfuzYf?O*c?PoF!2J|sn1t^3 zx1g{gk8xC0GOxU~f-53>t$49L$*fw(hzDk`vbq(_WWZ|dN*%}!NlwpzwN53fS4v)l z36*kF7*3!=K?7W|BYw@}&v~}xJweSAyuLH*3FbV*S#H=^aJh)!@uCaSfu87YQBUX>soW4UHE?>4UjCFiZ@ZSucv0Fy?6 z$YWmvroC)VF?LcSCSxzI$0lDZ#im4vGSpff+^-x}_p1kU$yysuflJ-ci&~~C#fs66 zQb*~w1$=sO1E>a{wm)ReObi{Ds$}xvp}2itol9z)i45XYHgnhE28DMAnr4_l-59+} z5TuQK6EpO83F;n#^8-Bf6barg4aac{lypZ*J`W1w4l;&ga5kNy;9bD*Ck=zrlA5Moa=AC1?kQJ7H3qx^ zKok$y1TZ$HyYxv{&y|s-k>$v{2WF2}IS<{vSB@?nUFp8DbFF*-yz{bcK2l|Zg}(Rr ztNhBVH^`ds^t|h`?egR;Z`VR}KDu0&^ZMtlw;y7B!KWCRA6bli=fLb}r7N)R?Od3e zpUQjvn%BS5n)3$dtyS0ym~2Qx&O4O%?$Nw^ZrF0(-*^b(+ud81A75$7b%USvG3UEq zFX1@?d0_AT5frFiDmTi|hN(>dU~J>TXAei9wyR1d7!a>!V#R27X;=)S>s85A_oivJ z>#j+-LxHI@ZYiV}CXm-Lqc+!}gwa7*wU)$g5AH9OCBs6ay~}w5SuRkU(?A7# zZwhey-*DRGnw6XeoCdG(AQ${MdKxRdHy7{$!XXn9k>rS!_EoW{)b)n|MR^ReUx8Vk zmP_fm^y1dv`|T{Zxi(*@1pXTMqPtO=N;g~5k9@-L0y%>#tdJT`f-8FpuCSHYQnPgl zC!iiHu%uUW_bzYEx%&(9+h*iB+2s zrf`I(G2TO{S+GXJDs=REtf-2Mk}<(~k`MqD9-54Cw4c;&I8 zHxA$nDO0?nP&-(|OD98sujr$h^D3+``nTop(uF}a@fF_-Fb+ObxT4k(0p^s<0yuQk z&7v_zf^tSxVW9jhJ7MbzvBCP1%B&Poy|v(tRRnk|x3boVKLd!TNesewdzusOw0vhAoBQc{HL5 zei8D(h;S0)=>k^ZV1&tx3Aq3aXPF5j%{cQ+G8u=NFHT876cN-Uz{MVTeUM}bfkY&r zEv!-kF}g2A>2m>w?Qwo0tinSFu7`FKNlg(M-a^jIKx0&?AlO7*_!@EC0i|pq{(vx% zk}36NOMig^ty(?OT}C$G5>yg~xV1bfK==qAViX}POcQv}NTy1CLE#h*ors4g1^A{E zlU7MM9fz&)2Tm7B0PfYG|MYSB1bx4GAtlQaIgKn6F{Yo0%91dZJckVk3g8gR)3TlR z6DbbnUseTR?Vun9Cxe5qUy2GZMfBYz5G8t+wt#xNrx;M*zCz#%a^* zOHJf07)YLi%sgio4BMH3rgT@iDX0SWmpBK5=ny6Uyt5GW?pdzC4)KYeHEsknCe?>P z460Dlj@c+=0rpg&4WD~tc-#nu^!iZ9*jp&`q0n!p!tp{6A5}s`ilhj5iz*G+lV=gi zQOI--R6o2M!E4QNY)pL~d?#4@ zV|ToaultL3EZ!3>tlf3j!P>V#nAG9^466P2tWdgxrDB`E*yb;`?J2hHx$E%SgLfaZ z*|**0t@fU~^;WxZx87!d^e#^`)_&lw!(ku!w?? DB*9Q+ diff --git a/rsl_rl/modules/__pycache__/actor_critic_recurrent.cpython-312.pyc b/rsl_rl/modules/__pycache__/actor_critic_recurrent.cpython-312.pyc deleted file mode 100644 index da5367145b34d74fe1aa9aa42a70ac898592d050..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11682 zcmdTqTWlLwb~AjA#Fw6wD9P4HvLsTL=-6rOwc=Q@SB`DPk{#RGB+7%PIb(@3IaKZp zZHu9H-G$MWiUhW>*jn0P*GfOS6k@=&-?UNGud$*hvsR$j&OzZS z#Zep`p`!FCO=H;*F^n42uQ6&GHRamOqh@F`Ml4b5s8wwXwlhi3TG+qhcIB za!hd_h)#w{I24R%?cszvRIv>6(HNNu6~I%JO{>sr{{V#_!OnPUl;$Yd4Lxe)44h#M zcJQaPmUAY~%vqq-$}wEU8CwO#HO)|GXs+^-W7Nc(Iol=b!yNQc3)JmJbt}{zMRkU2 z=A4)4(F)FWi5jissvx&@ibVvv1&Lych*DG?a1Vx4Ay;l+_^E&&5RP6;F%GsbNw`xba2MAUO=Wg>6)?_f0%doC0HC-*$>$8(| zvKo6P+jXcDiCS{_BCDg?z1MiDAP57Mdi1SMa};Z ziyFitX#>jG^&LvhdI|PoPdby1>$VSbT!wLZ3MZ*@*PFPd*8F@2XH6Q%3`t9pDS}X? z_Y1YfW0S^>#wAU#ZnZuuX(H89XTDaY^8?1z>a9YD1iaDGM2}u7d?)$}Wxb^HuozC! z$+W|MTlBGo@B7rx3_1pI{tD;I0pr=iIK5OdzCi1mGPG{cXBE?lb0)3p=$RL?Ni*P% zM@dad^Ex`!=vcy-CVkBRl1_OzMf-NazWqAdp9_-xaa`RlZPnpT9MVgLZ?d8Yx^fikGcT>B6FRJklX|J}{v`<;{tzSk`Ji})TFE%8xtZ^tQeo1j1fs>w-2(h(;`pKY5KMQ-hV+-EFo2q1QpvP zO-=Gjr4Ww>BEc!xv0^Kxs7+moJw7th&n9veP|5)_)CCfDO(D(o>os-2z5j%T2x=CJ zF%laaJ9OitpFYSY_I}m&vT#<2oflY90uBTT2Ru0!R?%Y-nSS=GO@8*ocq|^_kTu!Dsm? zQ85E;c%m>mSCmR1P+*M2;*%nv$_p@JJP1Uo1q3BN04IhQ;D|{9QQ`=yv2Odas68ka zi5EoJOy&22kvM;V5GYq13zA@zmjJXWCh$Nd0IgVo7r2IE=fXhU@aZ^Oa0InQ#Ut_( z5bjJc9+3j)fD#;15^kpGt5ZxOA338KfldTfImL#D52%CoA3;${wqtX(MzGvzgv%x> zHM#v3w^iay?Q|Wr0czXOUC@po&^AI{uckaG>B^IdWLV2oheY z$R83QFW_6CyiySegd#yv3O1!8)vDkMH)P!-NutIRa6FirO@RCDKy>0Q@q zqi4pk>giZKwBqTTIee$qeMOw#J@@@=!&bRrYo@k$nZ8xKJ#81$Pm3AIYqzi3J@Y3P zJqvGTH*J?UZC`GeH$5ZUp8-tGRFfY7H#N^Vzb3?{`Ou{(&9!hTx&5NNYT zjtqddS6@Cfdk7Cu*3QaycCikROKSfIo3m{@<+h#6?<^DwiskcPu%x-e+a+vo~M7c_i&U{Mp`rt{PpfuFqC) zk*l|)%(trh9+|pY*LZbkZfKs&jb!V(<+|>aEoD#FJ@c3Gznc2gz34*O{*OCj#>j!QeUOK!id>a2{;*&)B z_?wx7Z^=8~%Jc>@t*2m?{jQtcczGc9rZyzD=Pu*SGG3YSrs{7q+t%O`Q=z;M1}a#F zOHHq#k#!YxrwRC$rgFZT6Vwtt;xiC5CKOYY4+={(*#{L7L9*l+w%bsW#)JTNC(5cA z{-O~Q&PI86ky7c$$}TqR>XcobX{M9B1RdWN>IBt79eE49IG`_(mvQ+bYkcaL> zYgyTt!yN`(z`2j0-YvVj(@giHr~}0{s(%j(8&Vxdl_cG(Fo9Y^N#f}1dEWtW$2dYz zW-rl-5gsLq6ao^PoPq~89xdH6c?%{aTFUM@ z9#jf~&H^|Icybjphq{P}__EpwCP64Cn#xi{oqrMfMKr4}Qmc&pvSZefWtwHCdH%o( z(_YeYn`wXaDBytu#OXEEey>t^2ekAb&B;&jDVWK zZNR=K`t$G8RrMxG1 z0`VzH;KejtH|iGl8-U5QaoQyOW&@Ze73Ndl0H(QU4tl&xQqrtjh_L!~Depn%z&BQI zcVOjPO04|HQ^{B!PL4_~(2M7yBu(VKmxYdk%@3HqvH?_$PTQWaXKJFR09P;Rba>+0 zp07`(B2T$7Y0|k4cDzkr9jtTfh-&%~^iPsEFhlEtpwp0`FVAOJ^>Tv#SxrqxudHH; z3#Y>Z2SJPrlcXQ}p2Ow}sDiB+03&5DHljRHC8W=y$^x<;a8#l=cyMNAc#dKxtj=TA4crLtX&A7dD z#=B2YAwVTbF^J~y6qdu6vbRhM!5=Zqz*=TFd{)IV1{N@=9Gdw^ zdSMamC9{fe$9W!X@w3>ZDr6Qhh!4h+x#%#Pqe1a3SfRQz23--NVvWTm(9fbG5(;WN=!YHCZPsJK4qBjI(pbsMfn>XLrh&arQ#J#(lMKu5Zz} z_;PAEQ`4WV*)P}Z|9E?*=D>^z_(60L z3cue2pfm3Y%*VH`9P^9xF%ku2eP$xSr@%WB!zzj_P|)^a8RZT^EkML=0z))*IDqKj z^E<;skT%38+BafMVa%T)j3LOr28%K^Te3C1a!qgQrA*B>V2`Wza$+_y53FcU>)K1% zyLcTI`u(!v+q{b>s2n>KCtj+8=Y{NmeW*I2#u-%U$=i^XUJr%`A5SnqVYY~;4aJgr4boEXU29!vO*O7~cFY`J1+Uf7*`xE{U94CbxozKi-$*q*eb-fYd1`iQzHyPP zxO&n|&;Jt|t491C9IuvJ%QoK~timtk_t81#TjMKg3M6xIy*Kr@Y{O6`D3O z4a%P<{UnupXV)994x zX%q?@lBbxUI13WI9(c@4QaP{783Twe)p?;=u|$MPn!GA?UsNMK*1h$Kt__K?ZrmpT zQ*47wrNrgw+3BpSM|Sn325$^64QG1~%Do5Eu7hdj;A4)+zry^DXh3>l$9R}ZQjyZO zf%RDK9Zu$7Syja*b@X7&X+r3RW%7kbmC6@KDVG zHS$34_Ni}+uUOPoz!}U111bw|(E5w3)a8tYbpfCa&ULw=pTg2&5HjE<8W!ERY~8Dk zE!oB%xv?kpbf$3|3dE|cy;HV#X6?PQy*KsRihY+l*n7+71=*SP^va&z)QOB|`^=C! zxJ$NoW$k^ky)QL!+rAr|;P#xRR?JsC)ljF`yimaZfuUlfrCFf(d+?y3l6{@lZ9br= z!DkHiz$G~n3v#Rkk;zFtR3x&oGir-Uwmyww7^CDP{>n;xBrxrRP)mQlhAp|ETR0@K zUJi0!KYId|UluDME`nrAhnxe3v-E_4D4cmZ7K^~lSH^ijim*});9?JiMtK3;DhExcnAm!5FS1~1&v{G9m4w3xvv(#=@yH?d3_L47F6tu zwtid%TC#h@y9jMqig=1=z`FLCqcybC!#$m`W8WwnVJa!%%5EQ^6Dou+x?InB! zbSQ|q18I9%-aqE=1%8PVMAy~8uFs!e693?-@!}-B5FK#76c@q;qaC;$P+6{BJEgMF zL@vWXauhQC!d=vKj4RNT*iu$hmO%X{oQw_#h>1OH%0;eP7b};+_u0C_^us%%N=U%! zX&6ONrsiDFBk;G>lRh`bj0FgU1_DZDAfPcC%Jx9u?RYSf>#>K$Kt5bYwg3=?fj1FC z&e9Y?4>EZTGj!gn{z}4QNi_!JSUQhc0y9-Qsj}#gusVa;dzj-O@W7~&(5K)^?9 z#G#?3%^Ver#)jOsUn{ik&>77NK@&;a0!EOKN0RjpVD5 zdM(6n8CL&Ip)R?kX;+$U$L!GJv!{l8GIUr}BE bMh(c+z(dQBk*=HHac%d)?k_1UtK0c+O_lkv diff --git a/rsl_rl/modules/__pycache__/discriminator_multi.cpython-312.pyc b/rsl_rl/modules/__pycache__/discriminator_multi.cpython-312.pyc deleted file mode 100644 index c25a93e75600c291985543f5a95c36f09ab9dd30..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5983 zcmbtYZ){W76~E7Zetvcw+eyeDo0uddlz3#J2?^CGl!j%{BtQ$Hw1lG9hu=$L$A7|o z&jsA)mP$x-I@%DaT@tlYsP>^jpuwa`W1q%{Y3j5|%bU~LdZilEK1}RWR_I#w%XaR2 zes*x((rGu*{d?ZI_uPBV@0{a5R98C@l%F>LGyF3*LVqU%v)Brq`{U4=M*k?b zDrOn8P^52(qL_8es`sfV9kYaBK4#NvW{jbb1syC!>d+g@O@Xq_xAyyl14?V8j&HEHJX0 zqY0E|;agh@sM8L`I*3j^4?U;CMrur&QY+V)eYxK3MHYF`NcDOkdX*Fj{?IXTnK*Cy+`aB9+58wL_lzZe|f_0ir zN=}Oy6K&C0jtj@bGRJ9*Aifd~iJC)_d0FIya7<&9lE_6xJgqt7$rv||`Isna9zHhB zVeu@F1um2j$2EqZo)+VRRxM47AsO>gE}pZ;|{@v5( zF+s@G#;-#|xA_PJBBalu`tkyLz#xZv)!llfLuDUdWqa>7G-v2bRe9fLXx#axR%oj8 z{tjq7cU!tM^eWq&uW!sa?>4kt`JUR)m8tr~-<288yBcz?fa(goAG^Nq#?aNFn^U)U zWxEb%H$A)Rdaks3dTC;D;=S;ys}q>cjI7zw#_vC1QA5-Gsku{EhF9vh=jwaa`rey6 zv-P`RQ|BH3*6WrVj;oH9-oe{P)!t`5YRh&W%=!=I{LiWW=T?rqkoCW)pN@6SI%k|Y zwn=51R#|@@#-15Zjt!`6V3lpT+xmEh{+JEi2f# zreUW6=psdcED|8aQlzjd)=_koIvlj%Mj*fhc!D(E2#rRa)2K;}5+s7EL;-LcnYNE7 zFj4lDzrxEE);0s7M5+rY@AA$L&kSEa^OHjviVU8ed3IqS>)He(k2k|&LKa$oO7iNu zlnO`(Qc{(q6iQNpQ&Oy4@g_j#BJcyJLV?*LND-1s%)q5t6`Noof7j`21(zk*2-Kz-#iB5gqLV0n zB`^YY-EQt7*ivxr2zk!3A~2?u5yG&nm@?>&(5~3QakGapd5mL0N=K3)quCQlc{(Xe z8WWD|t&2nwoH^?XVUeIEOioW~E-cD`UWHLrA|5>tSVKO;M62(S3IY-<08X$P^eW2B zvKZH@F~DLNixT-z&{a}bPQb?m4ozxwA`Y-i6NEDWjwp*p6Jm|lq1HN`IIGbyk&g#i zLzkWnQCooazT|_zA+2)nx-Lm&h#;3n0a?0&gf0 z14(pI1rBIZ?G?%R+jiGeH~ad` z>kINyYB9CEE!(m)>v zy48~n?8yc8seygjz@S<;m|^pt#*D`>(4u+4;fY%620E-8C`3>YdqrTfyi@`oQ_UdG z8UdxY$VFpSEOIeMi;x>HE2e@=)TK=T3RPvxFqDg?S;-6)OQaZkl)g1h!fYvY=@3mo z=YQvqDw*_m+f$)N5$=Ff1)YYj+Y7!^bB3jG9Mm=*5-~vxOd?QB1i^iz>V-;U&WhoQ zN!fsJtD*9CV@}ejm_%Nyq`DKL3=UrkaRrKss#{L93s^`~PywXZZphg4HTCoUIe)IE zU9D;VOU;(dz#Wfoq3!*nOD7gjeC*kp_cqSE=iKinm(q*r}dViR@WGfrjEOMH3ScHkub2tbhfT zNqo;Bg=0(HB9{y7Py;)%p1u{f&$x#5q7cWVUqc_%kGh6yKuk(M(&Oi% zy&1ElHYs4;`bM*2t>`L=jem<(0tQsH8ORU{kCcXdt&;PE*<$`tI#Ve>DSi)GTqSE- zrvw#TD!5YXI*JXY9n2naxWr|e`w$06#sYoaW@3)+6d{kOlI6t`qBKNp8WDy|$ zhmfmaWE2`j3L~&7xNj1ce7hV9lY+xNS_{UpB((zDz`zNtBqai9=I z2)9VssM!l6Gqz$1K~>c{$st2FPl3%8I>gJE1VCDq?tF}tS!0a0LWos6jYT0GlDSfX zo!U|XSierTA9j&QZ~|$g7rEbNL6^*8Z$_XJ#Pp-<0!r ztN!j^*8ghbFE-vfa=ZPH^?z#qee=rk7gokjs>e@ehh9<#UQ+kGv~oJM+9RlbAu~cE zyY!psd|hL%u2Ze+T<-aET~|KPmJ4*Nf$rs3Z;j-3A5nK7Svh(vyZiZU;P~bJWPj(J z^Bq^dF);ty+-up!t;;pbBe|}WFRSqXqnFh#ex;HBq6N7FGu+x{ zRMWg5tKRm@kR^BIeBG+Adl_eay*E#)zNarUk7B*jIPgUfSj}>8a$iX#?09#udB5BG z+b0Jc(EQGA+uvw^D03q&Nq0-b(19N|zacCnz@ktDS^#)}vb-o39fDyxt>>pf%VB&F zM&N+S&lqNY${1>-F^1l06n;2U#YcfS)lxxY)~mMwL{b-25Z6?DFZUTQqgHn=?^j*D zH;?7|ht>Y!kB+PTqpE9kebO4V;>Y2;g30P>(mjx@5XV)K-;(0+3laR5*gqyD|y zVyn@^44qNXVIT`@pkX|2M4senwPtg!J1K{wQuo6@a@q7rOp;uUIi&ymqjS894DQ5D zY3y#}HlD)d^+|LfJqFd9m7=Imkn2-a|0(kS74>CN-vfrDsG5b^CEud&b42<&pZ@}Q C|L;!# diff --git a/rsl_rl/modules/__pycache__/rnd.cpython-312.pyc b/rsl_rl/modules/__pycache__/rnd.cpython-312.pyc deleted file mode 100644 index 96eef2504dbc20935569b5810a384ba243b0710b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10119 zcmd5iTWlLwb~AhqMe1QomPOH)#xy0HdQg@fKcYB6oXA;w?I@}3I0@r~9&tv}$kb4| zGqf#&E}XzXWgy$6A6v^C zI}biYIo_rp9m{vlz4zR6?>Xn5bI(1)e{OAUVc_}89seO8k21_}v0^`76JVdr0dR{E z8IjE}3v7~Q5q4!Kl+P*9YtCTMb2QK#Z=!~Oc{H;wr5g+-ajX(2NytLbb` z(IoM_KxB$Gl_d*8M$XfU(e0qtwWq7IGmOWquutv+@MplJ#3Wgf0mj&*JHS-`iJ>a| zLy*|I%*?Q&>qcwRBesd|8=j;Wg^=`#?P7=Mz2QyzDZ8BLht`0|LD>>u#4Ta~aD(Dj zu?5~Cu~Q7f8>n4nlWk|2c$X1ABPe2aVM@}jWXUBEfhK1%l->M4o?KYWNDGo;ij2?B z@Fp^EVe_@<{N6KDlX3pN@%LL=czDi8GZK-Mw4_eZI<_Se*w4R`Bcd?EzndfS2rp^8 zkV){-<5w3mSwd&#rVTmX&kay(E{34MSiL3wttWJy{8Q+%}-+$!! z=lH$*_l+NkC+NiKP>p_k0K*PppG2YjIeUxIoX(q}0`oK2q4%s7mMO3WCgrjb3NENw zyNh88Zm3zxhMFI0*6xN{3)FBX`V=^*`JGw-YMfK!$YHJ1sWI~oMZ1k*W?7N_mFu3h zj`?lX9vdn!^R+x$&G%Rnfcb(prm|b>vElh%8>+tNcUEB=vgJ`rrw||`TEC4k3wHb? zSJ+|G`jIPi=!mGFEjHCc@FTtugciqRXnUMK@Fj(y=nF%`FBsAPOSdg+!0)oL6u;=H zt!u+}59|_m!Bg#!yR9Ttj1%-D!E??(z`HZk65!1dKCnwSsfr!sCy9UlfmD9XA z;)j<|Js=#9#`Dfu(R_stCip%$e58r0H^ha5%uu3pvM7R3L>auTipLYUG>IP_=Ven; z;ORs;tz`+wMC3Jr%!1pre51;zv&xJ-n=_qfjeM2d!n8!tZ&Mko&RTG05O_`nQ44C* z71U_<7@4i{;W!our%aL){8YmvV^pg^G)w6|b>&qB+ElYS4fcduOV(=Q_e#KxDB{LY zVJF6sFO=OMF@uAeG6kLjfx}#<;c!7}8?~@u3QOko3pe%gN7AfUVm(12t;v_^Q8>XL zb6|cZr$8mEB=|{bM#yC}D&?q@$wlkOUyBZ*4OR4EO30>R-e-1X8wzMcwL$X>eV*Ct<6*nQeL{sYgqCkWNNvm~F zHp8`Qo60KS3&Fmx%*p9FJ}s!Gb8V!8h6)A_T2Qol@gdcd8ukm|Iq1$1$P#}=*5>G% zX0n+~_6q9%lC%UCTqk=RJjzGGjZ4x0OBOA<_KJb_tJdOe`PhQp$c!~s)ucrjcNSjH zJO;M8BoG*NngScgZD9U2OLI{=UE=`iMM1l^1RbTCo|#A==49~6&4oH6D?$bwBxM74 z4}!d)S*phU(oD-L_;kx_QVRTKA(Lw`8B0Xw47A_UUd;&G&Ku{bap_0~A4dq5{Oonv zI(lX?nB=M@Y%SRg8h(@Dq83fomI&T`j4chM0J$x< zC}J#VUaX0+y@5xPqL!r^eu!u=cr-d1F$l)hM!-3{4WD^*8XgG@Al_^SSo(bN!jP`~_8G%$Poe@+ul~P*)!B#fD zum9oj7~ZATMIkMX&E*yZB`b?5^%BToIt!sGc;qpnW>O?GW=7HK7*WK;;u68zOFp{k zjXYs*16-4uhWp*`uQT6gSSHYR)30+qYh3TTkBJN{dp>Ee^v38bGPvwn<9aH6P+j99 zmF^wT9IWgySt||XaA@@Lz|a@(eEv>(;E+CWs5Ec{hel!1{XAVBK%q!6I9M5;z@EJiW5cVH z_mADrel?>XdcAmXsyOvQaet~5yGR%ED7Kf*{W!d1CAk_Y9y;}#WGVbkIsBd;ey@1p zV)??Xeqpv0o&%oS9(IRUhVEV}9ytD0zSMoD-2Il`{Z{dOvV309&kLpQX%q}BAkg}0 z@Mf^g^?*db=6auiteLcv1%L|fb(%wHgVtM20g@^6h}7_iY#z2o+LdzJ;ST2H51dBG zGTgIn>enE{Fz;j$YA{@N7)~ys8OA(Z*BT%DMt2SF#Ph+%UdUHC^q)G~qd8fI`L>Nz z=AGMiwGVH`dB_LW<_E5MVHHl!qXyUOaETbs&AW>6QNjh%Zq_4u@V?(n3o_(z=AGMh zmDXRge~#H5ha+VR;5EsJ)Z!FyT~7X_N-Ga;2e|S=Ld`q%4Jz|X)^Nk!0$dZE{9iKh zHqr;JMk_eE6xh0Xc_Nr`AiJ>xJhUdH=SUP=a219RPGli7Yj`lcCYY8XyRg8d3yEWa z0*d>{0K5!090f+ep=@+mZqB(f8J$&sSZ(ZZt^z63b*t$2O=m_%kZ>2SxDhL#*xW@m z!~YN{ss9PZb>us9-&=cg0uZrLO0e z1CJv8cV4*t!ri4(WV{?XqDPLDA`?Y)Xwf^$ZDsY{Qs3cn-%EPmOQpV-i(KS!guiq2 z_R(@=M30OVM_(+yd{!SlTYNWJjEs~b@4-l*SdNVAk@4dGS4xprp#{B8M>)vrLB1Fq zsq~Jm#LB&6I{wM`&y^3J&<~y{_R7DRET5j!;jdR-p8VOVN4~PjiH&i;ZgzHP7o82@(!KBSYtstPDqx--R5rbK>k2Ae3xK? zSyja~fyzyN`{6=^^X7MLu3`)2CX7?j4_s#+w)GYxLaA-K$W5C}*ZNDG0RFR2DgdCr zY-XoV_l%H>ZJHY;5omWZtw7#-o`IT^gSE$iNg*}xR*laOQ#%V5KLy5!_6q?L5oE;- zA*hMdI08*L?xG4H7&53RB1`P#IKcV-%|%WreuAUULP5>w)3%#!Wp0Pg?N~Xt#ziZ^ z_HuCdgW&E8Y~|1{J+!MF+M|c|ta{c$qqH;jAQ+>aQ9TqbhxY2By{lbop+u#6^0$-EDT033j>=E^L%5z9V+Smu)+ z52N86^l7%m3NF$0D<@S9?0crj^ML)$f5Sue0TlPpc+~I-i;I#Xntt7#7BuKU@sJ}> z7#v3N8OT5xfoeSOfgZ3)OLoseEx%`T+0~sYD}b!N00juKbK5dk3Gckqf4hG*vKBtL ze6kYi`t*&PZ>+p}H*kCE>(KDJo9R9DxV`JsrJGADJ$K1k`%sY^+9+IZ%ePUPQa#lBwJz%(VkfK2 zFI-PK$C}~)6-=t)%wT|yx%KQQ_YCPhL#qc$J!2?f$NZ>;g})9RSoboa*hZP6bwD)6 zKEVTjGnt+>`TmkM&wJB@ll?G)9D|~s%P`!t%uG?^T{3@Q^X;?gpjDi}^zdMh<@TuF z9xZZFlapF4qAR1E>;(WgVGFKH&_>{kS~FjN7QUWYe33ta5pl0+56G3#Qq7(i!J0gW z$u62C8isXTZKK(R4a|-`g;|Wq8V6v2tfCgcW(J2=lMei4!wHjV$0GnP z3V<^GF#ylRG<+z!GznsH6+9qWQ zvU35*H#rBWr*!xmI#qo0T=DI+{$^T_i_08@j;H9-*Jz^1zB zN~z8+Vn0{Y{|+>moKVF!V76o4NP%25=8aM*GuHrcD3$s!CuFQ1e<~$r)2S3qK9CoY z;wxAj$0CLW^|fyxbOwv_SiB2`(P||@(o!aqO2MH&46XFgf0;&;1Y;~(pc7I51&ZI_ zV7_&Eyxe*(6X7d8kxDFH85pW`M=E=U*TYHH3m1az>s|yPD&H1f_fwE#y2I-M3brtO zd_73P5VLb|y_JG(%(kBOb_#YdTe{b`P;e{L7hUh9U>DQBdwm-PyO}Lr>)R>V!}JdP zj!ygJs;k?}f47hIrrGa(es9-zTUqa}?|dHb@OL4P_qp|u%ga6CSnuv9KGu7Z{VyNq zHMbvSFpb{8e$Y$M*u=-8XAt3{d6^-F0zepLd~`*0UGqTvKw}PwOdxEy=B?fZz|Osh zH;-<$;jtqW(Q^s5sOtYIe7r{Qh0n(t2#~(OY zb&l5UTHvbdstZ?Ca5wvbLbK=yuf%D0+GWy#=%_^)0b!hx)NJOm`Jn?pnZR*X{c>sL zr|UmA!W}r3o|!d2Q9vb2;Pb{p{lJ>9AJvZv5KNha0Fa3JHElCQD~q}j-^{Ky9VRK4 zO?&`kl2!0`G8PACsL9uSGsSNJ#AF>lxIQBjh9B?qz&lM8o3SPCi2Nv$tA2y$vr1-( zheYvWmcSRm@kzdhV>Llez?hosDCQ309>blH6at3{WL9&^6@nqEX!z;jC~DwQ{g&X( zfC{t;?hrUNS~1b=BpS$R=-7}n*lO`g@k|vH%mMs8BL58(*BP*XZp&vs(fyIT{?&la z?=SK%J?aTBw^qWt@19x;cL_|x+CVSW2>sdMB$`=E2Y7@95~m@Yxt zIbI2cSI*uI-~OPyb6np!em|=3JgA2b-h3S*uHNx=jv0A=+5cHU=XzI;>fGMS@WI;Z zBd*=-J5u6etM0#R{j#;Z_eCB4xEG zv>EMYRySdPn5UU*ni@Cr>Au(Lk0wmFhyo!8u)r*VdHd#$nojy9f?+n;3!^5WSa-85 t`@0as?*1(k`d4P#zcbT1GyT19&c(Lfb$t=|Jn%h(@RRf29=4Ox`yc#M;CBE3 diff --git a/rsl_rl/modules/__pycache__/student_teacher.cpython-312.pyc b/rsl_rl/modules/__pycache__/student_teacher.cpython-312.pyc deleted file mode 100644 index 48ba5bfac75dc0502679798afbd1edeb84b6ccca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10497 zcmd5iZEPDycC-AJxFjV~mP}ce)|PFNvQ5WP9LKWGAF-006Gch z?9z%DI&}jVm0Ab33iKr3HHgv{I0;D+AJ8@}P!u_!9|yEQ$WX4>g%2=r0owGJVmr7? z{bBt&O-2(zlV*N)=y|R6iR5IQ#$(Zh z-X71ep|J{BLhO2lyZ0|p_&wN#M8cd%0CO&E7CDg{g`NCKDHqM6MYN9C8;IC8Lq<5! zcEuI8h?_{LKP&5&2QaO-Ij+@iTpt7%b^ z)iY8wHYU+0gXT{NM+j{I;8i~F^-KJ_C3qPe zUn+A%wHa_`o3iB1<_av}A5)e@%By9uRvs;yFS)WN(c*{cgXH`nSk0|^3|Lt%>&Bk( zCZnbDHP#lb69GaL`_sUW2L5kwS=WQ+igtvWalvY?tA{NFuYI%*a^g$Y=h^y$4_?1&^@k0JmG7^IT@{w?1}xdH z9yH3Lt3Jj*B%=GR@_yd{9RT|*^SFB0m_I&Xlqz4bibJPCqH#>BO_88yICFBn5DrBm zaXGF=BEo1&71FZw>ZBA?B~ggd(R31owX#Q;8jHuqgsFHUAzYAz_^6zslGraiNlmb% zW}8eU;<4#Zu#$P~%$OvsUqMKw#B@RuQgUKiP|}l=DXJ>M{y`yiL6PW1U0a4e`7cPC zUDZ`X##MqLfZpsEGOm)!(5noM!(p0R-TYkQ?j6_QCW;qQ4pnzaRyURrzfR; z;ft*y;q+K4oe&ZE9$lsM2q`M`0Dzugmu8)$aaq;Oib^%hgfy*aRzOptn!})t=1!!d zq9jI2=xL3Bdt{WR(vu3{A;~afEDBRl188@z4Z78hZo5iy=rG@r4D z>c&Q#tTT(|W*l4I5#5ELY!g$swq}MC)I7MH0UMCe8p;Qz_ZdsDzDOjNh$>1Xq96f; znsoV#6Z@u8bV8Yo#-x2?>0}gE79+|8Z1+M6SS~J)?xRW~LKFLRR#)~JOP7`jO-@r3 zQJJ>W>nx!1#2VX>sI76Yos?S+daj$UDge3E^-9H7m^ zGQB4_+{&(hbMCm^d+fumLhlPRgSYsek9+sMA6O2o_8!jn9xn8LYi4i-8Xd0dt_AV! z#Erzl%Q^R+702HDwmNW&4*nWH%da~8d52%8(>GlQQ0>~wh+cGhR-K)BXXj$zBWLgJ zHvfEfF1tAV7iT{R|2X`ivv6Q2zhkJ-_Uz2yo8KpJ#f%j)qn`QvAD$Ij-? z(%e?1;8k(-{*Ie9BDhCpm2Jp;c1^cRyL8L1ryY{ zdw8km$uwE!hL=guOi}-8R*^{uSLrH(lvO1~!TE4YC@LY%N+pm4nU)&XX00#5fP1*^ zWdel12L%!<+v-rkGF#jYg?XYvuBSIR`YkAd59J$9sV9?@2?>>vBrnLY`{Y6P9S^ow9;ro9Hqs=g@BLtE@Jx#RJzdW5`3SoQ48dv@j=JOAg9wt>V4 z)wuy}CU;XZDuj%`KGx7B)kV4|@9D`odNv}B{KJm@MJRkJrEyeEGR)(8j-AN8S2laV z-82tV^s$yX%?w%v_?exJ9hiuwCZ|E)rIWRrr7ysQ4GdJ85r;lyCuHl+C6M<73ZC9o zPblvR-F!Ib2^Bn#<{Xded#>gcyajB}L%_9c&&a(sDa#q=;w5k)>+zb%F+;WkdM!1R zp*miAKB(@=mZ@^dEPv-qpqd!eeJZ-wERC0)F{1@tv~j*tGfD==U%9rw;`EB<`gnlR zYpFJR)rt#c0^`3#PcVebcutd`b(k51%zB}yNh!Kb!A6R{1Tzk35K3AD*tTFKMxqK8 zux^GrMPG&P#tg~Hm?UUn(uMBBhLf1#bOrm-G|;7dW2oQ_ zu6q0O-oBfI1@EER6A#oAA9rkDd}PU2=m4d4%i-V1s^#nX$AIV8U{wwdv(9C$VrCqH z7oa9pMu5Ld#>+1zm0{xzm9kMXgn%J-t(s>POi@KQt*#}UvBTJdwbr!eX+0L?wK*OdNA(}E_D~&A*egRrV#QD zVbQnZ*jcPdZWMwAPcY{QK7i0;u*Uxmp$(#$>33mGolxwagmJIcL~#KDz&9L`s3gav zOvgs@RC+JtM}Q0Ka9o8? zWB7s${|r&Yk<+A`><3Lh1zz?!fWUzX^{V+CnJQT@CGM$GzsCc&*2`*kg5-*A*BvPeut-Pf)H+B3F7}Y=w76bJkQ}E@aI(%1|5} zxw;u%1|N7v@N#K-udDz-y<3JUKK?NsPoIO*vPJhaZPCd|Nft{YJ{DD>0}VPR&NT=6 zfeG|wAA`2_GM(LsOlJ+B*NB!sz-egr0>OQoW*kNT*0;NEbS-UO@jpCsvdFhwdv5l* zg%gVnH->NVyVuO5z3*dB%eCp*>4nXUbj8z~bM$WHdE7hW`TtM!#TSPPeQ6A;z=% zt|t&dN8~CLFep-Sq-Md6d=vIi5n)WoA;vYSp8aNsMm7h%r<(NOIi&*$T4@{1gJ&%o zjmx^I>xKp<2MtHl0#7Fr?T6-)cX^rFSyYCW4ZbA<^cIdqoda(sw;iro`>gA_2p;{N z8V{do=I*?sTX%_X)w_RX0$v1oOy|J2HqpIbY3HEP{xwNkG0Rb77^Jf3k7H%stl4xM zvP33bh&h#tsk7b}Zz)-v>mD03wcJ&2(4WF?l}8{0UPntb_>nVMZ0}rc59QlKOG=@= zZ|0dI-?Ym2W`lR0ax$E%`A zEGv}(04G{-hQeLl0dt^ScaCNKoU8p7G#l#O4pgZ%2{19eXRVkIFtd(NHC}I~W&KRi z27XZ4-}@2_%KlRY4TH+;PS;!efpezqRpB8BF#pOY`K*dA8cVZwkWCOcf&k(=n*1N_ z%%+xuSYm_0d2evR#%MDz)DT1f1Qntilg7)rBBVxGOX)fa;9LqNr6P*VUGdm!8*$?*IYlK{)8%237dh7f1r5_qgsZHS1?Qy2)=hvMh#$@dL+6PK|uR6-ibY+|K# zktK%E%jmnNQ#Jdg)6np1?DQWI&gSw?n~vu99$AcC_b{8~qdrDPTC65wI>;}nbx8ke|o>_Mkyj>^*%+!`=s;;IC3Dm#@X z6A%;9Lvx;*^U%FK^mAx8W_S+lp(*HK{)pCCL55}#tk*D@D;?T4qYeEQL#s=)>f9O6 zXjXI-OZF3LqS7TT8BxCjEgMYq8hn>vj4}XP8d&8%grGcm_u-}X_q&$6R)YNn_u&=i z;XB*9b9`s5#954EMmtqiSDqa}PwH&w^`(Wb76S>FFkP9CE@W~t>SmCQJ4-7@+9y^!2FqY#FuJG|Y+jn6As0)k}ed^Xczg4dr z#>$>Nzh{LH6@8m;yPEP%N0-9yN0uWidyf>Fj;^?lLac4}sXX7k`1H+-Ilg;^Kk}=< zY{R#1l^k>5&3 z6Qv$Lu0*I5OH+7vQCZlK9!GFbL8dt-;q_Xg5D$JD+n&Yj1iBqEU@r(58_1>~4+ebh&=tlQUY#OD4Ks=Mx) zp|pmja$BI>7AUtJEVmuJYjavdciS!2-n+cny6tYG#k%t@Z?Wz!Wt_F;9<1T|%*NWx z)`4F%?y`otPYLGiVAwu_Jkow74O$M)fui8n-wGTo>K>A1x((VO3XMVdQ1$z+q6fY` zdayTSsCX9j*CYD2XW4HZ*h0D=dII62=c_pe^aDRa=b(xtR`Bt8&CGG!XFTD$e@^(H gkxf4%yZ)UVyl?w$6Z~-T`l#L zAxGn=6EY)Lpzn{}KJytrYcJ%xJ-?Cg^(?AaR}I zD2|R%aeADlF>Q*O#!brC9Jh>H%4ODZE0mdIwzz%Vu9R6~j(FX8UA%t0KJFZM(v*pM zisGzSD9*-PrdzbZD&Kd^YI@u?YYNyu#!P^go#8|xDTc*pG9ipX-Zqvb@o+3)mdz(d zpOc%Oh|fexG!l-f<_r<$Sf6(I3lm>Exv(;Nk}p~uafi8D>Y z41P?jF=yedoQ<<{4z7+fO*rc)Zu1;9L38z2T;mq5opWBHVb07IYTU-#IoB2Ho$^@Y z4#>L=`8vpZ4EcJ_3%#6N1H=sHgV@FSA$D_(5PP^y5PP{Mh#R;Lt{KMhah=N8Em!Ks z{ah>5G?q)+pk&h=^$I=Sbczae$<3$46vrpT(|kBG#gmhKBt;13pA48krf7;Hxb8|M zoB3F3@`pDt{QY6sCIAt`M7G2PF|IWD7@AWNPkCDSalj9#NZ;FGqp0uGz#DYNbAci| zL~N>rX_JuD;(aOU#^fJ{TiGi}%ED_@R&rwsKHlkt!(Vx3mUS;X>M zQWW%cY2}r#wiX(MG*oG6niAbw%Xxo>dZQca$_2o=)(jQWN6NS%r|cALkTF3{ADOdV z@@7n&y%lPXQfK#5)TIUyGfK$NnFh>Ex0MSF9}N@Ef%6ya&rn>QhQaG}#=B8pt{$PL zVa}JlR}WZVH(l}p7EUbvZ^gp*C9&`!7Gj4sJ;2_Xab=iynRm)-4m8twNCaA|A;VL`zk^nL2TsmYY7Pr7GWyS)*SXWTlpd(Yv%Z ziCB&CeaiSpHy+j5`VAGiFA&Eq2D$0R~3JK9PpH82i;@L2nOvOP+u`jQyewmF5 zEXU6fJ`xsrjtwU`_F^;^W6$v{0Revj%AyH2%ub}l6yezmJP|-q2(r(`_^`mUDfk&- za?v<|Ob8+$=7NfhOP{PEFoU5`G!Yd;A$Bq;vZ)0B$_yV-a3~`y46zrdqLC>bq3C2H zNqBCAeUhMzlWjA}STr*GF=o;a<2=_6#m19dD#o+PL~NE7QZqA2A`0xzqiph=z>^EA z$`kzfKOx9=z!>E@s67^Oc7~Vh6RCJ87M_I(%T6OfUFAX8lVf8eY+BEPvV_^PUZB-# zS*4aRh_*SFocz4|g~a(p@?wG&M4)+?aIgv&q6$hZjz7YF-VtO^O(jz?4i}?eRUZ1; zBw_mjK!0GX?6?>viD+U{kS&;!eMid+2`luXS|yujk~6Y(hC~yhY!*b&KF;&Af@}qH z@I-cKYaqL0$uP%rp)widhH@eVoS`Z>ay^hOG)a=F83E?NCjiJ)7@(kLAv@xVIv5fN zDz~y-P_-nvwVViHXIMVaEXx`s0S$+eo2u0VW>zstB2%&{k&qq7P~VD*v-%(+!82ZN z)o^?Ow%k;$3c$J($*91G1QBM6t6cY7n1tiJ2vwG3f`^Gg6+vkr+kwj1P_F-4IF{m{ zAS6j-hKm9bqvuknE)$f$FhgDpar{I$6%#`jV0s+vq!8>lAwjuLD3? z$yd0VY{BNX^T0a5K_0BoumE)BIwY#H+G?GKj8Yd0MPgw=2!#Y(2=>8n`STOoFDA)( zVI~~mw@;^7NIu z`+w5*c3W=fk$m@|1^2I<4XaE;$;++!HZ4qCKfieXFXDM$-zR3eVat89+2y|Ppz7T7 zcFECFaJ1d&?47d|9GgqsyYSn2*G6sLGH1E!DYb2$^ORb9vriXVhv$y1`kNMnrTvSq z6`Qw7&D-++p%wa;e`k(~=XS^Q5c_wQn9WNIrDjH9Yq$B6N89cy9mbhtv7yGk*#~cbuN`Kup8JiEQ6bhTk5}cIA9K3(T(Sn8Cox zqGHER$=R75xaAyNZEn4uSK(NOjC@~nVJNBS>)0^Uw}Te`F2(8 z`z23*&e8uM_QMv-zckP z2mT_-nORV3Qt|b(CL!oho43kczXOc~R20^%)PUp}$a@Bho}lCjuI$Qrf_cwFImbil zs|~;lrdW0DP-`g{89vo~TN#}^m&}P5Yd|$A zP`6j?;OTk;*m4s3qT0SLXm-~Zj@>Hiz8TJ39S^VoEk+AatoUh6Uude-~lSL!|K2wak2G=}6wg&RG;8 zt;^byuj%^0;y}^YC;9rar}MtS`6KHjtgrccU@@?C^sUhwqj_JT=-Vy%cCQ@G`}WTt z*-);2)ZUZbv+U2egOI=NXnl}p^$oiUczy|>xGVIv z2c*iO!sFoUrK<7nMxNVK6i8u32nX!4XA+DUK^329su6VYsZ!Z4d@IHCX`XOcq#P_Z zA&m0trYfHmmXYnrln4?azD6#~W;pfJn`lYX)_^=o^xJL(9+SecR{kAj69eR&ub}=7J+osz?+(lk=WH z&JoyPm6u`g{~xOi!gvCZ4Oqz}d@4&srZB7Q`)Nq6T>%#m85}JWJX}&l`1FZg20HF3&%4NYBbf!mr5 z`4Ek*cXhcA>YHVZAOk2m961ZAn-{5{^jRya zl_MhEUu(CoJMRA)`y*gjEdQoCN2#^rt*#qg%Poc0U314u;6HhG{@JA?*}5BJx0!8V z__Xc*$kTXjc7ArLB})pP!JK38!43X@lF-QC6+)8~W)%y{0x4b7Av2KMn0$uKwE~T+ zk*~GTy1P30_S7JsQ9DMSRnEsBL!y@9K!)J-FmgAv#ZXO8hL+V#cbxdTFkw1{C*+re z%Td@~H54uCse<-Ac@85q(|em{m8@;T(8m@sXlG zDEWgcmiJvhbrp9Umv$V_`H$zA<9F6p=lpwe%$^4sR@eN#2heL#y!t7i*#n9+s6tVK zwo5hYZ>Txqxjw5?7Kti1a=ZvP6>F(iIm)%8`&lFsuAk;1c@5I(p;|<)2J{~g#2$!% z&YG8GP;v||?=3iXRw%EzM}iU)Tu$9)_8R;mMh4P}dlf+9(*_Z~YM2KF_YF8LDk85I z8LKJEnpm$T)Hm*$qN9l z7Mqe`V7LvLOA&ijShX6xjHZT>CAR?v6>U_usU}~UUaDc>6e?!LTGh0%Y#^?=7OKso z)mr}o$y(-7JPQK1oPknXXR$3PwFQ@jeB181ef2hT0DS69Sz;M! zWCw7Nr|+d9R(MoDkU9Da@MkrOYW#Opf8?Sa1`9OliGu0?KMVBLl>!*094dX!f;s*T z*e{o?mu#FVV}(aQo8b9|Ib*H19`Kk-S0K=0=PcMmps)5pA4pg2V_Dnh>WHO=8a(aP z@T+_Q6VtoaiV6zcRmCk*wWsBQp6D7~7s&HKCHogr2_69GRI-)l?1VYnYs}fUZqD7x zoW1J?hip~M?Q1dT>_#UHP@swlF}>+Q*xD*9CsZ}6ZP;hlQF;8u)&22YNKS~Lz^Y;1 z0Vgeth8}<8YW^?H%wBoU#no%O=5;#Y3^%}57=4s-Yb+9 zF*d=A%9UfVz8>{B43ELp=7^rd61Z-M+gUCe5!nHZ10!&$2t_Pr;7&J8X0hONK~UE<(tx}H z1z%rAVQn8`6Iue~u&UuaJ=ClE8TI3Ym>06C&tez}?Ct zTYT-JZSM+01YF=(Ow5vQ03%zYA~+poJK<+y@YX z@q`EJ6w6w!ub@R1Z~&iX=v6A@ZmkS0a7}&TiGeE!YPqr%-6g#2FE1UsST(j$)yO(% zrS0yrGWmTds%?3$o7lfVW8sIOqyV4YZ>JcK=zbdEicWZ;t6K z?bwxLhD)6T#m=DA8T@7Ej>Y-~^TG=yKfF3vJX|_@EZ4X-EBxfuw_ja(_-7~c{f|nG zkCps`%Mo~{P&zl6YuvjMdH?)R&*vULll%JF{6pWo)fl>4M|JJITTi(fu8q!*7MTHw z8OQ|=eDGwB87MGD#S33DT2sb|+Od-g0o zvv6c-Q>nQp8@bgSD2+a!YuUDZjtJh6+rumB7iZBUVun;%)l)lPsCaB z5|(`(BfQ;HJ@zq7B``v7oT{wPU@C(VnuX*tMt_D8GN0@!Kktd~u~;Y+pjB2yP54pO z-a)P<7@-~|{2Zd+UjfHa^X1XIZmM^n)YC7wZ-ak8-BNQ~sdHhb|I|%edtgaz?oUj#b?ZGdq~OJlb*o-BpqCBkW&8B9eV^K# z*5IdYR_oxWZi{uxr;HsUpT)ZEQ>Gm4y~miX4}DgzWX;xtzpEd%j?CYI7?)@#r{D#`}Ys&K*YU{sI chb8LpeOuHF|AA!b+*{Eb(a$I)qs-~Q0M?NxHUIzs diff --git a/rsl_rl/modules/__pycache__/symmetry.cpython-312.pyc b/rsl_rl/modules/__pycache__/symmetry.cpython-312.pyc deleted file mode 100644 index 6e226f1fc0897692c143829e0695154b602bf4f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 642 zcma)4J!{-R5S^8bPq_~p9Iz8GWFeSDhManH1k$B);TQrgvMAa;ov^fPcBL~iS2&Qu zrK=>ROZ&H^vAUqjkSeJRhZ|R(J>7+*%fJk8M*C*oj`n-6*9NpNo4=E52jHho7KXVs zGF`{`6*zE`LPib<0q%T;1E+NTrC;`lPwGx6rOLxRQPMBY*4@5rf6?;Y<6bI zW?>q$Xc+I%edJ-HbdryqZ~{ z?;&+}aSbc5USc@t3Vf$-%{A99R*YR9rT2s2Sm{G^6h2XxD`l>h($ diff --git a/rsl_rl/networks/__pycache__/__init__.cpython-312.pyc b/rsl_rl/networks/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index d21790c6e23003ec56d4295b1f5c8aec34c8b701..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 404 zcmY+9yGjE=6o$|2ZZ<&@E3puZt%NPiT8mo6&IBuqB`}OT6CE;_<;(_29z>tPXR)zf z5DT%;N=awsY`mbS`1#I-|M1`SdTZe8diz!G6M(0t`KjO=mU|v=yy28#(CHNUb$r{IzMa`64teSfB_w?USirQ8;j z*0EmsW9Z-2p;jeE4fV`lCv5o{#$Ik==@mW>vnz{JSF%E6mov?bm7KaMtHilw$fdc+ z&?$;4Goo6ZW|TIuDeaf@x^wWA^9a|yTNBl7dU*>8A@5xvyU(!o02_}mzK1b({pu4a C5O`Gp diff --git a/rsl_rl/networks/__pycache__/memory.cpython-312.pyc b/rsl_rl/networks/__pycache__/memory.cpython-312.pyc deleted file mode 100644 index 1ea714a2756c828f305f9dd41b6d938ea7f6e3ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3639 zcmbVPU2Gf25#GJykraQdf5%az_##V+&rT@0DNx6}Ny1cfMpgVKPDxZ6OkkfIDS2R#^Q0@TfmQq^eMr_SE- zDA{I>!uPN{J3BKwyYtQL-e0285Q6W6?!Tw}5rqCq2j0iI1l~CZ#0rv;j8&Auaf~U= zs7!{9vkuLwz6=-VFk;YYB>OHSnN!#_*t`F`!MdDyV3860cj!RCc39Fh-IUCft{Ias zZnLzkB#S1I(n`YANlGyido9(B;GL7e{XV#+pg5LMoRM*ymHh=Yhh^q+Q`{#9WcD(S zbI|*s_sc<4Yar!XiY`OkPvW1)ax{`@n6rBB28a9u2eh^XLaKf1}31 zD9@z#>el$)yo~1$BSczEe{BM#gC4cpYt4J+9i;5eQEG`}P@c=xr2~qWkgk4REQx2Z4ORHA$Cv{Q(DRt zMVnLg1z<3mX0vc$wvV2c%@P<#Ju}Y93jy0_DC!((g7s~mxd?W`KoYtWI{*q|LN)9r z5jx2*MA4v*(g^eX@_&X0otS5(gff`4GLojJWYL(1kk0B7kyF~mL1L&PQ3pME3}zfG z#j=Ye3WjpcE~a9h=N-T_#-aIg8EpllR|2Krfz4po?QZ_(XMTF-XD_Xt+UP#^Ae^~hKLLu-;d)I3Ee^DMjT0(k0sq$>e@a-%QalU`6d_$-9aO+q&WUUteBfjIGQ z>hV@qhUgKxF@du~=OxurP7$IL(grL-7qXiy&A4PKN=_l|R4zbh)NIbds?B92W8QFP z?9i-aCXyl)48>-3%e3)c{P5%e)eAr&Spc-;_SLXZ8$SV)#+%SAqiSc@YG1K)pb+{b z+sDLe>eO0)xoxO0 zz8yp@?S;@bi=ICE%j7STYu0bxUVr=JOXa~A{`A$#_;hJ}x;${c6gdQn9he(13B~423+Cz1tR(-%j zgzDJ*hl|aIF@EQB!0a23fVtk?$|mE>{{ztz^^4WR$^VlFjq0Cw0pF|t2!0=LjCIHr z2<{RC_5XoPp;`Y>pBm^m*>!|wSL5{_VU>pQd(aMD*~vxUWTJLqQ8dspg`T6YAS&=J zglV}#7a%2;a2}`kN>XX~yd(bruV9R>?x&miHM@YdNrADM*bosw-k{*~ zvZv|fMMYEI%97Dstd>Epf_P8L#ZIbvLQ;*w`(3P?ZDWPmm?%4MC5Cp2qa59t}y6TG95L%Am&;_V4g zH$I1krhjuK+*1no6vINbxudXje~NDju~JMe&RnR>%x}!hmt$&KP_Iqg>KM2!JX;Zl zOTutP7%K^5Wnsc$fBLW*ar!2EbGmr`+kcL4%*2Z?#!JKBsSMAShG)ye@KV@%#cVgcK8zjIwFdfEJ>}+1VmBR6Qbxm@klQyk|VS^ z28|u9JtY&0s*0k3$z#AZC?vn`)M7&4#ZE&Xh(-@IUtUIEFg`ALH^OnfJNv?1`%YI2 z*Y>Zz2$#W*&yGAigu$md%d`miL}>i&ApCG7k}*vqa62M9;yy=XmYGrw_ch}`>ukcY zu#Q}7&buUd(%s%Dp(v#e8gwh$EXMe51mQ!Uq0Z0H(SQ1%#~EBW_rsYhGj|cC9UK1z Daz#t` diff --git a/rsl_rl/networks/__pycache__/mlp.cpython-312.pyc b/rsl_rl/networks/__pycache__/mlp.cpython-312.pyc deleted file mode 100644 index 941e41dafc40c267cda9e003453889e5acd296e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5308 zcmbtYYiu0V6~1@&y*uk{9<~zOz{GYUmiQrB=p)b&lLrZo9fYCKg)o`y%-UmSXO=s& zPQ1)YL5hSerAnec9Mq}=35n34DysS;Rr@19srqLvJ2V}EL`(aF{xWrMqwS4Eyy=Ttj&N<&Xvwu#flLW@kyZ@zc&Jgk$-UK7!6Lh5j#5_@nN)1w^IZBZh z3?V1*Yp58`h51^@h!mr_D5t|ltQgP5DG|t1M2(ywYE%nPZ3=`puUA4oolDLL+1TfJ zlciotF-^-+9NjYQF}RO1O)cj&pxT*cTgJ2|o2IPf9etWJ<(mQt3Hvv6WVCEJ`hcO#XiO?;EU%RuW|@P@qy*!+NlgMq$<&;a7JF6FZAp=A z?X|LI<~7MGNQQ1|3X}NbuxF}L(>JM5Ho;Lii9r;^1saO&_}sp%*5@T>QgI}!?3Bum z4L6#kYPJpsD5-i;Gi{J0l}$sl`DRnxIvP>9UDEP;LD$r}FNYne#H?u@gi4C4>S&o_ zNO{Yg)=V88vUmGl4@id#T>qr5s+!qUZ%dQPv<62otr%tQ|MBAk4^aS(txD=ZVwUoZH}%A4B$BsjpDt)N_dtH+%(R1xGdNM&3xZ>_mbc zld7~Db3^;dxp-cfAbH_f3i#u0Toqn@kPs%g)TD0wf^aq+Ce=hWK0&HUH#rsaCAZEF z+R4Wh&-CEVzq-_gNZ$rGR|J10uz`~G<+;%&hm#JZIU54XO$7UDxB?o3yZz+YF!+#K zU+O)qOt>ku(S<8W1?rnxe0E=$I2#+b3Qhx$0{>a|n^~v|+(IFc6ZlPu`I6HfXha-z z8$4C<|I#Jisq3EdIs!|YS579da^Wi3gRe0B133(YJs+z90 zxn!y<;Avjn^e>3p7HFm9Y-gCv(7Zq)+N<%CBpXw{hD14R0=y`OUeS1N0)*Cc5kYmB zO#sp$6$}mvGN9aUX@3Jm09_FS`JN9G0lNY?6xPZvY|>R&~zUI?}2`STTL9_Ee$s+WTT1Q0v|8+#WorF*c&VGx$w@swDdiR0f1}Ok5u*Jn z*0g5q^RHnr#HP;?HuH6q;&FuZW<+2ShItUlMUU{tz>6@&oY1_Gp_yJ%w{_EY6sWXbNH-mZ z{f?J-K9KJvG_zdP82IKTp+d_=OD!8(HsghD%_w+@Mw)oZW(s(zmgt7(8iC_&SQ8Ll zTXPd-dz;rV;39)pL%jADnKE<@&>PsA>P7t&@lyN>2dZkPytD%ipCN-qm|)zC2BC?Iek|x!6K{>r%Y)R+MxO zU9K!{+dbQM^Uw>mL;0oqzkT)S(*65uda)Mo`rDBg7Nr;eG;)3Xa{lVGzt-Q`c5QrN z``!;nYVrGO(u>PcvT4g~^k%&6TzW43(-ZUh1^rT@wrlV8jSIW>Eq3f*iXW(No;R1` zJO0rny}P-#WB20r$F3<0+n@NTYwzsHO>xt?(YeuI_Pm{aD_iUP{~e&357S#oB6gggdbf8ewet=+Jqvw5xK>&0eENns z0*>C-yU_dS)yiV`le43ruGPtTtBakZOX65lQ2$%~m%nvAxzstbBpv~0#RqDshw9t4 z&VeOy5af3CT{`$t=dQV@XZM|NzbR(U9i2OR(Y~l%(r$?TUxY|k|9}7eDB1oaYGY9U zCUdYayefUMS>X*-54@l^p@CAvvlJR|dPb}Y4rV(aZfgqxwV#)jP&MqpvvrQJQ*ZbQ znQCp(>ia>FvFX|EEu5IBYsM6`S!;(IoWg``1O*K5J=@XNH;`Qotui|I}T0v_rjpJ1+Kn`0>D!@OMow}?h z^yV9H01U%7$f0%*5WLxK?9NfA0m@C^3yl+DkvZz}hHae8QTSw{m6@kBhb!e4-Nlmi zy~4b|v+Uz4pv7+KFak;NB!`ZsL&%g3v2O6d3mdxacoF_RkQLYfu0{Rx@#}>TDPCei zb7a0=>BZya$R8Y41OFRL__=9shY9gUB)tQdfA~A~j|Z+ETk6?&{yP_YFFaNgkABzz z6T`+6K4!xcif)cUiGc50r_4+*QZOvV$tDd!v)0e&;d%QNn82yq zZPCOt^wuWQz2nkzzj|VB1aiOQ;|<#`?tHiBz3e;L8yosR&TN{GU5Nc9)4P!Ay>#%s z(RW6FpSa%phqhX#cPaB!eS5y`LfggZw_ktj^&6R8U&KiFE`B1b00I+<14JJXSl^+q zgfEJ!z>|*#2~wqJ#W2)<9$KRZEp`&hxm1E`5d^}$5`za0eoaDcwk_5S8HOJ?5HtKR z+kj?+FxiMajnj)k2BF4E3f0tarjQLI?KFYzq@#28z+W@%=arwt=Z2RyjKzlfeU#2UV!l%j2 zxOnG5!_HmR3cmCEfTi9^`9ZTq)f^>1DX+4lvK4i%w{sGnX9scOfe$Saw4GzwByOr@yFvX_+WJr&TREWcJ( zjQUPYmQ^b+%e)(5PoS7#oDSfG4WgH>_cD3SFl0GP*$}MS6sEm?AA^OF;e@rq&cXEM z8S;gYj>MNABwf;BZFakRT0L9 zZ~L72p@J_K`8!r(yj7*>de&4-Evp;-Q1|J2!X9Ur4pV;RS5SWwjzPFMuX3<;KJHh= z2`}uiMY!+J)Ma_+3rI?gUtAq2Rz69jGyLAuX8dm#$5rWa8Jc^W4dqsOaZqcN=Hh_qMJO({zyGoMQ{?Sp!(M;c(App2TH%bTuBlF-9dz0y`!d3u{PumlvBrq^c<*b(0%);6^dEL~~-Y)vHIsxt{l9D;B z8InSWB=|FOB%viWqIN(t7!5m{lqL$*jkO!JSHnR@K7+PoZ`A-v8Edx+Cm|}#Hp^tx zpEXI3Gzhcxa)Xs}3RN=1BoucL`ESGQ)5oyF`8=nV(m`3!&)^%UXL*dm9brta($G+OPis=K~ysw|BPs~pd zzpapa(Lw_5mdx{X8#EiZCyDOB5dl=W;`R-MkDa@4h6Ke$?m@KvYH_l-7;P;@TW4Dr zqPrHO?WJh@LbT(q*g;!h(06-Y`Z%zJ!=hWF5*Dpkd=HB9L;Asrip?9G5&HJxcnJ#o zh=s|*VpJ+crPdzf2DloODzJxLr?ikn2aB^mb-Af zkt@P~5l#ic15eSdJ2z^d6m2r*V|QusqXw=jIrkX?e7yjMSA1-P<2wBvFvq~Fy~r&G zx!SsGz0-YDeN&ree{l2A+@V|Md!zHCrJYZeqTP4J?njU9MH=0}d?QUZ_yUTf!lB=2 zu6*puqeotYJH@s5R1kK&?o)lJ)8i1$jPjL4Ip!_ObDmq9p$q#C)j=Jxm#cU$+a!2UaiR=RFF_PFo?Csm$@VgMtuadNM8Wu~6}u zm(X*sb=(&;b2tluhV5IygD}4eEoIRMxN~r}85`|h7>xNXzp7;{F|VK1FjlrgBZTCj z3CyG&4nk4i!EcHtRlsBFL(HX7ye`^|MLiUjAABy|iB;4AidzL|c#fLP>|!BiadZzh zhnP>6N38(5aZ7-iPzfSn!hb|1X;WeD+`x-MUkRTLxxsu+g*nBR4J_2|?t*E?Cs2Ta zirnUgtK*l)m#XWQs-jnWCVTEywa%(HQ*)_1y&v?w-}iCbGtl>-u6}yY)Sj8vsrHEw zIIydc%aLnCC9&m7WGUJ>({Q8tdUGkdb0Tnm^OouGRCuO!w)WP`ANJncd~`y%U)wm{ zJk>mNbhhi3daw5T6Tzh|+h^1psq3lImc0|`SlN2&OuW>%rzGxyMWHEtIecw!=H=PJ zn`h?Elv+CPG=F&hw}C%Keitb{d%Dy$P-+<{Z5b?ygIB`$8@3_F%$PR{*9!{`v5CNy z@T17`4FMJOtOBku*aRDaUdK8?fZm6_2&l!*Ub5efLWmpCZv4h_9SjSFP%}L2Ev&^} zO@}DZARA(@)87kA7`vgcueKJj_r%nR*}zS4PAoO-1#fe!_r1RPzEbqyUGX3jQOx(Z z*BjnF{+HsVXZvGOjQey5Lm}>q$g&*+16?i4=kiM0=?TfQnoYp@Tu z!HRjN*{d1M(uw*CKFcvx;Xc_}MHvQYJ5A8#u6rpYX*MIBQp{mxWRD@G;r#+H5l8?) zY39`ezUEJz`myumsI)?-G@xM`0ACf<=8(+m3ETp{g!HVwA0ARzRcN6Do-f!6H<-+) zB{S>ohOoyDE@E@Hq0?Cr9#mOipqPvc%Qi?JlLRCr*vm>fm)8;_@K%vtC6cPCFc?^! zEdYI9!x-OVVegamDH6@=2(=P1Of3V5hmSC}4-?WHC@__rw?*pp*e^BJ6S})i`=OWs z)Al~3;fDE@BoOP89pGTsCOb@T<`@r&kAUlp%ab0fj&00e=CHh(1#X`2j|C~}Ix{M! zah6CjIT_K!s*_i>=Gi?(&K zchM8ySQWje(g3Vupw7cn>AqOwK9v62)Ojm^C-LEv3r$DisZ?{nsr?qvU4X+iUxUN{ zgoQSW@aVzHIGpl?29up{HEV;hTBS&?;w&#~#Ii*d;=fAiN|hf0kfqz;vMY1)q8WF0 zZn!?w@TCF%0I$;WgSK0KxLX5}#`W(ER*PNV{Mwv|{0yvN z`~-@N96I^Kmk&RvYnU`OQmXQw*FrAZa`FM zJ3M)Kv8tt1)iTR}T-6G1VsQ5@*IO31?JL2re&5~deGlNF+-G|eKRpDDhR~rjy!$V3 zDIwYe4fKatJdMR+EKmV$FL4y9V^EZ%Kl(9>#gDK9gF!(5&wPR)ez7SaJn`@9Dq*V) z%{XWcG&{bVUwI6A*5!450LB3oGrEDePzZqGya{+-*@L}=m%CLdsU#rx;-FV~z=AYGFNq02g^)!C%KlJoWhZ3jxB^FCfcAMGJqdrb z4+Q8@Gz6s}ZW0(y;9(0q&ADoDi+wa0P% zF(_=y>z?esTeW>@Yx6|!TQ7ojLF_#`IXV$t_H#`;rbnknXLsFfpKC8|?YL!>wmyNQ zmLM_0#xN=N#{v}h@KrU-yLArQ>9rswW}fQwG<1PdZ)kA1+j-759L&)$^=CyIY6d34 z(99x&_<560*#1xygeH45+DS~rm@t_KRuCUjL?#yF(Ew0Z@FYu^+JPHdrM)=jqgEMO z;?s6a`3!v(I#3afBTy{+d7l4bGso}zl&k)PYj_y!;h&m0edEmaGY>f+9~Sv-{MI|Z b55n(1)8`x%jKhBcjBm&1 diff --git a/rsl_rl/runners/__pycache__/__init__.cpython-312.pyc b/rsl_rl/runners/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index f3c06566f6ecabce17643a3ac4ae4c48e2a5a7c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmZXQJxc>Y5QcYims|)*@T(Clt_^6e&@u?3AQoz{yk=N$H*v{+9D5ffcK!+N?X3JE z{(}P>#X>73ot3+1BBI0c^3Kl8!!Qr6Rs-3%UVG#B7@=pE%%d_h(=BUm5JfSfm{tUq zmAHbP_k<_?*e|^=sFd8CSy~S}bsda+O-hoU7lT76#=URn&5Sp=4%w*v}n~FGWv;@QVQe|_oOn<@W6-Ovmh)m(%kmEWi5dz>2O>K%kd(nk9 zqbwx}i_XT9D9tHkBa*XIO$_B~7@16f5z*`(5dc>_2k54gqTZ95js@E`Z3}Z}$X2bq adl=)7Cc?Y;1+71#P5Yi2`&WT!+Uge}8jA@4 diff --git a/rsl_rl/runners/__pycache__/amp_on_policy_runner.cpython-312.pyc b/rsl_rl/runners/__pycache__/amp_on_policy_runner.cpython-312.pyc deleted file mode 100644 index 7aa2f804b73557aaf559884c2fcce49f5a4e845d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30533 zcmeHwX>eP|nIKLQ1V92LK@z;dOC)&e;2|D5C{Y$AQa5!`2L$qgmq>tq0Cm7%Y^RbP zD&rYZlgxDpA8&Uni7%+}_$K+VIvC^d48XLd3*TPsJJ%GUha z@9T#LfYhTc$C+%ZcAqG8zwWQU?*6*_>*M?Sdw-Rlu2;bGXGQ-s9=M=T{2jhXk4E$; zpWO$~O$DQ1l#_}n<)BiDe$}LEP(@w!pqjdxK@D}agIemQ45omqo=lz64eDZP(gxEY zOf#vUG7K7Mn0C@Ql|GnG{V9_fQ>H=FlzGrRl{uI>l{J_(l|7hE<5DLrQ#pecE>RA~^12)UiBLq@Z=+G%>i;BsiVqPq-7A3BTl%$}p>Cw4=(|6eb1o zh}TJiQYbDdSetwdlY+UwnHfl#$WFjZJ|^{@7}b;tUyc+S(5_Px9j!^qJ25)02h|0L ze0gzf8YYd=Lz@_AY87LQx#{TM&v++BKWGtSJdHfXO_+i)U#w9m*j&inD#0h*iGqY* z^0_nCDJjJV`DRFv>%h^H#`0A$=5_H=IKXF;@cGIlA7Cz${0Ud0KBS4azOPD(mrnzf z%#2Ymi@F*J5pz~C*|8W4y6?n#+(fm6G2sI4c)v}ACY8q`g#u)qTZ?Tk)#666wviUGh!r zlza(y6vuzOZLTWpIR}L_=j5md_|VvtkRq}$A=5oG1w7d`?R8AMScl7fNl5oPf!mL} zeJ&RGn2;iJE+Lij|B+E44Lvb7o?!B$uUY=%-cSAC+c6*%I;W-`hjbOqM)^*v*;soa!5ppnq!Ku(OHqJ0m24;N9D_Aht=k#8bfG1t- zw4j^ux*X`Umka7)pie4L*?}ItNMq_xVO*EShh2giDQt;NmoqRaPT?f$7@eMBkvXy0 z&jj zvS}Eo@_B@GT4ibW7fb2&u`|OyN224X!7F5pdyzgqc6?~Y=VAmCl;1Hr=^1iPI-spD z3Py-^4ow1KJfmJA6@CGp6;duk?}I5n-RJQ+u#suc6Liw6P!L;*070EB&ZiE9r$i7^ zSTA6o6m$;SaU2ey)3}fJ#2GI%$+X97PZcy?*W?IWk9A6;lQNQ@?Zn#S7#Gw)7$}}= z#0i9VcwIiP7kP+HeEzk!>9U8t=$&>ByPC#krkrliICS=lux1(ZK>vUeHHpP&5+{Fe z6D79W5o>EPv~l_>786zkK8M3!@WLXZQHn#B;H`pQ_=X~yT{Ne8OBc6 zf{I`-DL}s|nv2nfXm&0_Jxx)hrSrNnqAOd`Rlr;MhP&1~R<3p%Dce4$S<#h5i_31G zx^*h(A;qm2Pyj%t?CWFK#`uhCl2IMWu+OQZR@=OP&KNaV7Bshww~WM48M3@vbgw8p zPV755dw@B)=7^#+7X%d7Gc7r{?RS8tG576nm8Wlpptg z*t>j=^qk=w7asOp2sYo{c4r%3vGZZYPHtp^A90frH|GhEky)&XEm~21ck7+4Tzwa* z*tV4UuwvIO=d$K!x=(c65hv*#LZHIOB{d-nZ|^4d?nuclF{rpHYBcl4N@9d!=5V>y z#8^W;4aC@p1;@Y*#8^!|^~Bf!p7{x4EPH0smFRzMRv6N+mCp@aKXdKOA2vPBgf>9J zS``*6pIuL~>qEyP*_&wl=oH0ed{GlAYI<07Al$<@_mbw`Fz6~ z4UK%-RSQ?S^2C=PNw{NPxkV=Al9hmzd{SvsM zazDQOpG{dOJawILT~b}X7PWox?aTx$>(@T(P**2yL1xo_Ps*YHJt>C)C|0|ciVDR9 z&iQK|Hn>jP8rKO=|2?T=#_vfvOh9Xy?GC%ew@D&3lLj!GrMHCpeFdX`TP=-P%0uN^ zttF8{@~z^#tx{;hoizu{0i_=TW;1>5QaFTVN*tj_F`shoZHUkE;mVIb6CIK-;YvBp zW+%}b_x5WZ39dx}hX{!r6%+9-SR#Q`V<>~>+}}ePEbH{w?nD~N2OC3;REOCdNGCVz z?3SPtZXjo!xkfI%3@?|;EV*Z4WiYFr&7IAgwFcDd;Qj(Uf!yC9P2N{e1Le!5v$tG5 zzi4*O59I#_x#cJIlAZ+hC0`(4qTg)6I&Jrg1e4b>`iI-*nzhqRiKtE1|c^^sD=Z_^2pehg!V}ib}rNx+1owV2pUEHd_)Xndp-s z60S7D*$aSQj!*R`U?tzgA<36;A+;^>Ciwy-*j|C$m@A*dVF^0n-cOfS0cW9oF+H~R zZ0W2mP#h?^qPilkmTdlX+!qY1Fm|2tA4_1Be86pT19<@}^dZ?&OUITVpnwBu*?O(* zdtW>%aCIw3Te+DV4?pE#2c*&`TsKGsX3GHnL?TS`y`h>{z4aGNM!*UO$Tl;NC4 z-_zV$3=#bSJ)@yMeZUYXranU;5B3B=?TIsqdPqLTd_Pkv*=#u+i=UHX5-wmb$2zYb zBNGX!LmX5Z857QUP}~!b8;%FXJyM)h5?BGG%9cd`yDtg!XQ)4p7Vts-ble!kazIPU z%_dB|VsVe(@Y|x_Y(+pPOPh*`m!d4x0!se{qhF>UdR6n9w~oGUl*<=-=zeKxH7?uZIy{6{wueUWYe~#ye8n6G&Pw@CPq=;U1nm!S%aalO7-DZP9tAoLp&M8cIyG+P7lZzRGc zA7ij)KE7NsOeFhgR}lwJEZug1eb6pzMb$(zJNVZlZyFNFWwwvu8(W-qv2%UxCAzY4`Bg;#l5rBs}TotsiIuYBu=SJbB!mz5w61LX+K z*!bue6$05+AUQyFggT(gFY%ZXqSYWx`6V)b)T5YkUYT^c0Zz;Ku7a!#0sZC2uDXZE zSdV+$kHY_HC(5coV&SDq0VoQw`PGe$jrWukJ~rf^?f`^(cU%M`q`IcZy`UW-s6kTl z3PkFhUWadHdeQ~?LinTuuuS_f^UI)MFeIe9UDLi9x68g;$U`w3WK3l(qbM?o30btD zb}&397@-s(x9~Y8$1l1>!AMptJCL%tM}1?Cp_vgu4In}mRMN3JlEv*m+l)V0V?i$p zRj^co1?5z0giT&5X%e(RN{?62q2LWfUS1YY2t?5n2w)^xTngxu3_E>7+Kk%^nkz0y zq(M0C2*?e6ES!0Qzy`#P>{c2&%|apBcJy0@Sdc6ZJH0+fg6QlH3I-J(bqN}jFM*`W zJ>ml0mSK>zL6NDDO;CeM1~fNFJq7LXq{rH;Noj5A m?ecAjkBW(+jQQ zbYErhphCz3S~KG)5Q~cp*)14ho@PD6F0YrOpqMkUtm7c}9FB<#y`mJV3Bgm{9yd}* zNO4{9fnX0Lhto5H7N`Z1qZA*gmw0`G<;B);cm>n2XKEUxZh+M{!-7OLWoDWI`4o%i z8|-n!o-*kfc20UlVWI}bxq=EBLo@C4jR~3w&$t`w4H7mMR-p=Z7p5^l&l?2^Fzwqo z;|Xb04vaEddo8;gL#VX%Mai63$f1%hrw@gHj0akL8oL&|2NR{nI*6cyP-qP(NE!%R zq0d3X2x$Np^JXF0P^>tdh|U0cpd@Ye&P)ke_`bS>3RVpBLdOSjV;lu067{fg^!|il8P-=<32%4V+5rGDmE+%XH&(5Uf>MSm!LtQ zO<5;M8QtKK#)pub{N}g}Qg{b6+7$aL5-A(KpmjA#!KrZFE9euQOwde$+*!~C1pid}*19_XvslLb9(P%$_I1*+KM zFsOWEUKS@1Z!P@9#G^0auzRsQT8h-4_oC#JM&KsSdrpU9&nG$~ud5`w$`##)&r=k} zvd6Ved~Fx0?TVT$H+463yt$5;>wa7jX5JfnFvhnYA+1Lun~y$MtLif5dan0g>s>Kc zKh>`lZt@xRB%?kw@a~y=XCfKxAnVi_ubZx!K=>BT%Dq{0qlV9FB3Vt5tW9&N(X9NN zwKr-P25z6ZbtaN!hY*8_HW`(dLQ)i&3j1mo+Tf@caZEI zSJhXCE6=eN#xKPf4{J$7sC_T;U} z(3a5UNJ-oL-Y1(|->Z30^IpS)hNaYx^&je&n}4?TldV77@yU+J&NGqCXNje0e)mFN zuqu@Mq^|MZ)_bkt%J=FX)J5vH#}jmZ(iz!#k~@8#KkX!^o!rm}KQu*#rXr`^!K#J4 z`Q1-Mi8qMENyW}c#jg4OXhqZA9d~wwo8Rkv&>5-NG2j2VVKesi#iB^VE1aeFNoCF5 z-aEZ~We2J3m_HD$Z4{eeFR9%dsol5SKx$8Mmg;ENApj0-BbIg;c<h? zzQ62Ve)SV}c@NoolpC1f2Yh6}$IV=d3|!{gu5jOY9r|@$w`#s;#ZvuQ*@nA?cM4&! zow|35Z`e*6wl8@<_J8PKDLXp9|4Cu#?X9=A@`am7;imso7CsWLB!z8@TS(#d`JQNA z;ms>Iu6)lQ?b`W4@B6)c*HO}Sl<#tpE?2b3CW_F*8dB5}zP#vrFv}e|&L25Vj-2L? zjFBT_Pt}TTBg$tA)wTj+ZC^+YYHpU{F}WudU;C& zu{4B+!)=RumRh+!CpSFG4__j~msTv7qwO8@J-nrwSgJ$m#M1g%tYg-Tjs430kHOz-EQR~7X@NqEgigu#6HZ;m@-A5Yt zk)nOK4nDzmpvc0(C)V1KlUSQLYi-oJF}$Bxw{zBwP}NF%Xw#i`zPyE$w=B+Zr& zJqxK(YZ-5?AwZOp#Z*x6_-W=+_mYL!U*YZhiG6>>-p|{Q68q7JeSo*0A@(y7`?(eC z`FJwU-nqD!*n56@WLdd104aF;L1I4`u^;B`Cy4z-#D0pmJBZy8u{&3+L($D!7BYgD zp?uNSHYj4KjaautYj-WRk=njx=4WG{jPVCA68NjV_;AY2O~1-by-MsXZ@)zBmm>Bn zy!{)*{*7k}MQyigAtP$7<*l2DbyK*J-@KP>-W%DxZ^gPl+O-Yp9L|IGj}!6-K*$Yz zQ3EMz2zeGqN#iR&eRa8P$wwOd`NqSf@o=Q^DBpOBG@gny4)Tpo(&&sd4zCn3ziG0z ztqX@jdr47qRP;rPnxA&76a{V1dsGT*(aozju73Bmc}=u@Ln!A?{k$n^Et=OtJ7wmG zQ(O%(*MzcH%niRjqEcklJUyXO&ztR5P&bgjwesNHH zt7vY2v~eTO5Oq&Lu_tgXz?rIXYUzHsvHOWQcm5#v`_>h6`IGFN1r2YlCGeLGJ?3F{ zQ&ePmKNz|_dTW$&wB~oa?sYA;E|o7?mX2}l`ywU#S1f%{#`K))2d^DmF%~^BWzG+- zm`Xz$zNUSprv0Y=ID*lo0B&tK`p53PFJ{fv6AmNO*&3CyPL>*>;93}}Zat|;ZSY6A46cUYCI0RxcwJC>uIrj3+}(bBW0JPF?f_N6rslTZ+r@<>puXbmvD26S<3r82Ry z)Rf9IqLO^Gy0`AaNn|1?n+z^p99qw$22w%x(E_bi{vxEDyboGQ4=qPk;U)PKoOPuH zQsJ~E?QIpPx~sphcw6;-4PyY=gc02Iw?zp>3Y@UmUOp#~%cNqzmCp}kFk+7CFUj#V zJ+0axQ#&|6T>HpMmCDvaCA7b04Za2`7Oq&xjXFa@b~I}Q2s8HQ3Eb$fdESY=!+3y_ zodsSpn>wri3iQfEioA_7ix=AGty)TjmuzD=^;AsAQEcOyBE##8d3}WvX9o-uctWz~ zktp%Ca9aXM?X~^%rTaOP1F6NUHD3oyE?^PmurFXq;yFqt4||0)E`h)FfK)-nDt&Pc z82J`qtP<~Im*gSj&PCZDeT6p4Pw2Ev*w!=@yh8m7lFR1$)gQiAUZr1}m+iMk)3Ptki%JfqelA-toQ~H#Oa;|&^?!w4Tlu9W zSxsSu+$t(*etUVoQT3%cucr5xTrJ;&GsDqL4W zn)E>W1RjI0dGK zCSbNCr}$Eut?D--hPN%V8BmIsnNNXa@%c1^X{IZOmbbrX<&g1KA*`cbp8KAAd2Yh) z4P-E_U#a%1IBbl5ZC?q)D!v(`pN?vaWI$=QekEM1MotWuxWYqR?OzGks$HqOO1xz{5@VO?Or|XKH#j0+``{R7x>mtyDOI;I z+ml0masASXU2AcGY<)*kJgh|4K9IM}o>SuLNt`5|+4*msD{uhuTCS4mM$U+u70fQW z--VSo#L`f3chgvakWwHd@&Sx5=M=z~yw?V$v!bdk?_<+I|qcH2Jfr z6Bnhy4L6rR=PEIGL0REf>Ggi&ZYn^fmkK+e**1>&b8T2GZtjfg}zDMN)+KkCv5PW2e6aS{y?kK;WzCAxuR%@04dqwS9$gR^a1eNT$9de zuM5hotLp8jI?z#d%&*xq(A~lQDYnM}{P_d;u~&-sQc;178fOqP@UACZlAHw5DJo2C z!x+?qXHgXqgx4;&*Tcq?1;#-x3(`ywzY8e?Gy+r{Tp)h-jyXYxVgysDK?8s)7=Vxy zx;{&V&w3{V_(6&dLTXe4VDYR6B%-1W6Nt^8#{ey~HQdZim)Zve&|qCcgP}qy$nKp! zA4`XTfD&CFNY%%OK|m{ZW%k>MB4rpXU$|Iy5?^$1r*#BLC8Uf_q06FvmE9!DzcnCN z2PyC6aR%h*>S-rjd`)y0N)6CAkSHD#w1P+*@5#DH1@%SORY8vuaFIl)^+5&Vf)?#X zxC9+mAIH3)hJh}qAst9(;p(i2z-xk{tdYP{X)6`_`*UPut)9_eLK^RPVHyERYe5NL zu#psO1fkK^2V40qJp}#=dL&6EDHxnHQE_a+LtVjwhv*s}!Hq-^o2Gv!eZj(&G?9Er zyJ#d^_K^HNM7QUU(%~q2ZTMUUidypK(m;edua6iCLBzALpX61Cl#lXi=hY~IJbUYG zs60II-kAqyR*Jjl_0iIb;Qp1;CXlD)6$GnEo*izcJ}#^Zj)sQsO#XP!d;JgkR|>ly zSJ&SS+zEtFE*^VuHheHry&EJx>RZOgc?An?!7aDC_`(KK*bvEU43|C3+Y+sA40-Pa z`08z>dfTEiQoS<<4&ti(JNcpgi>XWM_l+Nz-Z#PZQIKs$whnN0c#3^vKV0s#ZG4`k z0I90}mOj`Ku{O?YpJeAQRNUymv-7?ieY~ZXfaq*XSR3{&_B`;fSawA%#i8twHgqI7 z9#)1c!a1SKTvc26)y14e@1kL8GgsD2EWLlS@5w3*9gAvG*SUCnsb|^pAso6NLNw{& z;qdXrGHyfH;t@#fTHMQ3^?wY+=j40re_S9Y9Oj{nKNr&S7T-P0Plf_ii7jjcC# z+}OcqH<0WGK6^9C-WX~uENrV z*GNGfU$BW3Z2IehE!XzX?ViuU6ZpPseY~-p7|VmlLFQQ%+!n4{v@A037d$fdz=?Zd zGjy2ihHD01R|5Bcxza7rs;3^D;#>MiOW*PU2%kS0T+y9^JkrvW;aV4UAEdvZ{$EYY zmSyIXf)(As3m{G5>@c&~{9s~5*Nwd}DVLV8kLx->+75Du2e{*>$zjOj%o-Rd$znZe z*&EN`IY^I9bH4dLVzNaHWx?)@Am(wA9W85Hw?Ua z12J#ls&{>~|Kmd+9^wX0ai?8mU?kEr%8gHPZkCLDSIoYz0UZp81bgr7gR^`~FKOvr zu3YwgQoCY4{v?_FTt(|o^kPq4K0|s>N4BEo0q1m)vm-0U(N#^wRc!xA|FP*q6Q#}t za@ZN!Im9tzP-McmSB#!D&^te>{kZAFCTJ_}(NV)RE%*}SQg zm`a0f4^6de>bm2jf{#l-ECsPF2igNih9cd=oNJtSvBbr$n7yl@qn73$PrT=O;8|jp zJIO02Bdw>n!Snp!C>b0Db&kxer#eMuLueped2cY(zo=ZST=bKU{akzBayyrCd`|tD zA!AMZJ;|{ywA?r|e;}9@&9UAbyfGNm2ECD-I?hrb>WSu51iOR7!Q;U;&QcfJ6gn2p z4(;QhWA;QXRiCSs2b9&(>e{DjWxujMTGzIySv*U&?qA-*wH;daMROWL&agT(6FSLR zT0m@l<-OSlv;X<^B`??7x4b!;QxnR>F%e4TEE~hhAJ@Lu^q^^}lG}TR>^d9S47wan zZqpDq#5^@^3GtmG8`bAdk9QCuaYX5y7u8*-Vf7rsE6CM zpVaq}JP_n_$4-7SLGn()fXW$EKEp2ZOq)`k@l=sgW_oH=m@=Q{=+iBqtBq;;IsNkj zMOwz(#3NlkyAKv8tIQd|R!tPb9)*q8dO}!P5iF)$^Tb6AvX+))FL``17MaaB9IGjR zpqW*#GKR3G9Qewpq%AYcz}8v@Le5+Qd+_C9C%&lcYDSH#>UFTm2zS2(n+CD@GNo$B zUD<=nWpN`9^zwp?u$Iw+V6quXl`CONxZY|Am$QB)BZx&?XL3g+63uZtkN@l`2f;#} zS=6YeN|vGuq{TB0Ykl0zz)~AF5Mn6qB3RgApvtPahsXd`}6}qjp zXC!Z6lJKX+Q&PQU_Rlc|Zr4E@ITi!1AoL#64lV9)v;^n7KLhTvOfn1 z*zkbfFGH*1MMVePIf&bZH{i{m`{H&16Py9nckkZ!Dev(CA;d?H-@pNrGN8V%lEeq7&>^P=BFkmcB+Lmnpb>5DNe571 z!~%}C#77=87{kEBD0tZI@({8d1uv2$*IGoOkb|9&FDF$Jk;(}pTg84Opk)K=;FaSo z@_4BgzXaajas6c!S=OmPRaQxq6Y8#q))NmX6^?k_AS?}!BabQU`h%~)%?+b?=-_pD z+>=)Yor96W zGn~%)nITJ_Yru906nn4tU+d?MHDI|ZRLHgL0p-ONPp#%$U?&K@)FjVly6qSut~;1(SYuV+z?HTMJ72g#m3kaonl`W zlgvWqnBt!DAei1uL))d|Rhjs)n|L^vj`v~Ww`pL8!z&^5n9qy(#=3=gCnydUGmGZm z@ZxNPcZb%(y@CA(r#3K$20Q+YYZ)s#Yt)d*8H$moWAh)nc8E7(120@!F>Zjp>Gdnu zu3Ufp+UvZjo|x)6UA@>YarO<^keZaWZ-vm`*0PdP91yR4WLf?fEY859NP=m|E)ALZ zLW=Ez{<1bhc$bsau-La=684Lhd(z_Rp6VcFQNT#zJwwn^_CW7OB{&aUcEi5m#U_X? zIWB`*+y%2u81jphY>nt-SBl*M!o*{y_=S;|;uY9g9vN*g z_1jxOKlJ~14~_j8Bz>)RFsK|BS;)E z!Sj!F%`f(UoLDG(jKc3tMPlWfWYh&&=$eNC^$=+dDosOhc|1-JU_ZEKSo1JyRD1)t zI7WbznUK7>46LaWs?78)j*rnoTzo>js(tA)r+_`^SQ{smInJJ68jnsEPrQ6kC*zum z!z+a9WkciX60uC$570_*ypE7oW{nD>cUhdMBuW*kHY60SL0MN82PhO6*TVvey>TVy zZ{jZdrOSWuEcEu|+|fd{Olrs-jf}}q02Lk9jYfyU;6+~8QMxc9HwxsJ=&i6TO?5Fg z(MB@8I#03L=mq*}zpA>?Z|q0>Qb=vJZQN@cWnFmj+2$JqJqs|x4_`k(`M1YL^S7aR z8Zu1ZOAQmpk_sC4@bDy*K6S`Bd=al+YcLm930xE8yeL&>--I6!0K-Lo_a*kfV8~Vs zp_1=JHN}hQ>2U+j)X;Dp0_snoJa-Svf*hO*#qq*!+*ooJy(}%=7pI>{5Hv_7II@8? zu}Fj$CQ8&nms>{5nUa1RXMn$epEndBCnzodVfkJA9Xp6rDkA0EBPBb~wrpXvq?|9= zL`pV=yFm~^gWy2LY~fAi4^8FK@(OzSG+I%Kt}R+tg>DI0$z31iW5{ar-S#`}p%dX_ zk&5<6X$Rsh0&xLc-g&KaVNbA9yt~Gg zwQ>b*4-IXPvx=fw1q;o$J8yL^G;q}*h&#uho+PIyxl>czl!r44S7H$j3XomRnuafhxa_3l%^>OBzBrM|bQYkT%2Albc zR#MUW(9lY;lm<1tt(n-GL7>64IJgT8f5A&GcsYlU^IhW1mlIf?>*2VY*X0vkKCi1M zy87^mrAmI+39{=%{Nf>OT2^$IKNAnP#0h(W5gX@`Zu5)#%=m=uhDkJe!Ui@Fo7|wc z#2cH@Pi{bxy!W47xfXj2@&YlRBeAM2m%#ug$UK8N2m(nF%+SYOyWW+6Sk1n=2 zhOwl@!%Tk_EF)kxgUAyDsNiVDu zdm&*+2d8<)1y)z!q}0VU+75%Jm)B*3iPClf4x~IRe&}E|?hEuAJQu{zR>($s5x)<= z&4U%VAe!#KU<0Es@NuJ=n8F+?uDP5{qmZ{wK45r+e$(g0ECl`Vq!a9GxWO0@Jt1RL zfr$w!cG8~5;yR7Rc4ceu2iu3mvwlGh3l2dCCP?tR2VkuQeegx5SmoiE_@o%c)?$)> zp-HB}xQ794?1+kS-h`nZapS!aLGk((waOzF!${X3_(>)w7c z9IGI$jbKa;qu&y7&5i+KIUX4;$%0qX@n&V%pA`UWsns^}*QtW8B^gWV=%W?;z$*C}K%@u<~xx zohETH6P}Ee?~WArfO(C-=>#E_Y2I?vdc(R<50{lCgB;t4ecR`1RhK^6y$2O*sQ-WH zjY*#!&??e$U{#TmzhJptbgPKZx08H(s6LY47O`xdOZyZ)AW;q zqb5LHedJI*C!oR1S=aIbxJY z<;5tEs#^_F==_SNqRM1xV9>=%;M~?|jvdH2qVv!kj&)fAN3?oOWLM#t$GyW_FubdjA+= zO{za*w#G z^i@dUOuDr8`lj+K{b$ec7axXmNU$;Ir86d)3WGVyCK{T9kJ+KcF|-~g7(~@(`mG#M z1p<2-7__q-6b;rLo0*z&vR6;i?*RQ0hmBoL?V<_#k;xhF7<`du*elMAr^J+5C(+{F zs&s%aeRY6Jw5?VS6N!IcKP`(5Et%0%h7r$a9`kr$1yLQdpjTaQtEM*fs__J& zI;s5CjrJ@NEiSf2wF^3B!xtMxRSdMpbs1`1Abt|4amF`3>1~W5cF;H0hBH6a;ue<| z;)GA<|5`vRSXZT`C5ff1z?6B^h+hnIQsLN2EChUaQXy%6U`8yap;2p8J_aExzx~DI z<8^EYJ_9t~cK88d3>bIVIMG`?O^9V)F-G?{{$nC#Y2^hMZ+q zN?h}r?-^FIU9g%r`CAdB;w-HhL_!u2bBW=rot3Ed1>g>wi-hXW7U zVlk->2jdxbIM`mqb02?>9|91jq&)j`%nKVH@- z64J}!5e}=vA6#gNn*zEEFedW@@q2H$kf5J#qhEfbAz4^f#+`(fk%qK-zrW< U6eoY3;!Rb$mA_I@hi3490I-fRg8%>k diff --git a/rsl_rl/runners/__pycache__/distillation_runner.cpython-312.pyc b/rsl_rl/runners/__pycache__/distillation_runner.cpython-312.pyc deleted file mode 100644 index bd267023cfe7206973cd86587e4b5b8bf313d711..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8627 zcmd5hZEPDycDwxk5-Ew4M9KP~WGnJoU-6d|#j<0^a%{(m9DhWy>9aR0?usH!k<9GU zA1c+$6)7O0MlB=7Ny4R&3yexF5$gl`gY!dIA!lNc%^*_;A4W zPv7ivxUxeX?h3T%+Ty%<^JeC~H#2YEd-Lmx3Ks?6PaFRu_Me+5>T6sfpH&Cw?|%xw z9K};SouDS@Fim6FlrV)&T4@fOwbBx{Kxs}`Cv0I`k;Wdj1J06gOgO_%4QEZbCfs4S z2HO&ziHdN=gg5Nf>h^?hqB2~m!Hz`LM0L1&!XNh2l!-b_@y>TB-X&DtHtT$a16NIq z`&(RLXw}UnlPQUlVyUD!40S6nyp<6E@C?O7DV9iRO;ERu2$4g{ON>eNo{}=Wkd#gf zTx488YTc9dlR_ke5VS)l@A2eosYEPt^<*ZQ6cBW-MuM11ToTySsK`c9Dw7t~N>NH7 zfsLeiffa!lA)<4t{}$P%zwd$i93@a;nx{a8G^o$U+j;XlR#2dWck&j1tWFS?x4vTu z+pw5w^?EUME+&s2Pj;s@wn9kbD>RluAqjD@WGopCaY;TTT;dWLP4rM|EF|fYc8j{; zCDNYZNO7?>Rxu3R0;eX-Y1WjbSX~DA%>u{uxTIK`AOUA8;-H8AWi3Ealo_aLs98#F zRNPHqqr6=NX&5_hpi2;MF3w>is14{WMLPRBItPJ`GG1lC1R^|4StUQgWvv^`W5x6E zwkZ9foovdDCIl3vP!lhV5P|bLT0G!0+SS*@>GdTfc#Mo#$gRsyD%c@pJ1qmNt z%z$P$5VnPSH&H^Fh{PI_m;qG1F_GoLoS@ifMiO|{2LrOvL~4{vu!u`u1a@O0H<}RG zL@Fw(cK8K)q}ncnI>5qINGXYxViSU<3e`cXYD3Y~unB=f`0%orUgJu&ArVGTs1BAL zi^0$=FdhSifPu+~EE3Wwk+G>3QAms-yf0LzW&yB1P#YE&?^4waf{OUygtYIEhkGxl z&_yxLMTFk*%mkNA#dub{2u5o(#UUP8=+)=#)lG}oiv=fkl+}?v>8qG6FgFs*PHy_4 ziRvMZcr%L@*ywLl1%KnT<(i{V7h2jR*L4HzDKuxjJT$b$p&u2=C~_0T8S0)(DIZG*;jQ?tfHAy^O9 zio&)|Tu@DoY2ZY+gy+*fo+j4kZ|JNEy&;t>P~1akj1u@NSr-EvFIg1>x6XQ& z?-MX+2}C&g&~&>fUHSRgk}WUa9sffbE6J=F!OWJH>Z zQd1VGg;4P34=ts>)(!Bsb$Skwo<_NWT=+CU^+}rhhLQ0tPc=ccyks9*$!#@d|9`>< z+&}SWe#k%CDKP}g+kmFb4fHdLxBsJ=Xne!7(M`rNN@E3KWbBkZ^-Vdm&MZ9%*eRE^ zoiqZ@MfT?qHETY90ovWtPSPIVK_H_fGfjEc>3f&aP9Q?Jo1&?8Gg9$BLSd9dV&H*3 z{|WF=0aBE+3Ij$T0%c43n({)ga%XxEp*6~^cZ1VYuK%AnQH1j!fxVhCPx+=Qr>e5% z4aWWx<79pRgFcmic^}}#v%bj+(6l}4;9VNz$X4?1i$h>D z4w9LSlJ|UALFSvP&RXM#j5Y$nnALa|2<@o9HSz8-&HIlZChbN^c69uxfg@0zM@0Jr&6Mr#x7TJVk8E zh8FBqLpuZ_TAHf)i)XIQy1{cV-IQPj%IwgeNPr(Evl*p04)_yBoj}ql0uj#Etsl%6 zKTX<=auPfg*o_cnpuIrxhS5SG=?sD5VFDRt{Ja4Z2*xQR4e!mm;kNL>ZBabQ>$tPT zjS!a%v+j-uZ8J{ z`?WClB^cSkiL8`KCj^)SGtNSfw1g+R47r_A)t(g6QYI-dyVW{24Ur%RkuHo9#Q~%q za-wEDAa>AIuP!HB46aqz2&9IDLkOjiYK6pAWL)(CUl969Y$A41(1Xe9B6|>-CZp0g zJDM3&%|N17gV^koji?dNV}h};3RIUKhho+dz6w?Aq<~VQ>cGhW2+&2Z^e zHH~0c6vA*|6v3KAI8kB^#Ubnc-0gnGbD) zY{nR;mjA#JJyVacU#r$(z5Fl%AgVQ5fXzub$H1pRcWJyDs0(*;Kn1h_^E;7n8CC}r zqB^0u2znv207-(&5Yw|_W3R6r{ty+spvyg6yXgLzq zt`7!_$ET8lsOb)IknBhbm)RH(dVnlwTJ=RziG-l_(J~&uQwkP>kHO-&G$D1g7w0mx zyaraEh|YmfBHliFo}$bz!mhk()y9|%mdQD2EdzL*H(}eoO|3dSieroH*s|hyW))Hc zS7xqU|MAR^6<_y5U-zTx#$40Ie03t{NqlFcJgtwm^(xy2r+0ej`jMFZ;<<%$%Yl!Aw}J}uyv#iRtM*SjKk3Ykgq0Ca9^qD+Msr#! z#8aubn;*KHA9<>BfxbJE(tk|uKbGrz<-x&R^=m7h69rEt7DDkbvWHpmbV5p_wn3@c zCD-hlb`|_hxlq6C?+13P>*ifcN0jRB=@%abf^$c29DVoL^r3I6YL%*XxvKs1s*XqX z&GVAdvRiK1t)ov%ci-*4Z~xTwiR;0x z-#qu(bH91vvlsIF&*gXgNDlPQ9-OaRYG1Bh?d5-R!|tVDHF{BTArK4sNG>i&WI0hxJ8VP2M*m-Eak3Ug9sPUe}@3Uf|o&gGf&E5Qq;ZaHSpy~8py^ve?u z==-Okhr$fY%y6E0O<_i4W+cy?Q5aTc*gV6n1V;-ycFud3E(1%2zJB0pxj!G=xwf5Z z=zqM63O3GNy>a#3znZla+MZdix!E=AD+C*7t$)}M4>Hq7 zzwy`24cr)bchB@tp}uL!y3jcNa-rwh=^@w{_~>M3vN>PJ9gFhpfrrl?Sk+I+d$m6c zu2i|t za18cbMa}i$nc)>rG0{>A%csAt{+vMst7+)A)p4t6i|p9gpT&P=uTuDwN5{+_ifGzEKo zaAq*)-n=}dbnSiEwHNRHqccbUVRSBfBRZG3kyzUG!QjooJAL=ID9@jjpFf)$2nRqE(W>CaS zX(IzSW{zQ3unafv(|=FLA%uMT1L`uNe~CNu6H=B*68@P|T0lxm);vb@I2BiB#u(N> znepGJd3c11mmW3@{$n(jY03)WZJ8$-9zXb$(^~1F!x#xD%RJO*Yf^56GJ22Fw~MO{ zp`Y~>#n>j>55g;^_5u--#s%Ve>d$zkCy5@_6-jV#rzXLfMSfsOce^S|Zynl`2!4_% zo21omv$Hmw* z|3&vlYKm`*#LZ+i9#!8{Xzh_mZ_`}Vbx#x8{f=s_-_&fsGgRZ~A7SZrKqRO%VIo>}h7*Z1cG z&rLhOXlyTdHbKWGS7FQ6502eDwx+>5`#&DMJE-H2pHhx<@^S8WxMkW5UV{@{lc4Mf z7Ca55i?~VlG(ko2w8@?}V7O3Gb^XlD8Kt6Cu4v79+8(=YPS;u`Rq3C0pGQOequ2i0WU@LQJL;@{y(gw;YVgcs8DI*yKwv~E zgwp`%75wu8{HSij*A%r#19f0@mHvR$Q#vuCCY0(b;vp!|RHb^02wgl8VPlQ(=`FPs z-@uR)BRxg(3>+&_do$D>eNDs}+!rs3?xs2h^+ETc8-T-BU&JxQnwh5Q-+L%}^VgK~ rOUnHv<@t(g`kJcvirNOiZ>bl*qL|-Odmq~_m}vie^Y1BKYUBPl>b*{O diff --git a/rsl_rl/runners/__pycache__/on_policy_runner.cpython-312.pyc b/rsl_rl/runners/__pycache__/on_policy_runner.cpython-312.pyc deleted file mode 100644 index dada129324c82cbb67d8f206cadc2125f72f5727..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25885 zcmeHvYj7J!dLW)82!H@cfF$?^U!uULBt9gOlw?}e`z=u~N)!ZQKoSrL&;w8p7>vEC zy(%iFL{yTCs9XC4U2RHG>^ni1tus~GD${$;nZ8ur&InWnnblUb71yQqe#vX)wklQk z_`V(t0764s-c7dl>TU?m^z-ZPue-nQ{=TojzbY)$%Ha8vs{ih}e_ST}JM@T;N^r=( zih|>|jFM4upKMA#BA26E;Zuw#$X7X{Bwy8tihR{0YVyq+$%C)bmp`Q$(WLSej1)ka z%BP*ujp#_2+NYl?94RF3JfC68IAWYKjhLp2MvA72M~X>!zRx^m8L>>2jFiY_3fUeR zrTMOmD)>Ml)NrKqn!>LAIeP4JUgz}t10iR~6YvKQgI`U#-Vv}S_^Nb@R5H-gX6Z$EgO+jGC-^x zkZPn%l=8c3Kx+|IOsT-7rp%OuQiCgxDxpfLJaFYxWt5f52bTuIG~bm2iVMCg8_`l) zcAB0ro+3A_}iyRfQ=?0 zw($UM3(-!G-{YUKIsKH)ec9=oAsDs=#%)f-V!Ml=(FO>a3{b(2!!C7JGtmt#m49^u zy6mG>F5Zwwj6vACKm%COpyvpbuz@OAVmsa;FE(dHAFG5S10Dpd_PDz_ssR85ab+UB%r~MI%LDxD z#c)VlA-Xf)t8&^%a9@=P6J1onrOjA|!zhMCl;AUoJ__T~_@oNJSDcPRUm6!A&LYNz zT11z(O>||x6F6$pt-2<&YY+1UPTxcTM(^YlpC=4_zQ{i_1(Vf19dt~)X@}c?nJ)}F zfed*3AvX=9pU)G!Q$N|u?4urt*dCuu+x1XYM^b6U8A$rCY za%38%1SfCs1RazI;yhzBAveVv0kC7j7Z`K;9JJGaiPuA{bIj*<_yQ9_J|EtYp62te z0A}1YUl*5^dpZ!b=kuzd+c!?*5aA05nL)hJP1xv43{*mcp?dCdCt$=8bccdLoQF2y`NyHo zD*^gaaN6l|cTUbsIsE|-bi^efv||A$O+k%1g=%yPyqYDO!Xp9 zR#3=k>R3(Pilza4_3Q3i?^&6aZEW54In|1$CRtr~_spF$(EwZBivbmo$Y{Pfd3};I zG_!{0guyR8;W>R$XIW6))!)&xy2hC0Vbz1GxQDgB!PpNb^;S+_%j#>Rx)puf zv%0SM&PB_|*2mVRHg@X~=GbVW?!uhvrsle4zKYeiVtz!d)e;2%Y$t9A*@$9C6y;l(!_K!nMm+Hl#4qK>Ab6!BPrNvQkMlg<&;`Fpp-Pp z%gT|$)hv)pZ<^p|#iyKd*@SFX6-pC3O7WpmY{{&8jqtoIJXcGx&}@Q)(F@Bmb6j#TT%}_;JRMKX68$G zg>Wq(4~5&Bb)qlx{ho}Se8u?; zbx@4{CDfs4jq%!($s@WVX?)EVLoO+L1sk;7CsMoyrIO13HPlP2`FWt`v&vcXtYx+& zqFe*^*T9LGe}z1jZ=VP1XC@R^YFus2#S*MY%7{5)iIjYhTCLU`leSm$(xq$kV&AK2 z{cv2I#q_vi&w1We%*$^SQu&b*I9!xORNfsq3BVgEq4IHOBld?}=C@KBlEboQ4swXd zoZ#xQG0j3ng(E&VM~GkAo~{zN;^&tnBHBlrW9wOchA?v?OQuetO@ja zo@7llzxkR_T5LZK5^zEJ=z9Hr3o7OnH#WUfAk`Y^(cd^;-_c6eME{$w31Ac2pTaAY zk1oK~vP$-vtARCQjhMo~g_|vpSZ7UZ^vQu&M_Yz|6CN3iYH>wclhz!16^c04XDhyS zeJdkII83K^pwcwxJ0S=^{A#(uNeDmoYIx{`7$(*%Yc@f7-eEC5^9{q=1GHg192mDj ziqn}G(G@x)y1XNzEA#b^X569+;9ytgJXB%C2zw0!>@`yR+pOBDycwQWI-l4I=o4e4 z5bFqYNp+(F;7eeI#<=w%r%Q1+Qf6{*AdNY#*R zv>HZUHRKo<(`CM1PsS~}BGs7|->6l=-si@%2$yJOa4&@a^Y)VDUT&?P zyC%cG2H8egP<}-rmz{;3ms|#G=l8EF&&saI?Iz?St0p}YlRrnqA`^MadDZ9k1It%U zg{}dQ2sstu`V-guu1PxJ_k@vEI_;#Lzz=oPLBc{sHj6E+?C9v=^WD>)ATWe^CGc5( zjt4rzDB2B>HO`BPokabP+dMFY2A0da&bIC0*Sc+2x1GmOM5t?+2&5ZL(NWvFGgB+_PY251Y z3V=n}Fj|2(gO#FjQ^TvnZaNU;HOMOi=2ehakNX175M76nN??c4xSQwI(=-%7H=x@x zMhBdf%NYzgGRUEETh6PYvJ-Azg*+zUR{6)>z<_rF8w|_C4Ir-sMl&#rfGfwFrs;so z9SmZ*gips8VHqBO(BlHW(iPzQ1O*OKE#}Pk2mDxBKF@tM1l%%U$4$@hYN$AHe)2-V z><)%_qbo2q4ZJbPADW^4K|XJ0ngT8o-3TCP+!w zDsRB!=hZIcgVD%EpppN`>#%Qthe$@rIt(cwoJ3@Z*=uOr;PYjO!dLlcLB53Wq?{pS z(NO_F^E7rGjpqzr10FyVlwSZXhK>Y`@dXf_avFd!m%@c3I5Wi;qknY|^IAZ%m{se^ zxQ!&OfJl;=1YiQdED6v8tCJ29UZ21WBwRv;C&a5s3@>*8ztbf!Df5MlFhZz+-wl|B zZlIk(8fiHW=qnBn1!x38$uxF@&*vr?3BM7V6o8JVJka@Cf+Hc=fSg3aRYdFuX{5bD z9I+|B;2Ix#BbKQqfWnp>Rv!u7hJgN5jeNf68a=0JWHpT|nss01$@F#4S~|Iw0k&lz zX|mkb+|qESR@T(|gN8Wu(d45^uJ;(*dn~c(_)Ddt)i5`7bMN)NEBfXa+FWiPXJ}&$ zZL#5p=N_C(7`6g)PouwSylxa&Y?jhF%`;1R)VyM8nk&e1mDk*@yi*w+zJKQ48Lp|9 zZR-7@`{PTGFEPhYamPp5Dls_XBbyK@ehgVqOD&fdq``#x>@to4&t=F|vx z%E_K`u2hdPgvqEk-Q040%guq`8+cY;6LqbWx6JL&a@924y>#bNtnuOc2kW_(ezv9m z)1J@zKj~*qpW{xCv8TsYYFu+ip1oc&;2r8soUZ1HuI8D(h_Up>L)_*A?B)YZ@4@Ar zO!3hb{jnre2yn*f?X2FuqHl#fr4?LBH(Sy@r%jrx8QW&oycw`lT(+Q%?&XTx=iYo~ zvEJTyYu_IonA`P5Q7Ko{#1=I@Eoz2lEQGk)9=5iJEAO2jd|p$3ck<5UUEdvFtSeE| zIlt%mrrwX%KU)7$`=j=y{Li$XXqUTxyyf#PKi={Aj>H@15}VGmmd^Q|3uV!!Sn2cD zj)%PudgF~BwLWT1v~Evl=>NPw@y2Q9>;>+ulRfKX#>Tm^DRyisan>JgS}2>}`5daZ z_tsv{(#~4iW3Kq-#a&CiE0+Dwcklmf@{`HWd_VHVR1Y-|G~D_(pR9kAIdh(|G(9h? zydAj}c~aJytZaZuV?i`8jj&Eh#wmq%do~*8qj^A-)!h0SZTB+)LURis0 z%bhKE2mbXyvZ^(9g{|scI07!F^9{D@4Rp4|CYUYz*p7W{)xJB2pQE3x>RCAa+}aXz zver(<+5%{Aw8u8y+sf7Vu=PEQGfaI?qJGyxe$raUS=Y1H^|6}8eBe6&sAy?$$->%q zbN2nLeSgA!kh33W?Z*@LVa|SzwVzAaM^~&D(%Be$|Kc9jKJ=qw%krgR$idkUv-ZOY z`%%t*inX6g*w1iw2WxjE?9LVISaMU>f+2bZDwXWr4E2g_X02W67QP29T*p?1{ZQhu)R?VvcmqjH4saVgN z*2l~%ruJWs%4NmtUyR9R<<)mJcQkj6cZ>|Lr~vMilow(d*R>|e1QfI1eI+&p~!@QS|bxv^+|WW`t; zQ*rCJuB_ksMcqai^=*&ZmQ>suC)qbnChAVj8=jlXICBGQZeW`G6XyQ^WcZ1#V?hNC zzN@>VYELl_nbNG-wn!kAeQS6`J&$si&s4Q+42HN31`6ECn8UY#^r)IT;!2J zUMN9$q|$(}Svi{rF+^}ZFQ>;uH>E}%2=EK#qo<&b`5~x@@OTZPEAySe_N7I#neY$u z(x(lL2pP^2e+m!_fX`5HSx(DuXhLZbZ3IHG+)!G&;!O)|Gf+jC!K$oUNu(2@O~}Qw z2!`MRB*SSJ(|9GM7-B9;P0oyE>9jbOavmxp?iS0)V5}-9F?fNul4`3Lx1KqVG&aTh zj4MRftmejj;2>m5vgH5*Z$y($tpy%mKJfT@06y-ya~^L~P5^kJm$cAx0w*seTSI`adQuw;jLa^|feM})=UkYOqQU{5#@Y-XEGDmdYt_%i5 zmxzg93$`VK*v=iNuV1$k$SwBfuMVXYpa=>{uhb=r=Ey0+2^Hrh(3c_L2FmHb3Z>;Q z(mg_1McPL%NvH7kidfBj=|B4xqpCc#Rm|zNiLT5S+E;-iTCDr8h4ad9EUW6DtSsbR zNsVy{e0lK@pY!}$?X3C6cCH!^zgAhb-&mII8*wVZqoe9j4T}zci;-0SjWt=_YMIo| zMwIjuUiItJvxaXhc{L8cp=7F2>=9V=^1dJ2IH}>}`nkBTJ>i$*`R#Bo9 z{xn45Xn@-Ed@ES1#(fHwuxCP8z26Gfs*$`JmjCVl>&G4J$}{8cHTIl_8x^qgF+}px zJ+nD8^QbM^n1%7C$szK>tS+Rsu1cpSSnZ?wvqM9JVqWi?qAT;I2D0K|ZO?t6Y>88x zxTa>&fkJBAKX<7>0myy_zT1(mp^TB*K~9jcmlZ?d`v!@H6k-lciBwIh-tQL+JrGLE z#z7MX1sL$iL5gA@JWOL5bOc_!j2=ff?14dOB8U;>o%CB6CGX?q+j%((M7qa$xi+lX zCCEfN!-a?3PQNY1s@!^Py2W|f1UsXJV9lxVAWPC+gQXnzJ#&z=e! zj=JgbPS8^Y4cGDMVAw+Z8Sb(TB@N+X5}F2xVe|yn`2dz$e%wz9aFJu3Enc86t&YzX zKhQpm*JA!FZV()W!bQWVLN7oQ_~GI5tf1ALd3qA-upMEPW+f8xb&{IbWQDKJEX0@F zD$i}4vK>sR<=WC}xi)$l;pRd7hAA5_ZwM>2+OU2n5q}WH-9G4)4ddZb8&US{4BFOb z(AI}N)-@j12it5S+8`)s*M>_;zX?o#8|erDKq0VA<6$+4gi?{6-7d{OYSHwqfqc-p5??SnU<@-%22`#=`r0l`u;#^5oO zM(P0)CAU8qpi^QqPY}jU$O*D6K5v*rfYi?olA_?GbK1=p%=m+FKgAseVW117XS)%C zY6`|OjoVsY>!hF*5IccX2~V&z>cm5+ARz(#(nyDRHFWP(kS`Rw1SCcwkfvcMpFimg zIzu6vjAaO1^I(?`&w0afvF9A|vq#1xqo2O-#Q6a-c( z5QKp==Zc2{!IyH{3Hs!j5khbRB0LfADKQR+t5E&iKfx<6xv%kBlwApkLUAY&W$|jf zSmNe2*nFJCyb@+EuY`Oc`vHw~0YDJAce!F7BG(~Ox^Qu>R3|X;2r>lwVOao~PDKq@ zv4O4F5byqI%cCt^*ANT8ilLpDIUlQ!4}Wy-(YclC!Fg@6wjsKIr50rqWfjq8w#*Jn-_I(Wq7yOKJ>L&@eRS~A z!IjFvXU%Q*BljZl(~Bn_osS<*H1Axsx_8Q?10*~<1rSx3C?Nm*C2 zxg!?57vY+>vCZ2Sor&f*QpxMq-7mja9^1c|zodMu|JeB0xUBlI?sHvY%P`Z5eBN98 zUlhq~8(tR6K=P!$qmA|@tR3^}=jPIdhFg8eT|RK@0B31oEiJLGxH=wM9C{R9vAmhI zRL9IQb?jKw6PL#u;w7;wOw;E0TZ<)&!A0HDCZ=vLYuWpU`%pHLfzY?8WLx_ePc98D zTRs7{^brJ872_X z@56(Cb`YhUx?4KVw2n2cW18RmbpK~ZJ~_e+pJC3r+2QfT&;;X|V*E7g39gtzzeeh4 zL_pX_=O3NtdiJtCdzTxRL!Y;-m`*;=MnBWg`$MfTQkT!Md(S4ej4hV7qfKQn$}B-pvg9(5+(7-OhOs4`3WSM-6T(aafZSz~Q<^AlrBPVIMmTJc%! zC$-G6lMKjAkBub;U5wkqxoOr-ub6_XQYS6lKk$AOcobNomiyV=rxU$rn2`(I$OJnw z0ixid;ujiOQG0AS-uPf7c5qR?*ti&G`}Q+i4=itG3@7K5U+4@uJ@UN7y3lj$-29Z_N?I99TWly<(hwbtx}qnen;A=MY-8+1+#K7-z)%}XTAIF8$`8q# zlg%wJl=6e}wq)z(Mb+YYcFX?dE@tzQ7YSjHM@jEPnN)*+;W~{?1a6 z={>N#=|zF6=$ITvbz#{HlMF~wKU>qu)%376J%3%Z`IdHGIe#)~st{&E>@su-%xv?b zIzGYFZ(npSsh3=fZ!gy~HAh&}ky}~-RJ#QRovCb%Ay$zr-x#lp?~M1v%NXm{#iAef zeZ1rGj^%FV=y~?QXrliD;}~Q5T#U>8`YEC5B_8>gN@Y>;ycW=3Shnyx=|B z9>233Vul^;k&B5PAntWD+sB#lNv7fwtG{%8Kg{0J^52GzGki_Q!MKUlMYqO|#ey++ zY!6f49UqLl7k9?bEh(6qU94$Ws@ISfndd#MrWT~{n&M=UWq#&X<$U47@nlhE0GeWLB7ziP%?jI-nj7A_bvCU z?^VO-dnQmOT!RkXIusp_HQpPEH^w9ErXdyiX%+ zU4$hmA$FwHAY$u=T9t~BGT&e`giDATy%HitytypJq7ZS8$7uYs=N*Pi>Lx)85I5~^ zK?}+fn1&5Go;zSaoDRjUJ`KkJ*i{FGjlLm$BUQ*}MZZ*DyMo>UO(DlkyCM6Okp;gX zos;P6(SL$D@Q}va^zAFPtRo8T1b5C>Mc+be$`dC|E!Zwg>Tz`M>y}evx zFWcD5H4d_kgNeqSKr=W?7i;NCmQ;LMpf1HpZjkBC^WC_0TsXM`>Ip<6=b`@3^vTxF zhg%+OLA434^-Z?*%|z?a+kcm)S z$dpPP+Vpnm6kaKEvi|kcdZp`0bREQc!n9V)Aglm+fG6-e{BBS+)(dQbpd;Y-UE}prPEb1q%~w?Bq;Z`h z^PYZ)t`G4Q$Jf8Z7tUq7R$zkwlfX%3Q4=Hjvq)BkgwX#1^Zq%$>NVoZ?}O zGxf5j-uT$iLGBb0!-?Dw!l5$q3&)!kmyjkRP_I`2w^- zA}EK~K`DeQMibsyA+W}fnxLDLHK{9Nbk#`BvG0#uKf>v;cNZ?N=-0tP^XAp-S8u*^{TJJ-5t<{06F)e()%8IKstrgUWLX1kqZj^Qfa?#cP8$2 z9w-hIqGDHP5ezk%&>1rEKu-EHh*(Q)4)%aET%OZuBpVqU2eGU6LmHyXjkXU^&6?9x zvzqD^P3>zZ_mXYghMJHqA-XzdoqTno|u^=fbT^b^! zlOW&0QVOj&rBANJmxhPe`yFX|bxr-+bs?!bG(|qOP+pqgo5>NDkI6ipz059LbtI%Or^15uR zP#|>!(W77?N}G{*hSjba$_awgA!5AB)*Rm0Y@-71Ac%cKwkc=GHEDBSorY^%ZptPY zxFS~|2{aqI2-zG~Gs!Pa749B+?I{#a?xJZB?G9pqF5941F}bQI-b*C5|9BN*30NDj(-e^K z%3_F}{>r9P$_SVf$^T!%>s=Wz(`)O$Uw_|z&koXrhD81LM9mH~yi}R2spo1ovNapy zgTSpPLBM!0SvX_;6Jvd{z5$O0waJD?{MwRrP57-z>WkL~`3#EMbbsr;t+7+_6N!ec ziP}DdTLnDzDjQeX$yRnIQKH%owy)BTFY{G3T1-~)0yrzCoBh}O7j{J(h3iC2T`yCy z`H61xv*M~`am7OS-TpiM3++rZNR>vpvp)8$k2y2NOa&Oz^jAuS85pyQLIadXu8*<0 z8WGSFT@L|4E_!WZYd0}9T};;~b773T@D_XFEoPKv=n!L?$$}!VVQN`jZM2(f=w%yv zpXhoCl-j6@vvsq!Zs7VeJr3p~#a#@t7lVu=#Dp$0rYjjJFSWoE<}~H3rkvBXv6{B{ zsij8l%~R}~rx@1+=eo?gF2fdoMRVl~fe9q6)(e!-IZrj4UOkefSL;$(L9%aNqv;#!*x3CfhHkCatL*k7c| z|IS?26$c6S3#AODS4Xi-a)8A9z&Vd(Nh-0VckxoZMEyi7lTtikitvpg-|Lo^UQ>Ui z($s`Yoo!A*s@$@iW`mc?6I&^MXB7&xv570YRGYtMj7agY$CVx&Q4+QJzWfj5k{oWC zl+mwr?UY$|DkVpql-)^*R;r+ts)%x24$8ZdC3+A860_&~pc)4;cw0E<@yM}FvuZFQ zQgO43F4idi1JvHs2J-B@P1N4;B=Lr)IUiiTN>o*FQ{S#3HK?hNIlsQpz&WP z-2dN)8F31U%*{>&QY^GD)fiH9UY{LCjFUP@1YwN4!rHU}7|9KUFibLLKz|=-Ztx+z z$OZx~(>6YAd(gJ1Tpa%R%;PglK6cv)&>`VYjRDqw z(=d=QZextQlG_HcSZ23_-F9(ab=!E$$k?{Brat7A5k8rad|Se}oiXlCZh!OR$;Xq- zo{Q{urp~k`78S3= zZ)5G-zEmm(w8_C;sN_N1f3MYFub&&fdG7kTf7$s$Eh{L2Wxk|*!E(3iP8C;fXUpxe zwnX{ngk{TI!QYsx7S6NgY2iXRW1e2Jd{*^I6@)OOPHr^7js~8Xr-@Qg0hkc5vcMqJ zn^N3CPLB)<@CYhJP{(9p(akg0&v1oxY+)UvuYak_E6{@Sg~>dpef40Eza! zLIVN0&%FFQcsodf_|41TeGg(J7KV9w@DCtHiae#a37K*z1(mjf0~>+#VH+Sw99q2T zyNnqKB8T~#b5p8I>tX)9O{u9ov?o3*M?2tzm}bDGU1T|sXvK#b@i3ONago51l&NKC zS^5Z3C4H8^f}szL7OkUPS9^9c8CQLJC+7Uek_LtBR(f)rly?qwbR7L^3QSF z*p*aG+OaDrELNxs4mpcYqZOc4`5?4#b$~*!ee(~HCA{w-sf!Km&ydsk zcwok#S6O57Ykt50a!t-Pgc?YhTbuFfmM~j z6=-#ZU~ai%Cgkx2J5qoh#7BRH>4++?u;L4AFj7T$$)GWY)Xv{;Ra#RZ5fbj6T9G>eM#+3deuA5uQy7^pZMEeCSQ4r8Q(6Q_L zmxj2ZlZl~|&ubg*UcGbmZsblRwl_YNs2#$ibropzaFEIRVQX| z6m9ycY4B$?t??mLNM1g^Qgd`(^{;eal-ig&U!ru1F-^U&%XZ06$(i;2qQt}w0-n== zcG&(@VXq*0raR!ptI)I$8Cm}bs_9?^-*y;`1li@gp(`m&HH0rWNjeq?`0R?G9|h0P zaf$-8_b~Q<;_H2Uk;MNd?MiXz9T*UUmt8S@7&4{(^j~1i|Hc;zmg)aR(tzP|VgriC zMa^zP8nfRMo3Dz1#~17SvAp%SokQ zuC91_QE5`|m%nT*Qa66JPNi=DgZn+(6p$#zB(gsR-3;% zE-zL4oThTL1GDvZo%N9)YMI% zLxKc-yfzh%TsQ1>kd6?m@ErmogEX=}Xbrx|{v4?wm@X622vkFdk;ro;_@MJ)Hf9jT zYX$^J+i3$tz)z6Qf(ocy{ttSYyy5R;1wWA${Y0kwiA?_!nenHx>c5i}{!~^6j=z<8 Y5;D)<%C`M7?{c2pFaL#%dsV%3Qw+YxFvegeLMZ(-?( zA~7Kbm_Xf_u+wtG>All?_jLEtZa0yw$Ad3E#0b4R@?YKttd1;rLKMY_V(JN;da;Kc z_JyAYabR&kYf~?-&FkI9;_NET1xs0Gh~Y8|WfB@iGR+;+p*B*HDGL*+!qM==!p!tO zZ8+-Ua%UukkcGKji|k}?#BxFAmqn5=bzKOFBa;2=&ndfCOHK?BZiG7QIu!XNNE8|M4NyJ zfM?tQ^zwPRHAp2b1iMfzyUbmqJGOoM7~`K7!sj38_zm@6w>!(j(`B>&&`|F4f1-kM A`2YX_ diff --git a/rsl_rl/storage/__pycache__/replay_buffer_multi.cpython-312.pyc b/rsl_rl/storage/__pycache__/replay_buffer_multi.cpython-312.pyc deleted file mode 100644 index 5451aaf1e89c15a3f7b8ca3e080ecd6343dce211..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2588 zcmZ`)O>7fK6rQ!$P8=Ih5@HfN1p@`LNfWCCfka4XDpVArrX^~HFjS+Bcap5*-F0Wz zO>7K86$e+MD!JgG9>_6>R_&n&jy+cD#oM&lT{s}MhyGkzQmH?uzS&(nAuLb!+nG1- zy_tFQz4yM|v7-$E-R}KO{xpElPi)d4-W_c8!JvX9Bym}ka=u*3=R;?a@sb~SK;nT13jrZ$wNH~=RxG@p zpPeQ2&3x97i-T{-OGFyfui9`f)_4#NT&4ln{3H5CSWN8!vS`iHBLx zHHSFDtmT6XvXte|d)2GOmZYF@#j zlEbr9RA3(7vWpdz$Om$oSiEkKoE2uOI_Sb%2muh1?5x$I&FEN?6&i-EC<7ji0Ml8R z?Cbn@Ho2(L1wALGNpe1~h^i(_SYLqMXEc#YvO1TfdKS}clD)e)N0QXW33gASFr*CS z3RWw|GQ<$$Vr*;pY$Ode)>zQ>31HXJUu|elZ)IV5VRc~b+JnXQ=-FEIoEbg$&GGf< zbeX>yuJ3)N%zxh+ug4FwZ+AV~%fwH1cfluAkHhST*6s~(hSFO>#h%%QUN1;1sKjNM z$~Ka?9*|rG$an$rDget%Uh);070l$WU&;5Po#75IxFxfTBXyU(o7b8BB|eUfP~#TfXEGQhZL;@au-rzC zBG>qCO=#~MIq&1WUlC3U9-4*7PWzs5NlK_>5sDhiH&)&hqG+@*S*eLR2!Qz$EK?1->scbneZD+RgYgPpZR3+qsbNiZs<;E_2t@u zqvnC5)yQ!D#Hgpi*rn=;%Z^R8dw6Bf>ap6vVH18m!)D~zlfhxvUugA6HPZiceD8`; z+m|%=CBN$Y@toudlw)>{YJAQc;#WH*$Tj!OMseYOK8XdF}}~Z2L2%L zQ3kNTok~2bB^id0Z3OxZ9^0sK*4tc#?r%Tvi+>XZR zEFlt})#xG=e>_K2LPe8;7$NS_Aw3;O3edfXnpb?ssM;qv(7Snploj!=vB zo3Z}2W3_=1b717_WHr{m9-A&t);sr9MwUlbu9%$zway{4b7;-@;>x`%>zyO(kyC#L zP{)yfbyjOW96vHHa8p7HYd9)2MU1UBj1^7FXW2M{@%wo(>#nq5ENN+sX$K2boUy~e z?jajsb^=mXr)|a{rn79jfqvvqJmtfIm!8HVf!DZ=UVmWJv0%yqPuWSV03>>@K=;Ae saWcS047pBbx1MrNHAV+P#sF%2|HIF5+#?iuggX8X?&n6IB4D=J|4iaIIRF3v diff --git a/rsl_rl/storage/__pycache__/rollout_storage.cpython-312.pyc b/rsl_rl/storage/__pycache__/rollout_storage.cpython-312.pyc deleted file mode 100644 index 7274cec747370bbe16ffb16443463de5e5b8b4d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13863 zcmd5@eNY_7m7m!U_G=KZ+GUsJV;8Y}EC`SUvV|l|LPk=AWMmmhwvgo2vNNECWf#pX z!r-OixST37zQBMe6)iC`$k28TmzFu{igxNsuQ zM>rv@a8qJcz*iwZLvd6O%~Cx&(l`x`>(s{e7K&w9{abX8fz4+PZy9=wSiGJ*HVY@{%&Zym7PgSJ0Ie0CR(RURY=I(q$4l{OG(M8(21!Ew+$KqAMIic#dy^zA zNYSHbbr2iI^Z~=~N;dOoy1<8GLIe*(IKzT4!egeVDEi41VaHNQDo6@&L~<%M$z{ek zQwBAaK*}{4He)Wm70j=2Er~^O0MT zDA&)i!R$K|&y9w7R*;QsJjMxf-mOq{1fGQ%f?zb>9~|c6y|63@o8S-^ipfkSBSiX# zLUR6%2+MM@pa83e6CTjALExf&0v-q7@1AcSjq`)Ta45_*-xwJR#o`e*C=3E+ZyZ{V z#QK|gAsXbP&4My{&Ac*~z>Z92)9{#V4hADJSa!kjU0*m$O<9^kXjkZh=nX3E^G|lH zGUe$~?_|fFv+0_8oUO`cOUmJGXWCN%k27h19WJa)d#muXOs!j4J=yU~rs4@6X6>cm zhmsjs2m|WWV_GG=pLx>M7{%Kl6QKDLc<{v#DPzRrl_?v;Q7*)fd%w7YgrE#p!+fB3 zOyS)ZZolyMIo<>Lu7DAbQ!F&Z1%t9R7#xbTBT<}Yg29_3p{P=#G>*AEph!KXb!%?9 z^LRlOz@NZC^cMA4XSLKkanqJEg}xHrhSdcL8B9h#iSNULhH;=beiF$%$f@I!)I(1F zB$7JFsh>pB2sz-Ogaqy*2}#31em0$ZX2-RIx@)>?4(lUSSa&TzQCETcglWQ@G$n8! zNEpCNVoONs`{|^4z`o@*WPd>aIFF^1c@g?M^gFsc4F)Q#pD<6DKz3a}NIam*pR4;L z-TRq-Kp)7qjmx%^AgNdO7vIPU!NGJdHsbbQVHoLPQEdE)((1d?;9ITOqt-cU&xgI$5ne7~bqz}zM0vD~dl zm<#A+Yi}qKz7Z56;~ZZF)Su#A3U+6lmKio8BqGr$QKGVrkMj7d50uFH0C_+c#rlJZ zv0yr`KRC#T>FCz8DQD1zu@8&AQqrRKym0t0cE8?}@$?oq~&ooRoh!y)qd-sCzBio0z zWqbG1No4Ov`Q2jQ&1Hs9yPH#P(RDDr(=~Hw`p{=4L%9`mFZs+$xyxr>pMG7eI=JwS z#gO>o1?j~r=*26muB#vgv*s#OOGQ^?abnpO5wC@$Yc~-5ToFR!^vqnDz9Lp0Sg2cU zUt)hT`18S4XOH9zB4=>L83G#XdLG4;%yuo?>ep%--Wz;(P~3NV>3~@Cns_O^3~_W= zijE@q)x1WKibCJ9Z1b&oy!RXKHHbCOExfi=BzC_nb-#+bUtRSCF?dE^Tq<8K85DzIDL8=OS2BpP(pstS{-Jw^#M&c^=A}yU!Yk5+Ai5A-Exit< zXV)sK?hoD@6zkg;4=kM*U%n!}+>2iBU9AX1={e{N;4c{MUFqrodBPiK@ucBx%jZw_ zQ~3fdU?dV2fr6{!*G$`ha>z^$UB1=X>0}> zJceum9f1?Fb%U6Ad-wSi)3X42vJpLvS8PEITgqR6y23}WjlV%Xv=!WKyWKWxkld}v z-MVaRU9)>d-^~>}FETu+7fZfmElZx7@QJW|zd&A*OEL zBLxnkz`<4K5aC&gn94c-dx3WYh}kVM`w+7)bx>;QLM>fVOE+ri7B5^9U%9-}(zD83 z!7NYRJ~h=bb9x$NsE`;xV*GPGQuQHJeQ1?Aj0rk#cTOc{#-_)Bv&7UQrgq*c)g483 zM^~9+nUVJ)vv1BLRkoqZwpH^%^7gLf&n#2tZ}0eyg?JM-DSGVJB8vVch_NH#TShSK zXtIIRV^kvNMxa?8R>=t!CS+9-k86;!n40RNCv=VHpNQby{M6x78gIslKY;f-`c zb%G>RXpq!pw}R{w92ADIt{=N6PJ?U0kaKJdXsBFhOEs8tEowP1lwN`pLzgrVc%q>^ zi7UVWJT=r3b_Go9?wkSlhHe6Aa&8&Te(AIQ$v*$&{nCF;{Zhe`X1_G%>l*m1@rY>6 zx*`*lX{cc`3pm;mvNMz&IRXncfW#lfJ2Of--gmAWz|~01UYOU>4XJ>!>w&JTi&(OX zhQRtQkS|*!LL>&>+gO+bYjk8d%JKW5L^gs++t11RC>P@!;RWA>BfM()W*p(_{4+Qr z>g5Qe0!GDl#2deC1gjHlP4WoKbcEo0VkRm!;xQn4%A>2HMj}8byiIoew9PHrd~+|P zt}grA(~SKSf7ATL!g%=v!Ufe-_nqH^&0AWRea(4q;YL~63`=(ts2>? zMO))1eS@pfxHvqr61}zBH~K`ctKadjeWPNi_q)a!+q7-ALn>=TWsOo<3o2_#wWG3U z#KNl!&V`#Fx<9J;uwuFJs%ky1K)P@A5oUqSu$s@zR6iZ1|CIdguI~!7``;P=xMhs41(3j62 z=Rqvsjp}b}+jpGw8vbr4Jp2Z2_c-2s;EZaMTNuEPPFacIaUrV*e^49z;XKBFn+#D?`XoJJirIlq%eKq|N@y7G5fdijL&GX>OGxICjYReO=8CgkRoohQ zxG7BqLTV??N!>V%LDGa{+~yZagxezgEJ=%6OHb+=vvnto*f+1;BT`@Cuc5e-d559w z&YS_auT`$e70Uw(NgbP{fj>i7 zY?NyZHd@$Ja~o>ELcV^q|KOcH|Nfn^6Gs9z+01i*xCM^oPeCyO;fntb_eVAY*bgNX zS6>S*Z^ds7~mNJs=T$8*O1&=4HF>7$_`d1oxn4~3%O#SE%$KiQ%t z$;TzzV%%$LZ3oFjgdm*r2Ena36p4kRa019h6CnX>%d96)Md;!9(C|ot3uYXQPa*Uz z<=+Gzf?wIM;RCg-Im*6&ajsRWI*O`}u9P49*wH@O@vva$R6=sKAXiH&gj~-q7d)G; z2u#}^R_ww~qdr9?SbnS@r33mdVziDQyl^8T`1(d-3A$U<_p~lMWUhJDCYwU*tgL` zk4<3Wm2V1|iQihbUcx8YijAB5?g)0l9(Bcyj=#W+dCH%Gw6M`Gwe)i_jt_x%e%!r9 z4XGuwxY960|4IE1Xd?_GRAX+$(HIp>p8bteP zteYPhvL~sabu@LxfO?h&ea!^cn7hsbdRo)q*|r{>9BKIox2=aYv4%OYZ9NOr)AF=! z*FP)N)9|ou+nx>TVN|sxY+H|kdKFtLL3Z1E`H9MGt(;j5yG;+Qk{uh$RG&PQbwXJ| zvOrlQ6NSkFHjkhevSucV0EZMNi#*hX9k7@k?@SXrlcr3{0r*9E=L9UHq>7W)Ov;tC zWm4`WlS!2%^D`+=a%a+!bSB-&;-o8ClJu}9mg=)+cBr!k>dsYlmoj$|uiEcAzRWJg zbCDSAR$*?4izR&ciU&SC%w7-=1l3KS`)aaZx% zdK?9CgpD%57jPR3&J>4~iujw7O?_~3nE~(lS@dR98C{7TD;=&`|JG?svN)?n* zhz~3Q4@83Ga1_oDt>g%EI3B(s@P5p?AXiMzPGwUh#=?oJpcsam8`_&M7rt!a;6tUG`_2PO;F?5>%n z=_avqZ|WSXd|vE+MGSFLD2hVSRr}DS3A}RdGRakkTy>Ib4|45E6{Xr&T+d80prmGY zPw$rOHK3#BZq8S(*mqBw(nZdh(&8i&02&y_D7W&fO>bVH=?wMlJyBg+Cqg~HT6#xU5Z?0V2^SpTCqWChqPJyZ4 zu7k((x^wt(UUwZH`d*zKNFDBVliuZiLg|a#j{;P2BWU&F+Kg5|S$i_@1zNohH0K|# z)we^0u+M8Cz^rSP+^o_Q$CxG-B--DYz#9Mw8~6MvayM%tkJV3@Hc)KX`GErLFR*IO zNn&?SfHkUEU70-AdK%Jco=X|C*mlB}P_?O+6EtZ{6lUes()~YDFUd|YNgG_LHF~Hd z!x~tslwwVg(qsE}LmL`60I1+%=(lyCldRv?^%3W^DqlWmuY9cGCUzu&E1@$bcVy{6 z7wR?M7Q$F)Acv>h0gb7N6*MTZ&q4zoXi)A1N)0@qLFoWWUv_+RPYp^ZQ0~f6WK$ZH z#XyM_(Uzb==}IyKwOhy_3$v%5V8g6Z19eZ+fOFfOaoQh*xp61mnw-q69=KW5yLt?< zm0;??cVcf=wq!|mMQ6_2 zjmZ+09H7MLWXVLy21?IUlpdARGvUGb4ltM#(3mS}2UG@FtWTFL$YQbNjtSG0?vC3) zLA)DdvW%MeY^ zP|b}6heE=jz;l>h#|_KYz6cLaBuqds7C(qf;LcZkn4I;>`NYXI8WMu|I=JjoS;6&w z1@tJ8&qC;Msca){D203sixN`D^pY{jltLFF3 zb>h>I*om`ScTu(QAKXruKBwqPt<;L^BFX zr|8sPQo(U(;RWRATskOT3W|>Fzj9VgU!J`=SNTrxS5EJA&pTHi8FkKGfaie4x`iq# zTc>~}TxtAMH}TmHYHw@m%*vi)i|xyMPRtI@SN>&e>ClvIZBL8V>xaeOx$+bv9_bQa zye0<60FNf=Z(vy3sk^L8JG}5)H|nn_;M9x3srNB}Q!flxMeUzkA77&}NHtPB{hvR6 z2KxB=X#l7a)5PXD3MZQw;3_3}234VQ|4?&V-g+CFPIf>@gf+kivrxf#R>>=oXId;>d?1 z4~{T4;!W8Idmx1)5WPkHR=2}g_K1RLy@@JxJ}KC3EPis8_847{4-^?s(2r|qWASHs z#l`{p@jjz5Og|}XFuFgxNwls~vU1EVJN`;8!!&OvKW%FU>pEZu~e*pzx@=;`9*Yz|_|2IR?RllZ~f1@h@ SgKGJ0-Yq>{`2zcELoC{S8b8y>Mc_VdYr(fXG33M=V6rauT^z!;bGcJq?-Q+5Xx*;Oh zIbs*Jja0VVF1631THCOgt)1S+ltO$VOMltwD81KaZfll{^sX&gr6or@XN|bg%y3y1 zsj&);O4n;sL|MrOaau3BjpI`DR*4f6yH)-~J9g9u0PqtN*#9JZ?_|;sNVMArbc;R? HjsMgyvlUbE diff --git a/rsl_rl/utils/__pycache__/motion_loader_g1.cpython-312.pyc b/rsl_rl/utils/__pycache__/motion_loader_g1.cpython-312.pyc deleted file mode 100644 index fd5328a5e5dabde16a156482d096e7dd83ddd387..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24487 zcmeHvX>c3anONf@4mJpY1aDFVFOiTaikD_|OQfiax<(>3qth6OfJjgvL3M+YhyiWc zo=u?L$qs3IEpTl|7<*C!?Z|6R;>2{VE2a}CRyM8#ZP0F}&8?cPom8UDA6c5J-MCUo zzV9{q0NIjeCf?1TZ4$5F@xAMN@B7~OzW2TUk9NC_g6Hcse;s1JYy6d7l1r z@Z6;WR6rM|F5p`q){W~g7<6Ks@q&@0nJ$>fyWm0rd7Ceo$=h2do7yN=RRLF-tJRH5ky15e80rb%Jp$5}JUwBg*(E*sBp{+#d}^CRm6n9FL&aHx&(qN5ss|{liea z{1kxC0KcbS0fgM8qPd1q)L5=E84^5kN^1CwNd+mPQUw~tof2VI{pXGRIW4I;E17 zw^HkjO$Ax3KuEs0EnvbH#4UyJ)9mx}a_JzKPR^xM)#a9IP~J+b;)bXkJys@rWe<)* z3_>%7suDGDA20k2_@n`!bPk{NnpsgHH$nEq3*u&JuFu$2?a_j@$E}|Nmez!=lB*(`Ghetivr~J_%GXdPp!%mI{nb5F5>HrCids?J_gy5*CCjq5oUN)u*HH*g!P3(*MfpM z#14leEO^-KQ^8;?C|DyC!Caz&of;Pm{wu6t^c{RA zU<*e^uyMW!BNR&hiAYp1u)wArg8q63{5l28RmMMz+*vTPlc5QLW=A7TG{{CL{n1gu z5SfY!R&1MZ+|ORiHH3I|0l^vwu~9L>6!1mD0m0;-7zqaj8+nN-wyU8DXa>n+3_<0r z1A6Yse}3Dy^=5>*#!mW&gIh;> zYoD_xM<1B@*6m#D_Qi^9YhTuRfOj6_oX4`x6St49Dlp%3+;=QmvaP*YXCLnz;G6?l z=keP|S4zv1*15~K4DXtgHR|ZqBuhckbn!d$Z2{YYP5La|t%3aIP@f`_-a^X_a=peeBM$ zq$#DFx2AeiU8(C?+MCd=+Kb=5c;{lWH07GFOr1#`NEc`AZ3+EH06%bNVD|bSoJjV+ zyY0R0?`)qx#nGE@A6zXizw4dzCJ$zd8{Qq_ioLw6i*t2lT|Ku4R>~@pRdW;2H`^Ca zedH?RUGPo;1&c-;F>`KqApla`T;4+0 zxWxe%xFvyNaNWVuK*^hwBvf1|gRol_E)P}&N>$-X2$!kCRS+&$g{vW4p$gYPxKb6a zg>aQByaB@1s&L(Ojdz1UAMW(^o_OUrp{HTB5M%`u64C4d`0k<*Lir@DQovFvpDe7w z;zIc(Oj9EusQGeY2PKANp^2o)0w4*Y2uL0hl3PGRvM@zL*dhf|7JcFds1KGB@rfc2 zCr`9m@@YxIaf7s!Nbi1XR)2XLMO}iP%$R2^acWGiLn;Y`oI1rX`4o{rM@8%9SOdi8 zurgzfHcIhhO_EP}$E^X~1HBAwQ$cT3pd?@1Hr6Wnm3P2^T@M(q%Hk#n9-F1~=jBl_ zaJdWmPRDI=I$#PE{GR@*o-B!lN|-E-m|hSt=cVVua=JNS$xF|LG5xI`Bg~O8dE})w zYGXILMM@uQmwd`QUI;BU{hmHxMptZav`tDM>yUiP8){YpvhRUa5&$J^=&+6Q43q&i zXTXcv_&aQoh5)HKC(U{N&=9Y+as_PBZmFlAmVQv2Q|b&9&e#Ee8e+Fry^3fESTB6e z?6wh6(2l|%ijHIQ21IJL_8f{ADJ_Fs8e+XxtD?XS#OHZ!l1i4|zyn^E@&)L4(Ko1o zJ#GQF=vyWQW>){!b|a;t^X*a|7&oa5RohGWUOBIT|CVDo5tqaU-;Dg9<0ee%3s*I;K}OVve{8#>)9Ed3_*! zVBeZd5RA*CIpY9q_i8}MnwA+SKpfD5h+~Spb}A5Lo1Ke?J~T)v&n`z=Jpfu>Yn~1L zj9Q@@RXOrXrNWO(!n00X+nc49sn!~5OkVM&RE0P6Q zP(RsNyPnxUm(+`yVij*buCzw-#f#z;y%3F(z z#7h*6z4dvZd4)SvC@HE|o>a<{d{CQAt5TeTQ5bPq9+)`2Ypw9(VM~G zgb((N<3V<(r*}LuH4*hhu6i&THozV>I2@S>u$~;GhbRL>xW8%5WEo@4n9I*bL0*8pC10p*w^;O z%tT-fcw!E=9h#2yh+PqM?XkjFm>_NtVe{yT(W<7fv64ffi4Z#)3;;?bm`9`)sIPcb ziw+8K47TLHjzOfy8Z|O`#;3y3&}29W)gn&3Ugjl$6-qSlCDdsSzEI#+endDp!3xEyJU4@(kZ~}Jxlc@I<^b?bU5s@vJ zh$cfYv7>$%HWMg=M}iEB2q57hf|Hj1t6(1K{&Vec}+2 zT&`SIP%P_%mY~u^6(u-T0pCRMR#Y7$j=W$P_D6++8*J1+d`&2Y`Au3V4Kb

jo`L zFnpEF5oRxTlFs4!YA76Jne!M`jzgi*cLo<9K^Kzs>6k|j5PglLX-9$=N~K|h0F)OQ z4zg@8AQ-OtLPAmgR1ho#Z8#oQ39l^Zrhuwh!6w3D0Q@jFMHC7Sxy;<8#>H6T7wR|T z#fDpaJ|McN&lf9IZ#LVd=pF!KFT;Xyn|fSDmDeVWZxyaIzLYR7TRbaOO%QZEE}&}b zFrsp$z7gLWRy+;(u3f3C$9IkVZh#ESmb#TqZTN0kX>P%H!%EdgEa;;REf4hHv)#8X z?9R6C=Qs4;Ihp89ys<(%dAgpX>v_7Bqg&I}%k;L@s=5@NzLu@pnK-suRdcUpz9qGJ zv46R$7s5>&-#hZok@PjbeIM7pZ%N0s_bqj0wj9bf9nR48AC=c6FY>kRTy1;$I#=8I ztMaZ{!%FAw#lfY9rOu^>#n-sbgBfRYYB=4M?oD^4MmT5tO4pvnVMsl_)V(;$bsZMd zJJYAr-KkN|xpk#`?;^X@yL4r#e{q`YJ}RZ3N%y2eoU`Mx-sEyWwo(<{kKL5r#nZJM zUCYx=9NqNp+4nBJb7{rd{B3{Qb-y?j&%kME_<`N%3?SM|UmkU8eh1Dz@+y-CRZY zLN8bG(xQv2*ts~tl^;(WAtkG7vnWWwoXWg%n(H}} zZ692whd^^SdpLO{xijTkxVUg)$(1QS@GH9i@lhRB*QfjVq>gHAey{bN)^rcQxsTi2 zmu)!k(hK-eD@iy`%HGzAenEgb@lgd z&EHBJ`PSWB>+Wpb9*nM8ZEkt*+&kye0lxhR*M211e3Z;enAc4&qtCNa=->+*vxSY= z#$y@J%bB|Ei*%;=;4*#a!-^WdVi#AjE3Lde-sSLb*zL zXEWz)=ACUYhts}aI`=$wQDto(7gP4)jQizfddEl2n{c)-H1Sw$&Lb0g&U zNXwgttX?3xw1c#yo4jvgP#hErU?~hl#)X36(Flk`%mIjG4uS(hN&sFg_RJ$P+zP3z zwM%8JIgbH93+yhIw-ub*)JG22+tYWZc}E@RsFMjV)z3HW;+l3nHc(E_Q%VPeKcFF47x}yd6?+Xa9H61G#g^&;^B=b5Xksu+lRacSdc>?vD^Sq z!OR3F!*Bw@+=P%!5^!~UCLW{dD8Ka(xAhQE z^kWl2rrufgn9^ITo`cKx2M~=KN;nZZDsMP6 z)u3=@w)S~H7lFzd@ysr7&eseQdCeovWO3&J<^p6`+QnlrGm(v$Lr{=l828^|`oTvC zfVqWn=BbGgS{Qf@WQ=ms3y=_qol{rXD2wzA5{O6#M3NSzJ#hxaYV$^Kt;G0k$jlCc zbDR3m>E@lSoRcgC1Kj3;hugW$L!5I6j$a(bE3|#q$UB=jXH)7NEZyHP{z2sjm3(I( z*V&it?B|^QOH-Wl_)~+# zm4_7$#B5LJqf44(lg6X+9p0iygQgSxW&5=I2$4S%?UxSfLO3hWC4`N#l5 z!xPq6aUMEI+@C^0EN-K3Q!CDr*&v)(Mma}gDwqzu6XqOU8B5oCD!|bvjrD-%nN&d7 z7c^HFYdI5)PB9ZK;TmKnOrsiNtzH8bPo@`2l$iOKz^`WJLJ@8Y<=HeHtIq3Be&Pq1 zxda>-R!0f%XyzQvAd)+_W-MF99$^op2kCRMEuB)IFwgVgP4mMw|u`WlL|({6$jNl=rjp&lJp|Z@(jTsGkMnX4&6Y6>62N4Y7 z*~#E==xPXbLdXMhE0L9X6>1RK$$s>m21hV}szQ{31C2nvU-`e*el3Mm`u>^e@%_F!(PfvRwyf zD-!0FvMRo;g)3`GTez~$1@P@m6p}qkGv`K1fsKlVbMHu4SDdcd-Z@j!Jm&yD=qN#+ zmfW10UTDrbwr4Ec*HaS;D}&WobRy2zuf>3~I4+N^mXR>O=9 z*8d!J&6xk*YO*Nj_)4qccR^N8k!Gd8ePuwDaw_)H<3W1HfI$Z_xooT3nq ze;~F%6>AY=-)aZVyn@G?HHBloJbt8&E5#skOg?o|N`e#q945!=l~~CakWZZ81fCje zl;V{);dQiB2Jku~^A$)1IS6&n_w}NB5f3ei{@PnC@~^`faT1-cfdd*`mT1;Zz|6Y4 zm_Nbd|MXMFn$)o?Y2y|nWwS46!u5wd)J6GA07+9&OhaF1A z7SI`O>REVg@jSQfFt_PQhOSAT2adne+O@EK@hI1Qh-*EZIGNn>3%VIph?OlL+bFPA zqvz=4X|N_cn|f{Dw=ndBiyvJ4x#>UK|AT$m`6};xopZjP@egO60hyO)sy3xhE*{J} z`ZAWj^~_urjeh?hL?f7L(jzWS|1Y9ZR2GcTW?b4*k%McQ`AfACMz}^~iuK7N6R7#c zBdyOSH`!EjQ!i9WVsWodH^Edij4;C>ke_LceeprmgArz%AQjuy*%YcWr@_N+l5 z1lP&d8uMBuTQ{sP;elC_i^rPRr^4+u5IOK6mRK#KU+ogv+9{PWR@%Det%|UK}qOF$64NcbeO+f;?|K$#en`a}S(HxXomCp#w&_ z6w`rD3LO-qm_6Wl%d`Z1<{eD`HaZ%zy|_ns7gNdlyajwhF)o`-Bog((8Z64fsyGJX ziIIgO_cQ+t3qZXGxog4r!RK{<64~FaNcvb8(SU#fOMWcw0R$B!wqQLZ#;%2gBL)pf zjFtBl4613F`{Du4U3m8(R3QRNQx z_GPtwC*OX8Yd`UDEOQ>V9b?@2Yuw3jwtYN9ds3BPJFwceoo_qFwHPFX|OKHoEgfzGW75@uKldkl$ExZ z7H%zBxt9l^Ie*^!@OtL-;KS&{!(7`CY0EF*K;4DAtm03(%W_xT9he)))OBaJ9e!Av zb)QODSKUpS<{g<`Cmv2^-Dg3p)Pixi@f0`dO`Tt`W-50s>6cYvWsWhzyrd}PIjEv| zItmU^$E#Bz4j)u1;xmKihdG6Y7$dsMLOX-%I$7bSz`)WfUv?mB{ZeWFkvkkD%7Jz` z=6eG&F>j*tEpYCO=YFDkfEEw+z#t5*m9QN^m!cZLDysovW$P&uwWv=3ob3h&w)!RI zd`Sma(vdCcN?6G0OdD4Y0(`dor36jnaZ%&HKxZ9$GM3yqw;D13ztC2-7H#tGEgyH^ zg^n^m!Y=2bi5!9`hFuzmAWGujekz1*;vBHqS-7Ou1mu=EC`UEV#A{hFOGtFS!yJyv zd9;p0wC)>2NtXOlFr=lZ6^~5f>T5SRTQzdW$t9RUJaI?0)0#&$+fg8wRfr0ha@;wC zVz9hdS3|^&Dlx!tk{pLK{|F%9bn`sCltXZ48iSuh2hSvkSptJclw>9Hn#4tzm<$@> zyn$uNAt&nMSfoUT!}YBdu1D^#er3b*~`QnZd1G|SXD>QIg>XA0pCzEURYi`P7w zWop4G?X=I(sxqHTTQ7i9%5;!xL0VL3gl(GgJp`wsGt;`H6fYPnUDH7bN-N)(ys1=R zq_nvCGv`!V=TeS8#^lYXQdd;ohH4!%#N{g*QF(hi=23D>KHxQ6+)+x^PLh+bxd-DxB2H0KbZ zrDLb(P>5lpIVEZ|0)>0XJ^~Jyha(IVWGBf@2}o%r@t!@N4zF@c0t+K0b?)>Gl3Nv@ zs;JI2MUg6?O=x;A_w!;$o2T<~Tkhz48ZS!VNqF9e3?$&ZE+SM^Mv~I#oE(ymkKTzj zCE|UNXB5En!035Pm@V)UX!Ukf6@uNFXt;q_56E6MD2BxkG5Br-!-50z!rg&C4<>E+^d@d&r^$iofUjpvqq0p1tpeBsR0^xsg_4PCGPH{$DKimK|(cuDiE$erGDYaD1sbTYUsh8CPmN_Xg$%QZozVOJ}n+$1-r&A_sMCVHe0; z)rSBI&IO&NWG~sCZqMu-$Q-|%@m_GjAeZNIR#QuBmADmVJwePIuy&mCq9bE z04Mu@0vk#cpk^=-0A^%54Pns{8lG5la1uDQH!ZQk&$H zL_KJGGqfGPfuLo5k^7XB5%t4h_hcH=Jdl&h(ot`<7^#WEs4RRaRsnFYSQb09^wwX1 zZ&y4pN!y68>+b5Hvw!TaQ?A z?lKY@P36*=m!Tg-`FIk}`p;nC`P&oRh%m3Ca~vF@ZM7 z!$Q13<+X_Hv-Bt-TK&nevuw`MFu+hn3Z{1wh)# z=)Et@f8ke^TW5R8c4!M6+=De%MaP00uE5OpuDGl2o|rq4s{f_C={IJossm5yn=(z? zmz_KCQ!bn5HYcy9>z9k$vUJ;T%BkX(|H&e?e5r3|-)`Nony`>8aZmI#0UZH7kM0sI zi8J~c!;Epp1jkh-(MnWO%f|o|%>`!)0(wZ*%c%y*mXy40IK>t6)*|J+?@(DtO{5IC zs4d#1LK&nctEwDauGc*9!3_9pi>$6!Lu)VyFc8c!of#XclNdijKl0i?Q%HJe5M!Z5 z1WM{VF+!wOty1GN*E*1RwZDd?rB$%Nv4VX>Hu^zxo|8C>@!3zza&w`_KZadBo}LS4 zKQjzMNf+21hC`EJ$-@l8hn1LfkmfBC3CD<%ilDnC=%xi-OwipF^dM*GLBJ?rB4BoN z3;bPzc`^i-CVkTo=!Rd9V4C*d^u<_Ya}xE4Z;^&775T>(jrf97U9elJM8UI)ShYLee(la{vuBd-B%8%TSBu@` zgGYT|-I6drE8FvGb{5~Y&DoM&Nha%TNSNTmQqJPHZ{4|-EK8NAn==(1Sx0Ba(kZTk z>Is6Ak`UZ)a|9=;p910+R&?UmTY&rwGcWphp%Fg92gE2(=@)$raV@<1bB5|RLKE?c z8`hMsCTT5FTnn%M)Sy=R`ZeXNiC3%qweV=wXGRRft;8*eTgcmnE|M!){z(ud4w#Mz zLEuF~Mnu@+=?MJi^X#dL6MTNE!GF-ju^ByP&7_Q*221M11q z_=o$zn2*r70lwtMptK;8wL>Ip3)w!PgleMi2ZFy2Y1d;~-(M4{yF{dCAZ&$vW1c*^)_^#J6m13v)PU8EsowcrY0HA^e&NEo zMgIpEX9p8-Q4&e;)jO{OB{$9;&Du8r2|hFCVDiA+v8>&rz^sqyPj=4@WbJhdj1u#q z-7(t$+@ zenRy7!9z|{nfpDxQZ(vo$prz*(9Mon2gUi^!+}ymw!#O-D7hL4v9bV4iP1kddHn`R7=*U<) z)^{1ZOS%k(y7{ucrev>B#(mvMXgXS-GJWW*X}VBI1VZwq2R2rb*9BF|2sR%jY`k(; zJln&Uv~nfS93Q!{G-+&{d^5EQaD89Wz;|L(ejl9s@S$7Wm#VS(?;t)_nb#Dx;$jTi z@)Q?Ta&2U`ch;YnhAq^3oEZUz;0(2OI8Abv6z~C8>FXnd6jK7lNlh`4rl{MpfNiOJ zW?MuldIDSI&b7$>mD5RGvOZ~;ef7=^oX@Ur*K^TBFhdLz5Iu|=5f48qcW3I~V|$)~ z8{lGiH-;%gXEOk5U(0Y6{mnxBCQs9D6s4a{Ek+3WWjPCi0?^&`Tl zwb~(6DS9UyEwNr-XBTWbVlC?iN>tda1H`_m2TTv5hiUfQoiAi88`h&v9wfXbG;Bb;M2V;LoVKj}3b3hnQ6P>(Fnl`VLS3Gt2b*2H` zlS2lBY0ndz(Ny(>HkwZ9&^-vQ&eZ*+KyT`JVzHQZJaOAi2Xs$1SWW%9CuQa6b{S23 z5f-9pi|MfLiM!4;p?lJ&D>7|)ve9kodeT;8+Vt2%6%_qOuQzo*HGuCi`rr!%7^gFB z%Ef{2Lz4?*Dlw+`G5TO(EhxsgS~;#(j{Br=b#mT1Id4!WM+S9rY)Fj73Pa*OWP&?b zZE@ZYf!WpoyajWFZO3oPFvvxi6BuP22}iEL2i{ruO0XHeB|HKL1)`Sx=Lkc@)uPGi zD{wvxBuzfh%OJG~wD`^5b}PO?lf7O@E(GMxECELe7nGj%WX zG7K;LQ!G~Z*r3zteoa~bf~x;DRY?Ae{)(c1O%?wIwduc7O&^*ax0!!@>x;K$O<(>3 jZ?59ZRmnll?73}xs;}2M{^p48j7b;LJ)y{p(98b?h--;S diff --git a/rsl_rl/utils/__pycache__/motion_util.cpython-312.pyc b/rsl_rl/utils/__pycache__/motion_util.cpython-312.pyc deleted file mode 100644 index 078ba2c35d783aecb5daf7ad86a156a4d4b4f777..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2645 zcma)7%WoS+7@u9gV#i4nCuvIQV?e4JSB*n?s3^jfQX!?}KyY7C4QMs-j^lOg-SzCc z!ImqW3aQO0Ne?8JD!Fh4kh5E@P#fNyqf$4%YGNIUcGeD805 z^UY5^Jplyc>qCDifBF%6%zv7rt3m9n3ka)7axZO%d2Y!eMe3->SlZ9{$23x9D->Ll^~AS zfRe>|;4oJSb)##rdjW-A#f|gS%ro#ta4t_w%2e)TAw?O*xyS-O6UA>~Q^SmAwW|~s zIj_n@O%$-4r<{?)Q9PQ91OgaOusLHKR*ZAwc%t1N$jwr0m;`j0tQ4tiD>QsNI!`Gq zC}yhV-;M}*RecX^*wqi%H?D-P31;hb(ma7Zqa-v@Xtp?McZ+Nh^B=gJph#J<(kDF1o@PN3+UB6?aGJU#qW2q#x#@Ia3<;It&@xlhB;(l8(~`JlX@n73A*vDX04=qH{7Zdq@w2iKxAiYz zFYLBNRkvJZ*04lfvE0e5CWl4KV@G5KxIM|H4X%hW4#@l}|kK_-G+VmbNr?OU_HlTF??Hn|fZ(G0xh;!K6qf@Ia38h`oiEWmf&r?$w){-cTyj{WS z;%f`6u96E{mYbY_+tfr7)ElQL96U5g00Y^DL+&Itbed3-3V@0{-QR44Tx?TiRAU2l$DQ`lsNv36AfZakF`3o!j&Uaz+ep~8#k-GBtr3Ijqj9iowt!8;d!1047Vp+pI( z!DEMkcNRWl0;V$h`ylGsw|s8t+^s9M;L-B&N5Rm;;K*ihWMlY;H)_Fm`A+Zh)Y8<- z!g_ACc<1!Ssc*(>!ASXd-QQP^*boj6SUyRTwS**DqGnh=MdjyiT5g?~Da!{zsw7Jx zgIRba_|6PG6b#D?HsQD}kDAZw1*d31=T(W= zuZmqe`^~U#wVfy9?b!~|sYKTY&@b{CdWM|@20s(y7EIeNK@k4JNOrj!I zrI}@ENm@T&ovF-JrHZs6ZOq)4?n)1$u^(q1Mq``NSS@;?HhQsk_)@Lsayd|kI!!K3 zE>AB_ub-_>Rwk>{mFbPC+M(06;QM8N9f>!*i{2aEi`@@VXcL82POT`vAp97Gen&^2 UbOnX(Cr3iU;LeFYq2G@Ae?5eAf&c&j diff --git a/rsl_rl/utils/__pycache__/pose3d.cpython-312.pyc b/rsl_rl/utils/__pycache__/pose3d.cpython-312.pyc deleted file mode 100644 index 12646b28034c9784f70499a6548d3de6056d773a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9852 zcmbtaYi!$AmL{o(Mafp;w__(ZomZ`_?Zj~&jV4XuCTSjRJV@%ajc0caTBId9GG&vL z9~ya~PG>OI?qKaK1`~TaGf@@=k~9G}11zxJ2hjf9V*kidfCwBfP|Pl{lifd_dVbBX zJ?D}lX-j!bN<)|AeVlvl<#~MPT>nQ+jh}++XFLB_dbEe4{tI79m)m^U_#I7AHz|P{ zrUY7G;`A_0?#!@@p+wiXuQb5E3ofW9lj#7&XIQGB&um9p7GJtdESXD<2!B z1kc~oFPz)|oble57$11f_^Ou}U;Ui%zLywZBYMYZf&C#p92C8xZ;TQA@Vre3{E!;n zE>uHVE7U+)Csqp#v<9KIUZlsn%6AOtY=hQDVLSA1nqxw>sm`mK6qhtfRJpN)!u6lw zCKIa2MH2F~sHjpx?tO}M2A{g1yhrOEL7STtL$0SFCX|BTmlPqF@$B7NL3&_q$o)T1 zl1r(A8!61H^*1A?q>y%$MVxM95Q>|WMk=uwclI)C$30fF^^gYHeZZ8YFHNVQWu8eh z?@=Ka-vyjs5w)Zus~pcMyeuRpaNW~lM1vg|Q{a{7=Ltbb=3a??w2y$aMh+ zswN2HbcjogZJM5ZA3h%Erg=r;N8_T}OU6$m!`vm2mn-%ck=@b&vH)}G#@qvrg-bFn zJ2^2q$MK57&mnaMKSPMoJt}giyHzO#-(F|rN!@GgobEMt3Ti!MB|0;a)LEl(WV&BP z3b4vQzJ72fpOmC4exY}A zPOpETe3||$mipI>Y{Oq&4f~5XwVvTm>9;3T} z6WxuID7Xnvu{z{ds^O;8K%rNetI|D0B|;vh8QWU0&1cH^to1Z!J*89>EL;Jnpy$#X zHp%c+W3QABXzGKXIu8X1aT68XzIZKr?MY4hYE65tKR0#z*hBZj%fGJ5*R&UEUe9!` z)olB=9oWEPOSWa{_2mz9v-!G11-38G^eGL%K?Hu*WogD|uc#TCC0CU|Lx6gI`vbaU zzUN=T0SJuXa!{Z(^f=_I1XKi9ng%BUD^3B7DakzOuRRJyV#}4bdZrm1JMVJzuK;z* z`n%KgyvL!3+qX;ruc(D-Pt>XV6@ZWpw1}R5X6vaTq7EZJBWW=Xgj(a`q8!y?L^IEc zpnG75(I38?W)lkPWK2}VN>0Ua3E(RP2fd2^r3n@pHP*fUg@WKxLGK^(+e)akLWS<0 z;I){}OiJK{rW8$gMG~s+QYBew!{(~7c!JlC9SM1LHlC29l9m)iy&Bp_;=E?Q!4Uh6 zp4I6&oldDJrbJUsRju8ss+!URUEYG9`YR|vPq$mnreihO@#)Z=4{m>uJO7VI{^|Iy zkLQCOh2VEGp|#rjn>%jocv8E2wRU$-d$=cGySq?(EYrPK+lURz{kfeFt5-WuuC|>l z)V=}rUjL6{AH^2t3f>*d@2+~gGMyEwirzY?H+8sgiC?Z=3VnL`PWX2CG25GGdJRRk zj63jaDe}+Y<<-!Xr4qD5TrKqmvgUd;N;E1ws zXa+B-qKWk0M^a}MC830oFT*`L=8W-bX_>OfD_lxc5|z};F@)%|y|_Lg3QFuiaoqQ2 zySce;F4fJ=`~@LKfOqQjWqWrg_&H!3Jhe!sL>hLh^-;oHdMeGm!R;IC=BDtjf;&XJ zsC(m*%*Ug;8}(kVlGI2Xz?<&3*9#Dmp3=QjBiKVlH(PRy^%Z7PjzWj`;HSO|1?U?` zRR&EHV;T`IEMmTDIQzjx7W!-k6q zG_F&Or*Yj&`2!2@XIt}3y`ddAf9XOKPXOSr3^!+Kj%}_5`ucIOOF<3$kqhLR+APZ& zS}k%_f>Yy`CX{-CX}2^191ZFyxhlD*G&nPdR@q#*0j8B}6)kv7cVtto;JyT}gXaqF z>Ooa$&UrUzs$1|0benmfb>UwtrPW$|S8z*d3qFE_1U*kdpNKo+iZE@?k%&CDT#k`$ zfj$YXX*XC2mRI2O*xt>d2U?8RxTxjFP?@9BvL)-x}ocl9?r>a%E#R}jAUcSv8E#l^0nNBxTFE3p}Yy5uj7D!fr3~mFT2q2@wKJSAN(a4qg$DR%{;8L-vBL@Tbg9&STZG)l%UF#I> z>H2zG!;+f)e&)cM*GDkI)GwG@{uTdY@19I&F|aMunHk9Nnf*n^g%2O^|H+xf^V#!D z!-Z-tcVe}=dzI<_c>fw(Ly-T00=o-Ge6+X7`ZHaIO|+~EtOY$gzHeX&h~ZUjqHTXF zUDisN80wqCZ)EZHWt$6=M---coZOA7f#!P`oWV<@=gx^KP%=(jEZ%dWz* zi}P-wYTkno|2p_2<*y(ha5;9HE@36XTjqg$Y1bAH^xWP4HeA}?M?LJ=ZT8#HOuNd^ zAcQ(82(0JfKJQI4A9XmqVg>&Mp^=s06Z~ZVylHQMg0U_rU62FxV)A3Y>IGiPo3=fj zFYOZo#Cx(ppB=-DVKx7KP=nKaJd%V^0)r&9wUCQ~AG#H%)t(d;OoBk%15lKAfJ0(? z9W>DGl+(f5k_X{7(c0{$(0>eV#;Z<)JeEi*25vR@EJvkwE(UFXn1h6b=HRvrMH&uE z5XbmrTpK~g!?s^B7Lrsk1!|yeYYpg;5Xk~GCSaEP0OHR?QUm_+(qhoTq}pu&a%=r6 zH$z;EBC0Sa;YjAh*@!qvd^jL`%)Xk06p>dGGI4h>6|e%EZ_+im!`!we$#XXOZh*TH z9=j4>M80K-nh6wO_eOzP_|EV#$wya2PMk_g5EDRjU~Y<}Kzb5jg}E(PaUXyQ$Q<(% zqf!)Mwlyo%n!QykHZdo)S_mDISe_Ik(wHO)-6R|$9GD3*YlCMbNC?CCjl=+P%bc2w z#H_sqM1;|hBx7c6Qb`EOh{lnC0i~&YJ~<)x$YK;Q(zF@XKO1$R*p%Z9Sh^At=6z>ouf>t?oHj>fZkpv{YJ>;_y z6a$z=wTm1NBmyFi_Dq@a36R-}wy}9k@s%mbpm( zHzY4wv#m>~e^Ga9$I6ad?JMnf-Gv=JkiQ@x=$*BumYdQI>85-`&J7is4%|JHZ|p-v zbQlp)KT_%cnypzFU6@*ES=^P~^*0@vz2*3-Sl5*4DSCoSyB>Mki-G3MnYAka!pTBa z%ksdZs$FZn(PLy%k&u z=Cndf_gx|1+z))U909&+jsjm*Mgz zX}W0BOIyRt^F8Q@o*KqFkNCI*oO*KKJJbz1q*)^-Fp?%#8p*6J**jMXk_RPW51jU~7Ei=z5W7=4>@!S8i)}KlkfXaRre9Sq z0rh`_pNf$jSdG0D=w2urdg5|0}73+zr%ZfUt4lVyQHdo8#STdH->=tBP07vUF9^wHj@u7QR9TUm8&XPUBvvMN?;w$B?br+Zy zP*ZwBcY$ruUE-X$Sv|L3P)iDFaHf0+y#5Y;DxMU8TDG7wuU%};Hh=u*OP3#cnwC3n zy|(gNu5;x;ZtCZ~cL)CA+`V%T2ku?`y!Nxf|MWL}KJ~@Szs`I`=SxYCqK{qJz0e0g z^wpya`?G9j&zjG_$Y$B4JwH`$rf;Nkrwh$p1z!jn9KfJ$r7d^)llHs9W48YZdwi8W zUSLn;nG;0yL(O{C$cT`LjEtB7i6p#@lu#Hq95X;V`~KO}L*Kp9KXOQS!+&e2c152; z4vnl1AKFdV?8jlf{_@pXh&sIzlLJnMbq@}+bO#1H&oy^RNcCds%cz!izIL^ zFOS|DUm1T~9m=r9>UyX>mp!*Muzcm#@XGMx>MpEpF#7)8wcO~P@!Qb1?@z0R`IH*B`>E57%bp85ns{OUQ*YNr9H@k!M!Ebgq(hc7n pYNG?|C#l}!UuqlFA-ZuLP+(X0otfJ+8&m_`yYV5toeq;7_5j5ChM3`EV|Fj%WW(G{ z_#$ray{xY4R?~d;cJAU5IxDLx^X1FTmoMM1^q&d~?G&UR*8jcV`8q}Y3U9PzRtxl_ zMJUWt9L3QAYK-ouX)GH8hJFLN8vBjpYU($UtGVAyt_A%CUkg&_|i@=?=iwy+W!2s`5u~G9hYV@Pka6e1&R6os8Fg0ebfTQ0t!{k^v z19W8Kteg?9=1G&=rW75E27~_K@FAWJ_C`2g8VUhvf*kJYNB^2YjL_sMWTWZH(eCjY zH$?Q?QEZu$vHn1*0+s>pD%$zS8U?aT9 zL`HZFz<5x05UnB;iblfGh~Box zZBY!tuwst!LP%6BLxB(*dF5rrgp;9|LP1_sjDWdf_JyLsh})=`L_RR2SkfpeX4034 zs|mCF-IjF|A>o`DW_|p+k?0s34EZ^)cn;=zFvJQR%)>fC40wgWx@g275Y^lho>Z(} zuRrLIc)hWz<;3XGUX(!>;ugp*Qjd$MimKVYGkfRW{P>Me_J6cL;aWFsx@=o4ubAC3 zvt@4c$4#HC{b+5Xyd9oeQx)~ora!hnLW1yR6D<-9A?=T$FiT|-VszS=BZYS8{)>eN#iy&HA%URJp!Tz+uy`q z@QZGZGma43o_0Ra{>6yT^BV~0at&D7C=4)@9TyT74-6d226kqm?VwtScH0ufY1V>5l4W~EPun#L<{uka4Q>loYOw)}81^``7VPkCXHF+1Wg zYOjVe6}Q8kKH9hg?$Y>DK#4c$9-t^fcg4^|B|fA@r9eC%Wd+{tB=jcK;@djR>hZ0a z;{y>^u|+}=FQGHV8ssNbN;HmnSp~8|%r+_(e1T+X!-7hb*dZaxbVg`XsD`Jpro4o? zY!5U;J@GFfyGW%AQ>xLq#aK5b1Em0cORG&1M9UVE)pE^3}Ori!ZOUXqJiuvB`r>yrP2rt9u&?)mOS z?b;iCpY`ADmut5yyd>9l-*f#~y@0CkYpd}j}aJwXnb06VD^2z(?e0GsatKJT`wYt;s|)CD6YvT8;E zBedfS4`l8FGi_e2h=x$3+p3eAnPNn#S(6AbZ2$&(} zBC@UbsK*AIdBvkjlewFI>?|@js@1WQ4}N+AW`K!)v=Q<@S|;U8Q$xU}-?M?7V*s(s z1Tqc<0-y=&NZ2+;2EQi|!2W2$O3+k+p)w22m{;)SRsIz99KL|&&m-CO2 zNBhe4a&G1583EA&}8(Rx6S!8SV+@xf!zHfD> zs(^?tJ5$vR&{CnwZ1qhJO>nZw4Ez@;JpV`pc9MrpA}h{19`hW0&@LZKq(-4-ZbIgy z%{a09+LEvV5>%#}6y`qK`mEAE2~Ccn7!%EYCKIrF^f6Wd2Nx!Qf}*k{LY9>3l9lM|nx zxMTXldDkg#KA31fBv&1hj`Ssu49G_Y?!6; zQibSE8->@PmkeQn2pwR@!YX~K(wG2RE+y=QOtGUd=2fBDKmk#U`kqe{U$IVR>ibx2 zJ`z;lA#72A3}v#~+0!$p;b?JxCMlT>fd>VzKMW+n3&+>z>~)*L!Ze<+^PP++U9U zbZp_wy}rL0`0Ig0@hQoA>Pd=4(SY!kenPPz4(q}Myh$exRzIJB1+tPo)&`(jPp}#5f6pDd}c}|J|7gAF`P830bh@^Od)?3yyv-S+B zT`P|w=gCQ*rte2l{?19-JFPuUT|R9>1p!YSrFaZfDX%Kbi*U%f?Mm$s&vU4fPJkBX z9p;0)zyh&(fgm+)0Bb51)D}fPq8MYqAIa(c2K22IjEfQ0cMj+s#R|pf3r7`mFgg~V z1dd7+3#$+c1VFnKl|udkoOBT{sFEBnC>7yQ!0(&%BIHcZN=3R>&SPi#F~MGOrrVM` z5f?05@KZ|DijKZ3gjjvnhIryF4n@2S83=&Pnk3UDGhKIfN&UmgenIXRB&I9Dh>~?h zifKSudv~2y0&3Eb(+j``do<#NGWOc7x-J7U>Gg*B~u0EBh9{9?6<{KmR z(rKD(7C3O+fY&hmAg$U7(%5<^2pcd%IY$*7wqt28W+x$23e&=Yj}HXAUXV3jg{LC! ztevWi@G@4#41Z!RWWRfl`i;S8Uiqz^HV5f%3ykK|^#8D%&0ZA`QrVZiHw&Q=*|~2( zo-GmO+o_|dHRdFZRwMfJye;+Js8^$2uw|Yd`hqPTnZC5r3%0~r$w@ERQs)kt<}cXt z89d@SbNdVq@|>0^u;-)~#I`D@8Qi{LOVm?x(hIgkJu@e1EjjuF=M>llN0F!HB(2UA z$dGY*oD$4vK}0lqsR&AqByk|?{|HRSk(JsLIK`T^a^|W*(|Gfbs1J2j|6ST`iv9Ov zyci0Mr>&2yFXA63MqXwp8iborkU_%@vE896<+0n5>2(UjVh1VV9ke1ESO?PwCO)jD znfvmF-6$mh++}_!((7CRIFlXC=w1tMa48sDvJ|-aRh9Vm|OiE z*t7hR$=Hg0!EsRHxO`(^T4S}X9tLn;_Fe=tD%eQfWdb?X48fpc>^}IWV(KONF&@iZ zShgMH*>jWXgFb>?uZCT(j@=%$tsh$*BFODaBd3@Hp$Rzr?3r0pOaZ?bQB3FfNfAUz ztG0w{i36+tGCiY0P^rWl40<#3EW8bbwjX}tKSBnU&|0db;>v|f7p}xF#ovEtv7}e&@DD}Ka^f04oiSCI>_EU??$oJ~1Fjr$wnIk|>Hw)~G6G zW*=6%n0XAIKKHQ7BS=1n9p+J#p3@?q!ld(xya!<(bqSc4i7>Bc96NcLmwwpj%u7E+ zfO$RRSa>S)`p?UlS126ukD(dm1z46|_bH4^fB!6Y^+cuTGbz1EKGtLcLpsNfDQ3(k z6r(uIDHdT&3=5%ftXyL-84#_p8lA1^Pc^P$SGkIryiiP?1Tf~u6=Uz7U5e${p6mdOOBlI?!7X1m zQfu_tAz zm}`(Nt*PP_b3wVdU9z^5UFx~;ZWZP5SxtH|-pq0*fBFQw*q0LrrLxn>d=&^|yQyC}-d@8l>Sl zW_iP?BNsk6Y7E14MBn`Rw7y#=O@jozQ{Q$ONZM$?6Knpyfin&nP`(&4Xd4YQTO*9a zTuZ^#%o%E_v_r%6EZuO*9MLy%oQk-#{xn>K|K+EM7W)io3^d;Vkk)uDQ`W56L9=>J zid&yM|9NCE4o0S76gO+7DR5b2^@fJ#oCLi!Cu@$b%JXb>o@c1K<)pYxTUl_cSP1RH6){g!0Ic!%4Js9A zH7@^^iHhhFH@ag+*XDr5_GJ7KG{=(!GnzgEhYh!^`s@}2|5euP=37hZ|LIy%&s@v= zZ?o5u`u|}qVJ`trWCv%hA3QZ>%{BT+auSD*x2MuFykl}2!0j!YW_=t*D?m zf|iJDc<{X8%0Pv;@$70nAeYKBM*QLkCF*bLtekp^afT z!fzA&US{G9yBP|v47Ax%Ia-!Z4MN?)jZkM0Di6L4zs-2_%D{lCSYhqMBr2jK7w|wE z)I4>nPl)nuOm3d1vr{S>ZOnT34~-dU&Ma%bn+=Eni(K*nP?7!T0MG}SMj;mibr?@a z;1#uVY(0QelWCugyEcUM9CkuP${0~-fyglEg!oOE5mt?`8A8+G1g?EZ0FxjtIe59i zWR9Utu$~}9!lZyXaZr892ggNNPmc4WI?M*LY7dYV{x#D6um2HJIvzskga*Y5Xx}biI*BQz60%p<5^&XK#3xlI)NlHWJVR*jIaq5J9?T`Q@#SO9}I-C1VcgG z0$}0?GbZS7cE9Kkf-w{#c?47-pkjm{6T!cSTMi>=*a(0SP>d(OKZ1cULaLIW7sV)m z-syxb<`wz#Q63>7vWgM*mQnx_K(MD2TQmrsbe;$EtwZP{t#S8?iXD6Ks%0l$AaI6k zXr)ZwLz7;0n~F-+lh3?l9OckI2R32-J?uM$KBv&*igVes68&z{OSZG<0Bn#n(nBhwoJSm7o#_(S-K94yd9YH2l9Scdk}_YYm{PL-5i?4EA=gMAM`LT#LB zoG+d4Oq8xpmU`q;&kf=BhD7P+i~Apfn2J96(PYCWxna}oW4Cwx%>0Gpu0v|rlxR43 zZ#3C+R_-~Q=o$Rd$4NuOl5be<8Ie}_6Xl~952k8b=Gpl{sm6Wd;6nLAlhpRQT=TkQ zs!TPkPd03k8@41GUX@IB4=trvSAVefzNIDAz)04*l&g03;LO1v_FURE-8p^!VI6aQ z+qG@;=RbM-qqlEwxXmV7wT`#L47|voidL@>-u<=bJwGux0HhCq6py z$(fJNBwMz~EnAW;yX2N#cTOi-j!DkOd0KWhJ!sjwP%5`<`!}5n;vK_+xUgGp=^`(` zbT%!OQWedQ%cw?o%Gr41tz`RlxqbV>dAYswXPcxwXA)grsqO4zqoL6*Tk9U%sp=K8 zZ_m6v|4OoXlia-NcIWN$iRM?6&7E>{=be>zjwYJ-Nc+wvs@Q4ggR16_qh#Q({N%OI zU%NwpzC&sa+eNy+EU$zFMGhFh_IT`*#d38{(3Qjv8HZ)%!xi)e= zcrBQ0STEzGzb-etz7R+>9Dr$l(6HgpR^INtJqW+mGOPqVs&e1z{_Nn*gA1h#yZ`dw zPY>QJ{pn$;uRqzx%6;tpw!vguKyC|2W5GmQNUjP=CRfVQATjGD$9l=M{z;Vsg?FNI z{Cg;X%0cB*IY0o{BfRn}CD}Dln9D0d;bD)gAt!M}=IQ%!ja&E^0GhB);e9B%jl!S6MKO*0`3W^(!61Bs zPt59$N4}~#z{{e+CpP$x;r+jZpLhZ?*x**mR*|$e$kv9r^Ybec)>ZRwe&YSed;91D zooMNht=oQKE4<{I9=zI?EL$a)t(re7m$lxgPrA3s?rjTi$nG5p+sZsFrIXZ8#u!)UC*S!R|7g5y^Fd^ z4}g?DWwKEb-4l^7jv;P>_PUk`!D!}udgN#x5B6w@hC}8$RS?m2y3u^ny?GSfJvm8} z8AtPITT_GwSm|165saH$lB#s+o#lcoa#iD4`t2uj`1MIc@^n3qCYv z`Z|ei4@^)Oj3=oHx|!Ml9>EC%O`W7@_;-1``rKXIqZo(%kv{MjTX~2jjd(-npj;#J zEJQTG87_dC9fFYrO@D~3>J|2vip`c<81C}7bA(()FP%#}rcn*wb283`X6cajHlmfpv z9GD-mHGcnGBAcrW723kjH; zd+-xa!FdBwB4w`G)|uAXwwboM@%dX~o8ZTYHj>q2L$x(#A4Pux85 znfInQ(Y{@3+ac{ZA)OpZo@C{dZ1Uugd~!${9!Z?^OD9I9%5&3uQqICFdoJy{y63*L z{3(wt?= z{Em5o);N~i$__%u%lr&#h$*p5=uT3kdQdMzz=JS~m8hNC%e+2QY@qr}wQ9t1O6(xW zlg2=I6-Nk<8|0Z*wa*UF<^=y>6vFATeXHi2c!uBMIGR;Z2~OMS!9l6P&|DC#?(0=u zq^L<{JcS($0TB)2=RN8OiNlxx)nRFv5`~Gnr3_02f0IZT1!Z7s&DPTHq0)V#9GFq!Nuw}Cg zVyS8vc_g;0q!=K_Vgi9*9h-@GAiR5PsqhAp?QAw#eL&YG}GXdTsPZ z`Hw=NoK0J=x)+P8l0_{LU_L*RC|WmdT&!I=-#kB=sNFDK{LoQ;_3(Yis#JNCvZ{QVHbH~r5g2ekQ4M0@4Copfh8b=OpkdAY4~v+-`dt8=~a?s_wn zmwhOPzDZy|^rIt?!xwtsw8^JqeG1hujnhIm)S)2UqMoC{oDXguG?PD@g@JySW=b>_ zY{@V&2+Pog?`QX`wFl8nhxIPvsOmKd}Jr%i_W>c)G(!3+W~wzhp}d}cip-~%RMG)5z2 zk-nVew&B1O3@{u7k~3>o@@l-=2(xBI4_bVeHc^+SKnOKJe<0PvH!hZE&n>~F5^CY9 zu8v!9RaNK5b5^yG2>L}NFD7}(hZYSd5L$O4br5~~8LqW|QeUN{m$E#4je`?L_mXqv z87#7%x-w6nOF7`6(^ESYYmF z;rUtg7DQNxE=5yrR`0_?C<-4cVz+PtEJGy&!j7jLg7K==&oUDuyZ{FrFmViu!Pp#f zlMNXWC2+A8xDDXLM^X=Mn8bIGU0H>jJ{WOaPeUbrkf9>Nt4;#Db)+nHv+SdG2m!1 zPaDx&)gD`yO*Kpid=MH4XiJehW1@!Ah9)v^I)d6Pl0mhq=>1+mZ)(Cr9?;jN4i&!< z-gizN44itTS6sua^%7BM#Ao_Ik$|3tn!4^Fa)=;pXf3m#fo`c2s}?{-Zx;IkBDd67~V7NVYJnR%4&F94jNCRGu0mzd17)p z)J5OmVY} zE#xt4wX{V>*&LUL&06A&+zhbN3}qQe>>E9gS!^-WCVP!Lz*&iWl*qpshKl=4RWYI> zyc(P&l&XKS&8!*ZMF>R{^?7k;z~jD#Nya!xkYLB58i!+P&7cZGER06w(99q|!jAia z7lCYwK3o6rFz`i)o+LAuH*8Z^EdZo`1}tNL$DW~~0B&q>GIyoY{GT5SM<#(2ifMwu z!DQ-8brt=9bbM_uCyuPr^1SmEnp0!QgWH1oeE#UA31tbKkM<^p3N!)Bq z7x*?|a9aTKP|6~;SYKws z$gohvz$r`5l(tu%WNjitHYiTlvMI(zh;{s*LWL1kwdUnYo><}GtWDP%t3*TQG9pFZ z(Hir$W+I`rdC8)Z{5%hca~v2m(fprVxr2%A{e7)+%tIFNg*Qj+Q0wn=W5hU>e$wmv zH2V4HZiYL z!p{iMY?Cv?H~uZN6iA|J4h!lUa1S zt{l2_XvttK-A(_-Xf7;*zYS8pV#z?4GKGntxBa}{17Asl&1LmNN73~7``hMLe(3=Jf6d&Qxmu~2mC6REyD&=q zHMzJoRbDs03BqMlt}3bSyzDxEwF_M7pKNLt+o{_vE=}Z{9JWuaP?KmuL zKLX{W^!C*D9_jVNP(DKU(o1GzMK8{AX&D3vukTn`Bd_a9uG=fG+nZRo|CvBHrZ&DI zZRi3(yXY+`Fx$Vf7s~r`fYt!+kJnIU`}CUk4lkN)lD+l5X-%qrb?y?uNA1(x)xC4v z{O+%;FM<7P#kz+k8@77jr~m^!aTLQJJeWkHG=lYw{Oy8RG59}xBc5Q;bBO%K0;uta zP(%0@vv$a06>1Fvlv>yYT8D5Q-#fG4s{lRZbIDWL zFP1d~_Cjz{K@GrHfQSc)3J2`a0-3>1K>sUz7x?%16wL<0KS1`K3)P-eFvvC{vtemq zK=^O)DrQ3=jyzDoK7xntD*TfTs;uDk3Pyy3J~APN*`H&Ej(idnCHx(f6g#+-;LlEB zH)_CB0et%m%!J@09K*_JdLw_#fqXtlzEL6J=p^t*_%XgkUlXxKX}_rVun$mO3l`mV3(|wPyoVHe>)n^|>>Kzfev=uc zi>n}lh|ECpO)@{pey~Y~h+&K(OIBN zEg><%Fd8OBSRkySKn-OpF<8SfAIvWrCPfV=w&H_r<+tL6^XkYSvO959^xhe82D%Ik zbU2p#{}jJWZ;{^(@sTs&4E!?&cv#NLDL%^X)(@X2cWppBM-vghBnkw2=MjK`oFj+X csP-T_{Gwq}lq@oz!-4(~$b@+34EzEEukN2W5&!@I literal 0 HcmV?d00001 diff --git a/src/mjlab_husky/.DS_Store b/src/mjlab_husky/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..00f5a6c8249ae62495932f57f0130fc1dec6e315 GIT binary patch literal 6148 zcmeHLyGjE=6upx~jgKN&S#D(`76C!9T0;VYjgUgD%uAvqaYG&o7P9_Z&HwXCHU8Fh<@j4;n`7KtPL9fqpvv=DoldF0`x!mXm z4P^5#H{XMY_t)KNm91Y^_D>Dwdnq6XW#@E3!)`my5!H6U73EAVw|lEYt;NX43MS=E zULwhxp#gPm&dnA$<+#bWe-!cQT+Efs**ph09WB87Cb-T(J4#+_1$DqGJp>zRWJ{zP<=S|00m*3|LQi99gon;IrpSo}izF^_wFhI@XAHn9D# zyg`ALH#Lm6A}Xe2O#knA|W zC(B9r3SI9Ua1PiGWYyaOum6?l`+s|qd*>W*4$PDTB3G`KOSmMxwx%wQ*IEy)hQ`5l op+Y$YonDUR0WZb;zk&?pJU#&W8bgH`LAW0R(gxQ#2maK7A4EFKjsO4v literal 0 HcmV?d00001 diff --git a/src/mjlab_husky/__pycache__/__init__.cpython-312.pyc b/src/mjlab_husky/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 9717d474b1d5cd5c953cbc7ff49a769095836f81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 158 zcmX@j%ge<81k+?*WrFC(AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdWvgGFUzA;3keHmR zpHZ5dn3tcK5?`F1SdyBQpIDTVnU}6#T$HSzo0XH86rWLAoL#9OAD@|*SrQ+wS5SG2 f!zMRBr8Fniu80+A8Y2)FgBTx~85tRin1L(+S}G^d diff --git a/src/mjlab_husky/__pycache__/lerobot_numpy.cpython-312.pyc b/src/mjlab_husky/__pycache__/lerobot_numpy.cpython-312.pyc deleted file mode 100644 index 6c008037eda8954057da0778f8238e6c74567359..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1995 zcmd5-&2Jk;6yNpkddHhD8miE=5t%>(an!_w6b>k=lmbFjC8DK*Vm`K;-EqCPch;Sq z4YB1&PK1>D6eA&`N|ky+6(}6~e{hLqO9?{`NTr8xsuXeI0yB=+R!=PlB;H}(@4dJ4 z=FQvp-uyf;U`U+48~elER3z!45F+BTAo=JmzHUp7l$IRXi4|lomX65+D+R?<(`p;5 z1uY%tI<=subtmrVj`p>d1|>BK%I__O-iIY<`*;PB!1ZweGd_Wr(N%xhr*LH+W{BmX z08TAi)XpLYQlHpan1WueU@b$75UUg~z z()4)OHqg^WsB0UFG(8^K)?)ZEJrU_Ek`w!1#B~3On3Qfz3s30PuKu-d{{EXBeMz5> za+Et6p&sY|c{q}CBN6KHpR_2_HKn5DPW3Dia^m^VC5dPi<+u*kl&74iJoAaMr+A7z ztEb$VDB2!(^r-IF)QXxL>scb?fczA{SsxwmY7)_&4Y{IzBqfdV``4^If(Wlb7dvPb z-r&%5Z=8dff<-C9ym#T;J6!TDI{!MHin_dLky|06Ga2F{>=a7L*>YkPx&d7BF=BBC zQOnM&4&fFG8B<0gyeI~ z_}D6h=v_j5!VEWXaX>9>Bc`<{3yWVvIPeJ(jn1@otOT?%4!vTD=~*k_w;?Qsi%jQO zOdzIrF2z*rxE|993bIxaF*R_@i1l9$F?GG;Lz5blSDDt%4U;KTxCf^AAq_Ns3>5U& z#ne|m$p=NtMyYJ*S=e_SGss&ME&CR6TzoSX5IfbLfo3)g@}*P(5iv5&c9gR`(9zDs zG>`LOQvcXC=Mcyb5-k>-{iemMpB@+|>z_4^iCuN#KpU#d-@dWE@XbO?gZm(H zd$I<$hH67iFuDiE8(_Q%CicK&157r-t9xLk0cM&YwZ3?u58l32pK9r6{`IDrmJYxF zTAYp5IUOSErA7()cseF0*8uoaPb{Zvzoc#S|A) z7AJ^?1eXn5oW)MNI+6*do2KL2yl@TE%!K>{WST_0Ff7h1nE%gXH)4hpF{H?#SRh}1 zl*oj@qVX75@O&pMpbLa|qa@y{K)g4P6ChO4u$t({|h4nPz4eC?}hc z)FUT7lb8@={7bwT!U0!r#)F7$~|p+yNLN$iNf6W0Fu%URgrSHl_e30VovbE9Q5RfFVc2 zHs+iixCFD3BgoiP4)`aZj>#g)Am*jkdHNRw2T*Q5lq4#5l|YO6Rw721V=HxuV+7TA z`~+d_&K07yIQojG>wm|?#Q!sZ`ZUk8s~E}M_X~A?odCM5nV7}kCou-QsGfXZR>7M9|Qk0xq zSXnO4u0^Aj*|p+qY0ip?)P-`SqK0Sak;=L(8sfhf)Cu(?g|H}z@Rmynj?@MU-b$yU zY`Or@}r{y2@!goyJ%?ICouHT486stqPN*i?tZGWEPs-U T-=xe}DfL-OpCyLn%pmSJd*1+v diff --git a/src/mjlab_husky/asset_zoo/robots/skateboard/__pycache__/__init__.cpython-312.pyc b/src/mjlab_husky/asset_zoo/robots/skateboard/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 101653e1ecd336e7686a08d703d92452531a694d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 228 zcmX@j%ge<81k+?*Whw#b#~=<2FhUuhIe?7m3@Hpz43&(UOjXjMd6^|esi_L?h6)*_ zxrurCnJEgz*@-2oMS6akjJMe1<5TjJrAErJYtUf6} zu_z@oFI~U5C|N%@DWzA8UozbHQ`zob|npi6~F#1nZps0)DfcCC;tr)owvr%w1?k;KWT7a&d76|T- zelyD@sYjBcFSILZcINjzXXcx4zSZBF%?1R|AJzOObk>E?FUdf8?Agwf4|E9KLeq#t zUc{+5CZdWmUPeXw>WJE_rf-c`18+^FD5~{pDXl0{9MySsw6BfmqXw@bYV;cExHw{p zn!VkJzIQuS10xgc+;}Hz^L~-(h#Yuk^l#vrVDr-?;3qN*-|r z&UjZX_Z7@r4pu8T(;-FF#T zhqsX{^X}owy-k#|2RG#?&0K}Ig{$~<;hMbra3k*FniCM~i6Vr& zy|{VPxDU5(8t=#Lo5s&@O?V%X#ErD=p;`{)(B=a4*&NzhfF8`D?FHy_;I)ox!M*qZ z?g(Ga?~l?0G@Q}n&YO?D&Nbw$SAN=$d z;KwN5$L;63R-Va4=n$aB@LRHMVapxhp5+d5&vA#i!(2Z%z#ZW{FgnT&a>ux#cg);z zZeK{XGMv*i{I!~%mwC5IJP#5MZiIV*JHd^r;;?E<8xro4ED}*UOYvk^-RJDF?$kIAK$WS&|4!KguUF0ms=3x|AyXw zFGeqTFVOqzLS!a-(R-2hXCiF$CGSg7pVvpn7bBOVey^Y2Z5LraW6E8UTQd&O3*nz` z_Yia8pKV9Of4?2usxxP8d@IE}`?c0)|4Y_h0=<5YU3p2FSK(ZtpYw$Cy~C_rQg+lE z0-6tpIR6Tl!@@WmUf(`T;AA+g=vUqpNLtXAYf4dz%&?-L_z}CBepbGqY)vY4R&`xN74>= zv**S}6Bgkz$Xts1c#h(h$rGNbp`#}~6UPW$Dvn}dmJ9KcmdD~eAA>l|KjgX~rO=b( z(A+{H^r}kC(XXk*?6rRqm{xXvqa{<1LV^W)Ch`KT`z4{>aMTA^65Q{erb(Qw3fiJhGn zE-!Ra18gE5@8si`;-b)*XYUMl%fdXHkAb-b$q>Y1c4>Jh7~VxZsD~zr(zUyiO&g4L zQTvYeUGr`89bKxT>t6E%=3rWHO}0M)CgdlL^5oX-4*|J_Le8`Uy&dU2$B*Oa2&ovY_DgFFg;k<^}oW5&l61}e)ck3i=bUqyS$0hApcybQ= zB|QzjKq!JGwIK3zTj)yUcjSFIB?9mYBnVu5DYTN2tQ%QVzsa%GJM^{)1oj2Nxnv%ildV4!G5P190)Xp$Uqqq4F6~4-$MM86W%O zAK&`bC!h5Pr+)YPxgVS>*~I+?32yL6c4VVJYY%jhPdG&{2^pR=LHL7dA3Y*>(6eAg zvTw_hlI0Z~iTfexvXM|UBuXY62*i0&9%W^ZC`!&Ko!)HcC7*)qS30Ffg3g)K?nL;c zf}kT`ki+WMBrF}RERI}?Utz=XP)tme!&sP&&qp}SvvVQQKbs|#6%u&J%~{-8h%flE zdfY{l9Wpx!f|w7oFpICw@seH;LxDgH3xZV4`J!`>-FWh4E9rcE)CUX8kGfqIN;XsZ7@}SFKOCILuT$)(H*!li*a^AY3yXh zbw}VLw9)&G72%SWr;#OgkRaWx65{;By6^n<^oQTO|M_=sljqL{g8$lb;pSg``Hus^ zmk%4(-<$jSKu{+A_GWvPD|2>$$MEaqmuYPi@!W^s+$akGJVTzs*+ z{h66Odm}$Y_=l+`Sxh5YgBXglg2SqX3W5toCEXk!it~_;BwYaNJ5mACGb+by-Ultw zNfbjygchF6K=AiS+$3Hf2>$N<#~+PdI;BL|KMlZP)cdlfrXW=dafq1QX zyH)%cXxnsq5@Tf2PmV$ZMF`3J^vRcPqDH|2Dv|KhCtq@`sFW{4Qn)34G>k=;&lTXI zWF$2-t*uBU9S8;Id0P0oLbR;*;|_zt<(i-+r2sN9e8rWkg>y_U?DNmM#84Ew#JJ1% z3KVv(=)9QKBnb0ZaKReJK$SR)vzA;T!6nQ>waB@Aem*V;AOgktTm-wITJ~e11K8i! zNQMwp`=pE(v1G{Y0i;z@sjz{VWbzeM?Cv7I7-layPEAif&yI|bOtB}1&L5xfj15Va z?u+^w7e+pJj*{F(w51Gx|Rk?Y)*gPXosw=5H4{X{&{v1#XHqk*gF~25s11|zg>$VMS z-mtk>t!*2oT^sh6EgL8~Ay9hu9fLUV43a=$CkH*FL)r7cIw0v-HtGw-SQaiat$aq{ z;Oi#+IEKl0T@md8(J_U29ymv$b5QGG^7Sbd(GNr*ngAd~R$Q3FAYVj-PBP^E2e3HH zBEF_7-@W0O>g^mGODFvxU3}`Pkw01NQb_($%QIpjrpEN8he`Z=)C|Y81ImHMtFD24h%&V0q!eUF!JeK?f-u}72*_3sd%L=G9@2i^!|ZTy zYT834MdxCJK$VAzCtDEWLym`PFpZN5dq&yg6GNvZ^D)oZsgdzvc7miv$uc=La{T!C z&}4Rf?D?@`U@HU`3Z!Hcic;#MRT(K7HUl7UQaFe-tt9zovcEQxxRnV~T_uHxqHjA0 zI_-s=XFVg6Q=1{r_=(XWsje`=D3+!srUy^36L5)0l{?YTK0h=xDivn6oRd={qf?S~ z1g^7?FEU9kG&uVFGzrKTNpprY7f5rKG&7_jHw{m+70*H=6$K)3pGb=YN#%pPQBW>w zNrh>_EvW=rDoaJQz>!oja&|}`PKjCmUqQz!A$$eH|BABjlETB?kv-!+V$Ud3Xe!|^ z5jXq<&ZdZL8WMPpP)I72v=QHY%s_{ ziuqE9RPIkv*JA^+m8KcZ8_n07-`IP7?+yOul{c^4eC^HGmaeW@TkbU7YguESd!lA+ zOj3J8^8_&}CZk0<%Nws=fA#g(R#92`l44o{(|Iq*hrL&)k}MdxnfH83Qy+XWpH; zogJ5I%9&MEk}+s3Ows_evOkw*oQnxCYiE+?j80Pp^fI86A<#?v(@e!uJJ3skVEd0S ze4~ZOI%Ka;m)584&a}1UX>ra@W4g342U1(adJZiGRt3!~aAXx^Ovu)pG^K4NN%KQ{ z-Ez&Hm%qO&W$#K_KyaDza}KZsL7tbu7-LtPRW1^xV6UvU)(J!?|Qm zIrgl!97s71e0V737*1Mu26vEtRMMVdikuqo&s?20l%=gT>sEKl>P{Og){U(xW9w6= zQJz)>D(_C0o=%%;(}v2lb=SIeZ_2tiZLD55wx^8kPmNl-u8%F0@8Gxc*^sj{U3xfe zs#-doGSz|cC40(HmGj_^CuMEVxpC)o%Gd!moUqAuut8)8|5LIy)Nm-nFfeB=GTYMz zGudWyaRVr>d+b29&*X?UN^}6R7KoKVtl6I<5=ro<1}GJoGE~;Ncz)g4k#csVwG|KH zAl2=DSX-a2s7dc?d|2O()G<`SXukH-fq@p8j;1ZxbMb^r*3B%D05^w zP)*;`v+LEpsp{Ue){!>b2|4?Mc%mjBmmx=8+Oa#`-Sd~8l>T_y)P1je)!0Ygy{pCp zP{%pon8Hfcy3>xvw4rWUwW@C*E9O}>)}>9Z)dt|}13r0j6{uwILrrN~Q%--KX^k^u z)PjE&w5#E%U0=c^tzfje_o1dHt!YSWYBR+ev*xK~E$@Dy?^$JfR`oq;ZSfni z>#<}#>0G{k89D~zx}i2@s7>xk>kT(*SDCUKwVxPmNnd)`?q%QYt9R;eUt85wF4cde zsd=nMCVR$=47)R>NMpK@_()y(DdYMz)A9?({VCJ=$Y4v^o7e2_HA5@tW30)hH`=ea zFBaW0ylq&wHKc3}YqrJ*%$`S9XR54at*mv;+Lmn1Ga7DL-?pyXnp3vsHJkeZ)B4C# z_OY$uUhxClfsd=Z*Q)oW2B+5s&!+6>o))RC+N2xugweLBUv7C|Xd@$YCA9<^e_HfnAmo3=@yF#6_~wC4K ztml-^MCN$Rn$GdnMt?vr5f7EZ4je0k2Yih^FR+Y-? z?rF2N-S_+-5dvZ>YjE>QVbwm1Y{g9#CpvQMZ*w}5PPgA#6L?rt}Iw}m}J-fk}qXM}Ty^1AbA*dERw zD(EhtVMn-d$k*+o;mojqsHnSWsJOd$sHD4OsI7nbcq^TZA^|IMYYIhZr&*Z+R@2+MFm^^r{yJe^= z{3Yh8Gl&^XFdFO)2O|;D+!tcQOnaSPbe!hcX!O?EU{9EZS9A2%C>w!*>r`+!*w1nY zgAtZF6FxkA9U{FeA`kWT1LmwV?2RDD$f?TfS(Y2Zbimrx%MK%Ydsk>^EF6r6Muzci zlnV_<1EJx*5qNWT^z}u0IhGwh!wxen$8vQhu`n{$6Xk-v(Lf&;9AX0y9**>la6=K% zer+rm4FpF=!?#2m4FhU1XDAruLN@_?WGH~KfZ5r~_KfwP4@Jg;;m|nXHP#t_A$8=- z7CG>uh0DcmGSUalt0Q;T(KN z*2tPZ)GOb$+sv9+3v2B&6hcnc#u)a%a|KjJ8kA8J%AN*g(u8uPL75SXv2bmCiv zj>+lHV=YY1xRv&t3uABP)Val?qm5mI!6?gV4pA5b4BHnR3&S{q!8F3%5;I4~B3A<( z9k>xOD-vZ{PJMOLaTdaHs>bDUY(I=$E;LTZuM*!6H2_zzU$pn$>J7sX4Gpn?{VGhH z0EeIvF|&uoaTgedAsi7Mkub}R$}ibc!J*+$G%z|6k+HP{HZC~G27<%=VfKia!CnuB z14APW+b%k3cw{U(Iu;EuA+9c8v;`=(KtQwy5NZsn5fC#2I03>~Xdu9uppv4A3H3%f z9|nz)D97O%D>?&E%L;h6vZ& zFfn}c>gAnS06W={MpuBPn(yvp5_e&vsJT^xl?>;Bk?uG4X0n^6;7_7V&jIwpOsWD zv@VtGq~)EPal{Qfe^yk!P_vl4k;O8)vQ@TDqll5L^ zw~K*=g~?#u1M4(Op;-D%jPpHHx0lIT1uBEh9oV=sH{^3czI?_FWfm|Vwvg7xM9cO< z**>O%$z26!vi=f$-g{ZyekLC<6)^?i7c+(6moPr?OBp}-WlRzHL zKie*5aQ!`jAjbu7)fvT1j&7H9b%==BLt_IYy(59^EEj>*OU#6gc~pHf_V)FQ7KXhZ z>SaYoN=DHFWWX3V+>6VE;*idA!2hdk_^|hMF?~!IP||cU-GCZS1_lhtuoAl646w@X zs6@i^%`qrDr*@7241CEkSVOf2At)iBmP_B%)o-7*+72co zX4YJADD|^yDy4iNXPtsM`DE(dtHSDBXuo_@;u{$AG zhZ>~@QcBScbgI#6AenwkSC`k$;a2ci&*7*5PSrL%HiSfKgxdiTq8W+xu#{*NZ9@Ze ze-3jkm}o*yT-1%{Aa`AMIJvotw$t3mD9c4}iRSac@E9xV!{d4FV?#YGS2og@N(V&7 z-6&_Q5#U3{bC&@r(W03f=^2TN_9S8c*gzW~ zni&)eaLw3KBW;4*$O!WFlr08+MIuA`Mk94~QnCpvAi4sP&`t+%LM>(NF{iLLkfz z_eZbNw(dfNuE0oNUpO=jETSyz;Ic8^ay`V}fF809vC$wMC0W3CF#+J2P#O~ffe6rP zqsn{Wx+>I&)LBL7@NggPd)iEn7G)U(60jd+K2enMh&H9b2oQC;GRbkcJR=)Pn;96d zSd}=fS75e5j>sRv9GKL-%+cj|CR^V-masbBzj^!S`xCb(cxxFjHupj^p*hnd+e6!< ztcO|gy!uJz{j0aHzBiO8E}3jyGWim2FJwHC@Z`edhWP-En8f-Mx4AzO?Gh zzOQs9v*jzu>zv}=Yk!`Z|J_rQhZF9q$-_|5!pgY|e8Hy4!%L>zL`nJN;rCD6J~3@4 zCV#?TBEMOQsUTs>c>mPxQ@m%x{MnDLJifxacQ2XtsIfKk-jDo`{k(hYl4+Y7TRj(j zbo1fOM-vYx(o)vVn?75$-8$hnRY!lWlxO>&QjtmjXTSi zOcfXb4R`^c;p)s)A+L_))j_v(BwV@od^5h;qr_E(y^G&zU}j*h2>4%`Zn6ocJYvdQ zG8MeE=*r6ROBFpYseCYae=uRmct3PIBv=ZFrC=`a!?BOXACG@!G?YS*FPZ!RUr>o5 zZ({py=?hyOf;FF5^JjNIv#xvYsz_wJ?_RxgRmd(S*`;%5A6pVt&OrYx`EVGNI;id=%Xt8Frcyy5)AsVji^jOGIjtk zr#pbelG7Xr+hlfP0N1cI2YGNwNOMdo3!>KP8`V>Tbp0LGQ|*zIdTJ0CtF+h{m3s-u z50uNMIqf<#rVq^@;HCf$NCw2Nkpc13ss0P1{K2XQlKUxjc@%)Wykopx+ zCYk2mXdI|x{+rZ5yM&anQIS?9c!hCjO<~5>QU6WKPbul^%>#LzfO+6iYM4?!5dP^d zxB{rJTD=W1 zJ!1#OqvJaUN$msQLKUaP3@g%*`=pNMb1=G;d`Z?7^DR$P+0~SOQpcQvlkQv^Jq=$B zd6zUdfK+6fV^Dyhv7^Cg&s0|3|IgO|SkZsvK6rx~Fj;?Og0ZHqm%zGaw{s|+_$5+4 zzr?Ne2rzZX_e;D{7RO#23vp~D(9Z>#U*g&U>B!|uKtm83Ux&O2up$i{+lP89wt=eC zBGDj6^;m46r+#?!)_BbdIN+H;6P6BG|4W?VbrQeHVG+Pm=ONCfvC$RH)ul>#G1S)LF>8Byf!xXVzwXo<4Jkr7T}rcekebq5$^wKVpSl1Ee7JeuwGFR%+713$Ha^Fdv zE%U#KM2hE-ITr0G^ynF3LZH_4!1Gq1haC^W_mw$i7&t7#dD3v>h!MRS^@q^6!nG&ZVRPQ3yyPh=v`RzzfoeygqnS%X5W+OpTF~ycb0rz zg0GwSy7@~1;nG!d=_)@k825!SSN`+Dk_Z0#ekkk8!z&91ethoJ?oYb;(`SX#m&oZ$ z{NU(P;WY%zdtO@g;GO&L%r}3u_wnAv@_)DW&v*S~7k|E6I3FbEgZ#DYOQkmuu;{tJ z?7{B)yP+Py6fOjRJof4MC*%B?^TL@c z4_{j<xw0N-Ei#`Delob!d0yzeL^^3xifB^`pS$wz`DgsIXCGX- zf8|fBf4KhR^-s?J`K6y+;@@VMTz$x4e&H(>eCvsC{fA8-ZGXI--*PFw;d0z}Mew~% zd~fqXRtSbkFdX*{3BD-tMftH?f2Y%3*B>(cN_xE1=?)u?7~pZ-&}IOovj2pEhwk>5 z3H}Ylzv07!A02&slyANg-}tS#KOp#fh`)#L?H787N$+snKO*?A6aRJo#yf!brv5PE z#RnifYB+|E6NZzBx9Ak$El8AYSva)lUMyd9FB~Ohd$2o-6XjbMdKa4(4=gq<^po;^ z7+I1iZ(itHEdQip;Q}e&D@9doTZk+k_~hWi4N}p9QKes0)I8eoaKk+F(a_@|iP07I zbdo)teAik2N>I3Rm0Y>X-<(*g`1bQ8nN9JU}g|t&l*7G{@-s^$aSfWPa{4JYjE=-*mbO5TR>w)LOVq zjJY2u!xh+MT35!FRs(z;P*B4g34nabU-S~Cq?{W`|_Z?xKp@;2(eCN`TUHkGGd z6`Mw{LH}w{gIZj*M$f8k{VkNJ(!NTc0H?I_TdF()EinnIh6YrHiyDO%2zJepp{i*o z^gsnU+Q)76>jQLuK8x0I)J|rQy#+e-QJ{WU?m8P7;lL10^oH3!urwPU1hX1$BpeQa z;C4{VKXG?u+T9&qD z*;})i2ZR9F(?z+l-a)CN10!Hy2Q_7b+=}SB97(nf((++O*~GIYdWCM~Kod#3d{!R| zl5r&t=nc#e2}NPkcS=o7Cc2X?Mu-3)5j`pGQ6eS6sti7xXqT;{u)K1N$dC-AFsoEK z+KhY@u2bz(R82Ug27ZQbXh(dUL!|Nyq$ghwpe*hUG7l9@Q>Y=#&LEwHsDBeMe zcRXo(vWI`0;fr^~i&>y3a6!+XbtQ_*9<<$W6N;KhQ4_!KEPsB8-#ZK&$$mqtfiG%; zy`=cC0i@0dpEs*dlvF$zxj!P5>>wpOo}75H7j`|qWJkQD4@AcZQ!rbeD6f9xd*~C& zw~_K~&kjrgwhlv=fiK?%*ecEfHYzmt&vqs%);(JPaJ^8mom6c9Ql~ovTO~lBHC#0C z725&2@-2WaMZ86`!9-#41K)k0P}o2U8y2fRt^K4{*wsOHbzr*+8{&n~t~`|dPIo2> ziXPbS+l7L9Qc%C>{j}hd0%7MVvhx(|J$ylZyr7quP*mSOeIQX;^Qh%vi%_|bRPN*3 zdU)m?{&);((QY_n;4Ak*EvmYp7G+qA;``TDZX_t4qZ`SO;LH7i{+qVIog zEpw7NmC!e?rE1xcR!fZ~{%h7!Q@r#I`%>LD(rTHfv_JX$nzhu}M80ufs#{H3EmdL5 zYPE!|#7GUPBfUY`A9`+qPI26N>3I8*jw@w#*}$ZYf$Eg&h5Q_TFlpgKw4z7KBHN%! zS!Gs9ic{2ll8O^$I||d3mX205V&Mv-RW(0ulcF08SY@kKB|%v{GXeo>7#@%p(MgH@ z!b-2jel1f?J^+c24KddWFJ9QrO5BH1Wzal07 zc2c;VFWA95samaMgPOmZSU^RmEQ{yJ&U27o)_kSp-%JWO^97B(v*~lw7P{<-nW!-i z!nTQ2Mz2coLYJwF;1A#hmOzxsPzTd36;6(2z^GhVsBY>b`LHm)E{7qRW5N(KOak>2 zGYqK9sf-vNG1WwU!Wc6mTncU@SdpqFg9W3hU9<#1nGcruX6caw!XH$jUuA=6(i^w| z#)(#l2u7ov=u|pD)?|xDNU1Z?y_@^DP&oH7c%q4-LK+@q0Fn?u^rj+}%eXOKzxv*; z1^?dz$_R=RU~p!7rdtJXE%DaQ*DkciGj~4A4E^}Rle5C%x5(kQ_)D;rU5)PzfzsES zIXRKYawQx&cX!^|Iej_qsF-X8E%#&$VxK$Q)6IM#D0t(JCf?K}eQ65_f57l^**8YY zpKf&j0HLqLF@d3w)_2n6k`$S?^3>zzr?Dw&@;~aA2N^ZUni*M2%@_w^3~FzPfL&uV z)8a3b8nx=Kd>8mK*hp{cHF*xxI4r}xhJd;~LmAL@>FzZqLYhWO1FB>^i7iQ7KsmUC zrnv!>`=z;Mc{Dblf%3Er@Cw)~R>p#UQl7bb-9P_*6F@<)RIP+#lwMJ{bqcv|4M>lY zMjKT-0)s8Ra|uxsURN<#;)kW1ZiSILEzCut6H zsoxKB?Mh3A*8voHr#Ypju-fUg%=ENc2ax;I(7>zmC1bEgrp}*nn7tDqe;+uwGS6>- z-6ZLdD0VYoZ~bjx2je^407`Y!+}E#X(uh#g2vF%Gi>wJ#-sJ;Pigbv!&KfhvOyf|a zn1%XQ>gxjP=!ZIQPOC7yYMJ*jwhvYPpO$$aV^_bTE>Nf7jGj|MP|_JCoO~*I7{efZ z1+D|2E+kWx(As(FO0CWnB@eXXo2*%HCG#sGWt^|NBWP=H_bd66&(t-G>HxcwX-a6| zvJy%@uj{xP=mRb2KoUv`QOx_5x8xIai!wz-Tk>`?MG0w*RVM4hY~?G#I)RpM)IoiE zlyLG1u_u#nN(f?`m2mP2u{)A)N(f>ZC7gWrnSo|oDmCwCg-O`zo&6)jQ)Ih-3DesV5` z-X(;{!Ghs18>^_vL=RGdp<(HqokB2DQ&50{(=Q;MqsEw`T>%6;(;O)}B_;!p;sxNO z9LTBUL?wcI2TL~hf|VxAJ;thBL=S0j?nCr&&PbX-dw8Tj1iT#x1JTG854xhvNVE=* z45O_l7?55YW7%<51kEW{j2*sC#a>2G)xm*0DKf;|6s>S@6|;+3>~%QFqkbD_@Om%= zN8f;b#IFhjwREc01Lm>#z3DMTxf{3+tXMmEbzsSf;YIYm4<0fZRQCw|$o7d&s&h-J zaE_#JONq%&;RvxwxhpwHh>jo;iR3ssi81^PV#Z5Wr@-->X5!laWD9W} zK{e9zEFie=?U>mic-9fmy1&Y*nbHGqCRmD|S&HWLz}$TfMuHfDhJ%7-9kHxiveY8` zHgJ1Duv8FB#ge7!g~X1&KXiNOy%9ArpIGu|n;*2?Z{aQZpIbJ(%+h7ozI0R5LjN<9 zfA$)PvOWhhPqZ&Ad}b<~?F4z;=U`Tf5k=2TSOZfzXiOvsiTefH#5|vnyNTp(5^}ea z+^zB4?I=D2@u5)EOp2O?q83s_wM2ypf2rW#Li}3<|6by!%A|r9o_xVmO+3|tX9MwU zhXOx#Y_q zUykPUZ>?>K)lNb-Ux zkVMI4$^8;pa+ynol`TTyQBrvHsUco?0wo7|&#m^UX2DTR9K~^K$(-w%wF1BD$nBA7 zMu794d9_QHI`uotg8<*DOd^k1@|G+GFYqfCfZ?uSDuWL_*ZSz>!;`$JY{}I4!s?nn zE5M1;+!`TwJApszcE~Q7zzUWUVkwzxTWJ2_o{#tNmXamQe)`eya|xa;#Ixl|Gw<0F z_Z));oy4&hz8#k9N3oz#ga&-X;=_c&+k-D1mQ2e_oyBZfDK%A@$s*W9R%IsTjS}Xd z>aq8idtV#Q2a%xCgc6!c!1ia-r~Xh{-CAeXdCXH>Xj0MIWJ8=rFjBfN3{ z6L>&3WKhitRin75Hi(|6P@=&~L)rqZ4BOuX#z5Q{=0%b~I_{111MF&eGJH#uXo9VO z7>{j$happMI5aByCAvT=2rx>A>Lm(6%orP%H&5=r#TuEVE$tVGHyeywq5(CZn4LTj z59jI`F+-BIn=7A<&IBggpXd7SU75M^{jHDoJlvzmDi4v(hn}|n^u%W; zmI90r2$Mi~sd6ZuI}Gq(KLO_We{}f4@%zW;hT}zBe{B9VYCj znLPeHC-mh z=3ncK@=7AwlP5-&iKuX1;s2|v;HBweKwyKqn51}+lHb_rX)X;yT}P5&sSvl|^i(^D z03SiJ=$0wiWc4CqQSx|;bn%BgcBq(9EJ#UJ2p?)9B^w=aUjbh3zXMOUDUyZSU;&~V z-?;iPSXq(3M;sf$`+HTgT{1hqWLi&m`f6gTo*RIDg1(gl4KTO&>0>|GSHX|3Ljz7X(ei2##pA^o~qtN8D>L=A(~-*Fo? z_MhKniByBA3h|PkGp64IFYvuI_OG=>oghwyFERmFJivKRisPSWm~>PhB^rTgp$8P} zGU@!{L@0tXI>6W=mqJ+x?k_Mljp>PD-233cVKh8~7*ST`f512s-qs@;fh~}xma_E9 zlM9O(uUdU_CAp#h3m_wZ3tnlyF_B*|Wt=+t&oX~y z)a5j;r0Y;QNLPjb75uM5*GY34L>ns22V)74r8`Y^61{^6@?y!T#9MYzJk&dIu87Y{GE25?iCy#Q6>mcSE@!Jdd@i{_+8W@uhVZPjmZ-$S4q2&4+P`hYFqZeo;snQ?sp6M!m& z?JYP2GE=43BpZ5V7ZWp7YkUwSDlmA?oVHfXT81-?v*7*`iXC6S#`cyb$;<)`U*uiz zfN`jN-ruBRke@W9<~Wp+epnp#Oy zYrNnvwmuh5Aqyq-q@;cxES`mZ=gGeF@sbOeZ)OyREt_0<`Qtke@KyQ}JLS7WKcpwYd-9v_fj5xzd{YcC7AOYeH^uLzBCW(CYzq_e48Gk3CF%xzY4tMd(V zYtU?3_v2_$PE^_crumDM8YiCYmP^qLQpOk0Ss%KRa zMhT@H)3tq#+Nr911&=yMepj{2h^n+%%ofXlJB8FE+IS>o!ajk=Teaj78t^sb>70k1Bfs)D&VghhAi=}d`k z1O+at#-($ZXb(oU`LYbl z;*;?7_=R-IkqfqaF(*kL(^Hr?WU86N112d{^L4D}KgAa1B$MPEcnSt6?%`Wh#slIZ z79-5Gfp8-;UW%I%mK)@dRL62cvVw4SLpfJ^Lv+xvR19d!u}&bI4?qG9D+y|jp41#Y z5#({H5MTBrO*7oFts8GzgW_I=+%JZFk)H$o29$S6g;27Ylx!ACwvm!;@e+{ZAiW4T z0Cn8yh-Vg09)95l-LK%^Nc&rfMjDU(h3n~or(V+7 z&Ucd+TS`-)op@5ZpL4pq~MXLP&NCTZwDyp9B}ZKf1OsKwJj|*D>Nc_S6MR zpSI811kh-o4D!89TpB5P5~6j)y-sj%C+_V((l0junQPHb+=m2r8*#TiZG6i7bl|fA zp}m*1_wsCC+}%HU;(5vTMf;NzJRHEiay4EO!m~qd(+;pm7Oa)TS}9m-h_zzu2l&||&umGf*(`!jTVZSy3e42w; z4LK$5Ka3t_RMAulQl$rzNEm~B|DcA{4pS5v*Eo~{2G+Icf8ClWI_0mEkM6vywOFgJ zAlnE08ZE?DfC(|wA7rr5&yv%keZacF%&7OdA}0)HaF%vSGKE4Zs-&PYs$VF0z_95u zghexmV&GQ*VhBY|(u`pLCf&?;5#vxRD<5Fts&KhSeq6Co@az8iA+cv~V$Xj0@Hd{~zf9n7&y`nZLtQ|x>JX|P ztiQh=^on3U1e%2GN|IeU$24{tH&@`81UvWZ!u?Q-FsMDG>z?K`3 z>5G9d0;dkJ6Z9D~j6)t}5aVeR7-%Y6CnJOhnp>TwitjrfY361DJCbXhoFM5jd6zy*vl$pcn`LqWJw0uHZ8q)ti%z>co4G|0m%AnET0 zDETPZ_~tbT7R?+-pgs|#TY+3jp-ZT#R}IOl`GJ1Eao$AoHcfUs&-33KoPmq@98)6% zH2s#`Su=fROdY;GINd=km9MO(e9L6VOS>+A9b7b6`k0zlHGjJ2lRYHw=w!zWhmSY; zR*n!{4Cx5@E2z$@LNTpTLDkB1%m9jXr?vDlRPprs@uM{=w3Z~s2-il-HDkMm6G@FsGEFrEo&giH6cx=?4%p07S}Z z8Vk%8YrB}AR4_>9F^cfJAVt5hEQP8UI5UijU*deH%vGI3y0J9_V(Eg-p|o4#R`3`g z2S1$y`eMA$RJtw)W&PYO@an7*)8V1@P*eu8p1~{CAuTq087N$vvmB~N%uq7)aPSj7 zx|NrkdMF=9PqM3^P65mFC(-Go33ipwK1?PiZ z_jd{KQxg77eEl)vKMuEv;Dm+yHTm*p0Z6X$okDpFDQ|glW-0I3(?*ha5=?;di|&oj zjPn)Sgo@pyVz*FnfK(iK!Y$>uJ?$jbdT9Sg=QFn(Uy%}fqxsD3LGXQ;n@(rk#+B*pTQ)~!)ulYE{mbX; z^2FtnyFYz$_a`TJ|C>+l{;xN=`!7SoB0quVz~ol$Vc0^-xA4vFq`ZTNtA3O_7nO=C+z47$^;u}tHT(jxE{sIDDXJ6N1k0c5Zb{-8T3DWd%Qz2F8Q))nKqG`|% znv;*3B||wH`|GlD;7qebRU#b;JDfqH2Vqzqf#xQ~XWSp+>rwFFj&nS#LZwsOA47zE z5LvX*1s~!?yL_<;hC!(wi6Cu+k|`>d61~aYLVVRTKXEOz3eCqz^RcHDAU3Gr8&49~DUey% zz0)^|y>#+W!jV1MwsO^18G2R)<`v#l+4{bIY`ttK(Ai ziLWnJ(E$Q3S+lo#sYV!5^yh%6&9=rse^b(N0!g$t&95=gsh&tfJD^#<7BNXD5q=@G z@t5^Hss&53%kkGMH0!BBUNY)UDO3yfrnS@1BFQ7?cU3!E6nRwYVph=fm}3^@DqgDR z$(XRmGS)g|33?#=n(3LKzb#hENIn%NdRou@i{x*b43s4ylo0fKg%VCaA+{>{ri7Aq zWfNJTd0wMj?)%7iW8(pn#@vAmXXVo#lqt7Rc@^M92~kRY*C{0@pD}FzeH?0Cp)g2U0h#n(B%+C~2|mSdLm-2OJ%h zXM4;cx3f`&bjDl?q}rxTxlP8`_A}k=pjKM_43}m|_5`$_v*dm@zEM9@ylMSRbFR{d z+4Or$TkmT9EdBb81cUwDr1U}Z8Iw}6pPQ4hN=Qy!+0RmKm-n;O&h&n!kSYBvwaF2K zxzvZ-cTTEwa0CjL6nJ|vM9;2lMiLa)rKCPjKHc(J3!G`Oz%OfnfwO$N_Hj#!uFsz$ z9GXA`N6Iu0@e5KN*D;P(~;hhk|1EpYlMm1~P zKf$E$U@ZLnBfS*0cMR^76LU}z#&W}8>KC9FopHa1kne$42Uql97^PF(I`nG66SG3Y zDd!)!I!suP2@YzmCu`XKwh z6A0Thp&2ORss!H#;@j|p1M}DBTZylcxEmK(v>59k`#PSTrWbFXK1=qUBiqgq*LlAC zTS7NOx*7g0hPW6>nfDvx@RrQ^h<3s-!x84>v>ZipT9n_ z{LHDhM(}PT-c50D!{n)i)hk#Fh_&Ezs}IG`C+?h>H9g3>p9OY*gO3N7GIvesUpR8_ z?z^*Z$>C2FSIq4q#r3G&J4hN2KH2s3+LOJcv6B>c^5-s+qKi`prUveudM@8LtDj&0 zI18>&Jrpn6FQvTjl?c8j_z_Ni%L(E;`4q0-y*PD3Juk~QpCbPDr{l!mJ$3R$L6K0< zL<*V~c9DX8Q^(;5sZiKV3Y!3G%*rntC7SOMf{(HC}Ae{*d0{p1Yt~3duq5A#+c2Ak!fD%-vNnvZS$SPfo%(O&r zY0>cx6|owi_F5WzgqChD#a?R_8$8YWzA2aS+p8Ulu}w(bL{-iOY_@0&@M-j^h`doA zHDQga=Ppz@5Xd721#ZF?%~v`trv9A80D`1B<*TmA2V_K5Y8<{iYMauWqVt6_Tl;l7 z)yhmiP@V<|ufVJ5HMs(ZG#0>OZOwH2x!;%r?^29bDcFH`Kp_w~xNPd6N%uAC3)FT> zeQP8vss8Kg%UFPlm232&O`*e~9`$%y4eYiuWCh+JV5KCUXoFiY;3o)}K%H5%0cAoD zeu7+3{*h|9IRk&>9WGPBI~PSW+?~OOMMv;vD1zrGM&J^E8!QEnO%j<%X+@Q4d~6ae z@UwM5g29j8!cBE!aNh>a#Nls2)p@xdct|83B?S*)t;#Wu(t~s*`==O3i9(zWoC`f9 zd*Ecl=$L4h?&1gXXl#gup8#jMIZVsIpP-{ZV=v0bLT`$>f!4!a$Il#Y4IDXh;MC#3 zp^lRsXSzf?f<{JB|1Dc+NvByaVKq=v$(=`U19}_5TSmp*5SLMLH$<1xo7D6QI&nzU zheRV>NwSL36&`pU-?oNsL_MXuk}UgUs7~aE;K3T)xK(Iu2Q!{{<7r~5pFBCeX|DK1 z3he+FC8W|V+ldDaVdxd>i)P~4_oHi1ynhy4yh%LA1WyO?bnrmADOz*kOg}jT7Av8+ zXJB$5;Vrn=G1DPk>nC`367S9*l>-UnT7-N44hh~i;zgU9RN@J&alGv0U?R74E{o*W z|EVuw&z&A3_Ns(EZ+e2*t6!OoO}1a@jCr=n@K-ine(CHx^Lt6@&c(e?i@{_j4mWHJ zJ@>7j-y8Ss6nuM$4+1TM?=bNlj)OkPlKIl*2+>?H|L z>6~3CYa(S$3tRZIZP4_DZ^L|>uyG&RxNkASZ#+nRhY~rSgxfc}Wwx5nDSMu`;iW_8 zDE+n0;;_Fo>T(;t%F*SPXq8Y3V!wo|cy>TCd|UAFC2*6}JNtm@H%OG&GFy@-f zw34fUWf@n3GzVLdUh6Uo+-s1%XEHvtCl^H^Es)PlbH8C34kmM18Rj+1sQgCl$YQcn z%b*LpQ_~>#v7w$Socg6k{>K&$6{{eEPp;GIV$D@U5{HZ)q$gvOp&CpNgD2*saER1; zqt2c*?UlIoB`7h46s3zg<>hZ6mN6_vQcX*%E#46&Re+X)=eW3=;H^+DbN>Y(*W_T6 z3#Oucrd9kAV#8{{V(Bdqyz7W}o#5R_yc^@*df>BM^1TxHOO0TISo&}DbB%K*;;sf$ z!2BX1zlP-3%x8hwp=*I5wYwLu{6*u_;L~QZ_jDYTOK_(TEU>#~c1eqD6Y(^GD%g}W zog0TM%$wnYaCj!HuJ^xv``b&_{DdV(x}rDX^v}Z0=Jf>roDHCwNMysWHu_+-uoiu# zGh1yhGj%r4`|sXsZGBh?Y(mNtL0uamL8B2gM z+<;rj;A8ZK`bDc8&GjHIJpRgUKo7-`TnN2x^rGnXqKC2{(Q;P$iJ%{13@%w>&Uv;M zep|N-Ooh(C&$M#G+-H~u=3qJwNmmYxD|OOQ{J+Ey8aGQKsqGj#h~9DZI?$tZ2!kQq zkLehLAEK2mYVKv>X9xnoN74H&07kc8vX|b1py*)(B0C`E@4ut_nW6fHJ8yE+i`>H5 z-MqIZkyj*LjFj-^-8(*WT<}&CZ}qRNrfkz>Em#fNs^W%9_{gP|zsfkIH&(yMsD5e2 zz)OoRE9a$^25mZb?sRkph6R0vA(8ni3xi*_848S+FPm+~F2k?AZe#PyTB~u(eB`AL z9t&f5Th$YWY;Svo?=RgAM$2sOgQELIFLe-{uNKztC+qjW!j~_5^chCW%RHm8e!k_U z4jzk9xXZ*?|0{d8v2f1(DDz?Fe3X=Jd8vbUOoPXlW!c86FUv}dM+{%?*ISLQFWq*d z|H}fKvEWN@26%ZUqw~wm4aRL>cI#=I{6_mi;FS&@FL(KkwuREagRo5+aoCl;sjM9C zQgk=OMGJRB&^^-6O?NGIe)_yy##*1F~Hai)!8a z3x<>yunm&;*$D&u;Jty0m968j-^Hk{9PB4@EVZ07P~U_;))C5(VD(B2sHeeXOqrlOvwXmqn7h(bNFE9gEz{`Cbz2rJi4HgEW zFzH|i_8{2uQ5kF(LZV#)@B>$aaM`+mi5C1}XDYcA%~VdwT|u}`gtHBW7zR!VkK%=X zkukVt2zZPLPT?}@q^Wck;n7RLf8^gmD*V(j7-Z(nn(vIvc}P~(WNpG=pV}5TxTpJw zp$MK|SR8nN*L3-G*KB3nQX*I?!5(k!;5^vx#VwmA%U|TUrVq@RXUp%zX8Vb=8WK7e zT#wIz)oJjt6NuH?gT#68N$1n@pPYZn{^VQ4*)_TTh1n^XbBQ^3I!Md~vj?7;ixZw= z>0-FKVBAv+^zZ9IUSxQ9gGZw8Qb#n59%)4X@6ZDBBV3r0tRrBndJQ&{Jbn$Dvp~HX z45{3;>QAvWtn_m_=ybYNsE@)h*8G{uB3R#aBV1534+qrx;B)I>orgmmo8>| zBV4b6qPUFQ=9C`srFAvd+zjgi^ju8ufPeMN>r--Re52%7087_oj~A`PpOUH}T!Yf> z(k;|pw?gmOsP91Eshh}* zWv*QUg(zRy8z0C~-1kyiRk0E-rnYJ#3wkC?>LW_0WWyR$t&t-+vYl(=27FHJ>6jg9 z38kN7+4we*vv%H?Q!|$f6!wzEzYAz8z=Jf)ggXV-I*l5;)7CIg%pLQjwA2mnnex2J zq+cTgsElPtMJa6&2HgF>|DXSdxxf4O3;W;)!8i--`l5A^Wk=yo70@fg^@by1%A?AR z>&s6;Scp&(@nK0c|aNot4{}ZrD^1{myf=dy{hD2*90{0EW4`15x+TaKr*c^n* zjNt-eDahquMx1z(L_OS;h~q#43`cM_k%FiP9IV{{?Zp* zd@O;(y^io3Xb570durfjV>tRsTMTzETW_%VV`!U|KqH7&DPaX?KP3VS?|_Nm>Ml`O z%YBL7I*08?4}Y09@?T&%0j}6;dw=Zq*gv^3UH*HqL}u3ABX^EWH%)9S?WRA9%QX!GOODv*`NRvDg{UY`LRPWZ3WK-pT!L{`7%=R-|NZn!Yw$9?vY& zNJcfL;kzB6yM$72+`ch&?e_R|=kL8c+w@?^{T*`$?(d#=eXj)(9lLXEx^vd>z;@p@ z=eqBhgYw`m;urSp$wOb*vhHTw$(XK~=4NVUx%uWL+vdpwUx58~)=buH@cX$!c_S%r zTquv1H!p5k%4w1AYLOEkl|C$8aK+2FELJY%?1Ln@9I|kx@Q=1W*m-~FT<0I|nXd%X z^0=#h!9!fllgGcXI|aLs*nP82aeE0^`jvy}w@|f}RBc`8iC2NS`L4-BQyGG-kk|_0 zW=I>{oCZHy`P%~e8>qAoM=_W9M8Rl$n(y!ac|XBM#AZy4$W20j1cF#DbtJGynB~sF3q|BMPSOV)Tj%! z*P9k{Nzt~rd;4M)aqpY5Dv53L5MLj6H!MK>wkhiiSHW!4OwpWmzT#mfac!Kk$QHtL zdGlu$y-RufrW^@Z{_O5~4;(fou1!;x7rrvE6Yy<>+u_>oIA5r?nsxgAE!6$bVAgWB zu10j=j-RsxKGs)gl{oE5X=n9xu?-VTpiO3@?5jlo^7k$SkZupiD|(O)d~>%lvDd>*u=4pX-W#uG{-_-TwceJMwc~@y~TtKi92?n7=ca^>ADeykBSP^t%!T zCwN`%-{|er%}e@%uM9c*i+a8ie$WpCuZ%iF;Y%9#Rn>q+Z{y1vzXEnjZ+q39< Ls%X?ZX$}7$Dzg+0 diff --git a/src/mjlab_husky/lerobot_numpy.py b/src/mjlab_husky/lerobot_numpy.py index 361e8eb..d43b87e 100644 --- a/src/mjlab_husky/lerobot_numpy.py +++ b/src/mjlab_husky/lerobot_numpy.py @@ -8,6 +8,20 @@ import numpy as np import torch +def import_lerobot_dataset_class(): + """Return ``LeRobotDataset`` or raise with install hints (optional ``lerobot`` extra).""" + try: + from lerobot.datasets.lerobot_dataset import LeRobotDataset # type: ignore + except Exception as e: # pragma: no cover + raise RuntimeError( + "未安装 `lerobot`（LeRobot v3 写数据需要）。\n" + "在项目目录执行：\n" + " uv pip install lerobot\n" + f"原始导入错误: {e}" + ) from e + return LeRobotDataset + + def batched_vector_to_numpy1d(x: Any, env_i: int = 0) -> np.ndarray: """Take env index `env_i`, flatten to 1D float32 (LeRobot parquet-friendly).""" if x is None: diff --git a/src/mjlab_husky/mujoco_gl.py b/src/mjlab_husky/mujoco_gl.py index 41b958c..b682dc1 100644 --- a/src/mjlab_husky/mujoco_gl.py +++ b/src/mjlab_husky/mujoco_gl.py @@ -7,14 +7,24 @@ import sys def ensure_mujoco_headless_gl() -> None: - """If no display server, default to EGL so ``mujoco.Renderer`` works on headless GPU hosts. + """If no display server, set GL backends *before* ``import mujoco`` / ``import OpenGL``. - Override explicitly with ``MUJOCO_GL`` (e.g. ``osmesa`` on CPU-only machines). + On headless Linux, **PyOpenGL** would still match ``sys.platform == "linux"`` to the + **GLX** plugin, so ``PLATFORM.GL`` stays ``None`` and you get errors like + ``'NoneType' has no attribute 'glGetError'`` even when ``MUJOCO_GL=osmesa``. + + We set ``PYOPENGL_PLATFORM`` to match ``MUJOCO_GL`` (``osmesa`` or ``egl``). """ if sys.platform != "linux": return if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): return - if os.environ.get("MUJOCO_GL"): - return - os.environ["MUJOCO_GL"] = "egl" + + if not os.environ.get("MUJOCO_GL"): + os.environ.setdefault("MUJOCO_GL", "osmesa") + + gl = (os.environ.get("MUJOCO_GL") or "osmesa").lower() + if gl == "osmesa": + os.environ.setdefault("PYOPENGL_PLATFORM", "osmesa") + elif gl == "egl": + os.environ.setdefault("PYOPENGL_PLATFORM", "egl") diff --git a/src/mjlab_husky/rl/__pycache__/__init__.cpython-312.pyc b/src/mjlab_husky/rl/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index e44b5e9beb14f1fa9423fb8317eb7686ab81c91d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 503 zcmZXR%}T>S5XX0uw2eY3h#v_00KGIXAY%Oh4^>MJD!GI-*`{kZyJ3^GGJM*8Jg?(waHxb0+!6$vi2)(PzZx}0B4gj7|gd)sPf<25C z8^DHVXf}aO&(v%ITb`x4!5T@^Yie$?R$_ZLMrEsOf2h%2Ty41-bJ=N-5x3x+$g^<_ z*`n%0N+fL9*Y5t+ zaBazTccbbg4=fn1K|@WMVvHB)e1Qh~TrQCNhVEq>YCl6Ul!pKS diff --git a/src/mjlab_husky/rl/__pycache__/config.cpython-312.pyc b/src/mjlab_husky/rl/__pycache__/config.cpython-312.pyc deleted file mode 100644 index f556d57b7ef7c0380a3bb1e5c42cb312c937e827..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4778 zcmcIoO>7&-6&;J4w?hiBiXK9a*)LB3%^OuDBy=qa~M} zT{@O>A<(37Z}KH!iXL=z9pv0|Z$(jn4s%$~r3nf&P=U4z6tqCmL*JWSk+PKny>tP6 zoHuXY%cenMX1=(kY#NYW%((tH{*d{rWos$cf| z(DoaFYETYVLvqL`Q9`8GBrWj1qy--kH~xXwjojyXL*VT@uS!5^p5Q~uaaqL|#x-n?~v@#gh>*{p5ql?GF7-K>=ku&*NF5lyw#vY}cQ z4{qwz(4emGx^7dZ8a(($qi)c$_b%ecn}c-pYcM~Os3iMfjmVp@DzaZAa)1VDNb|n{ z-UzG-ynk2=j(`QVA+NSij?gIB;jSGMcEq!zazBmJgcjQ*USxmQlYGJx_dE%CfDS^R zNzWdTQ^FoZdnZ*)^W06#xM{4{&6mrz$yS+e>*dwW%5HI0EKGjWWO;jw=GWJ6{d&!2Ks$+LglXA~<)Cz5 zIs%mb_2QCP?Ydf~OIwYq3d^V|*0yTX4O3;BUaKrwth`ix$51zvt%kL|v&4)gcXyZS zJ1h*H2cs^q%X;ZFGeStrM=k>nQ+K=oW+Y^%(a#trWQKnqC;*tXXe^ur1eaP z%sJ#uXJ)=lMq96T$h<>-=*(Sell0!3t>t}x>)Crshg@>VvNM0VO)`75wYr~fUA<>^ z$YqB-<77sf3wO&MGUAY{PG;IQrX4cFrjIA~?S%RVxc?_YA;E&)HV?^R8Uee{v%_+f z#o2XYC143b|V;aTfxUdtRKPe}KJ>c1ca#GkS&rSomFh0|@2Zf#W>>Q9S4gMj| z9+or0KjPV=a#r}qJbPTu(IIF*f%eX1af)ZUY92Qz9pFae=~|L*Ab_3$^i+YV%7$u}x2RS5_x-OwdwpYlg~z)AAcvHPRZXn}RVWa6 z0YgCw?@*10EeJz8R6c#}%iYYUe_!FjidwCzl}+*dZ3U>VRx9si&~LBsehPcrtnVmg zlWuy|@2~KvMlGFDP1!bB1pALy)jLXsshUs%p1iFZ4XPNXWr@f)c&-7;Qa1E@9in>P z$}d00=XcO0I+eQ0)an&}=2%atqH4F*nypr-Qm5?1+jBqYn&8=5ahMIkB-k(l?i4nP za28<_VH{x|p@=Ysa2`Q0dl5?|ge8Qh5H27*jj)U`f$$8%MFeCwn?lGV2p*ilQsEdd z!1@CKQg!x=iODv}?pYl&>5vP~=vbSi_EtM&?2r(D=rAOWPBs_!vVzK^&bf=t`Ma;R zGjk97=LMH%oWX2!_AdCd4jFfn>E=wc(IIIhvMx%sOPj$sxZX2>4xc7N2gD!oJoaPahILOf4*Bu}_!& z9^>&PA7ebOB0fh2-)Y8EI6yVx@x(8OgdOngARMRY_YP7b!QtBXzVu1>0rZ5AI9Ow} zA2q-|JiQ|{s0l65V!(Xtp{E218Uwqh1W94{lmL!ZEhY3MsHMGmWPtA&)#HDfW_n|S zr>6v2nghG11VbXSrv$^o7D@mK!=yYyN43da&~AvD!tp>K@;oK<;T6@QHygDYWuOiR zf~f(EQtbe@G9N^~TQ;p*V(-)1ZT&o~3r`@~L`H*|kM-EMz;p)xj`{&0-Gfw5^Z$z6 zl8S+~qK}8_rlFU2u;d=HAP_6ilD&kG??QJ=^Y9BG2pgLE;_le#DVL6&uS2ATO3;+{ z-}}w>XXDodEAIPvQlpz{!?2Z2anSFScoY+c3NssZ>i{Xnt7suQhuI?S)EWG{OIhv2q@$1RRA8)beIlm2oKtrl(H-6yNYlP0aw)3ryE#W zN5FK0{QyA-(oHPgLU;o~$XXst-LJ=yoN_X zXwpJJmYk6(5Srxb$x z4(+YB(tB5b0m;W12l9`>90Y+<Tyc2R*XHlw|ZyKI0lFvB{D=o;{w*n2zi9wM!+{^2s->( zmjPfSWD2gf7aa23Bghb+yYczVdh={Mn(vTxhfF^z6x+|QeO|cHEVkp*9Y`uk!R-@! z`t~K=2hV?BQth!5Ktufx!;%U8*W7DV(7i@!5g9L0J)OrZl&AAr9P&FZ2#>h2HOEidjA)#O&qQ*B}2BD8rab@R(YyE2zI9R%{k)62~>0 zkfKmfjxeS)K-{{QJpFlvXRwW}TH6Yv??P_xx*-~#%u76pZqI`W)ouzes}{AF#C1Ta zH>@o_fQ_mqN{h0o8<1$d6!%Mf?0EP(qe72trq(RD^PvKf1IY*9m%=b5p2e1$Zk3sG z;s#aBeC6g^_g7d2+Wr54ryxh|-UuPxgA3(aG4v<)ob zUUM@p4NIE{LJhaDlt<|8^&g`t016fZfW4j_YM#4mb;yu2JmQMnh?5=$L4iBbu#=x@ zlUxh(sF@R~(rC%c(@M_lxi*<-LHIdH&)Q_Dm2Fx3tF1d9Php0(0M{wJIDHS~@!G@O z+~?$yI~|U;g~@OfagH- zx=(nfeLwOYN{F6MR9r>pHzeenZBHy6N$~4=p>#MPd=cN}cK*_l1i!-fHGz_t_kRFu CDaWI)@WwIBP~|9-?dg#<1IGHgK6eW`Qzv!Cr8Qj}zw3i?Pc zpv%kiy5}CAbMCoUe`#%Pp`d*n_*Z;?8%6yCU##Y=3^v|{!97Z(L|Ue1@ad50q+`ZG z6N-1eF6_2W6hu(QUl8Vep1ZK;E{8?+2S`8sHa0qsl-*DuE$1Esf) zvb#BDRpZt=dY|M=?A|I>Y?EwfGi8WQkp>&JZ<2$ydJYfh*-=N|EA{^5c{?}B>Hbl2 zOwGIMBy@{Gv0L(k_I_BC559XJjQc^~o}28^VR(Ye;6rK9D>p4vPKo9D?UIc!nyv48V9h#K@4|^|m6vEt>)IYZ6&lkjS*oNDvC% zs2pDqB2hJ-PASn!WEtNU$N1a$6OEpqqjQvCAh63h;0@qr%TWo|8rPl)t~Rztz*U_? z^;pn^kwdIE^VvkE4dvsC9w6JwI8+mE@8&u14Y;a$HsON2*4TgxV%}^`xH>F+?V0GR zjja(#s;&tn`7LLYDBJh zlpw~FI**f7JWTfq$+Q@cEm{N$W?bD#Bt=rgjLs^O9INT7ptFQRq5r7wry&G!l*&Ohz) z-y6R>zN|jDe*b!L=#VyaXuaz&VBRVX?s$;8pDGTX(gsf<^FS#yyw_! zBa)gSsc1C{0f=KGW+z|(F>CKHu#+{mixg=ESlB_(*>xrwn|nkPbSwpBhR8PP$q;ne zUK_&!c>rnAzn7?IeEaQ_w@w!MpvDIaY>*)CMeB**Nau-^5+#B@C6N$7>b2bg@7X~c zZR?l}o|@K2Zyhc2LmEF+V256w)}86gibSqNFy{(}PjAs3H|^}OjqY1mk+RP2#tKF> zex$&Tyu1~7D?x9s{3t3vWTSjN*O-moA&n0e*wAzIMmKEc*Om{{80D!*QW6BH>6oaj z)Om&85|ty0V&Jnu8y#!tanulZ4D{%>+mp8@3#|X=>SpG)5Q?q~EDXAAz($*Ds@{hj zn7+;Mg3r8k;W@he#Q1+yiT|$6sb9F9iEVLFoFY47qw`kYS5f&|lo`2UT;s-SzvhRrEuF2l?ZB2+M%nMOLqe~)WT7<-e!8ooTLhMf|Tq|8v`~H{R-qN=uKeE_U_x)Z(T38 z@7LP*7u%0%?Z?*JPb{5%%JnB&`A6p;o_fNcD6l7^7I;^=H_QAN$W{l}#QvlUD>6%JlzeY9S9AK8ANT2Pq zF_n?JF9BBB3mw=8itB54uVI0`?%$=czNNDt#Tt$C<+}y`bb&o>SV(tO4l0{s2^VO1 z<5SqPo0;myxLP%q-Er734hU*&e3LTWy5`C^F>0cEe6C5X#qKQVoOk4zJezmsxg48w z;w4qyrD6?Xv>c;W<0oNrFt(yVah#)mN7wPWehgpEmB2t~Xmt{7j>!;#SQ|mhyTPGw z`)z0mUjqPM6=%+w<3tB$BDH#q#*2ysI!r?Ymt?FJ=9rv2$Hmx-;E{Nf=oHn(1xfd) zai}5F8C4;_0B$%#qwNSPtUIAhOfDGi3%OU77W9_55>F{=Bo&o(&b)xs9co%<<#cM! zI63K^aYcyX@lz7!(pr z@#V#bqb1+qO26jYy>uSqE&BVz4<_$VKH{x*xLs;^YOKY>MvrqT~1@=IJKTz`a z7rg<^8~Eh#+VR!n1#e*8JF#>IHw^Po^H+YP6c|~Gee|Bj_5YLWxYM_yuH{$rE9ocP zt0m4;;`kEhEqk0^u1qQeJIGG>)I(1!UlcJfp;d#@R*q2JvKKK}34T$y<7Zw^U(@-@ zHjOt*aFyg#<|7crjGVrzv-9z}`LM^l|LQVsi6$~?TvjHG6Rbks136?CyI$;2FTEQA zdQO%$-BcO7I>Ev|T-m0ooSXM3OhDOPHhYxa{nrOTtl%dBmvHe`1ctzyzhMvlJv?4& z?JTwiwbo$4J5(QyV)5h!?c{~;8OLtUEC`>lqu+7V&WRG&@?q*ms>qFL+}Ok6Pxn09 zv%xraR;a$RyWMz!!C{!shlL9y$xRi`mIFexnVsyoucMuMOgmY1Q{>^>uS z%xf(A+y6m>>MIDu)8QQiZyWR$0ga!5Pb%nMLHHmOkEYBV{$s8k^vmM7f2c| z$<;-|!K<0R8AT;z2fktzNrtgQPtbYe&LAnN^J$UnMlc${*eJ*a98F+{2RNOBYfebp z1kWKl6Hlo|Rc&qqXAtI8GYhh$vsf$=qcFnkO)#Yp^eKV{dt@F=%U^K2_S;R(6DyKi~;?)U~m%fl4wEjwstu+Y8h-xNGI7z+ZFCN8vm zaK(53IHE&Ubf}6RUFkv8XcaYDMQvX`gQ)FQ)OJL*oLuBe8UOYjZvBjY_#wUSKdZ5`&Z6=4-@S-V+cJd z%$t=?3sGc&$P;jnLU0?vk_7)6Sr6LQ%MmPf17epXC_L!9OUmSc z3gpU_WOrh^b}QyJQ99{FuGOjC$&CHenM{A`nf3!9m;m0;V`idEt9~JoW}N1$=iJ2- z5G}b)rgwm|_dM>o@7Fo^-&`&`1L1Fi{}sE@!7#r@L=Pq{Vz?hRFw9LxWWtPS5ZSmP z!G>7_#*J}f*hs%zn1e4DHzmwrbHWm~P`)W{P1wRV8aK!72}jsL{#+!6ODn!-(q zKsbul?!hX1ELSU{bJ))COjYw zivEjz3`49?;{jdTz|0%xjiKg+u2WGGn^AZr#m~ee^Zd!0QkvC2w{M5l!D zNc5tV6fvReY)q6=C#7hLh)@Prts8>|os&+`wmBPGj!~F&rG&dQE2^(Nav1ec+!6afOPBiWX5s?d< zL=#}j-;k zx=xC9g?$w3rdUt7kz&0R>kIoS7NXdea1+J)DaMBbROdh$Yo^#B#a!Va#fB(09B!fM zww8IVl(&syPlVejZ+n@?Q{Iyl+YxRTpMu&xO|hNf4vOufSZ%nIrrS-iJ>f3O+eOt;nVzBDk-ThB2B{FBJ)Brl8{ulHdSB*(>bEn zR85+RSX@#ai4-m?0wN1`m`l1MYjQS$vr<;gqBIwaN~0<_o=U}4(;QBu*nh&{R*jHA zby9*5my(mpRBR^)^fGBmicL-_v5W;s4o&J(OjM>8YPF22%59E9aY;nwBC|?rp?{

4yPz>5}b$74CXmHd1Fg$kd2!lRk$RL3o|LAs5KfsNoJxNp-v;fN|BVJ ziZGRumD6gOn3|E2!Z=A?f*fK&n6qlVCavWF3n;D6&#>y{BxMe*);-)B)W&iKv_3*S zH7P(fp~>z+!!W8>p0!j}Tce`UY2#UQ_Pjilq9h%79csl%lajBrfYEs7jNRQ-siZl`Bkh!qqw^ zzNWh4Fg(Xoia_YP3Tt^8$BjufLdDe@os^n|jABHsO-#ol@En?Poqe^zTlc*j&L26DSmk8{F_i*=Cy*gbhWVt)k zivGx)B)}BFN)ZY2U+^pQBsGMa$L5DW<+*n(gH z!61Sm1j7inBG`uD2?S3fcnZPO2zDaag&3f};qI zAvg};edazu<_`ykFQv#uc_tE-hNor|kz^_+3NjuK##0d@#*&l6GKmgX_42Sx&pPrj zo^*z0=1DW^M1YMWe;az~4Q74ou1t5-%nZD~s^K478u{qOo9psvewcbqk^@cp_E&2kPfhA*}4OGRv zO^{a3vrWa8_DoOK4wPXPymK@>}oh&9V(k(|NXcy`>$aA|zUG z;j^rF2~_dLU>j1SdA4o6y(7yuEpMkq2Z6@rdJtG|11gPntatRFz>|5lr`X${Wm|7? zdA7gU+KI{(tY-&A1Iq_#mAf$+)TO%^Y$<2aQZh4vHYj9Qo^35Q^+KaSZoS3Uty$K$ z47J@_#PT*TpF*myRcnIQVh~DRzjPX_+g2%9R|rLc8UnRjhBI5T?w(b4h@8|rD2g*k z4|qF(1;lXw7ZAD0q?u`*Q}q=M^wj%;f=dkz3NCy!Oq=ysgVp)404-+ z`4co4bq)G=(U3L}R^(1GqUqFQ%6^Iona4;!6hpNfdj8al#}AxSYhFHZ?)ZVRLxN5) zYg>_OqN`}gq)o~$OvI*S)dI^%jHHsP5l*Ubel*fk?NmgD#ZSfwAmeJv^W+q!XuuUG z6^8`~S%S8kkUs;~OYoOp0FYsdChN8EmGIJ2E2gHkI^TQb=E6G*H!r_)`Bp67xOJuR z%uk>F_~=JRKOXyNEPL>D{>d}>y0e*MpEY(|-KqATqH=uW!;*8ISt zm&3q3I_E_Tv;pp^o5GdmN7K{BY20@=1))JFD7Y8XkT&RfFS^riv6e1^2SF=t)SH6* zG|+L|-V`7=+`=~nA)pf=6>jXC!edeeb+P1AkPbKhP2n-=s&m8B!lv+;R2#Xa&BXbb zTAMc7g%v%}pUh&-qa}$eZ6c1UzTPmyz)FKh78=sVG@GuuQ~yDEqj}Q@dGCFU#;sM_ z8uXrmQDVukM;RFF2Xw7Bd})j5#knu~CXQ zwTeMK@5Ne{N!RKt%%UebUM1NAlC2w(bs$++C3!-@?TCgnqqpH=JxJBx@F}=oX*^En zrP(`edhaYYq#LGj%i9!`Gddw{txCu1rGP$an0{H8seGyK@-DULfsxft>m5|i=`zp{ z9$m}yD>}9E)oW&8mJG>%1`Qo$4HtBoKT5;*_0bJ`&2>en)}cgnOxz!1#7^kBah+26 zzH1P>fFo8YdJH(-z?snT$`?32z?rI0^cZk@MI*pI<)Tg#Lm(U1vC3C(C6L~-Njh1P z(qmQ9{Z-NfpnFD_seG%X2SI{VIC>0n9RiM`57CN1J0Axx)^XOxALvb&Uq&E%-DiCIx9wYZ-wLZ%q*H) zIXo|qz%heSIkb_fX3avTa$vjwn-!7c0*Nm)AKWk!^JwDY!H&*Pq-K-iZhj&7%JXBR zFI?Eo9|sFG+*|pIaSXVJzWK22?3;W{=85#`Y>Y@Ee`!ie^53i^pawK5f;r5P6q>ul zArCYL5W(A^cJqTf6gx&O zMg+G6>?&%&!L_=4>9OvOM-t;=WMm;g4U>?~(B60o>{jv!L^R%aL0+~3$S}9K+lIfj z(TfZC4$8L0iBudaqB>Ez00y>6aAi=NiRvyl3EJ0y2qkbvI@s_*%0@K;8#ohVk#bs&WB+TK6I_(KpAZxY<<*v z*-*JwJ?biTgS}y###Ecaqi)T24EhwM?ft%ib}zJ!i8?-16V3@)waq3kQKySY`R{Yk z8B%;gHKxYF@uD?|AWubRB-MzvUd)~6Ga2o23g#{7lei!(bbno~hjg*OgVb_6fZxNN z)$LjH_-;18)4Y8C!_JQfKN>7NF`9c~bk%b>Yd=i;a_oKZWD!Z3CCLRRE%wl9a9E|t z`y4p|TF4jzJadq50eGJ!ix|hnnBJlFV%jSmKyVlNF$$nn;rCaWFIm=h#6YtnSAp~= z_j~K(THqu#%sP1|>&_v@>;(aryW>H7g_-CXq4y?5qj_?_@A=ZB|0zVOk7 z!j9*2JDy+l9?f}|*GnTcFX&p{7kc!}rNd6qaL%rzKdQodmg1g=w6shXR7s~y3 z1Jh$Iab_nXGceDi)Hh1-oQVDjFyE`z4Htm3+N`hIDy{%$wG#k*lCWZyy%3(V4IW9F z^`5qS+5B?O*%&!~k?>&P>@O;%)??Y9Ou+qCW=xB*o>!`>>MtQoH-oup;Z0WiZ zg-#UJS#6E~zglxc2D%|E1kZqXcm};BuzP5hC%)`3-OX!@_fZ1lxIApMv4q>O>Ws-k zWG)f|J3iRfYfuD@NYvI%%(SG^YhbXCCBUPtS@SJrI~U=RZI)gGCzY&vMqy7LODUtc zBR@w-im0X;*n1R}L-RSckgN8yk@&1cxdiPrs(B^?+n7Q+5HZP73EabIGSlql+b}jj zY}MKsN=Rp@FuTW+R?^D@OVmRN@a0GJ2zy4Yb5a?8`nrYWkf`B0HY1) zk&Y!NBq&o9?09rHAZR#JjWDB#4-+*a@FPHrfo8KG!q_kX)d<7q>kaoH5=;Jx)_c`? zayF?zUz8g-8c)ems1dx4;57j^PSx=MjhEE(ZlkV68nqufbK=A~;lvAvo>QHtPab%F z?72f4X2IczE*&^k$g|+>KsUTfM>I)+Q;4GWOJMpeAA-!d!obgo`(HywL^h zSWf>t!>qA^RW`WpaKF9##_kozu^&r6nf}pqp=V#NXWwei{_pht>`bQTuG7EVcuQGz z4rF@Q+)c~9d3Q%9v|dy9_WX_cY;Y)BGnCm<^tLU51%4g;6uszh=vPgiHE+{$&%5Vu z9m#nIZqFBnkL2LzJyK|RDcA5)zTs5nMA2d|SOPgqAZzJaYw$1E7n%lgO#}Ic!OX>? z%U^J{v6uJ$!|Tef50Cl^1tkmZl%-N&=m<7;L|*1hHSoqZzmc3c$zMq>% z)@{y$tvP3F&e}TH-2S5d#rsCK#$IAroBe^AY2RJ6df!u)4;F$$x!_Q~aTwU`)||N| zb8%^R$;#9=7i!vbHSL9(o?J~&W}@itEcp9!{=Q7=v$oJXJ8!2m(~x!1-gE0)*`B9A zy!6T3CsRKg|1|KaGkf|>)_V4Vk+oRC#ZVI{xcQu$&m3L{oA&uD=d-4kVhjJn^WQn2 zIehKJl@qI6;BG_T?Ru>2aM9GTY*{h2-8TNC?eA^iW+?=>=Yrcoa_q|3D%VtOZY?x# z%QbJ)C_m@?>nJmj3l3Dv{BZww_UpL+w|2(vTkiZ}=)0lYHJ?1a()a9djf|;w-R3G; z7^AynXUw*1=UzYe&h9nNRy3c1lAFxXMz)SylOIle+P`W&o9XN|_xrUwqQ!<;$z zpq8;UFK@qnt}u8oH+XQ>dMMNRnW?7O)%%kZKRQw98p(By6uJ)Ox(;QI!=Sh}e`S7Y zbj1?*k~3P47#=VPp+2>Nx6)T8zBC&xFHqtOlo)*L@|B6Z9{yHi-V@3k{mj+4Z2bP& zThHfPhCZBIb&X_3i=6%1u`9>kGcFy=*YdaLGRIcAr;4unf~zm*>if`~b@k<4PiIEg zp&P6#CVq|cEy>HHd9Jg-_2#(VUvQzK#kbtMVrgIF>X!~K`|=!L;JR{L*Dtsp%F0s~ zwuafcW)J?3X>?eNo`!;_bH&qHG6J^l^%uPTE8hN+2{0HU&`fpRD|Ou^3*uJB*Hq@) z0EgN(_)89mJ#aG3t;>;H^@aAmx%R!Qfqf+xlHE*GaQVQpTxi>wYumZ%-&LwXaxIdd zy=5%$yK?-lRsZgi2gs#5P*Mo&SPASX)g#`(@STOW{VQ$zOJ2l%Ok<$n8(i@XmKsrw zp9!`tzxw0OTanxKh0t@k&~y2Y(beX|r6!aPKqdwMwiW-jQZq^iq43KdoK{cdf={e^ zx0hN_td;RM7aBt=jiFK-;yg{(zvAyNwIklaw6+&ocC57QD0QNmE~c|b%WyOo8eQ!; zTFXDZ)F2gH<;Zg{bZDHCvZnfV!Rp=hcb&ssJ?l1MD*Z^oNbnIX0 z*k2k%F-x<_{N+=O#eOCFhIH5N%Qo%!FqPdgnr%Fsw;#!JN7h->wa}H&8~yYQ_s{_v z4<&GUGTcwX?GBy`f=xHhEHV|>pMHy9G$?o|rr}Z7nJWJg>f98j9h(S%R_yH}OT9%< z4h+=uz&_+N+VGhya{o&y+?o#1ZdCA8LxWxi1&=s1Jjz=n8qkp=n$j#2%Q=)IMWB9x>}3EM=DwLRdhfdZZ^bjAHNNX7)*o5dI4iDr1um51LbpSC z?#ZI74t)L(EL=O6V_V>Dz-3rLY68psd3R4Hbl2v3+keBq#<`2O#xbIpM+M4K z`;BIp*?X1;vyFW@Ti;!?{aWlwtYGGIX1-)(o#qD)tDVbq!3tyXF2(M$_Un6=xu3Hw zbQ0mi5iGJGz#R?E1KNv-D;UEwB~brKk5~AiA=M%XVk)Y=H<_4)p2im@IP%Fk1X%#8 zW8+aMbPFC<2~IuLf~T=~Y+O5^o1q7xVF|pZ$&5L1ZN_>L6Uz*DZ4g8 z&`dz^Rz~33$vpI$XNdA}vJyO8tL|#OBu?;%tp=)zwAY99Cu?-vRb40WxLsB^`(*eDxs14P0&GJEHddP>HA|2pm4?7;u>B(H}sxFVlg#zh^L;Va| zIlto=*KQklZlr@wvsS!>=t%^p0I1fz+Jm4Gat}~g9~t>A041ZrVEByL@*Ae(*Np9# zjQtBH^f}}IoC*Af+4C!=X_aaIHRJpxhD!{9`$mTKmk_-_Vc-l6_uYqBgZp=5T)@y;+Rbpz>kBK! Rrq5XO8$Bf>!v$zz{|D4nIQ{?t diff --git a/src/mjlab_husky/scripts/export_lerobot_qpos.py b/src/mjlab_husky/scripts/export_lerobot_qpos.py index 7cbe208..21ed84c 100644 --- a/src/mjlab_husky/scripts/export_lerobot_qpos.py +++ b/src/mjlab_husky/scripts/export_lerobot_qpos.py @@ -14,7 +14,7 @@ import numpy as np import torch import tyro -from mjlab_husky.lerobot_numpy import batched_vector_to_numpy1d +from mjlab_husky.lerobot_numpy import batched_vector_to_numpy1d, import_lerobot_dataset_class from mjlab_husky.mujoco_gl import ensure_mujoco_headless_gl ensure_mujoco_headless_gl() @@ -25,19 +25,6 @@ from mjlab_husky.rl import RslRlVecEnvWrapper from mjlab_husky.tasks.registry import load_env_cfg, load_rl_cfg, load_runner_cls -def _import_lerobot(): - try: - from lerobot.datasets.lerobot_dataset import LeRobotDataset # type: ignore - except Exception as e: # pragma: no cover - raise RuntimeError( - "未安装 `lerobot`（LeRobot v3 写数据需要）。\n" - "请先在项目目录执行：\n" - " uv pip install lerobot\n" - f"原始导入错误: {e}" - ) from e - return LeRobotDataset - - @dataclass(frozen=True) class ExportConfig: task_id: str @@ -56,7 +43,7 @@ class ExportConfig: def main(cfg: ExportConfig) -> None: configure_torch_backends() - LeRobotDataset = _import_lerobot() + LeRobotDataset = import_lerobot_dataset_class() device = cfg.device or ("cuda:0" if torch.cuda.is_available() else "cpu") env_cfg = load_env_cfg(cfg.task_id, play=True) diff --git a/src/mjlab_husky/scripts/play.py b/src/mjlab_husky/scripts/play.py index c98272c..cad1ae7 100644 --- a/src/mjlab_husky/scripts/play.py +++ b/src/mjlab_husky/scripts/play.py @@ -23,7 +23,9 @@ from mjlab.utils.os import get_wandb_checkpoint_path from mjlab.utils.torch import configure_torch_backends from mjlab.utils.wrappers import VideoRecorder from mjlab.viewer import NativeMujocoViewer, ViserPlayViewer +from mjlab_husky.viewer.rerun_native_play_viewer import RerunNativePlayViewer from mjlab_husky.viewer.rerun_play_viewer import RerunPlayViewer, RerunPlayViewerCfg +from mjlab_husky.viewer.rerun_viser_play_viewer import RerunViserPlayViewer @dataclass(frozen=True) @@ -40,9 +42,11 @@ class PlayConfig: video_height: int = 1080 video_width: int = 1920 camera: int | str | None = None - viewer: Literal["auto", "native", "viser", "rerun"] = "auto" + viewer: Literal["auto", "native", "viser", "rerun", "rerun_native", "rerun_viser"] = "auto" rerun_web_port: int = 8080 rerun_grpc_port: int | None = None + # Viser 独立端口（与 rerun_web_port 分离，便于 RoboHub 双 iframe / 避免默认都与 8080 冲突） + viser_port: int = 8081 # 浏览器直连云主机时填公网/LAN IP（与 --rerun-web-port 对应）；否则用 SSH 转发保持默认 127.0.0.1。 rerun_connect_host: str | None = None # If False, open the printed `?url=...` link yourself. Use `--no-rerun-open-browser`. @@ -126,7 +130,7 @@ def run_play(task_id: str, cfg: PlayConfig): env_cfg.scene.num_envs = cfg.num_envs resolved_viewer = _resolve_play_viewer(cfg) - if resolved_viewer == "rerun" and not cfg.video: + if resolved_viewer in ("rerun", "rerun_native", "rerun_viser") and not cfg.video: env_cfg.viewer.height = cfg.rerun_viewer_height env_cfg.viewer.width = cfg.rerun_viewer_width else: @@ -136,7 +140,10 @@ def run_play(task_id: str, cfg: PlayConfig): # Rerun needs rgb_array so we can log the same offscreen camera as mjlab (tracking robot). render_mode = ( "rgb_array" - if ((TRAINED_MODE and cfg.video) or resolved_viewer == "rerun") + if ( + (TRAINED_MODE and cfg.video) + or resolved_viewer in ("rerun", "rerun_native", "rerun_viser") + ) else None ) if cfg.video and DUMMY_MODE: @@ -181,6 +188,22 @@ def run_play(task_id: str, cfg: PlayConfig): runner.load(str(resume_path), map_location=device) policy = runner.get_inference_policy(device=device) + if cfg.lerobot_record and resolved_viewer in ("rerun_native", "rerun_viser"): + raise RuntimeError( + "`--lerobot-record` 当前仅支持 `--viewer rerun`（独立 Rerun 循环）。" + "请去掉 `--lerobot-record` 或改用 `--viewer rerun`。" + ) + + if resolved_viewer == "rerun_viser": + if cfg.viser_port == cfg.rerun_web_port: + raise ValueError( + "`--viser-port` 与 `--rerun-web-port` 不能相同（Viser 与 Rerun Web 需各占一个端口）。" + ) + if cfg.rerun_grpc_port is not None and cfg.viser_port == cfg.rerun_grpc_port: + raise ValueError( + "`--viser-port` 与 `--rerun-grpc-port` 不能相同。" + ) + if resolved_viewer == "native": NativeMujocoViewer(env, policy).run() elif resolved_viewer == "viser": @@ -214,6 +237,65 @@ def run_play(task_id: str, cfg: PlayConfig): lerobot_save_on_env_reset=cfg.lerobot_save_on_env_reset, ), ).run() + elif resolved_viewer == "rerun_native": + RerunNativePlayViewer( + env, + policy, + rerun_cfg=RerunPlayViewerCfg( + web_port=cfg.rerun_web_port, + open_browser=cfg.rerun_open_browser, + grpc_port=cfg.rerun_grpc_port, + connect_host=cfg.rerun_connect_host, + log_camera=True, + camera_log_stride=max(1, cfg.rerun_camera_log_stride), + qpos_log_stride=max(1, cfg.rerun_qpos_log_stride), + camera_max_side=max(64, cfg.rerun_camera_max_side), + grpc_newest_first=cfg.rerun_grpc_newest_first, + log_qpos_joint_scalars=cfg.rerun_log_qpos_joint_scalars, + log_qpos_text=cfg.rerun_log_qpos_text, + qpos_text_decimals=max(0, min(8, cfg.rerun_qpos_text_decimals)), + log_qpos_tensor=cfg.rerun_log_qpos_tensor, + qpos_env_idx=max(0, cfg.rerun_qpos_env_idx), + lerobot_record=False, + lerobot_out_dir=cfg.lerobot_out_dir, + lerobot_dataset_name=cfg.lerobot_dataset_name, + lerobot_steps_per_episode=max(1, cfg.lerobot_steps_per_episode), + lerobot_task=cfg.lerobot_task, + lerobot_overwrite=cfg.lerobot_overwrite, + lerobot_record_stride=max(1, cfg.lerobot_record_stride), + lerobot_save_on_env_reset=cfg.lerobot_save_on_env_reset, + ), + ).run() + elif resolved_viewer == "rerun_viser": + RerunViserPlayViewer( + env, + policy, + rerun_cfg=RerunPlayViewerCfg( + web_port=cfg.rerun_web_port, + open_browser=cfg.rerun_open_browser, + grpc_port=cfg.rerun_grpc_port, + connect_host=cfg.rerun_connect_host, + log_camera=True, + camera_log_stride=max(1, cfg.rerun_camera_log_stride), + qpos_log_stride=max(1, cfg.rerun_qpos_log_stride), + camera_max_side=max(64, cfg.rerun_camera_max_side), + grpc_newest_first=cfg.rerun_grpc_newest_first, + log_qpos_joint_scalars=cfg.rerun_log_qpos_joint_scalars, + log_qpos_text=cfg.rerun_log_qpos_text, + qpos_text_decimals=max(0, min(8, cfg.rerun_qpos_text_decimals)), + log_qpos_tensor=cfg.rerun_log_qpos_tensor, + qpos_env_idx=max(0, cfg.rerun_qpos_env_idx), + lerobot_record=False, + lerobot_out_dir=cfg.lerobot_out_dir, + lerobot_dataset_name=cfg.lerobot_dataset_name, + lerobot_steps_per_episode=max(1, cfg.lerobot_steps_per_episode), + lerobot_task=cfg.lerobot_task, + lerobot_overwrite=cfg.lerobot_overwrite, + lerobot_record_stride=max(1, cfg.lerobot_record_stride), + lerobot_save_on_env_reset=cfg.lerobot_save_on_env_reset, + ), + viser_port=cfg.viser_port, + ).run() else: raise RuntimeError(f"Unsupported viewer backend: {resolved_viewer}") diff --git a/src/mjlab_husky/tasks/__pycache__/__init__.cpython-312.pyc b/src/mjlab_husky/tasks/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 70eedb79b7fe03657c7f97e1a640fd56c65c4670..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 327 zcmXv{!AiqG5Zz5us7eXwO?uK(ddU8O6ttj7iwO19%dqV(>6&a3b`vC?{Dhute?j~d zFJ8PX2#O~?3B7r8lj^|Cyf?gg@2S(-Mz-#|Ut(eRM@|0A`vdEq1xv&b!wHgjf-zza zb0cTsIEZ8Ja_7<6Xfdk1p!Hdk02^P(JW~babQaTyYtt$Vk?315lq|mpJaZtg6FLo> zYiJSBT!aPHu?{z{Tq)Oj86QqY$AjVMe0&9$!?Uqz1Ef@PfCsd$(|!izCR4GtQJl<6 zNz+U)&@nCeG^2`%G$LBfND~P2QpXi(f(U>}MF9}zm1!TzjFk!Rs~zi!ZR@Tb-!8`Z b74_ft_E)FRyN7rFJMtIp6?s8DwPgjr0jgb$ diff --git a/src/mjlab_husky/tasks/__pycache__/registry.cpython-312.pyc b/src/mjlab_husky/tasks/__pycache__/registry.cpython-312.pyc deleted file mode 100644 index fcd5d88d7e230c8034aeff7085575945e081025a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3058 zcmcIm-ESMm5#RgZ#~mqArerG$;&`Pvv97kL#x06kO%pVZ_zIfnNHU|Di8pS-|eW0Rp53n!J<&0~F{}XZA?y!$sb@ z0C#6+XNEg7znR;=OiY*r#vfkz%KcY{kS}mE8WTfe@J}Fi$W6jXg)oJwp3+b&s)DrU zX$`%iOIr6*jdUd~>6DjgWGh*Pa9t%=2}^Ggmf2Gii+e-E6~iSJljnGz8#Q$fxO{?T zUxU>kY?hfk&vHD?^6;DBSvJWhnXw7CWCdo_@@#4=RVlCnpW=Dct*Pw9mR>o*o?}JO zoaRM1OV`qD2IQtCufVf4ot*@!nc;qD5xA#-d$MgJKPopOJ}6@lO2el!wvA~*1<<1<u zu$g%W4RZ0O6LMDeuJ{|5YjxnCDu-U#d)NO(;JMXyx#jy@z)2PDjfOTP1{kj$!bwGe z{HQkx*SO9USq|!{s7{We<`?hYSi+)P1|k+tmPq{x9#^^j+dN>6#PkCux-cpgGk0Dv~7__t{%0U z{1Y+&n*8VT@&|!f51USvFW+fFqyv}P;kpy?+kqn(M146F)#b)r&$(^iX@%?U@gKyi_k;pbPI{V3*$GUQl(T_EI@$)$c5U zP9!*Ax*EK?V3nl3AtY8PpmkgW)xfX0^_EDApap1_^%4ok^&xYK=3zvAo4V`M1(aQ& z04bj6Ciq*So@Ad>Xl=IlA@hSc$(q*;4d>RfQPW@+LiBq=uEbG#P6B19l0@T`b5G2f?Ty{~?ydb-e!PG3^`800 zzW&CO*)tC_AHVSDjZZgwvp?Ii?qB_@IwL`Sq*oxEBs>vBJ0u!HkC15CkqNmw1|f(H z$EZ*54dt&xGuu3U4ca0^#5)v?1XQbUg(XXi(_tVY&XVxx>%1L`A{b0cO*(FRV8TXs zLxDAV9)2O7+#~-m3J;8($qy%cQ?zHy@2m4tFE%8Mv8BX1Vxu~~IH--?^I&t>y^a#j z9=O;c_?fti(#OVugU?7~Wgi;{(hv@DEpXV88ax6_Sq6@bkF3kG>>Yq8ir)kPakEXM zfHno+z;8*Tm9$72)M#{v9}aP17Qt!h0+T5Ne1J1c6=#r3VG1!iBbenqxZ$Xiiv>9O zE{a@&=^pvUBxdpNix;+5@4H?7xkozf>U5x~rrsy2ra!s(qfb}67vFw#@p|{-^#gta zu`!Im)Hnw#usNKAXCnHo@rQ`IGNMrE|NlU0hXRQW{u4#UFhs|OMVgb*OWKck8Tja5 zh*dfa-UDlQwOf4UQE{nTT}JO)AFKYO^trwLdD+Z*zlMHk(%HKKWXADOoyE=BuddW1u?FukF$J391WuTQSjS!>yGX3x4gyaCTqa6h`grTf zzLq>wUKg)|Hs(2e3R7QG6eT8WPs!Y8Wd12R{~0;oPZKrwZ&g>G?-Q5?8f^P=J4F`G z-ZSohe_PxB<@Q83_hOH{)Sn`!=fBPsm7gn*bCZ1y33z$*Q%HcR6EF8OlFX9aeBY3y zNea{W5a~;`ex9Vw!Gt{dj;btdzqLF4;p%_@`L&r*^xeyS0?WflF8e>cHo$FvCZ#Oz iey>kp*?;NM0N4IhO0jlBrKch4^F)u$eCB7JFUTRW)Vo?gvRQ=+jWc}Q%oW!K~jMC!lO8t_=;_PC5u$m(M v`1s7c%#!$cy@JYH95%W6DWy57c13JJqd~4M2D$VDGb1D8Ee72p79a-zKjlHf diff --git a/src/mjlab_husky/tasks/skater/__pycache__/skater_env_cfg.cpython-312.pyc b/src/mjlab_husky/tasks/skater/__pycache__/skater_env_cfg.cpython-312.pyc deleted file mode 100644 index 43485df265891de79a18434ddfa53f73507e07fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11599 zcmb_CX>40ZdXJ|lULq+`)MZ)LO&?NxWF6OsdB9!JnzxA9C_p2 z%^qpHZPKKhcpVPYA8-x9F zd#YosgTW2*vQ+0-CxZj>@>JK@3I+$|?o`iM55OVOI$yR>QH$2G zckQu@aaE)T}r;uXMW(i2irb5c>w(sTkBn*SmI z*x>(9R|e6@R0~I_-28+oJMS}VVj)(^LTowvZRCbuO-`qdt1gK{b@BETGk)IE>uVlC z$T@Etqt}$`eUbxPq>i{Gi`u|3q@K8Q9xjD=bH3ZgENV&&a6YrHlI152rmS3mORXs> zWJ=&rm82jEeQrrjoZqagBsG#IQxfPGe&#+jn=&|*YrbvlbcD4n%uQ_s712mTA#UkZRThkGA!-$Sk1F-v4cxAYjZ|5XVGa=a%c&TE>36GOK^0X zlpI=uqnFc}^%5NYCMAcK;27j|W(^+Jw&ecR=4=0gd>vnqZ`qQ3E6sUv$XpT9Ip_PK z<+??M16lvg3S>F_*F~1+R@`>zeN|Ws;JR}?+yhDPob|eu^r^#~pY)RfG6>vJ9)%>! zbsMp&Ybz8h$tvQ&QOv%nbsTHfWc9P7r$-=bFoHgtN~f8y=7zX8=7w(@u^Ke9tIzWh zt4@Kh@IJF9&ylrc9a#_mZJ2}eAoo0pLs_bMBKM%n#>p9KT4i1?$@m45LgMFT<*k`R#cbR+~ z0zV)7yd!{5JzM@BF4e5bDbhyHLcFm;c@9?R*%J1doE##9xvy3%?);qny0hkr?&tJo zO-`AdHF;h#X*ooimRt`vU-2c^;}Dl>)}Ld&&wg(CM@+dKBC#bc<^q>b;&av|_Twm* zYu04^i;h4NbB^nEHFx5;DW5}2?n48g2ju=FCqC2iPjabdy`=oAeOOZdX;UtT^c;mu zn3|E~ob9@+=KRMvmswZaeNCQoCM}1^Q7Nk(S`OVA67ENuF^|Y1U+FiE)TY z%OP@c37!c~Y1Twuf@jjC=P6Ea))3#8il0?>x`emMrd$qv(Re{JbMEW) zYKF_HI(w#hpIJjTIL%{hO&)5}a)?~2IyY3_Yu1FC{#kX?s-@9olZ!)U23%90tDMfP zVFwl1!OESEsBl)eCE>b9eT7r0*DKJg93ojFzq@)!J(;daLBBuUNd11LGFDG;%ADylAGl@uKJw{_B#2R`g*1HD!D;;tl?-L_t_5jkJfjZ76HzRlPR1^aa`Fc^ z@%MLkIWPk6Nse?39-LSeznav?&W}Xxx>&sP0uCR9*boNiEiU2~h|nol0OcpIH2vw` zf4=tGT@H*G0)KVan9Q$WGA2C}8MUyEs7tFe)ZmM2v!^fYywOkUOUB@(FKih zSp(ENWBp-sYYaV#h1$)vx(GOb>3uHKwqWe}T3)@1(iFj64 zW0{EwMN})kx?z+a0O^K6=9HMGQhaH7WQPLZZH--zU8VS(5_Q5di%L8-4gJZODy762 z>{?RPf)ut}QQ^@U5n}qXq6M(4Vl3Fxz+2p9>WMg>1uvHrc%?--isk~(z?W$-c0;o! z9#eWpH9uGtdl=S-t}HP2eA4h@ z-9L$#xCm z;v%?=EK3S_3=r!0MRf!UU3k6#rPeYsqkdpfI3#+ch_A!a9f zBgRYutdxa#Fk`J=4{iwh8GVe;UGwX!S84lL*s3)Bx{kb^1SR0-8imcpy`=b!LbE9< zF)T}SUJ)^{v%R8U1FT^zKN*49;H$0XQzt>$$&5_c@mSrAU)2Rd{9@mRVii(;s$iZ1 zK_$YbZdwAGEgn8#|F|YI@hh>oEG5&LkG10xWpP03U`-x7jUx$8Dg<90b@6t z?(sdkoBZ{yg?~CYe&lZQd;k99_x}55Z~W8%{_Lj)aqm5?K{ug_-A3J73l)@>(^0n7Bk1V|S3soi*aRBHfQF&RgvWV3*v z^aI!et$sq3F1GCyb33D4B3GZ7}b$SZs>Z1U$8&}wjN-M5dI}lhJ{&0^T|rgY@(uR zSdUcvD&(tESUFSdODQ{=qYaoe!~8UF#{6EZ&{e3X%Wz~kIz{}rtFQ&)pJ8FY5?KmS zI4grEd6{og_8}K^&^?4V^~B3K*$k3qFgo~1OO4|p#%Z0G*_TxaOHxX6LkO2r`e_po zrzHgrTO;9#Dbd{&pA%^SbqyJt1UWF*n&lI4K*u3FvQA47EVJOTuoL=iKy#kZ^Y=mI z_f=v5Xd?3_S~500a$Heky0O#MI4FKXpzE>NfCawmQ+&v$8?nF#dC15|<}I0;yozSDISZaI*t zYaztc@K}nU#;FgB5EkeUn)@*Gv*YNx?(N5p?LHpcad_7midReOha&0)D`mN_kyU&3 z3|Mc`A0r*wJ^gbm{sN1?#Nw~8_-icw28%mb`~(UGr_fdP``;tOS2FaXG96Ed!;@L~ z_YoNhLKt3OS+s%O99C#zI5j25$77RO<>Jh+8dokV!;FR+6~+=w_lBltwB}U&q8LlA z)um95qCbbBan{Ozg3;U%7QI5CyX^7LdsY-YD@y*3cawQvf63dtSZ5D77u`Dc_cy=3 zx#VqlyZ4=eHwRunxahC}&LxBg80WHrXIaVLo?rG{-nX{w4ZYj+-m#)LTJi=K>m6Yy z4AHil5q1?kU5F3AM~dE6Af*kZgnAjWqu}Xy0vW1-G$q&B0gvi^pnUl1`>z%oHkN(M zKREW`nfK2WeZvb|mxt^(RzKch=?>XT&d_5U)W2-%|E#{C1*qr}ng=Udn5$sUxX#{; z!?mvZ$Tt7nuEKM>iY>d#k!VhtAKp9{e|`Wi|q%V!nLjbsAqoN zzQVeF#kLopqFgr2M(!?ny4fmj%==ydtvXSw&gY)Qy^!~9;<&DLPvX9k_iY4R7xF!g zGh{Mc^I-1S{F-frHQS2ek^kC>D|BP^;(m+J&|UJjlv+CQFWg*e>s<6W8=Q;&D#S7{ ziw5UH(8pGHP;dl_!GTh!wG{3E`zW<8gB=e!AKU5%>^BA;cM1+~(YLD9F<5G8FGbo) z&26PXQ`xs_-nXvcTUQEoltayEQg5`pyW-8Zx#NF)>gK8UUcPxI9~gj{_+5)Ww;!Z= zqxKtvi~U05ayH|Ef@h%Q36~<>4_z&#hBo+HaM;@2k8KWMsdqOz7n_B4-8TCQp1zVN z^ssH9)Ye-L_q@0FPNW##zEEce(mGeWb1@_Ymz6wCOmb(z(^+a?{!2#JQ4H@bh1*KO zj)%UE68^OTTcdlSz8;712c3(ZLa4I_?@Re@hl}BtIddCuKFtiq+bKkPYB2A;Q&$X+ z;(`XdV1*Xy^+`o=QZ*S~%5ObZ3?Hw}(cE2Yi2SDW#qb5DU-cMZ;=g%}3J@>3HFE zJU>41@#$p2HwnfB7U>oudIVr`#1|;}8{aAKXp*%bQLeWc89JXP3uDt{qS>?I3pBFx4&2yMDmU?2h+U}G!- zwe*?0Up>C^*7>}z7shXSmX;MNre#Nz$zt9NvO4>65lz&rr_HKvN~8C(Ie6F zf@eACQG=@0xe#q_wBJ~>_=3=dsXtC$5P2g!OhA@^LF04PCKYg~?b*|jplWUxhZY@N&-Xq2414esuA-ehA zu440x<#2b-Hb1zjFu3VnFZfNlX~hT1{J_S-z{Y!yV$+`Lc3rV)U%90(w|;)rNMY5; zeMhn7P&v|@>zQA+W1I<;jw67m!LSD=c4j2Uf;uaR2k zQ#AZe&xxE!)bSJ%yCqcQuynTjDUzyNMum{2!zzq?cmQ`x?Mvs(YnIp>~wHDT?=WRnQZyVC{HvI;o zFGdYCZh)TX`*pF##Zh6&s}|HdvvAW@hV&arBWOi`hN^jWq<$x2c`i%+ZV9xEG#b*l zw!j-?Ru;F>|9~bw*eQFVcxZp$Ed? b17Yt2VGZl=d+faAu-NlmTNVWb*g*dW%A-oT diff --git a/src/mjlab_husky/tasks/skater/config/__pycache__/__init__.cpython-312.pyc b/src/mjlab_husky/tasks/skater/config/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 8548d2eb5e39e875a60e1a25495913554a525b1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 178 zcmX@j%ge<81k+?*WrFC(AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxd6{ugHUzA;3keHmR zpHZ5dn3tcK5?`F1SdyBQpIDTVnU}6#T$HSzo0XH86rWLAoL#A3l31KwtPfUGq@SFh wmzJ5XA0MBYmst`YuUAlci^C>2KczG$)vkyYXage<7lRldnHd=wi>beXA;~X* zHh+>bDYBMQQH2T=g@TIfom~kjw>UcY;d6ZL(_Sw^5Jy`-#W_akhfP)yF3EKTeS5_ICck4n- z73VY?QN5U>y!cGhgAAGvZa1r{q-vpHXTEN|woZfAVX!Nc>EL54ICwSM0|~E^NfY}f zw3nfRnQ$p@)@^5HW&$Oxpov!EQoo5i=3cwWVXY_4&`RiGyQFHA3mcgGw+GrahDs{W zPIQ)rS_P1*bqn^xzN2eE^mmVZDb++*Ny^D#ohL;pSo;+FWullUGNM&V{_`P=i;&ib zq-Aj+GLnr5r2=-RbXYZ}vy+#smi(pagNc1?{T$$zi!r`LFD}u}jptzJCP4VvZ-D+q jv^|d=okfr58{6O5`Nq?EH$Lk=x#&Iovld`SZNlk)=!CyW diff --git a/src/mjlab_husky/tasks/skater/config/g1/__pycache__/env_cfgs.cpython-312.pyc b/src/mjlab_husky/tasks/skater/config/g1/__pycache__/env_cfgs.cpython-312.pyc deleted file mode 100644 index 3c114653515565dcfb0cc22d37c638574d0b21e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5488 zcmbUlOKcm*b-7%AMNtw(iTXIQY{{}H%cLDAu#?7?T{%*%*h(8aZa}Bmt~g6-rTwt8 zOF2?$Kq`!?ImHEvCjA_Y-U1nIt~ohCF9mv`5CUcfHfke54fjweiHZU}_08-qf05g? z3vlMmd-HxX^WK}`pWEBR9DKjr_c!H6FUNgG2kz%{fUkZEz?76y5r+ zVJcMlxh!pMO6-b^HAQDeQIKMJ5}7u1OUhc8BrCf}I-SvFSvL)A^LaH?0L4R0E%=`^ zuy$2ejjUp=P8*sg>Bv#7Ny{?zgc*%RSQ6%lP*9~+;>WU8#Cq1_Y%J|U@bkj&t5bfu z$ySZFMXB)J@E}jgv&7q@##XH15Wn|~C6L+riGqtk-o3d*_uz3zc*=~LT&u&C=Gtp@ zwPKL(x75Cvi?8=0JN$IKH+>00qV-$n=eVDEkpG)=h!Ix?`XBU&?Jk$~j0>TFwZ8@j zQ3!?av~AJdgRg-SLG5>zi!R_f!gz*U zI$$OlrWSQ24sTlLTsk_s<>X6{bD)Ne!Kk*jCy>SE9_>lGKaF1pat zybINPh~?w%dMMP#`aXD;UaQY@p{3VdsNSPz5$DQ)QhVY*?a9*BI-d)PwH+>bE_2D} z=9NiUi6?Zwo}8u4Nob(9`B*U};d&7zKBz^yO!`zgXNftPR$gorbtFQ~re%->%;EyA z@N0c8!swZ=(0ocoR|iuen%x}um#52k?t3u5SU z(vRth#v`<$XBN{0foW#IhoK?y1_d?QYu%{{@%o}Ht95;&q(f0vIWJKh+wiqtz$|l7 z(d&~+fT3Fr*DG-WmR4>inTtkIMd1IcbiJ7(s|&{UucNPH#e^%J>mn7u2$!Y?jO}wB zF`D!>c--P_Xo=qN#DZaxuw`J=V8#+3TZ4R#eP2!`qnPG75@XxCs1)i)m_!+FS(jy- zHNMwDC*AgbHnVov$U1M&p63ZaPj%Zzs-JuGnGs{KEb{}-hFIO|7gkqKA3Vy zpG`e}v=;yU>?3j)PlG6Y9vVmZM|UwzwRn~`-=PhgL9arXH7~o6BlXdgV_*3$zTnav zVUOhMk^tmi`d}hq z$SS&NNqSbs0XmP~GN~d-m5F~vk*~{`glL`>S-uQr?PrLIBXpv7MOh=kHPOiBOxYrl z1zA~xaHUgC(q^hME;xfoVG*8BB6Az^sAMi!LvO8R?V@iV2BcRAoJHEs7=y z$t#dPSn`h;3#N=$oK@?Rb!h==MHbvE7N{#YNUd+94ci{ND(({uQn|bWi)IpU!6-mp zm-IZ?7g&@f`k+j<;V5$^3r|5iK%2@HV4cuwMcc8&uudAL672YB!FIP~U-n$<+bfwe zN_R`{WXR51Iuh;ugQ`UzsA5(_CUt$d{{8Dp`nrLa%>pFV^kNa7j)sC7-nig*+Qiwk zwxmi6;$qQUUQJt)xooDHNSub#2QTY%{&?DXT$-uED(NrUukx?IJ@Jph0Qx2~XW$I2 zbN7Qop9F{3N2;-&^^wiP8;5TmtsWe;!Ohf0s>1KDj*i*m3V)y)?{%i7KR^p=0p2|HDuv`eL>1nQB{4HQG__PV6Qk z145ba-QC9x9;t>#UB-!Tz&Hqu31I9`Rzn9}##W8xW0%TT7b?+gi`IT%?g8c|b6d5R zkG)d9^2170YGHOwrkV#p^ZUw&eIFmLMCR^C2X2QxJoWKxC3=zi*wzJZ?zRVPZ82^r zRSoT@47qaZ{GZ;eL^71aU6y^d`9omcz;2AW#uVgsI4(Woh4xkNRqrmxwSV9HuvNiM zjO#m6J~CZC|3*1uz}z2s&hyaCz#mM|UE_7yB@Eu(>+T&t*W3TzwVz+RJ-FR-w9Lou zhq}x0(aP9MmC)%jf4UqxeSh#6;46IJ!~R3SH?c8M;d>uOyWUmbQQtqg9UWOu0@JMv zmHnqHp)+OvOgVI>I=Fv5xp{QsD6oC$@96cfk5u^B&OxrH?_PX#I}YdbW}5V*8Tw1! z-^V7zzld@E@aKa=?_0llVSTzH^i|`1>(e(cRZpB+pS~x=w}tqvH@1bLig4_qzq1k> zy)*KM$qyz!j#pwcJ6;~708<=%@&z~NHs&_xH|B4>bUR)V#sD|@{QC6f#f^)%65B%m zJz;oT7_JE8)lg)8;wuOzZa(RmN%oR}C?X>(isV}jA)As+$h+d2VWe1oF;n%7lB)kb z1hGtWyk)ZV-cV6G8)ZS9o7fK;NiRdEC?W-aNu``>!72nfXcHl$&_Bh@R70YL>cx;O z8Iy=T(XKhH0I^a;`Wlt8oMT&!S^6gXo<^po*k$MNQ>WO;%l$vhF2nFlFjSVFKHzW( z{KIS1$>2*+Sc;=4 z`Tj5LBx@Ky>BZ&9tS1+5!sMAjjFCZ1l5tHhGs<|@$TIcXaBOqnbnE4&6Z* zJg}pHA9EHl`*`B0`W5av0iaV4(5|W43I5>W;E_KFTxQRQvF}r^P4b*ZG576t8p&rH zqy_JV58{~-nF$Ia(r7Ytj__FUWY2Ri@+lhS*4qgOBu@&YNH$1`+#s8zOe$biJ>Pm! zUi+w#ZL&k^=Z#DI4~=f~?44syA>|Gb!;>)oKR-_Dc-jA`YcT^#voIj`#S?&)McufY zR}}`-B0pe)A`ua_i#3HP7z+Tohh27-ltH!Zgx3Gqsih`UMhNv0VV+RiYvq4qB~W9h zi8JTRFZB4j?|fE&dGg~~)L&?(7SK$F^Y`=Lzk4_y^i>fP9D)gTv`+oyZ+&Nyj9)M9 zp7vD%s)z=IN8Pfjo?k)Ggo&-=v1V>FG$Y~ItiQ1sI9d+D|9IY&M2B75Q-hI7@4 z)0>N~vP5ARptO;-5`P1Z))zobK|C>j-njF_joY%(k_UI>t-G@Is%*a|Yt2=;u$fSm>yUmh5Zj@)j@W=rn0 zS8wl^vP)y%%JSFYun)QgRX~UYHbSbHq;<&MYhHl3>s0klLYt14-_t_-;P*kO;=5j; t>;E1kjIHI0iM4{f;H! v1460P(otHpgFm@bhIfs)w^^`{Kf`4RKPkp|M9VL<7}3>;79Z%^o`u05{NP@4 diff --git a/src/mjlab_husky/tasks/skater/mdp/__pycache__/observations.cpython-312.pyc b/src/mjlab_husky/tasks/skater/mdp/__pycache__/observations.cpython-312.pyc deleted file mode 100644 index 626ce582be152c4842ae747482e046c071702e80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6365 zcmds5T}&L;6`sFcer9*sT`;g1V+@JS#s-Sr_z#B^V~jDF6yvDMCdy=B2E5Du<<0=Y zTGCCGsBvFxztlJ{AyK44V#$tFDd|Ji0 z!m?QB;#`7{@U+gyg@^!sLfn;bN8Gf}6|YECMk-lG<%t`{-0`YJbp%*(jOtb@-UptT zA|i}c#ytsd#GCL%e2JP!O~N1XCjyZG@Y$tQy(UD0O1)AIWi7NtDC?l~K)DM_FO>CA z`k-upvIfdVDE&}|pbS9S1Z5D)-AaQ}3uUt^#7^4ZFU@18N?UeaX;kW76W%WjMD{N7 z;m`wYg;`6KlgX4W>(NwFgQoY=b3YxF2F?u*j0}$rjX{rZVoFV_gGoKAFAhx4Ak9E3 zsmoLPgqqY+1Uf=PJrlFCu96FKQl3%CX<1X1i}AtayyYUP$&?;;S=BGk$+{$`)A2>C zlGYP*aVxmlD@Bv@D$!KSNBgEy$=KWsFtw@|h@6)6l$4H66$DOqf=3H~cHSE;>KbeDYI0t~wYGkHv3ns!X0^0DrFLJQOUTJoRFO24)?`X1IJsLRQ{9PJT%MFJ z&uO!Z-MXyJYTcBEbSIQ_cWP2o$-I3=UFk)uqVQSL0H46Wwhyu_vo89sow$19THn<^ zQ*1ZHc2n#$#Lm^pyx5ZydNzQFnJN{C3p%I>{jgXn!cqYvIiyTc3`-mcxpclwP+p+! z!eB|U#AmoyYxF9oE5l{@E1j5r4qTTBHW#ew&9fu{ zqz+<+%e88zQVBS4MS={~!V+;xp_9}iS2x*(lpcpNDJN8`(qT?{SuT~36tOA_G#U!F zl_pY;lfBNGC>?2sz9#rs9qn{#u8s+wgba_2m6eWoOpCbvbWq&fN+*PH1>1 zu+qO$cWcjT$Os+I`;KJQb8npKAI^&{IiZC#!a}xaE<%XH?kH3qs<}Y1Jvk-} z^pPD4bCwsZ1X>~~eHl!S<(i78k}9c#X}0!J0g?UCrrF~-fcipJ#T1(jvDp;c46!XQ zK9&<6+qqI)16}D$PwV#GQ34Pk~NsHMp8SwK<+2Q;39jmo>LTjNrEo&`i_?!_wcQ2f; zzm$FEtA@Sr`c}0&i))K_URis^>>e|^$MUVuJ5^qQEm4(!6DmJcwt$O)vz_Hy~gWd$pUm0su<$YFt{N zh_kWxBc$hTn^`~%kT{4zYKjjH%{JVpwk3TE8U!=Eo3vw)?KtgYhX2^3&U0OC$_Y*X zpK=EEe59Pcwq6y;tlBNja`Ns+d1Ka#@;+|MyRPBp*o`r>?toEuz^v;v>Uy&m?l&D; zow)Pd+H-eaTzk>%95p&e?@~#Rj|ToQ`n%D4;00aC2^V$( zxA;u>6)^mFUeF7mPD|_(2ag5TmKn&TuzQ$ZO!n9`kIs2H&j@CrFS0Fxzf_bY!FxI^ zSdG+K(uthZqWF%5ON2V-lUDVl8oi5QxcHoKY|do&p(OTay+mZLtGZkn(d_ zQ5tdqW)Hw$>w^pg8)`N~Pa2^o^PyvgP@f&WzVG@t7}+=38Vl-|CRTP0e% zMWQwZ8HZVjS4X5xd(5VzM$^%J({V#+$o6NKuFLCURi*{a;a9ubX?YTHA&ErSb`Rv zNm{OWYNjV`@$#e=cG-8UsL?=>L?5qiyJ0z_nmQn%i#Ys{p5pv91T`LD`_< zy6Dvwh7!Xj1bt@k*(iXj!4!k7jYnryf(k&6KxXlA6;P#JmeVSP3E7t|k*L$oJHV>O zM&Th)QFt6!Zo6X}l1LBo#OxycHN2~`%vZq%GuUPX+g5s3CRW>4dsf?4e)jgse6R~J z5vVr<`;5T8m3=D%tHITMtHG6_x1Y!dICLh%E@aU7uN=&+XcOd*E)xhu#mo`9pml2aH3fKNAOX!T?1U1P}2M zM$kdF+(gyqNb*|`ecp6kvV*C1K!{S1y-3DuIw4VrNP9$2A{P{@44vn=~L)Alvf`X%G}ntAGTX8#ww|8ws67rWY5hVr`( z<(S|J180lLD78~0m}|#Kt)m%E>Ki~QUplPdJmdz5+Q2rmRgi9mGze)U zl!qW~g0vdaW_Al(1L+p%Ukhm%uInIef%;95wnADDY1_Ox68;1iA2CX9CYem7nRG0b z6yVZ(_T1@X(IcNbcI3HdPn|dgCAMS9bSym&Ip4@Qm*nKjBNLO@`ba97X2#PaTvAB! z5v%07#51$ebSgRK z6yoOM93OpUejz%+rxHYo0LSS~EzQn(GA^ zv$P`n6UI{eTI8`x{T!dNpS7Z&Gh@#<7I0Ngj2B&+aRTLgP;(iV+mJD`)&+Pr3-0o| zM+_0$sRieeR3gD7+3>an&$cj-Xm}zumt=c+d~2T|^L>K3h*Po(NS2Z%$s{<*KQ4Cz znxd23|tmza#J|)=%AUU3s%-k!p5jP}2 zVrC)DN*;A!I*pSN1RQr_QnJ#lWLM{~0YA(!amoHllv1~3pI0*rO2(^BOB)Dq?}l(W z{QmhJms0$UFw2Z{JErE~Mp7|0Dj@xjr5K)#B`0?X{P>Qb=ImAbd8a))x&*PHUSw~ecPtCpLMr2Z+t#P)yiIF(Iv>H_)m<4+nJlA91B3s66u$jJ;@rKl8eg^|)RSS#(C8MyjX?tWlj$ zO-u7yG`yyV8|I8<9X41s?e%!mN_o?y!<*()Ja*u#Ac{D759V=3z7`@WQ06TP@RuZ4 znQ|pNa2LldaQr50Yi7m-z5#L(8^0MxXv9)uR5H&qmn2`gUlgw$5JlP8#DsuPOJ*0o z1AF)=fnzljk4+}|ekhE^BiWYh@ayJ# z_1*cAADzE>zSwh+^c*bq43nPWV$V6!bFMJ@C9&rvvHoRZs-=y6_q@R?=a$Yb3oGYI zU6<(X&TdzU@N>(1R)pN4>laCAtLW?~m^%3FF#Wjp)s~3x%Rrjfko2HMWylz*5VB=# zdW5-X&)D^dbJ3A;WKEwpfUt37OfyG;upJrm49v3tq`v4(mxPy|D9SkX$W0Ystm&5d zU4@wzT^R>!!SNT}PZ-nvxG_}%Ui83IKpLtDtdTW>m{UX(Yn_3Ix#)eujE;<#3Prb^ zK5qU6co*s#G^6UVjtB1FrrtjaszqPM2dma$8`h5VvyOcbLyKU=O~K`K&Bn5fMP13F zKjT;4qJRAp$XM~2Kkkke0~uE)umE$yJ92>#buQGLp;90`!Exy@6XV0_Sc2=7`S+$# zGPAJ+6{r6yuj#=7EKJ0bk~yB5T&Q`TOY^aDVF&b#V$W!TW0Da!kGG|8`}iEsbIEj6 zZK5!?Fqe?56Y&(2mb~M#iWr@p0;Nr|j>nnAtYqU}N%M@rSL0ClNFr4f0wlLURmmuo zvflI*&k0khI2*-PX)l>q_$KHkRjFdR)D%QG9ilW8%6xtZ8#_SS^GtG*lWekllx$D} zYnNQfxdf=DH1DKv0)k(Vym-gun+xM0pl&d6PC)o8%WNel#x2tzcjy6 z`)1?y#_u+Z{_WY$HD6WHw}tq&fUIrFnTfAGZz8_VY~)_B4h4Uqu{(c`H159B@j*Qq zdagJ$OooQVp%KzJA_mXieetE@i(}-)u{Dct`Ot68rY{;TzcH9XmTb=hn_=fO*|u+V zUhaJT1!4*ns=J7(TVHv&uP6_VR9y)EY2PxlJh=2sVRJWecNbj)#5Ewgo&wdw2D)4{S`+NLUqtwZI4p0JD}`RyDt9%9vQ&4Ad-|VeO%cZg5#z zpKK-AWsMyy9MA{!ph#^`@5Y=l1B^IMQBnU1S}RoYfAZfD_(8aweB(NXn+GV005X*I z0GAL;bJ5w95WOS?p!-u3RY2iO*;HoIM#|;o;c44h zhG*h&F3yia(^sEfIh-?o>p<0gZh>ijcI5r2R*I;>&$q|q*5Derb5^RlvYwE)ZUsgfoc?+ zR1w~bK)o5$I7b-!4JmDFb^$YM5IEW|_6+@k9qlCJ%z4xkW zuhn0zU)i7M?^gA%GNfvEwj1b7(`0Q^$6FWjRXKrd0;8s8&FOmmWYN=1Jk6pfOq}6D z+pc2UKGL>NY}-$q`wOQ1GD<2)0h4M_^4D;2%}};Z^pH_gi*!^G^wCjIoz(MeKyMv2 zPivECX>HM@S&J@CQv1Zb(LT9pQS1{0DA`DH$dvE`79fyoV9Gh2ih=q9I0g0weStCX zMB%k;n3y$&iIYuD$OgQCTmx7lBZ*(`SQ?3+gxZ(kC;SOSXn%yROfOA;uLdP?Zv5@o zjo91C8_8nFe$ui3gHZ9%2!a2O5wZR(SR8kKFM&K(EZb+7hjQlJ@C_RYb%@Sw1=F^T zZy0fna{MjGJdxu~PvCfy_ATh~IjvVP1+3jNe(2b`8MFRd0{LSBZiY9to}0meSY3wx z1jEk&N3k(Mkr2rgpWx|RY=B!;WPxlnf+{OWLD^y~@gVj@p->6^%A2N4j%^aq26Qgkzn#TH>(tdR$J>yAJt7AB2tf~$L zG-YBpcqYIwHcX>AQ4thbq^w7gVGhk)=(eZ`U^?41Ezns}5i)jlHSa3#cF~z}$~293 zp)8zsK`!Ezs!^Mn;u!SYMZp%q%S(J0Hlschr&@Q3Mw_ zjnAk{=PyA+#5O^M2;`YJQ1osk-mST~=pD#z2kYWWaw%E#wGdxR(bq+MUD?Q*)myOE z-}BTi&*z>-pNXgY^0E79hsdpZh_5Ff7kzuPC)V7-g1fn3X@*$tu=}09Jd+>1@eHZk zDSG;{$L`tO--um~eY=K=&ZoqxmO`KcO}y=S zGx2uG7SrCPz00rUs(x^+*giZ{j7{g;DS;(zdW{G;WLwb zca<}JhC4y_sR-+N7+hVCwRMma^|1i;%;~y{!QqlM-zvLcTxrx1XqZOUs|cD}bXQgc zn76s22rj3?8_S?&>&DV_32Ua?H`apXr=X3BK8tP^aG2`Pj*JUa*1iB8shi59S*sq( zY3tFES`jjCbru|r#XV0I>%5wg(aTv8G_{=-RdAW{s^hXQ)?FGCU25GMTd$`p)2h-H=2~l8xmq#>T-}#-`_IImOEUrvsDU{P(}e{qFBq_fP)t#6K@o z|L{NFS2G8cHa~w~%^cuS_{jS2=CZ8eSu9Oqgp#3B-XL#CwNV-Tc`m_#HViTgu4e^s z-a`^))BJwyNj>;>mYeM78CJ5TWuKcFcf?#gGs8u}&jwq$^EjfNx&jCKC36~W{PB2f zRnv*4F zK<=21#liD17h^eo8CuW5Pxw1f^gvv?yjOacdW)_m;%X|owh`C1cZ~VIyoI=Wz}xj1 zglVvG<%@YIY3dV${gRwXaD^~B!?pX6St_W{_<@#5Mx3lQ!Bc8t1@q(vM z^bEmX9XK|;OWtpD*Wy>>xq-KL-`HJjdy2F@b!Ytj>A#-7d-{dq>6gjrm+#h|7adV> zcld+LUn2f+?o58$jh9xBlC4h@-ySe}z)qT3npuhbfI}N;)%<5k(lEHn-8l=}HFC9V zMm1b-$UQ68_pVx2$3@=|C>j1>(I18(Pda4= zcYy{lD0_8gdcHtAid_AvHF^aRWse?sC%-W};!?yRYVgtpJH-hlFuG|3-%b@q%EBr80pMMpZMPKzfTRDT%NGfdb* zUJ2aIBkd364Y|@K?PIzm>D93f22HJw=8Jc@mV(~R+`fk6_SlSe8m{9mTc-r0sy%c8Hk5+3scI@-z4SLEJmK^6Jv7MSl+<={wBrs@21* z!>fn$Q^da;c9Nh4t!*X#)}p_k`1^17tΞfZtQZzvsP{J7<5|cc+v1j}`qViT~tB z!-dl$#nb1=>2t-?EIG{^-U;3f?E_z|A`@OM)^(D) z&OBeJ>lEwy3pM?#BX?SU`~_08S9I?yi-^VC#4WDayPx##zjIOSJubRWJW@m;sR6Yc zA6^88{eL4O{zw8LUE2K0*fOOpuSI(qfb}4@3{)A#3^WSEkz2a22D!B*EWLVu!+niMe65da!BDEzY&PxSy)@lv6TQXo@7Q98{S z{|1iXrn4&(e;-S{+E#F!h}OvtEsBE3U}Y411O)Q1_gI2JRLF2>0S*bHV&1;3*nW_- z9~9dU6H^PE^m%#t`TGD%&Z|xtns$)Dj@!l(z&S`@4{X=}uYv1cuwi9y-YW)oWqW=N z+w_^G%-zPFt4;4+_@M1)Js4XrXit|c|HYfIH@KMQD~c&4C*@8nT_FXf;T z>_O}5en@c@K{?v;soU%CKd6idnuJN-LAYtroJq0LbD})D5^gHnHwPilABAZB#!Z7p zP|k8mCZ3*G_YU|~?2bJSK?F3?(4;-G;q+hGytKLKY#`2t0zK>nF`QEegJbs%w;7<3W5(AR1bBt zasrH2sK+ySQuX>O*vA|LW}}>jQ~{hOv)Cq{n~dtV{RuSt5&Q)748itSee;|3*Xwfw z#kM`9ZO@$xck4k+*RHv@(KDWk$72E>vk$Xenj24ZYe%}LqJYQ z^M46#@nI`8djAhWql#4lqiXh!CxCJvg#y#K&WfN~ys(RlyOT8N?M-QF^>z>Scct|F z)jD=2@C-_ZilDa#t+yAY^&YO{GqYyR=nkp~kK6GCoUEy64wve)Lf`F(fggt=ICpLp zz>yT^N8n0kTWX9@QCpJK)R!k!vD^eR2TrH*{xcQ6n$r&Qo)CDpV8|$Xw)hDgm+~~e zA1Xa%u67IHAcMo^boRf2y7%CxoTEjZAFYDdmR=M6VURwL-`D!^@vULafn6JQip&u z7HWiBHaMWIntpmjZq8_1=r~Er+omO*5I$4^0Rcy8kk@4+yzDD>R{Dyc!A7V|>$+(* z2e;U5!K2C^E#F;ezw@~*1{DiMU^h7VI<^Usn zEC9w|K16)oKv2qQ#~RVUrK}e{(#JQS>$tVG*f~TxhwkhWI}eNQBU(A6Zm6PxO__(D z9g+{6!p9&ykLo#(cZDtnMEu`-=;tH7XDODe=h-Xze}KI{gec;ZeF?PwAeO$55vm&U zOovN3G$rKPBc-wrFh|F+XvQg*Iy9x_{#5TqRN%`kmDg9Xj&gmwJW|J`>{+P^4D5Ncjdw=%rui0Q6uB)H)_&lk($J`pP1Z zN4}Oe$$jSlf`nfD!iAZQ^(GiN-%B^gso{ah&ZEsiKR=KHnonV19RP56M^Z3)^w{x3 zFPuCZJv1_M>}>SN@e}l$7y5w!|2%dd#fZUZ5~Bo0moQq!=q-#8ps6~@qpYPumq%oz zlzY=a(MbkR$G$|0h40t)^S^+4Bm9K_g6M(SXf%FoX#E#M_*Vwc$A&E*8=m>t(D6Hq z&1m{h14O^^87x&Fo1XrS%~0E(J0aF|7YxA%0mGKQ2Tq@{d(9qxV21R*-SfZ-DIB(~ zs(xU{oWtO2q-`F!47T7yH&A4&3Ryj6#5109@sJGz?>6vE*#5^ zIgg=s^8+vDd^naLau0{>5LH`^2Oj!N#_bOs7P$0yAZo0EXr~XNunVG1X5+Sp9y8=z TF#AJ~#klpi9-A>hAI|>(up-R% diff --git a/src/mjlab_husky/tasks/skater/mdp/__pycache__/terminations.cpython-312.pyc b/src/mjlab_husky/tasks/skater/mdp/__pycache__/terminations.cpython-312.pyc deleted file mode 100644 index 84ee4853768fd9c69afe2b598569473ed08ad980..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1197 zcma)5%}*0i5PxsqcIi^S1PcOUkPzZVXf-B=1Y@Fvpb{GcTjk z=_p{LhVoKQtA=Vcz}Z+LTH}dGu3TxzdVC9k~;YbvWb0S8 zfiKdGnOrE^)Nyl$&cy`hT*{1`GoR!vleF^|ozoYJyiiJdloxn1z_6rkEGA*vcFqY| zN-UQA)|_RT^VHHaAvx?Ke5@HBeh*X`?f3M()81ss$>@1ELIQG0gO zp&e-m^nu#4*1g*O>h{Li`q*B_V5MX5w;=U5BGe8P*iMp&D$ zTmA<4u{mng@HH-2hd{QANPu>;Sv??u^+G@d`E^h*zxbuS_;4jY{3V_!rw-b?Hner^ zO=?@(ZNIhstkOPOp4yLf?#P`i4tPxoN?;1u>;fn#%wxX7Os~kCv!-BS60_kCB-dg4 z#0ytMhM3=|>)9gUOPB6xYXy6$2AEBmWH1w6U9ZI983c-mVO$2uZDSiQT C$rzph diff --git a/src/mjlab_husky/tasks/skater/mdp/__pycache__/velocity_command.cpython-312.pyc b/src/mjlab_husky/tasks/skater/mdp/__pycache__/velocity_command.cpython-312.pyc deleted file mode 100644 index 6deb6494098eabe95db9975fd2d1881eb2d458d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11600 zcmcgSTW}lKb-TaCoLS6Po>3>B$)xRv96FOs zzIx7GEC4|(*@>sU1kc{bx#vF5*4UtWj%Up>H4;9@(O1)6(6Wm5=v|)w``+to%VOn#?(&>yGlM|V=6t-wiJ|@TF$(ST* zw&8@3j zB(%D3O~zy{HZhT$;S%X9f+z_ZI}@AYFdvBTK6~imcuW?=3$b);L=c~kNdo_B@=W>) z6#7pKmnTPFPe_xoWFjkwLy+efpW-s<L?IguwN^p?La>8V}Zdr7CrHN#D7u0fB;CTSg`v{&Bmd7PH zZ~KhpiMn|&)bx|~>iL@YnW%@aeV>YYXX?TMt(lDdwRB=QBc@&#l9_lyo+-{J%;gIj zb2*bqYW`6n#wXGvTwI^`U+LYWC|IOqCMIjOqLAby8I}aGg!C1ui1MIctHCmZ02Vzo zbHyz~Z+8OChi48pnuYY+8fAC}QZ`J@Si|-|?0s^-dtZ&EM+7NbFHcS+g||o__lL;m zZOxfXq`525r|H3LqX|N2HZ+(?3!0JP6+rZVEVG!csnk!hfkAzz5WQe_NhuE9~o?Kb!M* zZ85!KE98sq05rQu=Ei&xE&;~Fi}1+y#m!6>f(Q9RwzdP8#YypO)WJKU)KyHoLFyp3 zhb$W#_I}~Ay`IRzt{eH=YZ(5mUkqS}*({eC9+m{TxTWs*Yc?q!O9~@U3gPp9EnsY~ zm?&nZxTKICkw*u!9_-qnK&HD5VwznalZe|Ji=^iOzzQ|&oNfUG5&796aZ(6}E>FrK8ECzub6al_>z&XD?jr5Nu6l4L1&?VtDF$5PhX^MC?SEQQ!-0jjoPb$F}hz#9Vw%L&^861ZYW%xfiHmnzz31i86wX_wda7Oi91nNsg@|$HCh{ zOS8O}cYw}z@+_oWka9xG&HD%~7tr!7P}6kOdyxwJ$@G(X(yZ~}kvMXX5paSDC!L0w z|7#kAfKfs%T4$-hgz{N>)?x?)8hH)Zs49#Vt3bU3mpV71Ocb4KxG6XPiH&g|{D2;vA0x73~AR`BmFxFti z%3tX5u5!wV@vuH)D7mY`ZN{*up@mXE|CAOUOTQ>pfj)|B68?|qmuB%#p)aFIAYlwe zMOApr=xi0UhZ2f}iq=tKR|TNdQI!QrW52*GmHJUZ=S#2iWBSP$JPGA!YE%XNDDnT; z9i8j3vg#%u=0nEfnKtbzWSCS$cqE%W}Jcbl(V&-y2)?CEgCMG%2Zki1Y zs`#j8gR@^o(%AH5iX3i=%@$QYcUkiq-4=!6$|UKSSc^^g5>n+UNOKXITrx2(XnrWG zJUwX+FcHK=TmmhNdSCDO5hUZ}IC4DZI0@x*C={C4_2U^xJ<6?^AmQHhTsL;)Q(8V9qsof5)z?Z+`gZ0`nKHLgSw6 zz1Mmn>Ib_QKJ2Hvo39721y_e|zJ25EyI1pv-q~PuumkuD!JuxU*+{ z&xZVqZ-4adtv%-#m@T&PNkz@=*T=7o7hJxj1B(ZKGI>3FEqk})=RH5``P^!0abfk~ z=XNTzS8a(XEs?v7dT>BFIFN5SjYUnLc7$)9zj6L9^KrQ-4_ojA znuUKuB7#w38T>VlA#4><2Z~=uU^=`SIVO$>a*0Kzo@mkbR%7D;q`d&+k+k?Bm)EUWp7SjRSfA=^T6d(V4*Bk(t;6N%OdqhS{9(G>&?X2J6Xm zcFZiMtQ%@r5mZW=uiT`eZ|2bMC~35EId7SUKOD7; zraTE|b_>)tv093|#j5X?S?j~;EZB#oQCM)EfC?VJtHKx_#H&ITC9r6-U{}ViaSMw5 z$Bc#fa$}KOjmChExsCB<*R8{#RQ^_u`YVp!;KHiWgTA&6imyYH;`0blor(hh!u3SA zXgfqfilrvNH8#S@32=204XrV#?lgxa3lkhKi)g)zxLL)s05k@jwcy{&WMt4|>2cyy z<)*aya*p9^n-X8aYUi+;HFjCjg#giL-gqVrK3|!efOEE>%ZWD-&uJ*tsiGRxYErRj z!~I3N`EU+)Q*)qe5A?sG9*f6iKQ}o6-4(c^X3h3INsLw@qgzaB1wjVlG}v;z?^>T4 zJgx+fZ?r1GXL9VWLa=9zzmr%`sJ;CP{DS?E`DtCpJ+@{!aJA*b7PWS-QoC2JJ*w0m z&9lb}tY^u;=vUb$g>6#Vg9>|a+h(ahyx@3Xr<`70)#O3dcxsnkTzqkvzncCq{nJ<0 z=+z;mVc*&t|M*~p}?>8jeNtUyywl_HXJ1Jxs^J`Oq$AwO#(QkP;o#2jx#y4-7Yz<<}4Np&S9k!mtLz_ z6L)M?__7lAhuCuITv8@7OY@dtL#CL~H<790BG59!HIo`9GI!qc67^j;pO2MvM!7${ zm9IP_n6+YZ1{nw041F>=Z_d(0UZOrTEk(&sbe^?hwXDW!@s{z^!05G58=Y8HVYY+^ z9)>@4G$#6>I2seJppQnEfBn!KUG&vs3}+==qst_lkeZOebqcOy-MD8b^-B>>x4ltn zi>S_Y!G%jqoOHT3YY>v!ZmuXpg_03dw8li@_d=V@5M&Mzt6YMYG+Hb>uS?aTph|kx zmT4Pi;DIN~uM({y`Uo^GZ36(4-{oKGS?u{ye5LEJl3=r+TtB(xIyN6Gv>#YKbo0cG z6JRw>uTS53Z~Z+ra!!ey%eB8a&woGto%He>E3xIMVhY$v{PaiDYX2n#eqEOq7?lkwY;fhQ z!gdr|I#*@2>x9yEV&j<7HK4SBnbvgA?aTSw^X`t-Z*97}!Jt=N-HNL_7k+ljb*j+W zb#v^-*xmZO@qFhAI5&)bG^(Duq`B{78rw0#JZ)>NL&F-Jl*1xIh=_M>E67|i*lj0;0L04?bs0Dv>T z^7vf{>{0_=N}x*(M3g{eV@EC!$p`w?z*!}5HXnF#q3fQvW_jlizPa+XP4AvUQ|S7+ zYv@@^=?}b*gW_;@h7)@a&fFR3X@+1|v!^ zq6UvC!DAoq$OVt(gJ;#?%S!O&eDD=CZhGLnzT{i<<#u*$vfVm2nQHidA>34Py;R=K z^msQ+J>0XvE~u;v!^U97Q)1{bxF*(snf9ND*2(@Zrt52r4o+aKdmF6l~UP-j1)F!9s?mpX*Suy(Zr z?HuNzej$pJoD9ToVlsvxf#5uXuLG#$J243vnm;}%ibA?{@F&j&;5o>Zk`}FEHp0o- zr@x%`GfogiR-1zsrJDdi3%Y#^Q$KiS<_(z4Qm;DB2UW1C#j&k3P*eqkLV+CHPQCj|%@pyWqr0wS%v({PrtRst_06?B& z<*-o)cEvs;Uj7=b%{gH-`^))8>^lt=6ihKn6kO@uGT!Erz*}aK;z1)_{$k%sa@7U% zeYjj^#KzD^P!(n^D9O;$;Td8j?Sv7bAHOQV46wXy*2UXzJ8l(EmNeKJ9d7 z-@!a*jORJ>ekSZ8#*yZN9v2_JX|~HT87@Mc624i&hX`>7B8c+fQ`M?=9a9#*(o9;seS3q?&E3W23U{9g0u@LAk1cHS?#{)OzZuu=`cYELgi03dd|A0rxwyl9l zFFykbTf`HP@dppNH0vD_(o#kQ)N#aGL4d^?gRko#dkGTR+ODc6mR$r%yat7jz9b=! zNX{S;c0$a+-JU!n4ndNzgD7AujbI7^vG{QxY4B8r+?mCNWRm0Hf`x26*eld?`rR4I z9rAhtg@|+s!0$hx3eK)P)Ad^>V7o{^*iYGgzkx?t&G1gd?!)I<&EQSb^Ulzv%ro?F z>|R@VyV+@blm1PE(-zqVaP)T#L7V3@3ILft{hiECd`vhz2@hxBF`!7(eM!WO@W0C&6YxM(JC27BR2A--Ofj0+&Hm?5IUAfhhUcIlXtav~}9 z;yc6GL;~u`qq@u0g@|rMtf_d0*ozOn^hZ8>^!is4!jvHPmbkAdAw+z`p+C|gvl+r< zD}puv8vBg?Z1R-&_mIS?m)->Mz)I8fCsf<7sP0dw&@ZXF|DZg-qF(r&g`q8XjrGXBX=ggO%zP<>(cG zl@&1q%~;9s8D!KiU;Xm@qU_>=#N<@{jMCi1y!_0Rc(9J7{KTS^%)E5{;-X}IBvbXl zrs_jX)i27?kB`sH%PfhH*DI*J#bJ}1pHiBWYFESoG@TKMi$Q+GB)k*SCc G$OQlnKuRe9 diff --git a/src/mjlab_husky/tasks/skater/rl/__pycache__/exporter.cpython-312.pyc b/src/mjlab_husky/tasks/skater/rl/__pycache__/exporter.cpython-312.pyc deleted file mode 100644 index 28cf1aa05bbe35330159925ee3467bb48bc9bf6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1658 zcmbVM&1)M+6rWk`N~_nwMtYPLQEN)PQx!6}8D`es%e8*=Ltn)hbjn>X*{_x9&< zxqyJaS^V2u(h>SYZc@rmg>weNGvp!{`=}+k=HozXX;{h`KNILJJuq5^+UtHc$hC49 z<<-wEy9PDgEX}();CLs8&}P;(zcN~dJaT8;Jk{I+Xr)Eb#ha#E0$p*4YqNjJ0Gc%% z&poh1yG_|Wn^X6Ze?Q!+X>r9C!ge}VK!xqv!nQBt7T&?;>?6QaaJAj*bM9*bv?$J3`ic!8r8B4`h}D! zlTpOxrsvbp4rrX)qHH7LG@k9Ns#?hv%DD7E6uo-T_#$FG-nSjv=nMioj6B!kJzLO? z$Y!n=wi}!|jiBq>8&+q)d&7pXd5<@g4QnvJk-Dw-hw);vNjxxX`moJa%Zog&h+fp>tC)v)yLvQzi~Fd@@?tI?oTVn^PlWBPPMt; zwCXQfbv!@bKG9as!3T;nG~9|!HV7>Rjx3{Mz(NN`l6^pTOCV+4bu?ZkK0ERdvXw|JjA-lpe}bT){H$irsy2?-*X`t?GAkh`qS z*MJiODXx*n9ZDX3DrJvFVL(GsSIRW?HS+n*djxhw7J@{KM503(@mw&4wD_I~@o-}lGw zA6AcN-rcL6m6yl-@Z;n1t-aMV?X7X`@X3i*XV>5g%-XABg3G$}{80_FWqDpwS;Pee z^*YpYA`Fdf-wqYFikGh8q=3M!ZH zkP0hF+57bC3OAm(_;0sF#Dod zz_rsW3zH1+=VozY0H0*h?6s$2KRD0H-6FcZj$dTTxN(}RP87eflJLfRia$57%Q6rR~1?`~=*xOJMiX{i%h8AD}Py=!LH zNgO#+DiyUppd<%K4_qoC0ZI>4Lh7}5F1SI&8d?ddTtFOJrBy3V%*=X~lvD!59Nv8I z&D%Hcy?O8L_lZOg;Oq0fzn9&0x2LUckBH#07QPl~sjRVZ~I4XF^l{}X7>BlcIZ=#V1+%bd zTmU(pr7p-zp7ochjyKJ;8V)8)-5BLHF-`@)@r44wXK)1sQY=U-)&!sO95-Q8S>>l< zLC^%1$i+}3sST&W5+sM3uwrZxccHDU!VubOnuB>YFe9yseBX!*_e$bfUU2{LMc?)> znrd`wwAcguk3t|EVW%mhX2F&ySINouX~EE&+CCmgxG zB*MozJuiClOipDzo=|nma}FV*{Fpgjb{!8_otz>CM3fV^VvS+yAkj3rT}-ES-r`K7 z+zYnD^(Q8HCkKt1!%u2%Ibe!UyXM4a!W=W~GLb;V?gWd4ea{APDewvM$0&+Ck&g4> zvF(7nD0-BLVMi?CtX21B?Tnm4`Q<(9|6s_TsQ<$t*vFDfVaB7YDnAkki1xiYVYQ@E4XtGYH8ifGU zDOw08TY=vU?-V=Z0UyiC=QIF?&-R@r)}=JYbF-0mbKX1>iwHu z{7U6gWohWzY&)6#?#Qj9H;>*rdGqA$*VkS<`%~=PgDfx)Zt!?{X`RWL%tT1M8bYiG zAzr9;SKv58AJ?%Po)FRJtPdgVd43>^gufV5X+UIlS)%)lCC7CU%0VI;nu=-r26>57 zA_lD_qE_fWPM&@OI!1a;Nk7zftLNboeFDV(P*NhAU+U+2NWZ|c#(D{L5pnb3+fpKa z8O)SbkSA{F<$*0lk(Wi04~v3p|t51dwL_0_9cH+W-In diff --git a/src/mjlab_husky/viewer/__pycache__/__init__.cpython-312.pyc b/src/mjlab_husky/viewer/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 32299afac3ba202c8f47962438de1b30f714d133..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 311 zcmX@j%ge<81k+?*Wp)GU#~=<2FhLogWq^$73@HpLj5!Rsj8TlaOi@gX3@J=0%;`)~ z%#|#ftS>=IHJNVl2c;I3<^|*=R)%G!mZuio5<(MjPD?Lh2C6M$0TOx?BasNlIYq;;_lhPbtkw nwJQPz2O|&{3j>J{%#4hT_ZdPSaLZhfGrGWK+{j+U0Tc%S6m40h diff --git a/src/mjlab_husky/viewer/__pycache__/rerun_play_viewer.cpython-312.pyc b/src/mjlab_husky/viewer/__pycache__/rerun_play_viewer.cpython-312.pyc deleted file mode 100644 index 9f574c475acb84f3f9a356d1e2050463f73ac70f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20452 zcmd6PX?Ppgwb%>>`_6qQhdaRqq$p~mHp?Wn*`ck)i)cF#!~sb_BtQ*7(}ckUa-Ep8 zwYQ5L`)&2ShAfN)7(C<^9NiF>oI#R#n+3nY!dk2Go zDA0EMwf!=p?t1RI`#JZXdk+4SPNyc|`|HyGYX5iZ2;y@T5k46oAn)FTz*&MJ7;=~x zA^S)Yg(bt1J_+`zJ_`HNJ}LHPeKPFJ`{dwL!-^4QpAwgm4y#7geQF$*4QocUeOerr z59>zseflI`L!S|s&+5y z$!V-kr^{pY*j-L{FT_=h)nn}+wz}OAkv?wq41rJWa!y<(IoXLZm&XS2qGMK1|B#Kb zJa6mwxLAwFWpR#;j83d$`jZXB?bCou5d?X+8#;29@R1C8yqF+ZiH|r$c#@qX2p{1{ zFNKQ|FeOD05{BR#p(W|ilhmZdlbmG0FY!@A=O?8;sV}`1DJX^#>+mR&jUXIKAtd@f zsR#88e?C%Zp%>DGaLP~iYf_emQ7e=ceILO{8L3b911jDBfn`o6Sa}QZZZ5%aXeSwoWM>%P~%|6V@0F~9lWm(<* zE~m#nIOZC2TUpjRVUk0!kwH!g@D{h7v2o-OM~-vk2q$%WY@=>DAgA&7<#$@gUF>o9 zsI}kLIy5$7b-L_~#eLlBu^n?+S;p=hY<08!ts{oLpFnEUudE8n43*2a!GEu+KM z2@9WM9-ZKFEsSg2>9&rH4%;k3^=t{$mk0md0dV}p4Vf;G_0n?*jo!b1QKg+aG;?J7 zNZ`cuvr$#?`K}OkNp(RrTX{hnEp721@K-L%Ra2Ey2mY1aUzw1}ry9~q9ymjtdHC#} z(|dxuF3LhfKQly1Tcf#cQRBL(s@-3+sM5q$Wf4_bu>V3`s5?}5xo)<5w(f5$!&U1e zB!ie7#}q(~ln_%TV+uwMF-;Oui6OPgmqIHvDYP0^Kx-a+NZ~Pc{rBeXTW&{I93Q04z;rg$BKc@p;$?>TsAIOieq}9eW(v*I9A@5 z%Tz#&#<9x2JRGaSvFg5jjHL$0YJuh<$T}R;1LZ?l>T#?A=pU-jl*}u}d5tJGp)fUZ z*$2@odK?zflX#nS4Gh`?6+n@60Kq9?|5!#{tmiN%J?3%^b6VG^&1pHty2jl$w)aal zY!2E-_qv=mPBqAm_G4hL+U;S5WG(CsC(uO8kjw4il*6t;%ZX8!o68qM7KaP4Sls>A zVJqw2&FNAZ9@`6^J7@uL*{FyG#V!0cA9izwG&rZ*#p1ooX)&bD`Mkx>yuhguOh0UG z*2*Fx!)5Ui3rdGZ!q&VaMd=1yOiZ==VgGZ6B($)PxGda9C(skO$1-4Np-1{*8|yme z@>p0~zl&u!tq@0CgQK7lXlW9}H3sdsvz#$)H(Q47&)c|M0R)wD+dLL0G}?X}I?oje zS(tKIMs2LcHfnbR4ddj85S?JfI-%@wE=wrjdfvv4vvz<$#F@(z()pGq`=5s2`n=8J za$3;fu{LOgsf1Hna7!!}PHnM_xR^1ZDi%&_v78vQ4x=_&EG&%&&|*nrn~+)ssca2$ zYLQcioOKMJ45lvPwu( zMVX!7DsZA7?`C9_HQAt^%v@9&i4fLa6afQD%&=HkXB=QTO zG=<9K75N*dn5d*EQBoR~|X2cVkr zL{UATU7sj#3rq5YfU+%7T_2W|1OZWfBEOukr#w+wnJiwJr~pg_!5oCC#)Lo*$0R0x zG|)h#h$-4fD)r~Rpe z6BI(n7u?Aq2%JqjRAoL=`3Rw-lcbMylM_VRF^P1fjy-@Z`7x5zVJ2BG3=h@o&0g2O z=~;6d{#)1C@GR40!W-|hl#{TqS=}}(+dsq;R@ory8XLVVVfO=4wh$aQI&cW%lV!n# zX4npU9#}jOVD=znKZM}WuJIF#a^)-bllHK&HuPdtzTRK?JC!=2F(7V!VCJdmrve9F z>wB#NR`6fY04exEX~m<$eh2!&9zZ=x?|Ba2bJS%E90x{Vh1l7{0JR4}BC?U{dq5q0 zBr0$CrmiFG!iUaXcx^lmfp4)kNJr{0ETDiRZ88W)RLK?UT`3=hH6ZQvYSY&M?kDUI zS-)${=`m4kFMt9|7&`_BN?<;tCJ9HO{_TVG`$nN02r#DHz^GiK$E^LwEp{iQp#vW( z&<6z&GItp`Fd!<;O#XC!z!1v4RD7ZMXQi|5`O27TTUfG7zz;KWjQh2@#<8K=^ zNrBkeh6gwbA(r!aISMekQS*SlB?%nxO}o?Xu~@u?tA@%f-^oQ20ZYOf{SjKejsOX zmxSciUnXKWDN&_Ogpc&3hBecjPavD@4wztk*K}xB*CK8f2Rcu#`h=-=7*=IKG?kaN zcHSiAT=C=x`RFeF7tyAl-`6Hds!iMOOHbXu*|ya9{l6Kb?|RVH4}Xq)Atd@R&$&X( zfjH5s?}}9D0g@Cr&=G6Z$Hf2&+AWFIFW8?xO zKSWd$o^m0(hF~d@I6_Pih5(2Bunk{X(9UGDkuwYqC&MBm zPBCP4TRk3@HKIfXM37K{Q($q!#tu&EWb7lH0?z1IDxvCmq2D%oC4%b+4se8ZCA zTM$QjI0_KB5yeCVDoLS?_!^cbr}EMdP%8e`m6BvV0~p*JfOZ4LQfU0Xgf1(gHO}mt z-WS(aMYL7^o)w8+mc6LT33Q*YeY^Qwb4*na5P`huN8;M@h_*bcrQ_PVh_)`OZHQ}| zBiiPu)*RP%M6?}I?Z#9p1JP%+F0{Lsq$qJZCO-X5!Y5n zwAE2L80Brj$omK)PG$8{YMUB}$PsBUvuzWL9K zn%qAUq%8Y)x~vrn{4d>Tp5N@*y1Rh*#{$*vYU&@WNO12H52g<2xj@Q+fPZV!jn1uz z&t@d`h%K!8yg=GP^tcID9+VVCaeyI%cn|T|OzH0?IW&kCQ|Dy3JWvdg%AtJJNnKhl z66d9Iogm9WMCCpRDFYH^p*)6=LG2k>i%%wO-$}Voe$wERJJ1GP^+7M;#)*2l#^~r* z6@Vza+mm8D4m546K1QCFpCXQf1Iru=UjJtTHm^|6n5T7@KE ziZyf-&%u54S->}`{>Ij8eQHMcrt~dIS}%}9W7U^wv<6dRb77oG;?xc)Frf(;WfJ~R zD#r_D(|1wQ_YhX8+wQetSq@xt zS&jlB8q1;#xz&xiBG1Jz^IG4|i+|9852WdKC%z5y(jc_YIK^R*@jdDqX+k$wsZL2fAo?BN~D^ zAn)*^h&+pjoxt)yegHr$1~qwWA7F-=z+8aQ{RYnwHHMij(_7-2vIR|9Laqf4ImOJ_ zr|ofleFXj#^|R|16wQ#cWGo0u-&UPdg|^N~KUBT1iW)op`xez%0aNgaPq@^>Xn z&9fD!HU-uPu zf25AKbVaMXWAdI zb8A%JcDXBb;!JIje4`<_E_fi=6+98F3ujeE3{|sTep^`Ic3;(<7dmJA-y6C-G&lCH zD_XrHCf}K=wSGZSznEPT?D}i-bPalVSX8QKE2gk#P`YYM*6<;p?;EDNl zzkGUbf7G<|y7HkfC?cnFg(8#<|MEu#VOWPBSw6`s+$)pb$79i@249kt{3R(f&rH$} zfK7dWsXUgEVm@90lwF-_Y%1W+_N1;AOp0T|%c?y67Ciy7L7(L7E@G3w$&% zaXw0*z>_kcEYo$vlj5hDE*rQfo6_pYaNY39KtiDwP6&*~3n$x5$NR6ZNo=RcGkxf! zks3KM;Go5@5E7^~JY=SLt0My!WwdGeh>))O#8Zrbkpzkl*AYZE@dwwDW;i8fp56Sk z*;V+Yz;BgrrRHGuEFXcl{7LEeq!UShoJ?JCbUa_d#FW#pHYWq`3?T4_cO7={m<8WJ z#t>cGv@=qDBqR?v?YTt-;&}NdaVqfr-j2@sl(#E1qmG(Z&Gbuxg zN%770R;aX(`+Mf9eK!ZdqeL1BiR#mtOJp`V%vw#?>673R67 z1;uu__QiGdf{tEP>Q*GAWalc*lGHm)v@sLO+60?$Gm3LcX8p5D8lC`ETGB*H!vtPZ-Py6TWEqHBt4*GIJL zKX~}Vz3=ahYPb9Q7oog6)O~5+g?;gw?U9=8QS}ag*P>MZ%I=f9r*@oge|z(}&7rP0 zw_lf<{(P^1(B}P-kV2!?Ax?WQInkyj9=HqHhli5;Pfldl9uhGBFfpuN|(M7s(%ylM!SR!m)ELh31=dw^CBuQMU4#%qYqUGV zNP|F$x>6$SBi!B&0gLE6Ks>ejOYL8!2Pp&K&!_Zs3SnUm*c=bi4!~cUNTL)AgJeD> zQ-(IElk_PkRZQ8WdX2X4NTNvd0upu?Q^u4J(<7CW8W@uvAro+9=mmog<{8q3-cM>p z3}VQqWoS_h##D;oNu5sz0yD8z9aAOd`Zbf^^(fS)vo@SXz#sNZ+|5EA_>wZb|PS}6FFM$a<>T-q3GnL4Ikn6VUWmghMEmT5@qVd}M$G5NB- zRtqxQgU*-ua~u;wqA#rWHC87hLJA>N+ZxaV0*vS*Xop>l3n8K2$sAvfXBCHT{8$3u{frA;Wn2vKh0_FaTm}ru94pVPh)Xc3$FnQ9vxVmRScUz4mvL z=Qy}>rzi?j2COa($`IuaknD!EU73-tej~lj6ETP(U*4MZHs6clq5y;^ z1t%*#(?Z*Q1#9#Y-OE;eSGEcL#JCDYEZ<-2H1f4rCX0N9Cx7ZIIvL75BaYWsH2|V( zwDw0CAPap(%#QP^Gc8&vFOZJ!JqXD(cy*wECK1;-)RF;Cq?>EBBJdz?VWr}C=jJu~ z`9_9v$s`brthM`g3b^qe$iTN9KYkGJ_j83(lf`N0lHv^KlCHFKNwH9hc?h3NHu#F4 z1z{FElax%BPL@rU!yXs8WhQ=DDq#2qxH19Gs{s5JP6gBiCq!IR1+GWrdS72-iGay* z?!j>o_<{g4NvF}MpF9Z9>`Kc&|6sn4&h#t-qw;NATak7S_}35O7taCTsyCHstF^;Syu zBhHmmxq7B>>oS~>9DgIM1<@DiC`o)n{ky+?{b_4#kGLMkMIEPoy|W@Cw@#4KV;d~=E)51*V{E^TZUCbZurWZo z&8AN8`upIXIQS?%3O05i&G)#{?Y0^OEpm?w3hfdKJqCs-v}Xw32T(~n3ft(UVG7;A z+D?qwSss_$W}}gj1&x|w>KyweG*ZBF=atuQ|NP`1uKwgGR%bs--+ATKt)IO7x&Ktt z=l<81KmNg;3%|J;e2;EytZjd|vGEUAr*2<5xitUkowt5^=Z$}R^GEOe;p+6~{+DjQ z_RjM8+2!+d%WwH_hGuR~zqb6tb2p~~OTYdheH0I2Q&M-{Y&IYL!_`wqn~u^;SATN* zr|&NR_6-2Mb@|sfLsyqS`n#L2zC-WkA2nbY;OQD(OrV2M#~8Xj3GKf9-iON&4_Dt@ ze(mF1FMYK9`j2nDc7^XPZsR}ue-52qy7~*gYfz`xBvjZ1&l9YbZrMQ}O@BE#3Ui9h zXtsMb&9q=Av)yZe=NP1aKqqtAm!a`vf0f>Y*WO7rX=(mPw?7U*J887$Xw>ZUzlNqR z&A$$VzZ^KVH1A)YnS*J9QNDTp3gq6LzJ%Zp9okFZ{_U0JnbR;ePuY$&;Wf83e{pI4 z7cef%Ge3g~g7IDs{Cs)p9E^C{;11ZpViR>TSLh_dW_tnK&UJzjlY5A6>7`p9rM;TY z)Ee*fN^z2w9)s2V5NjI&15i#6MJ-*A_4f939p3j??-=5R0#=ye<>}X<<+!zI*57+; zY5s4ZH~d0^?$WrfM`>7#00*mY>9?WVul!>9!}lTc)|p@3e)%-a4BuzyO>$h$&))v{ z9E{7-uTL+{{{{vHmH`igdPNKMgAG@5Uzm~gR_e`ctOXhg^8{-NE!bCQ`4uHLD!H2I z)N%k7b{fiH+a69W?n|ur$SYlwoc>YUL1Yls&08{MQqS$+TH!wk0JWNlW4Uta?*mKKn?!2ox}%8o!2IR(<%wlJQ- z7jys3p8w)Kvy(SEcnk8pg+c*@8|<85&cV>a-AQ|k=?0ipx}_zl``wZpeI09k1CcSt0=cg@@1Cm7$SGrBGrtQN1>6HRt7i<9J2@%qa(Os4Z(fE}X*e~s4oqhl8w^LqGs$i|2a21* zC??-Z(LVLc=?1{J-79ORohNwDx}76USZf9-y3!3q63i{ZG{rvXbg{^cnX(QKn@AQ3 zGE9=*UK7~!Ae%Dwebm3d1;^VWR(l8x#uz$jj7EbpDhn^MZe&H4Of#A9z;%Z38em$% zTorWf8Ow;x>h$VR0s~fqVB=x!_jZan(~HCVZt^i1o1-Oh-)Es_X3;s@YZeha1q%JJ zo;v9PyVE*Mr{hbme3<8&wctUONrWmEp)fMYgeIY8o7pa?*7OvMZsMF2srs`=A;=r8 z)u2SR$m@6uxIUDRWRcj$iQh;_0E(?!Sadl_GlN@)qUeywNx)W80mEfyVc}4qewEyP z;;!bTXj|h)Y0k*kip(~7n_&Ei&B^cpvZDy0%rgeI!JwxFW!9qvI=R^KR)g)}HVc*y z$^F4zL9Ibg;+*y%ShCnhY&|Rs7MF<1VSOuJjSH=RUT0Z4%DNbK442^7yaii0b24}& z-`K&cJfn9&Xc^eXFf3XopiDi^h{k?c!Pzwo#*uPFs-ZCt zSYylC5f7*ifK>+Gq?~NzII^{rgFXQ=lH7nAheWJwKdK8BzsG`l0(%*hqmj-Ri^R6P zO`!^fnh~7J$}nh=0{7dB649drr%hi~yb@X^N|&O=jUW11#Og3{V1p?14$7vi&IxR; z2%8GkEOQUrY@@K@(MpBIWvb%Uw`J}@%klB#3SzOr&wf}Y zzVnI5&L^U|Px_B1ipt(DI#(1gYK;`N#)~#YiZ(=xHihN+OF2ymO-aI#Gcz$g5jTLQ zTxBA=?4ClJEB8NoPfcW1g&L#AmIdR1*t+g(&qUW903|`GDy(jw+aHteOenGg4y@6q z32S#<+ZvNUhI7C3sV^pd7J{Ifp%_7`88Z1jB_UHMQ*SBZiP0XEKjI&1BcHZIejFaS0Bl% z59c<_YUa8>-1q*zc*pKY$L{d@Jy645QV&`qQ_lo7p?#sPv%{b|uiAD`2FPJV)Y_Ti z>EfC4>GE)4Lri6wtNGCMzA4_mccFbR)*ptav*XhP3tM}qOo3gmHir*C2fCLUEhuCPOR6HOsQ662Ybm=ZSo5~&oGJ7S-`;5U zHotu_tMGh5sA#rlt|MBsdA{yiO*Ctde<-2K3ag6FKNWg<_W3zav}{|rtb0*k94>i0 zsy`4`9smo^xQ32s=uqLMvI}KVO{@P{LZz81#z0+BeRo*dy{OSou`@4BzYrv+UyN!> z{Kp{W6gx9MJ${Bg>pkranc`KOBUPK@RXZY8JO2H#|1k9TL$QL#qK3z#ngalB%sVp} zw9gjJbw~5JAm!A-MXez~o>_lpEO;oi>D+T*m=?_e(~TC;!hta=JYD!kagcf2e$E~* z-xPtrtWEPZADcch#XFylz+cwWaicY2v__5n{=tOGh^8>6DoTxfYU#+ zZwhxk7pA`(HCn=I%c4?0qnp+R24k>-X@qu%Zp?3O^Nd6cscx< z#QKCW7fn~(XpR`o39aFVseS&~{1cxZj#L~>l-JL-&(&V5iRAA2LPlk)VFBo~0tK%< zo5-OPIfW}ZL`BD4LS3QsGckF|NT(a|82-=6kN?U1P5?pVo#S zKlrI9eCTjk`$SCsmR$3Q?Q~SblI@g^31aX^W7is z`)FT$+rh}TgR$I0Q$0{xpA*)Vhc<+(JD{_1ZDmAT$s5$%C@e)P&GABWq|ltGZ3Of9 zFBDRP@g61D=~wgwXkBS*7qqprSrKjHb!{!oecaFxF*IE1;+3G|ZHFRlhoa4g7Yv6J z`s}#AI-;+R>)RswwwS&>(bzP_#Ps!x(16^on7$j@ipbU39jAAM>Y_QODdnOjFEDgb z271v~*3Z81?)Ldz3r*V-?VS;I>s0lWJFq{bn(Y8epxLyTlXtfBbZ5{3<*q#Z-u}z` z=O&`fyCONern^Clt*Q}J({PdQV9knDsxmC*6@xli=ef>M|C>AGC9RQ?)@aGPXkPnN z1L(5N)J=n?bJfL%FYUjue@;2;jn;I=)LU1SsAPW0`G?=$e{O%s^FjNEo$q(XN_WId zyCS7s(bAr1{_ZJL5<#GXH!hmB&ps2a+7eTDLQ7P-xUwputO|9+lnn``Hm)pwMYtTR<->rU4NbKhu==_+mr;}+JpCbIL-nogVI*&7zJH$Wtw-4w}gnzb%u zx2C`{5AWkK{ec8Ooo^e?8KQ-zseSig;gkmVT^x&LwR})LXaA-7+OCE69#FS76b8vx zhi~MRqNQV8F+!sQYnz&q-G)McLEfUeo(7d zl}A+NVO8UbL<*XJSZ}ZnnlziWpleK9{pN_)9MiV_%c7wW)wP>kR2#zC4_(W;Mum;t zadl5b-GllCIbW_wDb==wpt2uVH$>D8*VT=GPL|zhV94rYoZ)PA#lUT^Tlc57epY(j3uQkocjsZjT|F#PN2eY z_l0`_e^W0?6TGBux!CBI#aF%(RH_0wk03|9p5x0q8qifU!zs7%zlyVimBEgLKf;ab zhy6~HpAl7`6RO{%Ki%&M&F_h<&xp#;h{Df^{LhH;&x!2c6IyVWiJf=lDstl&g@i;C zsELsUf07iCjVlB=cPT None: + super().__init__(env, policy, frame_rate=frame_rate, verbosity=verbosity) + self._rerun = RerunPlayViewer(env, policy, rerun_cfg) + + def setup(self) -> None: + if not (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")): + raise RuntimeError( + "viewer=rerun_native 需要 **MuJoCo 自带的 GLFW 原生窗口**（仓库 `NativeMujocoViewer`），" + "因此进程里必须有 DISPLAY / WAYLAND_DISPLAY。\n" + "纯 SSH / RoboHub 技能容器里若未注入显示环境变量，此处为空就会失败；可选：\n" + " • 只要 Rerun、不要 GLFW 面板： --viewer rerun\n" + " • 虚拟屏幕（常见无桌面服务器）： sudo apt-get install -y xvfb\n" + " xvfb-run -a uv run play ... --viewer rerun_native ...\n" + " • 本机有桌面 / 远程桌面：先 export DISPLAY=:0（或实际编号），再运行。\n" + "当前 DISPLAY/WAYLAND_DISPLAY 均为未设置。" + ) + print( + "[Rerun+Native] 单后端：策略与物理与 `play --viewer native` 相同，" + "并额外把离屏相机与 qpos 写入 Rerun（见终端里的 http 链接）。" + ) + self._rerun.start_rerun_servers(app_id="mjlab_husky_play_rerun_native") + super().setup() + self.reset_environment() + + def sync_env_to_viewer(self) -> None: + super().sync_env_to_viewer() + self._rerun.log_post_step_to_rerun(self._step_count) diff --git a/src/mjlab_husky/viewer/rerun_play_viewer.py b/src/mjlab_husky/viewer/rerun_play_viewer.py index 1aa9a02..d264393 100644 --- a/src/mjlab_husky/viewer/rerun_play_viewer.py +++ b/src/mjlab_husky/viewer/rerun_play_viewer.py @@ -15,7 +15,7 @@ import torch import rerun as rr -from mjlab_husky.lerobot_numpy import batched_vector_to_numpy1d +from mjlab_husky.lerobot_numpy import batched_vector_to_numpy1d, import_lerobot_dataset_class def _downsample_max_side(img: np.ndarray, max_side: int) -> np.ndarray: @@ -164,8 +164,9 @@ class RerunPlayViewer: img = _downsample_max_side(img, self._cfg.camera_max_side) rr.log("camera/robot", rr.Image(img)) - def run(self): - rr.init("mjlab_husky_play_rerun", spawn=False) + def start_rerun_servers(self, *, app_id: str = "mjlab_husky_play_rerun") -> None: + """Initialize Rerun + gRPC + web viewer (no sim loop). Used by ``run()`` and ``RerunNativePlayViewer``.""" + rr.init(app_id, spawn=False) sg_kw: dict[str, Any] = {"newest_first": self._cfg.grpc_newest_first} if self._cfg.grpc_port is not None: @@ -228,12 +229,21 @@ class RerunPlayViewer: connect_to=connect_uri, ) + def log_post_step_to_rerun(self, step: int) -> None: + """After ``env.step``, stream qpos / offscreen camera to Rerun (same timeline as standalone ``run``).""" + rr.set_time("step", sequence=step) + self._try_log_mujoco_state(step) + self._try_log_camera(step) + + def run(self): + self.start_rerun_servers() + reset_out = self._env.reset() obs = reset_out[0] if isinstance(reset_out, tuple) and len(reset_out) == 2 else reset_out step = 0 if self._cfg.lerobot_record: - from lerobot.datasets.lerobot_dataset import LeRobotDataset # type: ignore + LeRobotDataset = import_lerobot_dataset_class() unwrapped = getattr(self._env, "unwrapped", self._env) sim_data = getattr(getattr(unwrapped, "sim", None), "data", None) diff --git a/src/mjlab_husky/viewer/rerun_viser_play_viewer.py b/src/mjlab_husky/viewer/rerun_viser_play_viewer.py new file mode 100644 index 0000000..3e0c442 --- /dev/null +++ b/src/mjlab_husky/viewer/rerun_viser_play_viewer.py @@ -0,0 +1,65 @@ +"""Viser（浏览器里的 mjlab 三维面板）+ Rerun：单套仿真，两个服务各占不同 TCP 端口。""" + +from __future__ import annotations + +from typing import Any + +import viser +from typing_extensions import override + +from mjlab.viewer.base import VerbosityLevel +from mjlab.viewer.viser import ViserPlayViewer + +from mjlab_husky.viewer.rerun_play_viewer import RerunPlayViewer, RerunPlayViewerCfg + + +class RerunViserPlayViewer(ViserPlayViewer): + """与 ``ViserPlayViewer`` 相同的主循环，额外启动 Rerun Web+gRPC，并在每步写入时间轴。 + + - **Viser** 占用 ``--viser-port``（默认 `8081`，避免与 Rerun Web 默认 `8080` 冲突）。 + - **Rerun** 占用 ``--rerun-web-port`` 与 ``--rerun-grpc-port``（与 ``--viewer rerun`` 一致）。 + + GLFW ``NativeMujocoViewer`` 不监听端口；若要在浏览器里并排嵌入「三维 + Rerun」，请用本 viewer 而非 ``rerun_native``。 + """ + + def __init__( + self, + env: Any, + policy: Any, + rerun_cfg: RerunPlayViewerCfg, + viser_port: int = 8081, + frame_rate: float = 60.0, + verbosity: int = VerbosityLevel.SILENT, + ) -> None: + super().__init__(env, policy, frame_rate=frame_rate, verbosity=verbosity) + self._rerun = RerunPlayViewer(env, policy, rerun_cfg) + self._viser_port = int(viser_port) + self._url_host = (rerun_cfg.connect_host or "127.0.0.1").strip() + + @override + def setup(self) -> None: + self._rerun.start_rerun_servers(app_id="mjlab_husky_play_rerun_viser") + + _real = viser.ViserServer + port = self._viser_port + + def _force_port_ViserServer(*args: Any, **kwargs: Any): + kwargs = dict(kwargs) + kwargs["port"] = port + return _real(*args, **kwargs) + + viser.ViserServer = _force_port_ViserServer # type: ignore[misc, assignment] + try: + super().setup() + finally: + viser.ViserServer = _real + + print( + f"[Rerun+Viser] 单后端：Rerun 使用 --rerun-web-port / --rerun-grpc-port；" + f"Viser 三维面板: http://{self._url_host}:{port}/" + ) + + @override + def sync_env_to_viewer(self) -> None: + super().sync_env_to_viewer() + self._rerun.log_post_step_to_rerun(self._step_count)