From 5cc2455caa23b7c5d36ebf3e660c5d96cec2e1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20J=C3=BClg?= Date: Fri, 11 Apr 2025 09:55:17 +0200 Subject: [PATCH 1/3] fix(sim gui): increases max geoms in render gui fixes Objects disappear in Rendering GUI #168 --- src/sim/gui_client.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sim/gui_client.cpp b/src/sim/gui_client.cpp index 8a2e99bd..b5fe7c63 100644 --- a/src/sim/gui_client.cpp +++ b/src/sim/gui_client.cpp @@ -65,7 +65,8 @@ void GuiClient::render_loop() { mjv_defaultFreeCamera(this->m, &this->cam); mjv_defaultOption(&this->opt); mjv_defaultScene(&this->scn); - mjv_makeScene(this->m, &this->scn, mjFONTSCALE_100); + size_t max_geoms = 2000; + mjv_makeScene(this->m, &this->scn, max_geoms); mjrRect viewport = {0, 0, 0, 0}; this->platform_ui->RefreshMjrContext(this->m, mjFONTSCALE_100); this->platform_ui->SetVSync(true); From 2821296b16b4f2a5ca25cbea4210c4575df9c724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20J=C3=BClg?= Date: Fri, 11 Apr 2025 10:12:08 +0200 Subject: [PATCH 2/3] feat(sim camera): allow camera render on demand Currently the frames are rendered with a fixed frame rate, this can however slow down the simulation loop quite significantly. Therefore, this feature allows to render only at the time when an image is request. One can switch in this mode by setting frame rate to zero. --- python/rcsss/_core/sim.pyi | 9 ++++++++- python/rcsss/envs/factories.py | 6 +++--- src/pybind/rcsss.cpp | 5 ++++- src/sim/camera.cpp | 16 +++++++++++++++- src/sim/camera.h | 1 + src/sim/sim.cpp | 22 ++++++++++++---------- src/sim/sim.h | 3 +-- 7 files changed, 44 insertions(+), 18 deletions(-) diff --git a/python/rcsss/_core/sim.pyi b/python/rcsss/_core/sim.pyi index 9357b0a3..eb50d591 100644 --- a/python/rcsss/_core/sim.pyi +++ b/python/rcsss/_core/sim.pyi @@ -159,11 +159,18 @@ class SimCameraSet: class SimCameraSetConfig: cameras: dict[str, SimCameraConfig] - frame_rate: int max_buffer_frames: int resolution_height: int resolution_width: int def __init__(self) -> None: ... + @property + def frame_rate(self) -> int: + """ + The frame rate in which the cameras render in Hz. If set to zero, the camera frames will render on demand and without fixed rate which takes away compute effort. + """ + + @frame_rate.setter + def frame_rate(self, arg0: int) -> None: ... def open_gui_window(uuid: str) -> None: ... diff --git a/python/rcsss/envs/factories.py b/python/rcsss/envs/factories.py index 865f0dcb..95c413fa 100644 --- a/python/rcsss/envs/factories.py +++ b/python/rcsss/envs/factories.py @@ -241,7 +241,7 @@ def __call__( # type: ignore render_mode: str = "human", control_mode: ControlMode = ControlMode.CARTESIAN_TRPY, resolution: tuple[int, int] | None = None, - frame_rate: int = 10, + frame_rate: int = 0, delta_actions: bool = True, ) -> gym.Env: if resolution is None: @@ -272,7 +272,7 @@ def __call__( # type: ignore render_mode: str = "human", control_mode: ControlMode = ControlMode.CARTESIAN_TRPY, resolution: tuple[int, int] | None = None, - frame_rate: int = 10, + frame_rate: int = 0, delta_actions: bool = True, ) -> gym.Env: if resolution is None: @@ -319,7 +319,7 @@ def __call__( # type: ignore render_mode: str = "human", control_mode: ControlMode = ControlMode.CARTESIAN_TRPY, resolution: tuple[int, int] | None = None, - frame_rate: int = 10, + frame_rate: int = 0, delta_actions: bool = True, ) -> gym.Env: if resolution is None: diff --git a/src/pybind/rcsss.cpp b/src/pybind/rcsss.cpp index a4d996f8..1ba93d27 100644 --- a/src/pybind/rcsss.cpp +++ b/src/pybind/rcsss.cpp @@ -486,7 +486,10 @@ PYBIND11_MODULE(_core, m) { py::class_(sim, "SimCameraSetConfig") .def(py::init<>()) .def_readwrite("cameras", &rcs::sim::SimCameraSetConfig::cameras) - .def_readwrite("frame_rate", &rcs::sim::SimCameraSetConfig::frame_rate) + .def_readwrite("frame_rate", &rcs::sim::SimCameraSetConfig::frame_rate, + "The frame rate in which the cameras render in Hz. If set " + "to zero, the camera frames will render on demand and " + "without fixed rate which takes away compute effort.") .def_readwrite("resolution_width", &rcs::sim::SimCameraSetConfig::resolution_width) .def_readwrite("resolution_height", diff --git a/src/sim/camera.cpp b/src/sim/camera.cpp index 1d1dda92..6f70233f 100644 --- a/src/sim/camera.cpp +++ b/src/sim/camera.cpp @@ -24,10 +24,13 @@ namespace sim { SimCameraSet::SimCameraSet(std::shared_ptr sim, SimCameraSetConfig cfg) : sim{sim}, cfg{cfg}, buffer{}, buffer_lock{}, cameras{} { for (auto const& [id, cam] : cfg.cameras) { + // if frame_rate is zero, then we only render when an image is requested + // this mode is useful when no videos are required as it speeds up the + // simulation significantly this->sim->register_rendering_callback( [this](const std::string& id, mjrContext& ctx, mjvScene& scene, mjvOption& opt) { this->frame_callback(id, ctx, scene, opt); }, - id, 1.0 / this->cfg.frame_rate, this->cfg.resolution_width, + id, this->cfg.frame_rate, this->cfg.resolution_width, this->cfg.resolution_height); mjvCamera mjcam; @@ -57,6 +60,9 @@ void SimCameraSet::clear_buffer() { } std::optional SimCameraSet::get_latest_frameset() { + if (this->cfg.frame_rate == 0) { + this->render_all(); + } if (buffer.empty()) { return std::nullopt; } @@ -73,6 +79,14 @@ std::optional SimCameraSet::get_timestamp_frameset(float ts) { return std::nullopt; } +void SimCameraSet::render_all() { + for (auto const& [id, cam] : this->cfg.cameras) { + mjrContext* ctx = this->sim->renderer.get_context(id); + this->frame_callback(id, *ctx, this->sim->renderer.scene, + this->sim->renderer.opt); + } +} + void SimCameraSet::frame_callback(const std::string& id, mjrContext& ctx, mjvScene& scene, mjvOption& opt) { mjrRect viewport = mjr_maxViewport(&ctx); diff --git a/src/sim/camera.h b/src/sim/camera.h index 8d61594d..a9960d69 100644 --- a/src/sim/camera.h +++ b/src/sim/camera.h @@ -66,6 +66,7 @@ class SimCameraSet { std::unordered_map cameras; std::mutex buffer_lock; mjtNum last_ts = 0; + void render_all(); }; } // namespace sim } // namespace rcs diff --git a/src/sim/sim.cpp b/src/sim/sim.cpp index 45ca9702..b33caefd 100644 --- a/src/sim/sim.cpp +++ b/src/sim/sim.cpp @@ -164,17 +164,19 @@ void Sim::register_all_cb(std::function cb, void Sim::register_rendering_callback( std::function cb, - const std::string& id, mjtNum seconds_between_calls, size_t width, - size_t height) { + const std::string& id, int frame_rate, size_t width, size_t height) { this->renderer.register_context(id, width, height); - // dont register off screen in normal callback, but special gui callback - this->rendering_callbacks.push_back( - RenderingCallback{.cb = cb, - .id = id, - .seconds_between_calls = seconds_between_calls, - // this is negative so that we will directly render the - // cameras in the first step - .last_call_timestamp = -seconds_between_calls}); + // in case frame_rate is zero, rendering needs to be triggered + // manually + if (frame_rate != 0) { + this->rendering_callbacks.push_back( + RenderingCallback{.cb = cb, + .id = id, + .seconds_between_calls = 1.0 / frame_rate, + // this is negative so that we will directly render + // the cameras in the first step + .last_call_timestamp = -1.0 / frame_rate}); + } } void Sim::start_gui_server(const std::string& id) { diff --git a/src/sim/sim.h b/src/sim/sim.h index 9a899d30..b3e272f8 100644 --- a/src/sim/sim.h +++ b/src/sim/sim.h @@ -98,8 +98,7 @@ class Sim { std::function cb, - const std::string& id, mjtNum seconds_between_calls, size_t width, - size_t height); + const std::string& id, int frame_rate, size_t width, size_t height); void start_gui_server(const std::string& id); void stop_gui_server(); }; From 21164c8a745901ceb7ce7d22198a0c0eb213955e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20J=C3=BClg?= Date: Fri, 11 Apr 2025 10:36:46 +0200 Subject: [PATCH 3/3] docs: updated readme and deleted unused config.py --- README.md | 25 +++++++++++++----- python/rcsss/config.py | 58 ------------------------------------------ 2 files changed, 18 insertions(+), 65 deletions(-) delete mode 100644 python/rcsss/config.py diff --git a/README.md b/README.md index 9fadd6f5..9590d6fe 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Robot Control Stack +RCS is a unified and multilayered robot control interface over a MuJoCo simulation and real world robot currently implemented for the FR3. ## Requirements We build and test RCS on the latest Debian and on the latest Ubuntu LTS. @@ -29,17 +30,27 @@ Import the library in python: ```python import rcsss ``` -The package includes a command line interface which define useful commands to handle the hardware robot. +Checkout the python examples that we provide in [python/examples](python/examples): +- [fr3.py](python/examples/fr3.py) shows direct robot control with RCS's python bindings +- [env_joint_control.py](python/examples/env_joint_control.py) and [env_cartesian_control.py](python/examples/env_cartesian_control.py) demonstrates RCS's high level [gymnasium](https://gymnasium.farama.org/) interface both for joint- and end effector space control +All of these examples work both in the MuJoCo simulation as well as on your hardware FR3. +Just switch between the following settings in the example script +```python +ROBOT_INSTANCE = RobotInstance.SIMULATION +# ROBOT_INSTANCE = RobotInstance.HARDWARE +``` +and add your robot credentials to a `.env` file like this: +```env +DESK_USERNAME=... +DESK_PASSWORD=... +``` + +### Command Line Interface +The package includes a command line interface which define useful commands to handle the FR3 robot without the need to use the Desk Website. To list all available subcommands use: ```shell python -m rcsss --help ``` -A sample config can be generated via the following CLI command. -```shell -python -m rcsss sample-config -``` -The command will produce a `config.yaml` file with sample values. -See [config.py](python/rcsss/config.py) for a description of the config fields. ## Development ```shell diff --git a/python/rcsss/config.py b/python/rcsss/config.py deleted file mode 100644 index 564bb727..00000000 --- a/python/rcsss/config.py +++ /dev/null @@ -1,58 +0,0 @@ -from pydantic import BaseModel -from pydantic_yaml import parse_yaml_raw_as, to_yaml_str -from rcsss.camera.interface import BaseCameraConfig -from rcsss.camera.realsense import RealSenseSetConfig -from rcsss.camera.sim import SimCameraConfig, SimCameraSetConfig - - -# TODO: this design might need to be adapted in order to -# configure each camera separately -# For this one could add a dict of camera configs -class CameraConfig(BaseModel): - # kinect_config: KinectConfig | None = None - realsense_config: RealSenseSetConfig | None = None - # TODO: we can also add sim camera configs here - - -class HWConfig(BaseModel): - # Franka Desk credentials - username: str - password: str - # path to the urdf model - urdf_model_path: str | str - camera_type: str | None = None - camera_config: CameraConfig | None = None - - -class SimConfig(BaseModel): - camera: SimCameraSetConfig - - -class Config(BaseModel): - hw: HWConfig - sim: SimConfig - - -def create_sample_config_yaml(path: str): - - cameras = {"human_readable_name": BaseCameraConfig(identifier="serial_number")} - real_sense_cfg = RealSenseSetConfig(cameras=cameras) - camera_cfg = CameraConfig(realsense_config=real_sense_cfg) - hw = HWConfig( - username="...", - password="...", - urdf_model_path="path/to/urdf", - camera_config=camera_cfg, - camera_type="realsense", - ) - cameras_sim = {"human_readable_name": SimCameraConfig(identifier="mjcf_name", type=0)} - sim = SimConfig(camera=SimCameraSetConfig(cameras=cameras_sim)) - cfg = Config(hw=hw, sim=sim) - yml = to_yaml_str(cfg) - with open(path, "w") as f: - f.write(yml) - - -def read_config_yaml(path: str) -> Config: - with open(path, "r") as f: - return parse_yaml_raw_as(Config, f)