From 67b696d8a224fa1ebd2b9240a0c3b07f529a3dac Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:52:59 +0800 Subject: [PATCH 1/3] fix(gen-sim): restrict profile credentials to the service origin --- embodichain/gen_sim/services/README.md | 3 + .../gen_sim/services/adapters/_http.py | 56 +++++++++++++++++ embodichain/gen_sim/services/runtime.py | 5 +- tests/gen_sim/services/test_services.py | 60 +++++++++++++++++++ 4 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 embodichain/gen_sim/services/adapters/_http.py diff --git a/embodichain/gen_sim/services/README.md b/embodichain/gen_sim/services/README.md index 5855940a2..079fd6e44 100644 --- a/embodichain/gen_sim/services/README.md +++ b/embodichain/gen_sim/services/README.md @@ -60,6 +60,9 @@ budget and adapter options. Fields can use `{"env": "NAME"}`; `api_key_env` is shorthand for an API-key reference. Only references are resolved: process values win over the selected dotenv (`EMBODICHAIN_ENV_FILE` or the default GenSim `.env`). Unrelated `OPENAI_*` values never fill missing fields in an explicit profile. +HTTP profile API keys are sent only to the configured origin (scheme, host and +port). External mesh URLs and redirects to another origin receive no profile +Authorization header. Unused roles require no credentials, connection or health check. The example's optional job-control profile is only needed when using Gradio's remote panel. diff --git a/embodichain/gen_sim/services/adapters/_http.py b/embodichain/gen_sim/services/adapters/_http.py new file mode 100644 index 000000000..0d66a845f --- /dev/null +++ b/embodichain/gen_sim/services/adapters/_http.py @@ -0,0 +1,56 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Keep profile credentials on their configured HTTP origin.""" + +from __future__ import annotations + +from urllib.parse import urlsplit +import requests + +__all__: list[str] = [] + + +def _origin(url: str) -> tuple[str, str | None, int | None]: + parts = urlsplit(url) + port = parts.port + if port is None: + port = {"https": 443, "http": 80}.get(parts.scheme) + return parts.scheme, parts.hostname, port + + +class _OriginBoundSession(requests.Session): + def __init__(self, base_url: str, api_key: str) -> None: + super().__init__() + self._service_origin = _origin(base_url) + self._authorization = f"Bearer {api_key}" + + def prepare_request(self, request: requests.Request) -> requests.PreparedRequest: + prepared = super().prepare_request(request) + if _origin(prepared.url) == self._service_origin: + prepared.headers["Authorization"] = self._authorization + else: + prepared.headers.pop("Authorization", None) + return prepared + + def rebuild_auth( + self, prepared_request: requests.PreparedRequest, response: requests.Response + ) -> None: + super().rebuild_auth(prepared_request, response) + # Requests permits HTTP-to-HTTPS auth forwarding on the same host; + # profile credentials remain restricted to the exact configured origin. + if _origin(prepared_request.url) != self._service_origin: + prepared_request.headers.pop("Authorization", None) diff --git a/embodichain/gen_sim/services/runtime.py b/embodichain/gen_sim/services/runtime.py index b8e425a13..ba7939cd5 100644 --- a/embodichain/gen_sim/services/runtime.py +++ b/embodichain/gen_sim/services/runtime.py @@ -181,10 +181,9 @@ def _http_adapter(role: str, profile: Mapping[str, Any]) -> Any: kwargs.update(options) api_key = profile.get("api_key") if api_key: - import requests + from .adapters._http import _OriginBoundSession - session = requests.Session() - session.headers["Authorization"] = f"Bearer {api_key}" + session = _OriginBoundSession(profile["base_url"], api_key) kwargs["session"] = session try: return cls(**kwargs) diff --git a/tests/gen_sim/services/test_services.py b/tests/gen_sim/services/test_services.py index efa1b97d5..cca2cca0b 100644 --- a/tests/gen_sim/services/test_services.py +++ b/tests/gen_sim/services/test_services.py @@ -861,3 +861,63 @@ def test_invalid_default_user_file_is_not_ignored(tmp_path, monkeypatch): p.write_text("{}") with pytest.raises(ServiceConfigurationError): load_services_config() + + +@pytest.mark.parametrize( + "mesh_url,redirect,authorized", + [ + ("/mesh.glb", None, [True]), + ("http://service.test:80/mesh.glb", None, [True]), + ("http://files.test/mesh.glb", None, [False]), + ("http://service.test:8080/mesh.glb", None, [False]), + ("/mesh.glb", "http://files.test/mesh.glb", [True, False]), + ("/mesh.glb", "https://service.test/mesh.glb", [True, False]), + ("/mesh.glb", "/download.glb", [True, True]), + ], +) +def test_geometry_download_keeps_profile_key_on_service_origin( + tmp_path, mesh_url, redirect, authorized +): + import io + import requests + from embodichain.gen_sim.services.runtime import _http_adapter + + observed = [] + + class Transport(requests.adapters.BaseAdapter): + def send(self, request, **kwargs): + observed.append(request.headers.get("Authorization")) + response = requests.Response() + response.status_code = 302 if redirect and len(observed) == 1 else 200 + if response.status_code == 302: + response.headers["Location"] = redirect + response._content = b"glTFtest" + response.raw = io.BytesIO(response._content) + response.url, response.request = request.url, request + return response + + def close(self): + pass + + client = _http_adapter( + "geometry_reconstruction", + { + "base_url": "http://service.test", + "api_key": "test-service-token", + "timeout_s": 1, + "max_attempts": 1, + "options": {"health_path": "/health", "generate_objects_path": "/objects"}, + }, + ) + client._session.trust_env = False + transport = Transport() + client._session.mount("http://", transport) + client._session.mount("https://", transport) + try: + client._download_glb(mesh_url, tmp_path / "mesh.glb") + assert observed == [ + "Bearer test-service-token" if value else None for value in authorized + ] + assert (tmp_path / "mesh.glb").read_bytes() == b"glTFtest" + finally: + client.close() From 7f9887c0678d7b2b25eb43d0c692d3041b4cb910 Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:22:02 +0800 Subject: [PATCH 2/3] Fix locomotion benchmark checks for DexUni --- scripts/benchmark/rl/locomotion_ppo.py | 22 ++++--- tests/benchmark/test_locomotion_ppo.py | 83 ++++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/scripts/benchmark/rl/locomotion_ppo.py b/scripts/benchmark/rl/locomotion_ppo.py index 2e98c14f9..e7987c058 100644 --- a/scripts/benchmark/rl/locomotion_ppo.py +++ b/scripts/benchmark/rl/locomotion_ppo.py @@ -31,7 +31,7 @@ BACKENDS = { "default": "Dexsim Default CUDA", - "newton": "Dexsim Newton/MJWarp CUDA", + "newton": "Dexsim Newton CUDA", } TASK_DIR = ( Path(__file__).resolve().parents[3] @@ -90,11 +90,13 @@ def _check_backend(env: Any, name: str) -> Any: from dexsim.engine.newton_physics.backend_registry import get_newton_backend backend = get_newton_backend(sim._world) + solver = backend.mujoco_solver if ( - backend.solver.use_mujoco_cpu - or not backend.solver.mjw_data.qpos.device.is_cuda + solver is None + or solver.use_mujoco_cpu + or not solver.mjw_data.qpos.device.is_cuda ): - raise RuntimeError("Newton/MJWarp is not using CUDA") + raise RuntimeError("Newton MuJoCo solver is not using CUDA") return backend if not (sim._world_config.enable_gpu_sim and sim._world_config.direct_gpu_api): raise RuntimeError("Default backend is not using direct GPU simulation") @@ -250,9 +252,13 @@ def _run(args: argparse.Namespace, result: dict) -> None: env_config = yaml.safe_load(Path(config["trainer"]["gym_config"]).read_text()) env_config["num_envs"] = args.envs if args.backend == "newton": - env_config["physics_config"]["solver_cfg"][ - "iterations" - ] = args.solver_iterations + solver_cfg = env_config["physics_config"]["solver_cfg"] + solver_options = ( + solver_cfg.setdefault("mujoco_options", {}) + if solver_cfg.get("solver_type") == "dexuni" + else solver_cfg + ) + solver_options["iterations"] = args.solver_iterations env_path = args.output / "env.yaml" env_path.write_text(yaml.safe_dump(env_config, sort_keys=False)) config["trainer"]["gym_config"] = str(env_path.resolve()) @@ -313,7 +319,7 @@ def _run(args: argparse.Namespace, result: dict) -> None: if not all(math.isfinite(float(value)) for value in losses.values()): raise RuntimeError("Nonfinite PPO loss") if backend is not None and bool( - backend.solver.mjw_data.overflow.numpy().any() + backend.mujoco_solver.mjw_data.overflow.numpy().any() ): raise RuntimeError("MJWarp capacity or solver iteration limit reached") result["updates"].append( diff --git a/tests/benchmark/test_locomotion_ppo.py b/tests/benchmark/test_locomotion_ppo.py index 0d9d03882..eaa51be38 100644 --- a/tests/benchmark/test_locomotion_ppo.py +++ b/tests/benchmark/test_locomotion_ppo.py @@ -120,6 +120,52 @@ def test_config_rejects_incomplete_minibatches(envs: int, steps: int) -> None: _load_config(TASK_DIR, "default", envs, steps) +@pytest.mark.no_sim +@pytest.mark.parametrize("solver_type", ["mujoco_warp", "dexuni"]) +@pytest.mark.parametrize( + "use_cpu,device_is_cuda", [(False, True), (True, False), (False, False)] +) +def test_newton_backend_checks_active_mujoco_solver( + monkeypatch: pytest.MonkeyPatch, + solver_type: str, + use_cpu: bool, + device_is_cuda: bool, +) -> None: + """Both solver routes must use GPU integration and GPU state buffers.""" + from dexsim.engine.newton_physics import backend_registry + + mujoco = SimpleNamespace( + use_mujoco_cpu=use_cpu, + mjw_data=SimpleNamespace( + qpos=SimpleNamespace(device=SimpleNamespace(is_cuda=device_is_cuda)) + ), + ) + backend = SimpleNamespace( + solver_type=solver_type, + solver=( + mujoco + if solver_type == "mujoco_warp" + else SimpleNamespace(mujoco_solver=mujoco) + ), + mujoco_solver=mujoco, + ) + monkeypatch.setattr(backend_registry, "get_newton_backend", lambda world: backend) + env = SimpleNamespace( + unwrapped=SimpleNamespace( + sim=SimpleNamespace( + physics=SimpleNamespace(name="newton"), + device="cuda:0", + _world=object(), + ) + ) + ) + if not use_cpu and device_is_cuda: + assert benchmark._check_backend(env, "newton") is backend + else: + with pytest.raises(RuntimeError, match="not using CUDA"): + benchmark._check_backend(env, "newton") + + @pytest.mark.parametrize("status", ["passed", "failed"]) def test_report_does_not_rank_training_as_task_success(tmp_path, status: str) -> None: result = _result(status) @@ -220,9 +266,20 @@ def read_text(path: Path, *args, **kwargs): @pytest.mark.no_sim -@pytest.mark.parametrize("renderer", [None, "auto", "no-render", "hybrid"]) +@pytest.mark.parametrize( + "renderer,solver_type", + [ + (None, "mujoco_warp"), + ("auto", "dexuni"), + ("no-render", "mujoco_warp"), + ("hybrid", "dexuni"), + ], +) def test_creation_failure_retains_resource_samples( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, renderer: str | None + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + renderer: str | None, + solver_type: str, ) -> None: """Write failure reports with samples taken before runtime construction.""" from embodichain.lab.gym.utils import registration @@ -230,7 +287,9 @@ def test_creation_failure_retains_resource_samples( from scripts.benchmark.rl import runtime as benchmark_runtime env_config = tmp_path / "input.yaml" - env_config.write_text("physics_config:\n solver_cfg: {}\n") + env_config.write_text( + f"physics_config:\n solver_cfg:\n solver_type: {solver_type}\n" + ) output = tmp_path / "result" monkeypatch.setattr( sys, @@ -243,6 +302,8 @@ def test_creation_failure_retains_resource_samples( str(output), "--envs", "32", + "--solver-iterations", + "37", ], ) trainer_cfg = {"gym_config": str(env_config)} @@ -280,7 +341,19 @@ def sample(resources: ProcessResources, label: str) -> dict: failure = RuntimeError("runtime construction failed") def fail_creation(*args, **kwargs): + import yaml + assert kwargs["renderer"] == renderer + saved = yaml.safe_load((output / "env.yaml").read_text()) + expected_options = ( + {"mujoco_options": {"iterations": 37}} + if solver_type == "dexuni" + else {"iterations": 37} + ) + assert saved["physics_config"]["solver_cfg"] == { + "solver_type": solver_type, + **expected_options, + } raise failure monkeypatch.setattr(policy_runtime, "build_gym_policy_runtime", fail_creation) @@ -292,9 +365,7 @@ def fail_creation(*args, **kwargs): assert result["resources"]["samples"] == [ {"label": "before_create", "cpu_rss_mib": 50.0} ] - assert ( - "| Dexsim Newton/MJWarp CUDA | failed |" in (output / "report.md").read_text() - ) + assert "| Dexsim Newton CUDA | failed |" in (output / "report.md").read_text() @pytest.mark.no_sim From 68c73017165cd98dbcc402fc2504803ae9d792df Mon Sep 17 00:00:00 2001 From: acrlw <13927622+acrlw@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:46:57 +0800 Subject: [PATCH 3/3] fix(data): fall back to official asset downloads --- .../topics/data-assets/data-assets.md | 7 ++ embodichain/data/dataset.py | 19 ++++ tests/data/test_dataset.py | 91 +++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/agent_context/topics/data-assets/data-assets.md b/agent_context/topics/data-assets/data-assets.md index c0acd6e28..ce52d819b 100644 --- a/agent_context/topics/data-assets/data-assets.md +++ b/agent_context/topics/data-assets/data-assets.md @@ -51,6 +51,13 @@ Keep the lock outside the directory repair may delete. POSIX uses `flock`; Windows uses a polled byte lock. Direct Open3D subclasses do not automatically receive the project's locking/repair behavior. +For `hf-mirror.com/datasets/DexForceAI/embodichain_data/` descriptors, the shared +dataset base appends the matching official `huggingface.co` URL after the +declared sources. Open3D tries it when an earlier download or checksum check +fails, retaining the descriptor's revision, digest and extraction subdirectory. +Custom hosts and other repositories are unchanged; failure of every source +still raises a download error. + The CLI honors an asset class's optional `cache_prefix` when checking readiness and ensuring extraction; otherwise the class name is the cache entry. The shared registry merges core category modules and `embodichain.assets` diff --git a/embodichain/data/dataset.py b/embodichain/data/dataset.py index 7fb2f844a..b24980d36 100644 --- a/embodichain/data/dataset.py +++ b/embodichain/data/dataset.py @@ -35,6 +35,24 @@ from embodichain.utils import logger +_HF_ASSET_MIRROR = "https://hf-mirror.com/datasets/DexForceAI/embodichain_data/" +_HF_ASSET_ORIGIN = "https://huggingface.co/datasets/DexForceAI/embodichain_data/" + + +def _with_asset_origin_fallback( + descriptor: o3d.data.DataDescriptor, +) -> o3d.data.DataDescriptor: + """Append the official source for known mirrored asset URLs.""" + urls = list(descriptor.urls) + for url in descriptor.urls: + if url.startswith(_HF_ASSET_MIRROR): + origin = _HF_ASSET_ORIGIN + url[len(_HF_ASSET_MIRROR) :] + if origin not in urls: + urls.append(origin) + if urls == list(descriptor.urls): + return descriptor + return o3d.data.DataDescriptor(urls, descriptor.md5, descriptor.extract_in_subdir) + def _dataset_cache_paths(path: str, prefix: str) -> tuple[Path, Path]: """Resolve one cache entry and reject paths outside its configured root.""" @@ -105,6 +123,7 @@ def _dataset_download_lock(path: str, prefix: str) -> Iterator[None]: class EmbodiChainDataset(o3d.data.DownloadDataset): def __init__(self, prefix, data_descriptor, path): + data_descriptor = _with_asset_origin_fallback(data_descriptor) with _dataset_download_lock(path, prefix): # Perform the zip file and extracted contents check. If the zip is # not valid, the parent class downloads and extracts it again. diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index 49f2808e5..79d5de269 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -18,10 +18,12 @@ from __future__ import annotations +import hashlib import multiprocessing as mp import time from types import SimpleNamespace from pathlib import Path +from zipfile import ZipFile import pytest @@ -34,6 +36,95 @@ PROCESS_START_TIMEOUT_SECONDS = 60.0 +@pytest.mark.no_sim +@pytest.mark.parametrize( + "urls, expected", + [ + ( + [ + "https://hf-mirror.com/datasets/DexForceAI/embodichain_data/resolve/v1/model.zip" + ], + [ + "https://hf-mirror.com/datasets/DexForceAI/embodichain_data/resolve/v1/model.zip", + "https://huggingface.co/datasets/DexForceAI/embodichain_data/resolve/v1/model.zip", + ], + ), + ( + ["https://assets.example/model.zip"], + ["https://assets.example/model.zip"], + ), + ( + ["https://hf-mirror.com/datasets/Other/data/resolve/main/model.zip"], + ["https://hf-mirror.com/datasets/Other/data/resolve/main/model.zip"], + ), + ( + [ + "https://hf-mirror.com/datasets/DexForceAI/embodichain_data/resolve/main/model.zip", + "https://assets.example/model.zip", + "https://huggingface.co/datasets/DexForceAI/embodichain_data/resolve/main/model.zip", + ], + [ + "https://hf-mirror.com/datasets/DexForceAI/embodichain_data/resolve/main/model.zip", + "https://assets.example/model.zip", + "https://huggingface.co/datasets/DexForceAI/embodichain_data/resolve/main/model.zip", + ], + ), + ], +) +def test_asset_origin_fallback_preserves_sources_and_metadata(urls, expected) -> None: + """Only the known asset mirror gains an origin, without changing its revision.""" + descriptor = dataset_module.o3d.data.DataDescriptor(urls, "a" * 32, "mounted") + result = dataset_module._with_asset_origin_fallback(descriptor) + + assert result.urls == expected + assert result.md5 == "a" * 32 + assert result.extract_in_subdir == "mounted" + assert descriptor.urls == urls + + +@pytest.mark.no_sim +@pytest.mark.parametrize( + "mirror_content, origin_valid", + [(None, True), (b"corrupt", True), (b"corrupt", False)], +) +def test_dataset_download_falls_back_with_integrity_checks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mirror_content, origin_valid: bool +) -> None: + """The real downloader recovers missing/corrupt mirrors and rejects bad origins.""" + mirror = tmp_path / "mirror" + origin = tmp_path / "origin" + mirror.mkdir() + origin.mkdir() + archive = origin / "model.zip" + with ZipFile(archive, "w") as bundle: + bundle.writestr("model.urdf", "verified model") + expected_md5 = hashlib.md5(archive.read_bytes()).hexdigest() + if mirror_content is not None: + (mirror / "model.zip").write_bytes(mirror_content) + if not origin_valid: + archive.write_bytes(b"invalid origin") + monkeypatch.setattr(dataset_module, "_HF_ASSET_MIRROR", mirror.as_uri() + "/") + monkeypatch.setattr(dataset_module, "_HF_ASSET_ORIGIN", origin.as_uri() + "/") + descriptor = dataset_module.o3d.data.DataDescriptor( + (mirror / "model.zip").as_uri(), expected_md5, "mounted" + ) + cache = str(tmp_path / "cache") + + if not origin_valid: + with pytest.raises(RuntimeError, match="Downloading failed"): + dataset_module.EmbodiChainDataset("Model", descriptor, cache) + return + + dataset = dataset_module.EmbodiChainDataset("Model", descriptor, cache) + model = Path(dataset.extract_dir) / "mounted/model.urdf" + assert model.read_text() == "verified model" + archive.unlink() + cached = dataset_module.EmbodiChainDataset("Model", descriptor, cache) + assert ( + Path(cached.extract_dir) / "mounted/model.urdf" + ).read_text() == "verified model" + + @pytest.mark.no_sim def test_default_data_modules_remains_a_public_export() -> None: from embodichain.data import DEFAULT_DATA_MODULES